From 4e308b56a311d185438b5c177c1167d4d7e105d5 Mon Sep 17 00:00:00 2001 From: willcl-ark Date: Mon, 10 Feb 2025 12:19:35 +0000 Subject: [PATCH 01/51] bench: add shell.nix --- shell.nix | 101 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 shell.nix diff --git a/shell.nix b/shell.nix new file mode 100644 index 000000000000..b1914a75d99a --- /dev/null +++ b/shell.nix @@ -0,0 +1,101 @@ +# Copyright 0xB10C, willcl-ark +{ pkgs ? import + (fetchTarball "https://github.com/nixos/nixpkgs/archive/nixos-24.11.tar.gz") + { }, }: +let + inherit (pkgs.lib) optionals strings; + inherit (pkgs) stdenv; + + # Override the default cargo-flamegraph with a custom fork + cargo-flamegraph = pkgs.rustPlatform.buildRustPackage rec { + pname = + "flamegraph"; # Match the name in Cargo.toml, doesn't seem to work otherwise + version = "bitcoin-core"; + + src = pkgs.fetchFromGitHub { + owner = "willcl-ark"; + repo = "flamegraph"; + rev = "bitcoin-core"; + sha256 = "sha256-tQbr3MYfAiOxeT12V9au5KQK5X5JeGuV6p8GR/Sgen4="; + }; + + doCheck = false; + cargoHash = "sha256-QWPqTyTFSZNJNayNqLmsQSu0rX26XBKfdLROZ9tRjrg="; + + useFetchCargoVendor = true; + + nativeBuildInputs = + pkgs.lib.optionals stdenv.hostPlatform.isLinux [ pkgs.makeWrapper ]; + buildInputs = pkgs.lib.optionals stdenv.hostPlatform.isDarwin + [ pkgs.darwin.apple_sdk.frameworks.Security ]; + + postFixup = pkgs.lib.optionalString stdenv.hostPlatform.isLinux '' + wrapProgram $out/bin/cargo-flamegraph \ + --set-default PERF ${pkgs.linuxPackages.perf}/bin/perf + wrapProgram $out/bin/flamegraph \ + --set-default PERF ${pkgs.linuxPackages.perf}/bin/perf + ''; + }; + +in pkgs.mkShell { + nativeBuildInputs = with pkgs; [ + autoconf + automake + boost + ccache + clang_18 + cmake + libevent + libtool + pkg-config + sqlite + zeromq + ]; + buildInputs = with pkgs; [ + just + bash + git + shellcheck + python310 + uv + + # Benchmarking + cargo-flamegraph + flamegraph + hyperfine + jq + linuxKernel.packages.linux_6_6.perf + perf-tools + util-linux + + # Binary patching + patchelf + + # Guix + curl + getent + ]; + + shellHook = '' + echo "Bitcoin Core build nix-shell" + echo "" + echo "Setting up python venv" + + # fixes libstdc++ issues and libgl.so issues + export LD_LIBRARY_PATH=${stdenv.cc.cc.lib}/lib/:$LD_LIBRARY_PATH + + uv venv --python 3.10 + source .venv/bin/activate + uv pip install -r pyproject.toml + + patch-binary() { + if [ -z "$1" ]; then + echo "Usage: patch-binary " + return 1 + fi + patchelf --set-interpreter "$(cat $NIX_CC/nix-support/dynamic-linker)" "$1" + } + echo "Added patch-binary command" + echo " Usage: 'patch-binary '" + ''; +} From 341dddfd2e872eae6dd1a39095955a924401d58e Mon Sep 17 00:00:00 2001 From: willcl-ark Date: Mon, 10 Feb 2025 12:19:52 +0000 Subject: [PATCH 02/51] bench: add uv + python deps --- pyproject.toml | 15 +++ requirements.txt | 28 ++++++ uv.lock | 251 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 294 insertions(+) create mode 100644 pyproject.toml create mode 100644 requirements.txt create mode 100644 uv.lock diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000000..26605fc84930 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,15 @@ +[project] +name = "bitcoin-core-deps" +version = "0.1.0" +dependencies = [ + "codespell==2.2.6", + "lief==0.13.2", + "mypy==1.4.1", + "pyzmq==25.1.0", + # Removing in favour of packaged nixpkgs bin which is not dynamically linked + # "ruff==0.5.5", + "vulture==2.6", + "pyperf==2.8.0", + "matplotlib==3.8.0", + "numpy==1.26.0" +] diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000000..c9b220b6fe46 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,28 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile pyproject.toml -o requirements.txt +codespell==2.2.6 + # via bitcoin-core-deps (pyproject.toml) +lief==0.13.2 + # via bitcoin-core-deps (pyproject.toml) +matplotlib==3.8.0 + # via bitcoin-core-deps (pyproject.toml) +mypy==1.4.1 + # via bitcoin-core-deps (pyproject.toml) +mypy-extensions==1.0.0 + # via mypy +numpy==1.26.0 + # via bitcoin-core-deps (pyproject.toml) +psutil==6.1.0 + # via pyperf +pyperf==2.8.0 + # via bitcoin-core-deps (pyproject.toml) +pyzmq==25.1.0 + # via bitcoin-core-deps (pyproject.toml) +toml==0.10.2 + # via vulture +tomli==2.0.2 + # via mypy +typing-extensions==4.12.2 + # via mypy +vulture==2.6 + # via bitcoin-core-deps (pyproject.toml) diff --git a/uv.lock b/uv.lock new file mode 100644 index 000000000000..090e5f1cb4f1 --- /dev/null +++ b/uv.lock @@ -0,0 +1,251 @@ +version = 1 +requires-python = ">=3.10" + +[[package]] +name = "bitcoin-core-deps" +version = "0.1.0" +source = { virtual = "." } +dependencies = [ + { name = "codespell" }, + { name = "lief" }, + { name = "mypy" }, + { name = "pyperf" }, + { name = "pyzmq" }, + { name = "vulture" }, +] + +[package.metadata] +requires-dist = [ + { name = "codespell", specifier = "==2.2.6" }, + { name = "lief", specifier = "==0.13.2" }, + { name = "mypy", specifier = "==1.4.1" }, + { name = "pyperf" }, + { name = "pyzmq", specifier = "==25.1.0" }, + { name = "vulture", specifier = "==2.6" }, +] + +[[package]] +name = "cffi" +version = "1.17.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycparser" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fc/97/c783634659c2920c3fc70419e3af40972dbaf758daa229a7d6ea6135c90d/cffi-1.17.1.tar.gz", hash = "sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824", size = 516621 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/90/07/f44ca684db4e4f08a3fdc6eeb9a0d15dc6883efc7b8c90357fdbf74e186c/cffi-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:df8b1c11f177bc2313ec4b2d46baec87a5f3e71fc8b45dab2ee7cae86d9aba14", size = 182191 }, + { url = "https://files.pythonhosted.org/packages/08/fd/cc2fedbd887223f9f5d170c96e57cbf655df9831a6546c1727ae13fa977a/cffi-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f2cdc858323644ab277e9bb925ad72ae0e67f69e804f4898c070998d50b1a67", size = 178592 }, + { url = "https://files.pythonhosted.org/packages/de/cc/4635c320081c78d6ffc2cab0a76025b691a91204f4aa317d568ff9280a2d/cffi-1.17.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:edae79245293e15384b51f88b00613ba9f7198016a5948b5dddf4917d4d26382", size = 426024 }, + { url = "https://files.pythonhosted.org/packages/b6/7b/3b2b250f3aab91abe5f8a51ada1b717935fdaec53f790ad4100fe2ec64d1/cffi-1.17.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45398b671ac6d70e67da8e4224a065cec6a93541bb7aebe1b198a61b58c7b702", size = 448188 }, + { url = "https://files.pythonhosted.org/packages/d3/48/1b9283ebbf0ec065148d8de05d647a986c5f22586b18120020452fff8f5d/cffi-1.17.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ad9413ccdeda48c5afdae7e4fa2192157e991ff761e7ab8fdd8926f40b160cc3", size = 455571 }, + { url = "https://files.pythonhosted.org/packages/40/87/3b8452525437b40f39ca7ff70276679772ee7e8b394934ff60e63b7b090c/cffi-1.17.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5da5719280082ac6bd9aa7becb3938dc9f9cbd57fac7d2871717b1feb0902ab6", size = 436687 }, + { url = "https://files.pythonhosted.org/packages/8d/fb/4da72871d177d63649ac449aec2e8a29efe0274035880c7af59101ca2232/cffi-1.17.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bb1a08b8008b281856e5971307cc386a8e9c5b625ac297e853d36da6efe9c17", size = 446211 }, + { url = "https://files.pythonhosted.org/packages/ab/a0/62f00bcb411332106c02b663b26f3545a9ef136f80d5df746c05878f8c4b/cffi-1.17.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:045d61c734659cc045141be4bae381a41d89b741f795af1dd018bfb532fd0df8", size = 461325 }, + { url = "https://files.pythonhosted.org/packages/36/83/76127035ed2e7e27b0787604d99da630ac3123bfb02d8e80c633f218a11d/cffi-1.17.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6883e737d7d9e4899a8a695e00ec36bd4e5e4f18fabe0aca0efe0a4b44cdb13e", size = 438784 }, + { url = "https://files.pythonhosted.org/packages/21/81/a6cd025db2f08ac88b901b745c163d884641909641f9b826e8cb87645942/cffi-1.17.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6b8b4a92e1c65048ff98cfe1f735ef8f1ceb72e3d5f0c25fdb12087a23da22be", size = 461564 }, + { url = "https://files.pythonhosted.org/packages/f8/fe/4d41c2f200c4a457933dbd98d3cf4e911870877bd94d9656cc0fcb390681/cffi-1.17.1-cp310-cp310-win32.whl", hash = "sha256:c9c3d058ebabb74db66e431095118094d06abf53284d9c81f27300d0e0d8bc7c", size = 171804 }, + { url = "https://files.pythonhosted.org/packages/d1/b6/0b0f5ab93b0df4acc49cae758c81fe4e5ef26c3ae2e10cc69249dfd8b3ab/cffi-1.17.1-cp310-cp310-win_amd64.whl", hash = "sha256:0f048dcf80db46f0098ccac01132761580d28e28bc0f78ae0d58048063317e15", size = 181299 }, + { url = "https://files.pythonhosted.org/packages/6b/f4/927e3a8899e52a27fa57a48607ff7dc91a9ebe97399b357b85a0c7892e00/cffi-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a45e3c6913c5b87b3ff120dcdc03f6131fa0065027d0ed7ee6190736a74cd401", size = 182264 }, + { url = "https://files.pythonhosted.org/packages/6c/f5/6c3a8efe5f503175aaddcbea6ad0d2c96dad6f5abb205750d1b3df44ef29/cffi-1.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:30c5e0cb5ae493c04c8b42916e52ca38079f1b235c2f8ae5f4527b963c401caf", size = 178651 }, + { url = "https://files.pythonhosted.org/packages/94/dd/a3f0118e688d1b1a57553da23b16bdade96d2f9bcda4d32e7d2838047ff7/cffi-1.17.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f75c7ab1f9e4aca5414ed4d8e5c0e303a34f4421f8a0d47a4d019ceff0ab6af4", size = 445259 }, + { url = "https://files.pythonhosted.org/packages/2e/ea/70ce63780f096e16ce8588efe039d3c4f91deb1dc01e9c73a287939c79a6/cffi-1.17.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1ed2dd2972641495a3ec98445e09766f077aee98a1c896dcb4ad0d303628e41", size = 469200 }, + { url = "https://files.pythonhosted.org/packages/1c/a0/a4fa9f4f781bda074c3ddd57a572b060fa0df7655d2a4247bbe277200146/cffi-1.17.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:46bf43160c1a35f7ec506d254e5c890f3c03648a4dbac12d624e4490a7046cd1", size = 477235 }, + { url = "https://files.pythonhosted.org/packages/62/12/ce8710b5b8affbcdd5c6e367217c242524ad17a02fe5beec3ee339f69f85/cffi-1.17.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a24ed04c8ffd54b0729c07cee15a81d964e6fee0e3d4d342a27b020d22959dc6", size = 459721 }, + { url = "https://files.pythonhosted.org/packages/ff/6b/d45873c5e0242196f042d555526f92aa9e0c32355a1be1ff8c27f077fd37/cffi-1.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:610faea79c43e44c71e1ec53a554553fa22321b65fae24889706c0a84d4ad86d", size = 467242 }, + { url = "https://files.pythonhosted.org/packages/1a/52/d9a0e523a572fbccf2955f5abe883cfa8bcc570d7faeee06336fbd50c9fc/cffi-1.17.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a9b15d491f3ad5d692e11f6b71f7857e7835eb677955c00cc0aefcd0669adaf6", size = 477999 }, + { url = "https://files.pythonhosted.org/packages/44/74/f2a2460684a1a2d00ca799ad880d54652841a780c4c97b87754f660c7603/cffi-1.17.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:de2ea4b5833625383e464549fec1bc395c1bdeeb5f25c4a3a82b5a8c756ec22f", size = 454242 }, + { url = "https://files.pythonhosted.org/packages/f8/4a/34599cac7dfcd888ff54e801afe06a19c17787dfd94495ab0c8d35fe99fb/cffi-1.17.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:fc48c783f9c87e60831201f2cce7f3b2e4846bf4d8728eabe54d60700b318a0b", size = 478604 }, + { url = "https://files.pythonhosted.org/packages/34/33/e1b8a1ba29025adbdcda5fb3a36f94c03d771c1b7b12f726ff7fef2ebe36/cffi-1.17.1-cp311-cp311-win32.whl", hash = "sha256:85a950a4ac9c359340d5963966e3e0a94a676bd6245a4b55bc43949eee26a655", size = 171727 }, + { url = "https://files.pythonhosted.org/packages/3d/97/50228be003bb2802627d28ec0627837ac0bf35c90cf769812056f235b2d1/cffi-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:caaf0640ef5f5517f49bc275eca1406b0ffa6aa184892812030f04c2abf589a0", size = 181400 }, + { url = "https://files.pythonhosted.org/packages/5a/84/e94227139ee5fb4d600a7a4927f322e1d4aea6fdc50bd3fca8493caba23f/cffi-1.17.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:805b4371bf7197c329fcb3ead37e710d1bca9da5d583f5073b799d5c5bd1eee4", size = 183178 }, + { url = "https://files.pythonhosted.org/packages/da/ee/fb72c2b48656111c4ef27f0f91da355e130a923473bf5ee75c5643d00cca/cffi-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:733e99bc2df47476e3848417c5a4540522f234dfd4ef3ab7fafdf555b082ec0c", size = 178840 }, + { url = "https://files.pythonhosted.org/packages/cc/b6/db007700f67d151abadf508cbfd6a1884f57eab90b1bb985c4c8c02b0f28/cffi-1.17.1-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1257bdabf294dceb59f5e70c64a3e2f462c30c7ad68092d01bbbfb1c16b1ba36", size = 454803 }, + { url = "https://files.pythonhosted.org/packages/1a/df/f8d151540d8c200eb1c6fba8cd0dfd40904f1b0682ea705c36e6c2e97ab3/cffi-1.17.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da95af8214998d77a98cc14e3a3bd00aa191526343078b530ceb0bd710fb48a5", size = 478850 }, + { url = "https://files.pythonhosted.org/packages/28/c0/b31116332a547fd2677ae5b78a2ef662dfc8023d67f41b2a83f7c2aa78b1/cffi-1.17.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d63afe322132c194cf832bfec0dc69a99fb9bb6bbd550f161a49e9e855cc78ff", size = 485729 }, + { url = "https://files.pythonhosted.org/packages/91/2b/9a1ddfa5c7f13cab007a2c9cc295b70fbbda7cb10a286aa6810338e60ea1/cffi-1.17.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f79fc4fc25f1c8698ff97788206bb3c2598949bfe0fef03d299eb1b5356ada99", size = 471256 }, + { url = "https://files.pythonhosted.org/packages/b2/d5/da47df7004cb17e4955df6a43d14b3b4ae77737dff8bf7f8f333196717bf/cffi-1.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b62ce867176a75d03a665bad002af8e6d54644fad99a3c70905c543130e39d93", size = 479424 }, + { url = "https://files.pythonhosted.org/packages/0b/ac/2a28bcf513e93a219c8a4e8e125534f4f6db03e3179ba1c45e949b76212c/cffi-1.17.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:386c8bf53c502fff58903061338ce4f4950cbdcb23e2902d86c0f722b786bbe3", size = 484568 }, + { url = "https://files.pythonhosted.org/packages/d4/38/ca8a4f639065f14ae0f1d9751e70447a261f1a30fa7547a828ae08142465/cffi-1.17.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4ceb10419a9adf4460ea14cfd6bc43d08701f0835e979bf821052f1805850fe8", size = 488736 }, + { url = "https://files.pythonhosted.org/packages/86/c5/28b2d6f799ec0bdecf44dced2ec5ed43e0eb63097b0f58c293583b406582/cffi-1.17.1-cp312-cp312-win32.whl", hash = "sha256:a08d7e755f8ed21095a310a693525137cfe756ce62d066e53f502a83dc550f65", size = 172448 }, + { url = "https://files.pythonhosted.org/packages/50/b9/db34c4755a7bd1cb2d1603ac3863f22bcecbd1ba29e5ee841a4bc510b294/cffi-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:51392eae71afec0d0c8fb1a53b204dbb3bcabcb3c9b807eedf3e1e6ccf2de903", size = 181976 }, + { url = "https://files.pythonhosted.org/packages/8d/f8/dd6c246b148639254dad4d6803eb6a54e8c85c6e11ec9df2cffa87571dbe/cffi-1.17.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f3a2b4222ce6b60e2e8b337bb9596923045681d71e5a082783484d845390938e", size = 182989 }, + { url = "https://files.pythonhosted.org/packages/8b/f1/672d303ddf17c24fc83afd712316fda78dc6fce1cd53011b839483e1ecc8/cffi-1.17.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0984a4925a435b1da406122d4d7968dd861c1385afe3b45ba82b750f229811e2", size = 178802 }, + { url = "https://files.pythonhosted.org/packages/0e/2d/eab2e858a91fdff70533cab61dcff4a1f55ec60425832ddfdc9cd36bc8af/cffi-1.17.1-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d01b12eeeb4427d3110de311e1774046ad344f5b1a7403101878976ecd7a10f3", size = 454792 }, + { url = "https://files.pythonhosted.org/packages/75/b2/fbaec7c4455c604e29388d55599b99ebcc250a60050610fadde58932b7ee/cffi-1.17.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:706510fe141c86a69c8ddc029c7910003a17353970cff3b904ff0686a5927683", size = 478893 }, + { url = "https://files.pythonhosted.org/packages/4f/b7/6e4a2162178bf1935c336d4da8a9352cccab4d3a5d7914065490f08c0690/cffi-1.17.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de55b766c7aa2e2a3092c51e0483d700341182f08e67c63630d5b6f200bb28e5", size = 485810 }, + { url = "https://files.pythonhosted.org/packages/c7/8a/1d0e4a9c26e54746dc08c2c6c037889124d4f59dffd853a659fa545f1b40/cffi-1.17.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c59d6e989d07460165cc5ad3c61f9fd8f1b4796eacbd81cee78957842b834af4", size = 471200 }, + { url = "https://files.pythonhosted.org/packages/26/9f/1aab65a6c0db35f43c4d1b4f580e8df53914310afc10ae0397d29d697af4/cffi-1.17.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd398dbc6773384a17fe0d3e7eeb8d1a21c2200473ee6806bb5e6a8e62bb73dd", size = 479447 }, + { url = "https://files.pythonhosted.org/packages/5f/e4/fb8b3dd8dc0e98edf1135ff067ae070bb32ef9d509d6cb0f538cd6f7483f/cffi-1.17.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:3edc8d958eb099c634dace3c7e16560ae474aa3803a5df240542b305d14e14ed", size = 484358 }, + { url = "https://files.pythonhosted.org/packages/f1/47/d7145bf2dc04684935d57d67dff9d6d795b2ba2796806bb109864be3a151/cffi-1.17.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:72e72408cad3d5419375fc87d289076ee319835bdfa2caad331e377589aebba9", size = 488469 }, + { url = "https://files.pythonhosted.org/packages/bf/ee/f94057fa6426481d663b88637a9a10e859e492c73d0384514a17d78ee205/cffi-1.17.1-cp313-cp313-win32.whl", hash = "sha256:e03eab0a8677fa80d646b5ddece1cbeaf556c313dcfac435ba11f107ba117b5d", size = 172475 }, + { url = "https://files.pythonhosted.org/packages/7c/fc/6a8cb64e5f0324877d503c854da15d76c1e50eb722e320b15345c4d0c6de/cffi-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:f6a16c31041f09ead72d69f583767292f750d24913dadacf5756b966aacb3f1a", size = 182009 }, +] + +[[package]] +name = "codespell" +version = "2.2.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e1/97/df3e00b4d795c96233e35d269c211131c5572503d2270afb6fed7d859cc2/codespell-2.2.6.tar.gz", hash = "sha256:a8c65d8eb3faa03deabab6b3bbe798bea72e1799c7e9e955d57eca4096abcff9", size = 300968 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/46/e0/5437cc96b74467c4df6e13b7128cc482c48bb43146fb4c11cf2bcd604e1f/codespell-2.2.6-py3-none-any.whl", hash = "sha256:9ee9a3e5df0990604013ac2a9f22fa8e57669c827124a2e961fe8a1da4cacc07", size = 301382 }, +] + +[[package]] +name = "lief" +version = "0.13.2" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/e2/c4125c279eb2a23ecc86cdb188ed06e9d81a9c700e9412f9be866afc2c7d/lief-0.13.2-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:0390cfaaf0e9aed46bebf26f00f34852768f76bc7f90abf7ceb384566200e5f5", size = 3424746 }, + { url = "https://files.pythonhosted.org/packages/5f/d6/72235d648c6630c37ef52b9f6f4e2f3337842bc4b08c75abcae3052b2c17/lief-0.13.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5581bf0072c1e7a9ea2fb2e2252b8582016e8b298804b5461e552b402c9cd4e9", size = 3249141 }, + { url = "https://files.pythonhosted.org/packages/d7/cc/9895dff094cad3e88636195640b4b47caefe3d300d3f37b653bd109348df/lief-0.13.2-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:dbbf2fb3d7807e815f345c77e287da162e081100f059ec03005995befc295d7f", size = 3793938 }, + { url = "https://files.pythonhosted.org/packages/0d/1b/f4bf63bfce187ae210980bdd1a20ea7d8e080381eef09e7d26c585eaa614/lief-0.13.2-cp310-cp310-manylinux_2_24_x86_64.whl", hash = "sha256:d344d37334c2b488dc02f04cb13c22cd61aa065eeb9bca7424588e0c8c23bdfb", size = 4045328 }, + { url = "https://files.pythonhosted.org/packages/2c/2a/abac2e42c3cc56f2b5020e58b99f700c4d3236d49451607add0f628d737b/lief-0.13.2-cp310-cp310-win32.whl", hash = "sha256:bc041b28b94139843a33c014e355822a9276b35f3c5ae10d82da56bf572f8222", size = 2493454 }, + { url = "https://files.pythonhosted.org/packages/ed/14/34a12787dc4328227e0e84a97db8142aa1e2b33e0aabc538e93abf7d6e5a/lief-0.13.2-cp310-cp310-win_amd64.whl", hash = "sha256:01d4075bbc3541e9dd3ef008045fa1eb128294a0c5b0c1f69ce60d8948d248c7", size = 3089949 }, + { url = "https://files.pythonhosted.org/packages/2e/95/9d7377095fb7cf195aca8f64d9696705c71884dcba16663472ce17139b9c/lief-0.13.2-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:6570dacebe107ad60c2ba0968d1a865d316009d43cc85af3719d3eeb0911abf3", size = 3424752 }, + { url = "https://files.pythonhosted.org/packages/00/2b/7ac8e15ca198a5c50397aec32102e81ef97fd573a4285ee889ec9084d110/lief-0.13.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7ce2e3f7c791efba327c2bb3499dbef81e682027109045a9bae696c62e2aeeb0", size = 3249263 }, + { url = "https://files.pythonhosted.org/packages/d6/8d/b50cc4ad91278015e5ac18fc76f32098ed6887c371bef6f4997af4cb97c9/lief-0.13.2-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:11ab900e0644b6735ecdef2bbd04439b4866a527650fc054470c195d6cfe2917", size = 3792343 }, + { url = "https://files.pythonhosted.org/packages/6b/bd/ea25e9c8ff0a55b5534e5881fa6e5eeca0ed3eeb7c772a276984b8c182d9/lief-0.13.2-cp311-cp311-manylinux_2_24_x86_64.whl", hash = "sha256:042ad2105a136b11a7494b9af8178468e8cb32b8fa2a0a55cb659a5605aeb069", size = 4045112 }, + { url = "https://files.pythonhosted.org/packages/d9/06/ddacd724f65fa8e7eca438c335aa77878a260fbc714cdba252387c33a4cc/lief-0.13.2-cp311-cp311-win32.whl", hash = "sha256:1ce289b6ab3cf4be654270007e8a2c0d2e42116180418c29d3ce83762955de63", size = 2493336 }, + { url = "https://files.pythonhosted.org/packages/82/95/1de9a497946fed9d15f847d8a4a0630dfda6d186c044f8731f53d0d3d758/lief-0.13.2-cp311-cp311-win_amd64.whl", hash = "sha256:eccb248ffb598e410fd2ef7c1f171a3cde57a40c9bb8c4fa15d8e7b90eb4eb2d", size = 3090328 }, +] + +[[package]] +name = "mypy" +version = "1.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mypy-extensions" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/28/d8a8233ff167d06108e53b7aefb4a8d7350adbbf9d7abd980f17fdb7a3a6/mypy-1.4.1.tar.gz", hash = "sha256:9bbcd9ab8ea1f2e1c8031c21445b511442cc45c89951e49bbf852cbb70755b1b", size = 2855162 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/3b/1c7363863b56c059f60a1dfdca9ac774a22ba64b7a4da0ee58ee53e5243f/mypy-1.4.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:566e72b0cd6598503e48ea610e0052d1b8168e60a46e0bfd34b3acf2d57f96a8", size = 10451043 }, + { url = "https://files.pythonhosted.org/packages/a7/24/6f0df1874118839db1155fed62a4bd7e80c181367ff8ea07d40fbaffcfb4/mypy-1.4.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ca637024ca67ab24a7fd6f65d280572c3794665eaf5edcc7e90a866544076878", size = 9542079 }, + { url = "https://files.pythonhosted.org/packages/04/5c/deeac94fcccd11aa621e6b350df333e1b809b11443774ea67582cc0205da/mypy-1.4.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0dde1d180cd84f0624c5dcaaa89c89775550a675aff96b5848de78fb11adabcd", size = 11974913 }, + { url = "https://files.pythonhosted.org/packages/e5/2f/de3c455c54e8cf5e37ea38705c1920f2df470389f8fc051084d2dd8c9c59/mypy-1.4.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8c4d8e89aa7de683e2056a581ce63c46a0c41e31bd2b6d34144e2c80f5ea53dc", size = 12044492 }, + { url = "https://files.pythonhosted.org/packages/e7/d3/6f65357dcb68109946de70cd55bd2e60f10114f387471302f48d54ff5dae/mypy-1.4.1-cp310-cp310-win_amd64.whl", hash = "sha256:bfdca17c36ae01a21274a3c387a63aa1aafe72bff976522886869ef131b937f1", size = 8831655 }, + { url = "https://files.pythonhosted.org/packages/94/01/e34e37a044325af4d4af9825c15e8a0d26d89b5a9624b4d0908449d3411b/mypy-1.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7549fbf655e5825d787bbc9ecf6028731973f78088fbca3a1f4145c39ef09462", size = 10338636 }, + { url = "https://files.pythonhosted.org/packages/92/58/ccc0b714ecbd1a64b34d8ce1c38763ff6431de1d82551904ecc3711fbe05/mypy-1.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:98324ec3ecf12296e6422939e54763faedbfcc502ea4a4c38502082711867258", size = 9444172 }, + { url = "https://files.pythonhosted.org/packages/73/72/dfc0b46e6905eafd598e7c48c0c4f2e232647e4e36547425c64e6c850495/mypy-1.4.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:141dedfdbfe8a04142881ff30ce6e6653c9685b354876b12e4fe6c78598b45e2", size = 11855450 }, + { url = "https://files.pythonhosted.org/packages/66/f4/60739a2d336f3adf5628e7c9b920d16e8af6dc078550d615e4ba2a1d7759/mypy-1.4.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:8207b7105829eca6f3d774f64a904190bb2231de91b8b186d21ffd98005f14a7", size = 11928679 }, + { url = "https://files.pythonhosted.org/packages/8c/26/6ff2b55bf8b605a4cc898883654c2ca4dd4feedf0bb04ecaacf60d165cde/mypy-1.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:16f0db5b641ba159eff72cff08edc3875f2b62b2fa2bc24f68c1e7a4e8232d01", size = 8831134 }, + { url = "https://files.pythonhosted.org/packages/3d/9a/e13addb8d652cb068f835ac2746d9d42f85b730092f581bb17e2059c28f1/mypy-1.4.1-py3-none-any.whl", hash = "sha256:45d32cec14e7b97af848bddd97d85ea4f0db4d5a149ed9676caa4eb2f7402bb4", size = 2451741 }, +] + +[[package]] +name = "mypy-extensions" +version = "1.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/98/a4/1ab47638b92648243faf97a5aeb6ea83059cc3624972ab6b8d2316078d3f/mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782", size = 4433 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/e2/5d3f6ada4297caebe1a2add3b126fe800c96f56dbe5d1988a2cbe0b267aa/mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d", size = 4695 }, +] + +[[package]] +name = "psutil" +version = "6.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/26/10/2a30b13c61e7cf937f4adf90710776b7918ed0a9c434e2c38224732af310/psutil-6.1.0.tar.gz", hash = "sha256:353815f59a7f64cdaca1c0307ee13558a0512f6db064e92fe833784f08539c7a", size = 508565 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/01/9e/8be43078a171381953cfee33c07c0d628594b5dbfc5157847b85022c2c1b/psutil-6.1.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:6e2dcd475ce8b80522e51d923d10c7871e45f20918e027ab682f94f1c6351688", size = 247762 }, + { url = "https://files.pythonhosted.org/packages/1d/cb/313e80644ea407f04f6602a9e23096540d9dc1878755f3952ea8d3d104be/psutil-6.1.0-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:0895b8414afafc526712c498bd9de2b063deaac4021a3b3c34566283464aff8e", size = 248777 }, + { url = "https://files.pythonhosted.org/packages/65/8e/bcbe2025c587b5d703369b6a75b65d41d1367553da6e3f788aff91eaf5bd/psutil-6.1.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9dcbfce5d89f1d1f2546a2090f4fcf87c7f669d1d90aacb7d7582addece9fb38", size = 284259 }, + { url = "https://files.pythonhosted.org/packages/58/4d/8245e6f76a93c98aab285a43ea71ff1b171bcd90c9d238bf81f7021fb233/psutil-6.1.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:498c6979f9c6637ebc3a73b3f87f9eb1ec24e1ce53a7c5173b8508981614a90b", size = 287255 }, + { url = "https://files.pythonhosted.org/packages/27/c2/d034856ac47e3b3cdfa9720d0e113902e615f4190d5d1bdb8df4b2015fb2/psutil-6.1.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d905186d647b16755a800e7263d43df08b790d709d575105d419f8b6ef65423a", size = 288804 }, + { url = "https://files.pythonhosted.org/packages/ea/55/5389ed243c878725feffc0d6a3bc5ef6764312b6fc7c081faaa2cfa7ef37/psutil-6.1.0-cp37-abi3-win32.whl", hash = "sha256:1ad45a1f5d0b608253b11508f80940985d1d0c8f6111b5cb637533a0e6ddc13e", size = 250386 }, + { url = "https://files.pythonhosted.org/packages/11/91/87fa6f060e649b1e1a7b19a4f5869709fbf750b7c8c262ee776ec32f3028/psutil-6.1.0-cp37-abi3-win_amd64.whl", hash = "sha256:a8fb3752b491d246034fa4d279ff076501588ce8cbcdbb62c32fd7a377d996be", size = 254228 }, +] + +[[package]] +name = "pycparser" +version = "2.22" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1d/b2/31537cf4b1ca988837256c910a668b553fceb8f069bedc4b1c826024b52c/pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6", size = 172736 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/13/a3/a812df4e2dd5696d1f351d58b8fe16a405b234ad2886a0dab9183fb78109/pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc", size = 117552 }, +] + +[[package]] +name = "pyperf" +version = "2.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "psutil" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/02/2a/758b3c4cc9843bd385bc595b777345fbf4cd00733b7830cdff43e30002c0/pyperf-2.8.0.tar.gz", hash = "sha256:b30a20465819daf102b6543b512f6799a5a879ff2a123981e6cd732d0e6a7a79", size = 225186 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/f7/bb8965520a9b0a3d720b282e67b5cb7f3305b96e4bacaee2794550e67e94/pyperf-2.8.0-py3-none-any.whl", hash = "sha256:1a775b5a09882f18bf876430ef78e07646f773f50774546f5f6a8b34d60e3968", size = 142508 }, +] + +[[package]] +name = "pyzmq" +version = "25.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "implementation_name == 'pypy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/64/9c/2b2614b0b86ff703b3a33ea5e044923bd7d100adc8c829d579a9b71ea9e7/pyzmq-25.1.0.tar.gz", hash = "sha256:80c41023465d36280e801564a69cbfce8ae85ff79b080e1913f6e90481fb8957", size = 1224640 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/53/17/6a70f84b79e361af34f6c99064ecf9e87112c4c48b9c7ea78f8e680b57d8/pyzmq-25.1.0-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:1a6169e69034eaa06823da6a93a7739ff38716142b3596c180363dee729d713d", size = 1826810 }, + { url = "https://files.pythonhosted.org/packages/2f/53/fc7dbdd32e275aee0961e2a5bed1bb64223846f959fd6e0c9a39aab08eed/pyzmq-25.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:19d0383b1f18411d137d891cab567de9afa609b214de68b86e20173dc624c101", size = 1236489 }, + { url = "https://files.pythonhosted.org/packages/04/0b/bff5b6c1680e248bad2df8248a060645709fe2aef9689e9f7c81c587bad4/pyzmq-25.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1e931d9a92f628858a50f5bdffdfcf839aebe388b82f9d2ccd5d22a38a789dc", size = 864304 }, + { url = "https://files.pythonhosted.org/packages/5e/9e/32074bd8bcf2a5cf282d8817458fd5479c68b487b6c3a5d4627711ad38f5/pyzmq-25.1.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:97d984b1b2f574bc1bb58296d3c0b64b10e95e7026f8716ed6c0b86d4679843f", size = 1116061 }, + { url = "https://files.pythonhosted.org/packages/fa/fb/a114ba641eb873c165106d3c8ee75eb49d6ea3204168808708d866de360d/pyzmq-25.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:154bddda2a351161474b36dba03bf1463377ec226a13458725183e508840df89", size = 1065090 }, + { url = "https://files.pythonhosted.org/packages/ca/db/f9976803f1a660e753d0f2426065975bad5db8272fd5284efaf488dc0ce1/pyzmq-25.1.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:cb6d161ae94fb35bb518b74bb06b7293299c15ba3bc099dccd6a5b7ae589aee3", size = 1062464 }, + { url = "https://files.pythonhosted.org/packages/94/3a/c3964c0a86c3535ae240799d3b7c8e13527e7a092080dda9012b1401fa86/pyzmq-25.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:90146ab578931e0e2826ee39d0c948d0ea72734378f1898939d18bc9c823fcf9", size = 1391159 }, + { url = "https://files.pythonhosted.org/packages/a1/87/92556ffa8fbe7dc497d847e39d5c46134f9ad047b23f5bcefc8fbd0c2c9c/pyzmq-25.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:831ba20b660b39e39e5ac8603e8193f8fce1ee03a42c84ade89c36a251449d80", size = 1721009 }, + { url = "https://files.pythonhosted.org/packages/66/96/129706be681649f43bde93811416f566acfefcd3fb18156d5df349c360ab/pyzmq-25.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:3a522510e3434e12aff80187144c6df556bb06fe6b9d01b2ecfbd2b5bfa5c60c", size = 1611290 }, + { url = "https://files.pythonhosted.org/packages/64/db/e19f69fe9b1a4e53f6382274f553358e2e7305d2a2b9d9db36087bf52d5e/pyzmq-25.1.0-cp310-cp310-win32.whl", hash = "sha256:be24a5867b8e3b9dd5c241de359a9a5217698ff616ac2daa47713ba2ebe30ad1", size = 880070 }, + { url = "https://files.pythonhosted.org/packages/32/e4/ce4f94009f84c2a688082c2674d490d2e20e0c9058087f5358a2bf29ddf1/pyzmq-25.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:5693dcc4f163481cf79e98cf2d7995c60e43809e325b77a7748d8024b1b7bcba", size = 1137827 }, + { url = "https://files.pythonhosted.org/packages/bb/80/ae792378f98d6d0e39c975c334603d3d2535f7897707fe91f31d37f94fdb/pyzmq-25.1.0-cp311-cp311-macosx_10_15_universal2.whl", hash = "sha256:13bbe36da3f8aaf2b7ec12696253c0bf6ffe05f4507985a8844a1081db6ec22d", size = 1816147 }, + { url = "https://files.pythonhosted.org/packages/5a/b6/3c2ddd09aa24352e4f6aade53e9b9a1816c0774c844f11b1a2f508ddc0be/pyzmq-25.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:69511d604368f3dc58d4be1b0bad99b61ee92b44afe1cd9b7bd8c5e34ea8248a", size = 1230845 }, + { url = "https://files.pythonhosted.org/packages/26/bb/80535157e8811095901f98688839092afb6dcaf2ff154aa8fa2e575f540d/pyzmq-25.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a983c8694667fd76d793ada77fd36c8317e76aa66eec75be2653cef2ea72883", size = 866042 }, + { url = "https://files.pythonhosted.org/packages/7c/65/bccec1eae7c0e089d90648f350e6c2ff40ccb8c6d1b929548f4cd304b1f7/pyzmq-25.1.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:332616f95eb400492103ab9d542b69d5f0ff628b23129a4bc0a2fd48da6e4e0b", size = 1116285 }, + { url = "https://files.pythonhosted.org/packages/b7/cb/2a36d3eed310efb342fbb7b4adf6b05f46401c4b937154bd1c9b703314e0/pyzmq-25.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58416db767787aedbfd57116714aad6c9ce57215ffa1c3758a52403f7c68cff5", size = 1066280 }, + { url = "https://files.pythonhosted.org/packages/66/f5/15db4c297957f049cd4dcd35eb7fbe9098a72489e0abdb289c529d7327cc/pyzmq-25.1.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:cad9545f5801a125f162d09ec9b724b7ad9b6440151b89645241d0120e119dcc", size = 1061673 }, + { url = "https://files.pythonhosted.org/packages/fa/40/7729719e38324e5e9f2e77f6131fc253f063a3741eab170ef610196098e8/pyzmq-25.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d6128d431b8dfa888bf51c22a04d48bcb3d64431caf02b3cb943269f17fd2994", size = 1393337 }, + { url = "https://files.pythonhosted.org/packages/fd/12/0324dcb2554cd3f2ebb851ddbfbac27c4bb384394ba4a8978dec093fe71d/pyzmq-25.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:2b15247c49d8cbea695b321ae5478d47cffd496a2ec5ef47131a9e79ddd7e46c", size = 1723679 }, + { url = "https://files.pythonhosted.org/packages/04/15/b8ab292f0b74e0440547185fb67167c87454a2b3be429d64de569f7142a2/pyzmq-25.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:442d3efc77ca4d35bee3547a8e08e8d4bb88dadb54a8377014938ba98d2e074a", size = 1612761 }, + { url = "https://files.pythonhosted.org/packages/22/3e/3670e36c6f42e124492ddd2af550ca13bd4a9f1edd562e1ae7c35a1f230b/pyzmq-25.1.0-cp311-cp311-win32.whl", hash = "sha256:65346f507a815a731092421d0d7d60ed551a80d9b75e8b684307d435a5597425", size = 878704 }, + { url = "https://files.pythonhosted.org/packages/a0/db/4e586c563b48dec09b8f7c2728b905e29db61af89b5c58e4eba9ad36fdec/pyzmq-25.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:8b45d722046fea5a5694cba5d86f21f78f0052b40a4bbbbf60128ac55bfcc7b6", size = 1135692 }, +] + +[[package]] +name = "toml" +version = "0.10.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/be/ba/1f744cdc819428fc6b5084ec34d9b30660f6f9daaf70eead706e3203ec3c/toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f", size = 22253 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", size = 16588 }, +] + +[[package]] +name = "tomli" +version = "2.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/35/b9/de2a5c0144d7d75a57ff355c0c24054f965b2dc3036456ae03a51ea6264b/tomli-2.0.2.tar.gz", hash = "sha256:d46d457a85337051c36524bc5349dd91b1877838e2979ac5ced3e710ed8a60ed", size = 16096 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cf/db/ce8eda256fa131af12e0a76d481711abe4681b6923c27efb9a255c9e4594/tomli-2.0.2-py3-none-any.whl", hash = "sha256:2ebe24485c53d303f690b0ec092806a085f07af5a5aa1464f3931eec36caaa38", size = 13237 }, +] + +[[package]] +name = "typing-extensions" +version = "4.12.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/df/db/f35a00659bc03fec321ba8bce9420de607a1d37f8342eee1863174c69557/typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8", size = 85321 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/26/9f/ad63fc0248c5379346306f8668cda6e2e2e9c95e01216d2b8ffd9ff037d0/typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d", size = 37438 }, +] + +[[package]] +name = "vulture" +version = "2.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "toml" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b9/18/e51a6e575047d19dbcd7394f05b2afa6191fe9ce30bd5bcfb3f850501e0c/vulture-2.6.tar.gz", hash = "sha256:2515fa848181001dc8a73aba6a01a1a17406f5d372f24ec7f7191866f9f4997e", size = 53777 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5e/9d/3c4df0c704ddb5ecf07fcd92cfe6d4a5dc000b7f5459afcb7e98a2ffea1e/vulture-2.6-py2.py3-none-any.whl", hash = "sha256:e792e903ccc063ec4873a8979dcf11b51ea3d65a2d3b31c113d47be48f0cdcae", size = 26494 }, +] From b22f43202b221560eb8c232cda0400a1acf6a77b Mon Sep 17 00:00:00 2001 From: fanquake Date: Wed, 26 Jul 2023 13:55:00 +0100 Subject: [PATCH 03/51] guix: add fortified glibc 2.42 https://sourceware.org/git/?p=glibc.git;a=shortlog;h=refs/heads/release/2.42/master --- contrib/guix/manifest.scm | 31 ++++++++++++ .../guix/patches/glibc-2.42-guix-prefix.patch | 47 +++++++++++++++++++ contrib/guix/symbol-check.py | 2 +- 3 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 contrib/guix/patches/glibc-2.42-guix-prefix.patch diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index aad03a20f5e6..7ec2ced5bf3c 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -495,6 +495,37 @@ inspecting signatures in Mach-O binaries.") (("^install-others =.*$") (string-append "install-others = " out "/etc/rpc\n"))))))))))))) +(define-public glibc-2.42 + (let ((commit "71874f167aa5bb1538ff7e394beaacee28ebe65f")) + (package + (inherit glibc) ;; 2.39 + (version "2.42") + (source (origin + (method git-fetch) + (uri (git-reference + (url "https://sourceware.org/git/glibc.git") + (commit commit))) + (file-name (git-file-name "glibc" commit)) + (sha256 + (base32 + "1pfbk907fkbavg7grbvb5zlhd3y47f8jj3d2v1s5w7xjnn0ypigq")) + (patches (search-our-patches "glibc-2.42-guix-prefix.patch")))) + (arguments + (substitute-keyword-arguments (package-arguments glibc) + ((#:configure-flags flags) + `(append ,flags + ;; https://www.gnu.org/software/libc/manual/html_node/Configuring-and-compiling.html + (list "--enable-stack-protector=all", + "--enable-bind-now", + "--enable-fortify-source", + "--enable-cet=yes", + "--enable-nscd=no", + "--enable-static-nss=yes", + "--disable-timezone-tools", + "--disable-profile", + "--disable-werror", + building-on)))))))) + ;; The sponge tool from moreutils. (define-public sponge (package diff --git a/contrib/guix/patches/glibc-2.42-guix-prefix.patch b/contrib/guix/patches/glibc-2.42-guix-prefix.patch new file mode 100644 index 000000000000..9111fb5b476a --- /dev/null +++ b/contrib/guix/patches/glibc-2.42-guix-prefix.patch @@ -0,0 +1,47 @@ +Without -ffile-prefix-map, the debug symbols will contain paths for the +guix store which will include the hashes of each package. However, the +hash for the same package will differ when on different architectures. +In order to be reproducible regardless of the architecture used to build +the package, map all guix store prefixes to something fixed, e.g. /usr. + +--- a/Makeconfig ++++ b/Makeconfig +@@ -1074,6 +1074,10 @@ CPPFLAGS-.o = $(pic-default) + CFLAGS-.o = $(filter %frame-pointer,$(+cflags)) $(pie-default) + CFLAGS-.o += $(call elide-fortify-source,.o,$(routines_no_fortify)) + CFLAGS-.o += $(call elide-fortify-source,_chk.o,$(routines_no_fortify)) ++ ++# Map Guix store paths to /usr ++CFLAGS-.o += `find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;` ++ + libtype.o := lib%.a + object-suffixes += .o + ifeq (yes,$(build-shared)) +diff --git a/iconv/Makefile b/iconv/Makefile +index afb3fb7bdb..5acee345e0 100644 +--- a/iconv/Makefile ++++ b/iconv/Makefile +@@ -65,6 +65,9 @@ CFLAGS-gconv_cache.c += -DGCONV_DIR='"$(gconvdir)"' + CFLAGS-gconv_conf.c += -DGCONV_PATH='"$(gconvdir)"' + CFLAGS-iconvconfig.c += -DGCONV_PATH='"$(gconvdir)"' -DGCONV_DIR='"$(gconvdir)"' + ++# Map Guix store paths to /usr ++CFLAGS-.c += `find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;` ++ + # Set libof-* for each routine. + cpp-srcs-left := $(iconv_prog-modules) $(iconvconfig-modules) + lib := iconvprogs +diff --git a/posix/Makefile b/posix/Makefile +index 3d368b91f6..d79d8fb648 100644 +--- a/posix/Makefile ++++ b/posix/Makefile +@@ -590,6 +590,9 @@ CFLAGS-execlp.os = -fomit-frame-pointer + CFLAGS-nanosleep.c += -fexceptions -fasynchronous-unwind-tables + CFLAGS-fork.c = $(libio-mtsafe) $(config-cflags-wno-ignored-attributes) + ++# Map Guix store paths to /usr ++CFLAGS-.c += `find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;` ++ + tstgetopt-ARGS = -a -b -cfoobar --required foobar --optional=bazbug \ + --none random --col --color --colour + diff --git a/contrib/guix/symbol-check.py b/contrib/guix/symbol-check.py index 27483aa03756..b6012b98301e 100755 --- a/contrib/guix/symbol-check.py +++ b/contrib/guix/symbol-check.py @@ -40,7 +40,7 @@ # Ignore symbols that are exported as part of every executable IGNORE_EXPORTS = { 'environ', '_environ', '__environ', '_fini', '_init', 'stdin', -'stdout', 'stderr', +'stdout', 'stderr', '__libc_single_threaded', } # Expected linker-loader names can be found here: From 9d98a1be18e37d763eadd9fd20d7140ece2e7a9c Mon Sep 17 00:00:00 2001 From: fanquake Date: Tue, 23 Aug 2022 12:43:33 +0100 Subject: [PATCH 04/51] guix: build x86_64-linux bitcoind fully statically -static-pie Produce a static position independent executable on targets that support it. A static position independent executable is similar to a static executable, but can be loaded at any address without a dynamic linker. See https://gcc.gnu.org/onlinedocs/gcc/Link-Options.html --- contrib/guix/libexec/build.sh | 6 ++++-- contrib/guix/manifest.scm | 4 ++++ contrib/guix/security-check.py | 4 ++++ contrib/guix/symbol-check.py | 9 ++++++--- 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index 48301841841d..512530e21ba8 100755 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -142,10 +142,10 @@ export GUIX_LD_WRAPPER_DISABLE_RPATH=yes # Determine the correct value for -Wl,--dynamic-linker for the current $HOST case "$HOST" in + x86_64-linux-gnu) ;; *linux*) glibc_dynamic_linker=$( case "$HOST" in - x86_64-linux-gnu) echo /lib64/ld-linux-x86-64.so.2 ;; arm-linux-gnueabihf) echo /lib/ld-linux-armhf.so.3 ;; aarch64-linux-gnu) echo /lib/ld-linux-aarch64.so.1 ;; riscv64-linux-gnu) echo /lib/ld-linux-riscv64-lp64d.so.1 ;; @@ -178,7 +178,8 @@ make -C depends --jobs="$JOBS" HOST="$HOST" \ x86_64_linux_AR=x86_64-linux-gnu-gcc-ar \ x86_64_linux_RANLIB=x86_64-linux-gnu-gcc-ranlib \ x86_64_linux_NM=x86_64-linux-gnu-gcc-nm \ - x86_64_linux_STRIP=x86_64-linux-gnu-strip + x86_64_linux_STRIP=x86_64-linux-gnu-strip \ + NO_QT=1 # Don't bother with static case "$HOST" in *darwin*) @@ -225,6 +226,7 @@ esac # LDFLAGS case "$HOST" in + x86_64-linux-gnu) HOST_LDFLAGS=" -static-pie -static-libgcc -Wl,-O2" ;; *linux*) HOST_LDFLAGS="-Wl,--as-needed -Wl,--dynamic-linker=$glibc_dynamic_linker -Wl,-O2" ;; *mingw*) HOST_LDFLAGS="-Wl,--no-insert-timestamp" ;; esac diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index 7ec2ced5bf3c..de27ff44d4f3 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -594,6 +594,10 @@ inspecting signatures in Mach-O binaries.") nsis-x86_64 nss-certs osslsigncode)) + ((string-contains target "x86_64-linux-") + (list (list gcc-toolchain-13 "static") + (make-bitcoin-cross-toolchain target + #:base-libc glibc-2.42))) ((string-contains target "-linux-") (list bison pkg-config diff --git a/contrib/guix/security-check.py b/contrib/guix/security-check.py index be2e0cfbe2af..ac943e33aabd 100755 --- a/contrib/guix/security-check.py +++ b/contrib/guix/security-check.py @@ -122,6 +122,10 @@ def check_ELF_CONTROL_FLOW(binary) -> bool: return False def check_ELF_FORTIFY(binary) -> bool: + # no imported fortified funcs if we are fully static + # check could be changed to include all symbols + if binary.header.machine_type == lief.ELF.ARCH.X86_64: + return True # bitcoin wrapper does not currently contain any fortified functions if '--monolithic' in binary.strings: diff --git a/contrib/guix/symbol-check.py b/contrib/guix/symbol-check.py index b6012b98301e..71d4743d5823 100755 --- a/contrib/guix/symbol-check.py +++ b/contrib/guix/symbol-check.py @@ -29,7 +29,7 @@ MAX_VERSIONS = { 'GLIBC': { - lief.ELF.ARCH.X86_64: (2,31), + lief.ELF.ARCH.X86_64: (0,0), lief.ELF.ARCH.ARM: (2,31), lief.ELF.ARCH.AARCH64:(2,31), lief.ELF.ARCH.PPC64: (2,31), @@ -47,7 +47,7 @@ # https://sourceware.org/glibc/wiki/ABIList?action=recall&rev=16 ELF_INTERPRETER_NAMES: dict[lief.ELF.ARCH, dict[lief.Header.ENDIANNESS, str]] = { lief.ELF.ARCH.X86_64: { - lief.Header.ENDIANNESS.LITTLE: "/lib64/ld-linux-x86-64.so.2", + lief.Header.ENDIANNESS.LITTLE: "", }, lief.ELF.ARCH.ARM: { lief.Header.ENDIANNESS.LITTLE: "/lib/ld-linux-armhf.so.3", @@ -89,7 +89,6 @@ 'libc.so.6', # C library 'libpthread.so.0', # threading 'libm.so.6', # math library -'ld-linux-x86-64.so.2', # 64-bit dynamic linker 'ld-linux.so.2', # 32-bit dynamic linker 'ld-linux-aarch64.so.1', # 64-bit ARM dynamic linker 'ld-linux-armhf.so.3', # 32-bit ARM dynamic linker @@ -209,6 +208,10 @@ def check_RUNPATH(binary) -> bool: def check_ELF_libraries(binary) -> bool: ok: bool = True + + if binary.header.machine_type == lief.ELF.ARCH.X86_64: + return len(binary.libraries) == 0 + for library in binary.libraries: if library not in ELF_ALLOWED_LIBRARIES: print(f'{filename}: {library} is not in ALLOWED_LIBRARIES!') From cb32524055065283d17e366d23e15096fd1ed4d4 Mon Sep 17 00:00:00 2001 From: willcl-ark Date: Tue, 28 Oct 2025 22:07:46 +0000 Subject: [PATCH 05/51] clone guix into bench-ci To update in a rebase simply do: ``` mkdir -p bench-ci cp -r contrib/guix bench-ci git add -f bench-ci/guix ``` --- bench-ci/guix/INSTALL.md | 814 ++++++++++++++++++ bench-ci/guix/README.md | 430 +++++++++ bench-ci/guix/guix-attest | 263 ++++++ bench-ci/guix/guix-build | 474 ++++++++++ bench-ci/guix/guix-clean | 83 ++ bench-ci/guix/guix-codesign | 384 +++++++++ bench-ci/guix/guix-verify | 174 ++++ bench-ci/guix/libexec/build.sh | 403 +++++++++ bench-ci/guix/libexec/codesign.sh | 153 ++++ bench-ci/guix/libexec/prelude.bash | 102 +++ bench-ci/guix/manifest.scm | 574 ++++++++++++ .../patches/binutils-unaligned-default.patch | 22 + .../guix/patches/gcc-remap-guix-store.patch | 20 + .../guix/patches/glibc-2.42-guix-prefix.patch | 47 + bench-ci/guix/patches/glibc-guix-prefix.patch | 16 + .../guix/patches/glibc-riscv-jumptarget.patch | 57 ++ bench-ci/guix/patches/lief-scikit-0-9.patch | 21 + .../patches/oscrypto-hard-code-openssl.patch | 13 + .../winpthreads-remap-guix-store.patch | 17 + bench-ci/guix/security-check.py | 293 +++++++ bench-ci/guix/symbol-check.py | 335 +++++++ 21 files changed, 4695 insertions(+) create mode 100644 bench-ci/guix/INSTALL.md create mode 100644 bench-ci/guix/README.md create mode 100755 bench-ci/guix/guix-attest create mode 100755 bench-ci/guix/guix-build create mode 100755 bench-ci/guix/guix-clean create mode 100755 bench-ci/guix/guix-codesign create mode 100755 bench-ci/guix/guix-verify create mode 100755 bench-ci/guix/libexec/build.sh create mode 100755 bench-ci/guix/libexec/codesign.sh create mode 100644 bench-ci/guix/libexec/prelude.bash create mode 100644 bench-ci/guix/manifest.scm create mode 100644 bench-ci/guix/patches/binutils-unaligned-default.patch create mode 100644 bench-ci/guix/patches/gcc-remap-guix-store.patch create mode 100644 bench-ci/guix/patches/glibc-2.42-guix-prefix.patch create mode 100644 bench-ci/guix/patches/glibc-guix-prefix.patch create mode 100644 bench-ci/guix/patches/glibc-riscv-jumptarget.patch create mode 100644 bench-ci/guix/patches/lief-scikit-0-9.patch create mode 100644 bench-ci/guix/patches/oscrypto-hard-code-openssl.patch create mode 100644 bench-ci/guix/patches/winpthreads-remap-guix-store.patch create mode 100755 bench-ci/guix/security-check.py create mode 100755 bench-ci/guix/symbol-check.py diff --git a/bench-ci/guix/INSTALL.md b/bench-ci/guix/INSTALL.md new file mode 100644 index 000000000000..f9a79f66349c --- /dev/null +++ b/bench-ci/guix/INSTALL.md @@ -0,0 +1,814 @@ +# Guix Installation and Setup + +This only needs to be done once per machine. If you have already completed the +installation and setup, please proceed to [perform a build](./README.md). + +Otherwise, you may choose from one of the following options to install Guix: + +1. Using the official **shell installer script** [⤓ skip to section][install-script] + - Maintained by Guix developers + - Easiest (automatically performs *most* setup) + - Works on nearly all Linux distributions + - Only installs latest release + - Binary installation only, requires high level of trust + - Note: The script needs to be run as root, so it should be inspected before it's run +2. Using the official **binary tarball** [⤓ skip to section][install-bin-tarball] + - Maintained by Guix developers + - Normal difficulty (full manual setup required) + - Works on nearly all Linux distributions + - Installs any release + - Binary installation only, requires high level of trust +3. Using fanquake's **container image** [↗︎ external instructions][install-fanquake-container] + - Maintained by fanquake + - Easy (automatically performs *some* setup) + - Works wherever container images work (Docker/Podman) + - Installs any release + - Binary installation only, requires high level of trust +4. Using a **distribution-maintained package** [⤓ skip to section][install-distro-pkg] + - Maintained by distribution's Guix package maintainer + - Normal difficulty (manual setup required) + - Works only on distributions with Guix packaged, see: https://repology.org/project/guix/versions + - Installs a release decided on by package maintainer + - Source or binary installation depending on the distribution +5. Building **from source** [⤓ skip to section][install-source] + - Maintained by you + - Hard, but rewarding + - Can be made to work on most Linux distributions + - Installs any commit (more granular) + - Source installation, requires lower level of trust + +## Options 1 and 2: Using the official shell installer script or binary tarball + +The installation instructions for both the official shell installer script and +the binary tarballs can be found in the GNU Guix Manual's [Binary Installation +section](https://guix.gnu.org/manual/en/html_node/Binary-Installation.html). + +Note that running through the binary tarball installation steps is largely +equivalent to manually performing what the shell installer script does. + +Note that at the time of writing (July 5th, 2021), the shell installer script +automatically creates an `/etc/profile.d` entry which the binary tarball +installation instructions do not ask you to create. However, you will likely +need this entry for better desktop integration. Please see [this +section](#add-an-etcprofiled-entry) for instructions on how to add a +`/etc/profile.d/guix.sh` entry. + +Regardless of which installation option you chose, the changes to +`/etc/profile.d` will not take effect until the next shell or desktop session, +so you should log out and log back in. + +## Option 3: Using fanquake's container image + +Please refer to fanquake's instructions +[here](https://github.com/fanquake/core-review/tree/master/guix). + +## Option 4: Using a distribution-maintained package + +Note that this section is based on the distro packaging situation at the time of +writing (July 2021). Guix is expected to be more widely packaged over time. For +an up-to-date view on Guix's package status/version across distros, please see: +https://repology.org/project/guix/versions + +### Debian / Ubuntu + +Guix is available as a distribution package in [Debian +](https://packages.debian.org/search?keywords=guix) and [Ubuntu +](https://packages.ubuntu.com/search?keywords=guix). + +To install: +```sh +sudo apt install guix +``` + +### Arch Linux + +Guix is available in the AUR as +[`guix`](https://aur.archlinux.org/packages/guix/), please follow the +installation instructions in the Arch Linux Wiki ([live +link](https://wiki.archlinux.org/index.php/Guix#AUR_Package_Installation), +[2021/03/30 +permalink](https://wiki.archlinux.org/index.php?title=Guix&oldid=637559#AUR_Package_Installation)) +to install Guix. + +At the time of writing (2021/03/30), the `check` phase will fail if the path to +guix's build directory is longer than 36 characters due to an anachronistic +character limit on the shebang line. Since the `check` phase happens after the +`build` phase, which may take quite a long time, it is recommended that users +either: + +1. Skip the `check` phase + - For `makepkg`: `makepkg --nocheck ...` + - For `yay`: `yay --mflags="--nocheck" ...` + - For `paru`: `paru --nocheck ...` +2. Or, check their build directory's length beforehand + - For those building with `makepkg`: `pwd | wc -c` + +## Option 5: Building from source + +Building Guix from source is a rather involved process but a rewarding one for +those looking to minimize trust and maximize customizability (e.g. building a +particular commit of Guix). Previous experience with using autotools-style build +systems to build packages from source will be helpful. *hic sunt dracones.* + +I strongly urge you to at least skim through the entire section once before you +start issuing commands, as it will save you a lot of unnecessary pain and +anguish. + +### Installing common build tools + +There are a few basic build tools that are required for most things we'll build, +so let's install them now: + +Text transformation/i18n: +- `autopoint` (sometimes packaged in `gettext`) +- `help2man` +- `po4a` +- `texinfo` + +Build system tools: +- `g++` w/ C++11 support +- `libtool` +- `autoconf` +- `automake` +- `pkg-config` (sometimes packaged as `pkgconf`) +- `make` +- `cmake` + +Miscellaneous: +- `git` +- `gnupg` +- `python3` + +### Building and Installing Guix's dependencies + +In order to build Guix itself from source, we need to first make sure that the +necessary dependencies are installed and discoverable. The most up-to-date list +of Guix's dependencies is kept in the ["Requirements" +section](https://guix.gnu.org/manual/en/html_node/Requirements.html) of the Guix +Reference Manual. + +Depending on your distribution, most or all of these dependencies may already be +packaged and installable without manually building and installing. + +For reference, the graphic below outlines Guix v1.3.0's dependency graph: + +![bootstrap map](https://user-images.githubusercontent.com/6399679/125064185-a9a59880-e0b0-11eb-82c1-9b8e5dc9950d.png) + +If you do not care about building each dependency from source, and Guix is +already packaged for your distribution, you can easily install only the build +dependencies of Guix. For example, to enable deb-src and install the Guix build +dependencies on Ubuntu/Debian: + +```sh +sed -i 's|# deb-src|deb-src|g' /etc/apt/sources.list +apt update +apt-get build-dep -y guix +``` + +If this succeeded, you can likely skip to section +["Building and Installing Guix itself"](#building-and-installing-guix-itself). + +#### Guile + +###### Corner case: Multiple versions of Guile on one system + +It is recommended to only install the required version of Guile, so that build systems do +not get confused about which Guile to use. + +However, if you insist on having more versions of Guile installed on +your system, then you need to **consistently** specify +`GUILE_EFFECTIVE_VERSION=3.0` to all +`./configure` invocations for Guix and its dependencies. + +##### Installing Guile + +If your distribution splits packages into `-dev`-suffixed and +non-`-dev`-suffixed sub-packages (as is the case for Debian-derived +distributions), please make sure to install both. For example, to install Guile +v3.0 on Debian/Ubuntu: + +```sh +apt install guile-3.0 guile-3.0-dev +``` + +#### Mixing distribution packages and source-built packages + +At the time of writing, most distributions have _some_ of Guix's dependencies +packaged, but not all. This means that you may want to install the distribution +package for some dependencies, and manually build-from-source for others. + +Distribution packages usually install to `/usr`, which is different from the +default `./configure` prefix of source-built packages: `/usr/local`. + +This means that if you mix-and-match distribution packages and source-built +packages and do not specify exactly `--prefix=/usr` to `./configure` for +source-built packages, you will need to augment the `GUILE_LOAD_PATH` and +`GUILE_LOAD_COMPILED_PATH` environment variables so that Guile will look +under the right prefix and find your source-built packages. + +For example, if you are using Guile v3.0, and have Guile packages in the +`/usr/local` prefix, either add the following lines to your `.profile` or +`.bash_profile` so that the environment variable is properly set for all future +shell logins, or paste the lines into a POSIX-style shell to temporarily modify +the environment variables of your current shell session. + +```sh +# Help Guile v3.0.x find packages in /usr/local +export GUILE_LOAD_PATH="/usr/local/share/guile/site/3.0${GUILE_LOAD_PATH:+:}$GUILE_LOAD_PATH" +export GUILE_LOAD_COMPILED_PATH="/usr/local/lib/guile/3.0/site-ccache${GUILE_LOAD_COMPILED_PATH:+:}$GUILE_COMPILED_LOAD_PATH" +``` + +Note that these environment variables are used to check for packages during +`./configure`, so they should be set as soon as possible should you want to use +a prefix other than `/usr`. + +#### Building and installing source-built packages + +***IMPORTANT**: A few dependencies have non-obvious quirks/errata which are +documented in the sub-sections immediately below. Please read these sections +before proceeding to build and install these packages.* + +Although you should always refer to the README or INSTALL files for the most +accurate information, most of these dependencies use autoconf-style build +systems (check if there's a `configure.ac` file), and will likely do the right +thing with the following: + +Clone the repository and check out the latest release: +```sh +git clone /.git +cd +git tag -l # check for the latest release +git checkout +``` + +For autoconf-based build systems (if `./autogen.sh` or `configure.ac` exists at +the root of the repository): + +```sh +./autogen.sh || autoreconf -vfi +./configure --prefix= +make +sudo make install +``` + +For CMake-based build systems (if `CMakeLists.txt` exists at the root of the +repository): + +```sh +mkdir build && cd build +cmake .. -DCMAKE_INSTALL_PREFIX= +sudo cmake --build . --target install +``` + +If you choose not to specify exactly `--prefix=/usr` to `./configure`, please +make sure you've carefully read the [previous section] on mixing distribution +packages and source-built packages. + +##### Binding packages require `-dev`-suffixed packages + +Relevant for: +- Everyone + +When building bindings, the `-dev`-suffixed version of the original package +needs to be installed. For example, building `Guile-zlib` on Debian-derived +distributions requires that `zlib1g-dev` is installed. + +When using bindings, the `-dev`-suffixed version of the original package still +needs to be installed. This is particularly problematic when distribution +packages are mispackaged like `guile-sqlite3` is in Ubuntu Focal such that +installing `guile-sqlite3` does not automatically install `libsqlite3-dev` as a +dependency. + +Below is a list of relevant Guile bindings and their corresponding `-dev` +packages in Debian at the time of writing. + +| Guile binding package | -dev Debian package | +|-----------------------|---------------------| +| guile-gcrypt | libgcrypt-dev | +| guile-git | libgit2-dev | +| guile-gnutls | (none) | +| guile-json | (none) | +| guile-lzlib | liblz-dev | +| guile-ssh | libssh-dev | +| guile-sqlite3 | libsqlite3-dev | +| guile-zlib | zlib1g-dev | + +##### `guile-git` actually depends on `libgit2 >= 1.1` + +Relevant for: +- Those building `guile-git` from source against `libgit2 < 1.1` +- Those installing `guile-git` from their distribution where `guile-git` is + built against `libgit2 < 1.1` + +As of v0.5.2, `guile-git` claims to only require `libgit2 >= 0.28.0`, however, +it actually requires `libgit2 >= 1.1`, otherwise, it will be confused by a +reference of `origin/keyring`: instead of interpreting the reference as "the +'keyring' branch of the 'origin' remote", the reference is interpreted as "the +branch literally named 'origin/keyring'" + +This is especially notable because Ubuntu Focal packages `libgit2 v0.28.4`, and +`guile-git` is built against it. + +Should you be in this situation, you need to build both `libgit2 v1.1.x` and +`guile-git` from source. + +Source: https://logs.guix.gnu.org/guix/2020-11-12.log#232527 + +### Building and Installing Guix itself + +Start by cloning Guix: + +``` +git clone https://codeberg.org/guix/guix.git +cd guix +``` + +You will likely want to build the latest release. +At the time of writing (November 2023), the latest release was `v1.4.0`. + +``` +git branch -a -l 'origin/version-*' # check for the latest release +git checkout +``` + +Bootstrap the build system: +``` +./bootstrap +``` + +Configure with the recommended `--localstatedir` flag: +``` +./configure --localstatedir=/var +``` + +Note: If you intend to hack on Guix in the future, you will need to supply the +same `--localstatedir=` flag for all future Guix `./configure` invocations. See +the last paragraph of this +[section](https://guix.gnu.org/manual/en/html_node/Requirements.html) for more +details. + +Build Guix (this will take a while): +``` +make -j$(nproc) +``` + +Install Guix: + +``` +sudo make install +``` + +### Post-"build from source" Setup + +#### Creating and starting a `guix-daemon-original` service with a fixed `argv[0]` + +At this point, guix will be installed to `${bindir}`, which is likely +`/usr/local/bin` if you did not override directory variables at +`./configure`-time. More information on standard Automake directory variables +can be found +[here](https://www.gnu.org/software/automake/manual/html_node/Standard-Directory-Variables.html). + +However, the Guix init scripts and service configurations for Upstart, systemd, +SysV, and OpenRC are installed (in `${libdir}`) to launch +`${localstatedir}/guix/profiles/per-user/root/current-guix/bin/guix-daemon`, +which does not yet exist, and will only exist after [`root` performs their first +`guix pull`](#guix-pull-as-root). + +We need to create a `-original` version of these init scripts that's pointed to +the binaries we just built and `make install`'ed in `${bindir}` (normally, +`/usr/local/bin`). + +Example for `systemd`, run as `root`: + +```sh +# Create guix-daemon-original.service by modifying guix-daemon.service +libdir=# set according to your PREFIX (default is /usr/local/lib) +bindir="$(dirname $(command -v guix-daemon))" +sed -E -e "s|/\S*/guix/profiles/per-user/root/current-guix/bin/guix-daemon|${bindir}/guix-daemon|" "${libdir}"/systemd/system/guix-daemon.service > /etc/systemd/system/guix-daemon-original.service +chmod 664 /etc/systemd/system/guix-daemon-original.service + +# Make systemd recognize the new service +systemctl daemon-reload + +# Make sure that the non-working guix-daemon.service is stopped and disabled +systemctl stop guix-daemon +systemctl disable guix-daemon + +# Make sure that the working guix-daemon-original.service is started and enabled +systemctl enable guix-daemon-original +systemctl start guix-daemon-original +``` + +#### Creating `guix-daemon` users / groups + +Please see the [relevant +section](https://guix.gnu.org/manual/en/html_node/Build-Environment-Setup.html) +in the Guix Reference Manual for more details. + +## Optional setup + +At this point, you are set up to [use Guix to build Bitcoin +Core](./README.md#usage). However, if you want to polish your setup a bit and +make it "what Guix intended", then read the next few subsections. + +### Add an `/etc/profile.d` entry + +This section definitely does not apply to you if you installed Guix using: +1. The shell installer script +2. fanquake's container image +3. Debian's `guix` package + +#### Background + +Although Guix knows how to update itself and its packages, it does so in a +non-invasive way (it does not modify `/usr/local/bin/guix`). + +Instead, it does the following: + +- After a `guix pull`, it updates + `/var/guix/profiles/per-user/$USER/current-guix`, and creates a symlink + targeting this directory at `$HOME/.config/guix/current` + +- After a `guix install`, it updates + `/var/guix/profiles/per-user/$USER/guix-profile`, and creates a symlink + targeting this directory at `$HOME/.guix-profile` + +Therefore, in order for these operations to affect your shell/desktop sessions +(and for the principle of least astonishment to hold), their corresponding +directories have to be added to well-known environment variables like `$PATH`, +`$INFOPATH`, `$XDG_DATA_DIRS`, etc. + +In other words, if `$HOME/.config/guix/current/bin` does not exist in your +`$PATH`, a `guix pull` will have no effect on what `guix` you are using. Same +goes for `$HOME/.guix-profile/bin`, `guix install`, and installed packages. + +Helpfully, after a `guix pull` or `guix install`, a message will be printed like +so: + +``` +hint: Consider setting the necessary environment variables by running: + + GUIX_PROFILE="$HOME/.guix-profile" + . "$GUIX_PROFILE/etc/profile" + +Alternately, see `guix package --search-paths -p "$HOME/.guix-profile"'. +``` + +However, this is somewhat tedious to do for both `guix pull` and `guix install` +for each user on the system that wants to properly use `guix`. I recommend that +you add an entry to `/etc/profile.d` instead. This is done by default +when installing the Debian package later than 1.2.0-4 and when using the shell +script installer. + +#### Instructions + +Create `/etc/profile.d/guix.sh` with the following content: +```sh +# _GUIX_PROFILE: `guix pull` profile +_GUIX_PROFILE="$HOME/.config/guix/current" +if [ -L $_GUIX_PROFILE ]; then + export PATH="$_GUIX_PROFILE/bin${PATH:+:}$PATH" + # Export INFOPATH so that the updated info pages can be found + # and read by both /usr/bin/info and/or $GUIX_PROFILE/bin/info + # When INFOPATH is unset, add a trailing colon so that Emacs + # searches 'Info-default-directory-list'. + export INFOPATH="$_GUIX_PROFILE/share/info:$INFOPATH" +fi + +# GUIX_PROFILE: User's default profile +GUIX_PROFILE="$HOME/.guix-profile" +[ -L $GUIX_PROFILE ] || return +GUIX_LOCPATH="$GUIX_PROFILE/lib/locale" +export GUIX_PROFILE GUIX_LOCPATH + +[ -f "$GUIX_PROFILE/etc/profile" ] && . "$GUIX_PROFILE/etc/profile" + +# set XDG_DATA_DIRS to include Guix installations +export XDG_DATA_DIRS="$GUIX_PROFILE/share:${XDG_DATA_DIRS:-/usr/local/share/:/usr/share/}" +``` + +Please note that this will not take effect until the next shell or desktop +session (log out and log back in). + +### `guix pull` as root + +Before you do this, you need to read the section on [choosing your security +model][security-model] and adjust `guix` and `guix-daemon` flags according to +your choice, as invoking `guix pull` may pull substitutes from substitute +servers (which you may not want). + +As mentioned in a previous section, Guix expects +`${localstatedir}/guix/profiles/per-user/root/current-guix` to be populated with +`root`'s Guix profile, `guix pull`-ed and built by some former version of Guix. +However, this is not the case when we build from source. Therefore, we need to +perform a `guix pull` as `root`: + +```sh +sudo --login guix pull --branch=version- +# or +sudo --login guix pull --commit= +``` + +`guix pull` is quite a long process (especially if you're using +`--no-substitutes`). If you encounter build problems, please refer to the +[troubleshooting section](#troubleshooting). + +Note that running a bare `guix pull` with no commit or branch specified will +pull the latest commit on Guix's master branch, which is likely fine, but not +recommended. + +If you installed Guix from source, you may get an error like the following: +```sh +error: while creating symlink '/root/.config/guix/current' No such file or directory +``` +To resolve this, simply: +``` +sudo mkdir -p /root/.config/guix +``` +Then try the `guix pull` command again. + +After the `guix pull` finishes successfully, +`${localstatedir}/guix/profiles/per-user/root/current-guix` should be populated. + +#### Using the newly-pulled `guix` by restarting the daemon + +Depending on how you installed Guix, you should now make sure that your init +scripts and service configurations point to the newly-pulled `guix-daemon`. + +##### If you built Guix from source + +If you followed the instructions for [fixing argv\[0\]][fix-argv0], you can now +do the following: + +```sh +systemctl stop guix-daemon-original +systemctl disable guix-daemon-original + +systemctl enable guix-daemon +systemctl start guix-daemon +``` + +Remember to set `--no-substitutes` in `$libdir/systemd/system/guix-daemon.service` and other customizations if you used them for `guix-daemon-original.service`. + +##### If you installed Guix via the Debian/Ubuntu distribution packages + +You will need to create a `guix-daemon-latest` service which points to the new +`guix` rather than a pinned one. + +```sh +# Create guix-daemon-latest.service by modifying guix-daemon.service +sed -E -e "s|/usr/bin/guix-daemon|/var/guix/profiles/per-user/root/current-guix/bin/guix-daemon|" /etc/systemd/system/guix-daemon.service > /lib/systemd/system/guix-daemon-latest.service +chmod 664 /lib/systemd/system/guix-daemon-latest.service + +# Make systemd recognize the new service +systemctl daemon-reload + +# Make sure that the old guix-daemon.service is stopped and disabled +systemctl stop guix-daemon +systemctl disable guix-daemon + +# Make sure that the new guix-daemon-latest.service is started and enabled +systemctl enable guix-daemon-latest +systemctl start guix-daemon-latest +``` + +##### If you installed Guix via lantw44's Arch Linux AUR package + +At the time of writing (July 5th, 2021) the systemd unit for "updated Guix" is +`guix-daemon-latest.service`, therefore, you should do the following: + +```sh +systemctl stop guix-daemon +systemctl disable guix-daemon + +systemctl enable guix-daemon-latest +systemctl start guix-daemon-latest +``` + +##### Otherwise... + +Simply do: + +```sh +systemctl restart guix-daemon +``` + +### Checking everything + +If you followed all the steps above to make your Guix setup "prim and proper," +you can check that you did everything properly by running through this +checklist. + +1. `/etc/profile.d/guix.sh` should exist and be sourced at each shell login + +2. `guix describe` should not print `guix describe: error: failed to determine + origin`, but rather something like: + + ``` + Generation 38 Feb 22 2021 16:39:31 (current) + guix f350df4 + repository URL: https://codeberg.org/guix/guix.git + branch: version-1.2.0 + commit: f350df405fbcd5b9e27e6b6aa500da7f101f41e7 + ``` + +3. `guix-daemon` should be running from `${localstatedir}/guix/profiles/per-user/root/current-guix` + +# Troubleshooting + +## Derivation failed to build + +When you see a build failure like below: + +``` +building /gnu/store/...-foo-3.6.12.drv... +/ 'check' phasenote: keeping build directory `/tmp/guix-build-foo-3.6.12.drv-0' +builder for `/gnu/store/...-foo-3.6.12.drv' failed with exit code 1 +build of /gnu/store/...-foo-3.6.12.drv failed +View build log at '/var/log/guix/drvs/../...-foo-3.6.12.drv.bz2'. +cannot build derivation `/gnu/store/...-qux-7.69.1.drv': 1 dependencies couldn't be built +cannot build derivation `/gnu/store/...-bar-3.16.5.drv': 1 dependencies couldn't be built +cannot build derivation `/gnu/store/...-baz-2.0.5.drv': 1 dependencies couldn't be built +guix time-machine: error: build of `/gnu/store/...-baz-2.0.5.drv' failed +``` + +It means that `guix` failed to build a package named `foo`, which was a +dependency of `qux`, `bar`, and `baz`. Importantly, note that the last "failed" +line is not necessarily the root cause, the first "failed" line is. + +Most of the time, the build failure is due to a spurious test failure or the +package's build system/test suite breaking when running multi-threaded. To +rebuild _just_ this derivation in a single-threaded fashion (please don't forget +to add other `guix` flags like `--no-substitutes` as appropriate): + +```sh +$ guix build --cores=1 /gnu/store/...-foo-3.6.12.drv +``` + +If the single-threaded rebuild did not succeed, you may need to dig deeper. +You may view `foo`'s build logs in `less` like so (please replace paths with the +path you see in the build failure output): + +```sh +$ bzcat /var/log/guix/drvs/../...-foo-3.6.12.drv.bz2 | less +``` + +`foo`'s build directory is also preserved and available at +`/tmp/guix-build-foo-3.6.12.drv-0`. However, if you fail to build `foo` multiple +times, it may be `/tmp/...drv-1` or `/tmp/...drv-2`. Always consult the build +failure output for the most accurate, up-to-date information. + +### python(-minimal): [Errno 84] Invalid or incomplete multibyte or wide character + +This error occurs when your `$TMPDIR` (default: /tmp) exists on a filesystem +which rejects characters not present in the UTF-8 character code set. An example +is ZFS with the utf8only=on option set. + +More information: https://github.com/python/cpython/issues/81765 + +### openssl-1.1.1l and openssl-1.1.1n + +OpenSSL includes tests that will fail once some certificate has expired. +The workarounds from the GnuTLS section immediately below can be used. + +For openssl-1.1.1l use 2022-05-01 as the date. + +### GnuTLS: test-suite FAIL: status-request-revoked + +*The derivation is likely identified by: `/gnu/store/vhphki5sg9xkdhh2pbc8gi6vhpfzryf0-gnutls-3.6.12.drv`* + +This unfortunate error is most common for non-substitute builders who installed +Guix v1.2.0. The problem stems from the fact that one of GnuTLS's tests uses a +hardcoded certificate which expired on 2020-10-24. + +What's more unfortunate is that this GnuTLS derivation is somewhat special in +Guix's dependency graph and is not affected by the package transformation flags +like `--without-tests=`. + +The easiest solution for those encountering this problem is to install a newer +version of Guix. However, there are ways to work around this issue: + +#### Workaround 1: Using substitutes for this single derivation + +If you've authorized the official Guix build farm's key (more info +[here](./README.md#step-1-authorize-the-signing-keys)), then you can use +substitutes just for this single derivation by invoking the following: + +```sh +guix build --substitute-urls="https://ci.guix.gnu.org" /gnu/store/vhphki5sg9xkdhh2pbc8gi6vhpfzryf0-gnutls-3.6.12.drv +``` + +See [this section](./README.md#removing-authorized-keys) for instructions on how +to remove authorized keys if you don't want to keep the build farm's key +authorized. + +#### Workaround 2: Temporarily setting the system clock back + +This workaround was described [here](https://issues.guix.gnu.org/44559#5). + +Basically: + +1. Turn off NTP +2. Set system time to 2020-10-01 +3. guix build --no-substitutes /gnu/store/vhphki5sg9xkdhh2pbc8gi6vhpfzryf0-gnutls-3.6.12.drv +4. Set system time back to accurate current time +5. Turn NTP back on + +For example, + +```sh +sudo timedatectl set-ntp no +sudo date --set "01 oct 2020 15:00:00" +guix build /gnu/store/vhphki5sg9xkdhh2pbc8gi6vhpfzryf0-gnutls-3.6.12.drv +sudo timedatectl set-ntp yes +``` + +#### Workaround 3: Disable the tests in the Guix source code for this single derivation + +If all of the above workarounds fail, you can also disable the `tests` phase of +the derivation via the `arguments` option, as described in the official +[`package` +reference](https://guix.gnu.org/manual/en/html_node/package-Reference.html). + +For example, to disable the openssl-1.1 check phase: + +```diff +diff --git a/gnu/packages/tls.scm b/gnu/packages/tls.scm +index f1e844b..1077c4b 100644 +--- a/gnu/packages/tls.scm ++++ b/gnu/packages/tls.scm +@@ -494,4 +494,5 @@ (define-public openssl-1.1 + (arguments + `(#:parallel-tests? #f ++ #:tests? #f + #:test-target "test" +``` + +### coreutils: FAIL: tests/tail-2/inotify-dir-recreate + +The inotify-dir-create test fails on "remote" filesystems such as overlayfs +(Docker's default filesystem) due to the filesystem being mistakenly recognized +as non-remote. + +A relatively easy workaround to this is to make sure that a somewhat traditional +filesystem is mounted at `/tmp` (where `guix-daemon` performs its builds). For +Docker users, this might mean [using a volume][docker/volumes], [binding +mounting][docker/bind-mnt] from host, or (for those with enough RAM and swap) +[mounting a tmpfs][docker/tmpfs] using the `--tmpfs` flag. + +Please see the following links for more details: + +- An upstream coreutils bug has been filed: [debbugs#47940](https://debbugs.gnu.org/cgi/bugreport.cgi?bug=47940) +- A Guix bug detailing the underlying problem has been filed: [guix-issues#47935](https://issues.guix.gnu.org/47935), [guix-issues#49985](https://issues.guix.gnu.org/49985#5) +- A commit to skip this test is included since Guix 1.4.0: +[codeberg/guix@6ba1058](https://codeberg.org/guix/guix/commit/6ba1058df0c4ce5611c2367531ae5c3cdc729ab4) + + +[install-script]: #options-1-and-2-using-the-official-shell-installer-script-or-binary-tarball +[install-bin-tarball]: #options-1-and-2-using-the-official-shell-installer-script-or-binary-tarball +[install-fanquake-container]: #option-3-using-fanquakes-container-image +[install-distro-pkg]: #option-4-using-a-distribution-maintained-package +[install-source]: #option-5-building-from-source + +[fix-argv0]: #creating-and-starting-a-guix-daemon-original-service-with-a-fixed-argv0 +[security-model]: ./README.md#choosing-your-security-model + +[docker/volumes]: https://docs.docker.com/storage/volumes/ +[docker/bind-mnt]: https://docs.docker.com/storage/bind-mounts/ +[docker/tmpfs]: https://docs.docker.com/storage/tmpfs/ + +# Purging/Uninstalling Guix + +In the extraordinarily rare case where you messed up your Guix installation in +an irreversible way, you may want to completely purge Guix from your system and +start over. + +1. Uninstall Guix itself according to the way you installed it (e.g. `sudo apt + purge guix` for Ubuntu packaging, `sudo make uninstall` for a build from source). +2. Remove all build users and groups + + You may check for relevant users and groups using: + + ``` + getent passwd | grep guix + getent group | grep guix + ``` + + Then, you may remove users and groups using: + + ``` + sudo userdel + sudo groupdel + ``` + +3. Remove all possible Guix-related directories + - `/var/guix/` + - `/var/log/guix/` + - `/gnu/` + - `/etc/guix/` + - `/home/*/.config/guix/` + - `/home/*/.cache/guix/` + - `/home/*/.guix-profile/` + - `/root/.config/guix/` + - `/root/.cache/guix/` + - `/root/.guix-profile/` diff --git a/bench-ci/guix/README.md b/bench-ci/guix/README.md new file mode 100644 index 000000000000..7f6b8232bba5 --- /dev/null +++ b/bench-ci/guix/README.md @@ -0,0 +1,430 @@ +# Bootstrappable Bitcoin Core Builds + +This directory contains the files necessary to perform bootstrappable Bitcoin +Core builds. + +[Bootstrappability][b17e] furthers our binary security guarantees by allowing us +to _audit and reproduce_ our toolchain instead of blindly _trusting_ binary +downloads. + +We achieve bootstrappability by using Guix as a functional package manager. + +# Requirements + +Conservatively, you will need: + +- 16GB of free disk space on the partition that /gnu/store will reside in +- 8GB of free disk space **per platform triple** you're planning on building + (see the `HOSTS` [environment variable description][env-vars-list]) + +# Installation and Setup + +If you don't have Guix installed and set up, please follow the instructions in +[INSTALL.md](./INSTALL.md) + +# Usage + +If you haven't considered your security model yet, please read [the relevant +section](#choosing-your-security-model) before proceeding to perform a build. + +## Making the Xcode SDK available for macOS cross-compilation + +In order to perform a build for macOS (which is included in the default set of +platform triples to build), you'll need to extract the macOS SDK tarball using +tools found in the [`macdeploy` directory](../macdeploy/README.md#sdk-extraction). + +You can then either point to the SDK using the `SDK_PATH` environment variable: + +```sh +# Extract the SDK tarball to /path/to/parent/dir/of/extracted/SDK/Xcode---extracted-SDK-with-libcxx-headers +tar -C /path/to/parent/dir/of/extracted/SDK -xaf /path/to/Xcode---extracted-SDK-with-libcxx-headers.tar.gz + +# Indicate where to locate the SDK tarball +export SDK_PATH=/path/to/parent/dir/of/extracted/SDK +``` + +or extract it into `depends/SDKs`: + +```sh +mkdir -p depends/SDKs +tar -C depends/SDKs -xaf /path/to/SDK/tarball +``` + +## Building + +*The author highly recommends at least reading over the [common usage patterns +and examples](#common-guix-build-invocation-patterns-and-examples) section below +before starting a build. For a full list of customization options, see the +[recognized environment variables][env-vars-list] section.* + +To build Bitcoin Core reproducibly with all default options, invoke the +following from the top of a clean repository: + +```sh +./contrib/guix/guix-build +``` + +## Codesigning build outputs + +The `guix-codesign` command attaches codesignatures (produced by codesigners) to +existing non-codesigned outputs. Please see the [release process +documentation](/doc/release-process.md#codesigning) for more context. + +It respects many of the same environment variable flags as `guix-build`, with 2 +crucial differences: + +1. Since only Windows and macOS build outputs require codesigning, the `HOSTS` + environment variable will have a sane default value of `x86_64-w64-mingw32 + x86_64-apple-darwin arm64-apple-darwin` instead of all the platforms. +2. The `guix-codesign` command ***requires*** a `DETACHED_SIGS_REPO` flag. + * _**DETACHED_SIGS_REPO**_ + + Set the directory where detached codesignatures can be found for the current + Bitcoin Core version being built. + + _REQUIRED environment variable_ + +An invocation with all default options would look like: + +``` +env DETACHED_SIGS_REPO= ./contrib/guix/guix-codesign +``` + +## Cleaning intermediate work directories + +By default, `guix-build` leaves all intermediate files or "work directories" +(e.g. `depends/work`, `guix-build-*/distsrc-*`) intact at the end of a build so +that they are available to the user (to aid in debugging, etc.). However, these +directories usually take up a large amount of disk space. Therefore, a +`guix-clean` convenience script is provided which cleans the current `git` +worktree to save disk space: + +``` +./contrib/guix/guix-clean +``` + + +## Attesting to build outputs + +Much like how Gitian build outputs are attested to in a `gitian.sigs` +repository, Guix build outputs are attested to in the [`guix.sigs` +repository](https://github.com/bitcoin-core/guix.sigs). + +After you've cloned the `guix.sigs` repository, to attest to the current +worktree's commit/tag: + +``` +env GUIX_SIGS_REPO= SIGNER= ./contrib/guix/guix-attest +``` + +See `./contrib/guix/guix-attest --help` for more information on the various ways +`guix-attest` can be invoked. + +## Verifying build output attestations + +After at least one other signer has uploaded their signatures to the `guix.sigs` +repository: + +``` +git -C pull +env GUIX_SIGS_REPO= ./contrib/guix/guix-verify +``` + + +## Common `guix-build` invocation patterns and examples + +### Keeping caches and SDKs outside of the worktree + +If you perform a lot of builds and have a bunch of worktrees, you may find it +more efficient to keep the depends tree's download cache, build cache, and SDKs +outside of the worktrees to avoid duplicate downloads and unnecessary builds. To +help with this situation, the `guix-build` script honours the `SOURCES_PATH`, +`BASE_CACHE`, and `SDK_PATH` environment variables and will pass them on to the +depends tree so that you can do something like: + +```sh +env SOURCES_PATH="$HOME/depends-SOURCES_PATH" BASE_CACHE="$HOME/depends-BASE_CACHE" SDK_PATH="$HOME/macOS-SDKs" ./contrib/guix/guix-build +``` + +Note that the paths that these environment variables point to **must be +directories**, and **NOT symlinks to directories**. + +See the [recognized environment variables][env-vars-list] section for more +details. + +### Building a subset of platform triples + +Sometimes you only want to build a subset of the supported platform triples, in +which case you can override the default list by setting the space-separated +`HOSTS` environment variable: + +```sh +env HOSTS='x86_64-w64-mingw32 x86_64-apple-darwin' ./contrib/guix/guix-build +``` + +See the [recognized environment variables][env-vars-list] section for more +details. + +### Controlling the number of threads used by `guix` build commands + +Depending on your system's RAM capacity, you may want to decrease the number of +threads used to decrease RAM usage or vice versa. + +By default, the scripts under `./contrib/guix` will invoke all `guix` build +commands with `--cores="$JOBS"`. Note that `$JOBS` defaults to `$(nproc)` if not +specified. However, astute manual readers will also notice that `guix` build +commands also accept a `--max-jobs=` flag (which defaults to 1 if unspecified). + +Here is the difference between `--cores=` and `--max-jobs=`: + +> Note: When I say "derivation," think "package" + +`--cores=` + + - controls the number of CPU cores to build each derivation. This is the value + passed to `make`'s `--jobs=` flag. + +`--max-jobs=` + + - controls how many derivations can be built in parallel + - defaults to 1 + +Therefore, the default is for `guix` build commands to build one derivation at a +time, utilizing `$JOBS` threads. + +Specifying the `$JOBS` environment variable will only modify `--cores=`, but you +can also modify the value for `--max-jobs=` by specifying +`$ADDITIONAL_GUIX_COMMON_FLAGS`. For example, if you have a LOT of memory, you +may want to set: + +```sh +export ADDITIONAL_GUIX_COMMON_FLAGS='--max-jobs=8' +``` + +Which allows for a maximum of 8 derivations to be built at the same time, each +utilizing `$JOBS` threads. + +Or, if you'd like to avoid spurious build failures caused by issues with +parallelism within a single package, but would still like to build multiple +packages when the dependency graph allows for it, you may want to try: + +```sh +export JOBS=1 ADDITIONAL_GUIX_COMMON_FLAGS='--max-jobs=8' +``` + +See the [recognized environment variables][env-vars-list] section for more +details. + +## Recognized environment variables + +* _**HOSTS**_ + + Override the space-separated list of platform triples for which to perform a + bootstrappable build. + + _(defaults to "x86\_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu + riscv64-linux-gnu powerpc64-linux-gnu powerpc64le-linux-gnu + x86\_64-w64-mingw32 x86\_64-apple-darwin arm64-apple-darwin")_ + +* _**SOURCES_PATH**_ + + Set the depends tree download cache for sources. This is passed through to the + depends tree. Setting this to the same directory across multiple builds of the + depends tree can eliminate unnecessary redownloading of package sources. + + The path that this environment variable points to **must be a directory**, and + **NOT a symlink to a directory**. + +* _**BASE_CACHE**_ + + Set the depends tree cache for built packages. This is passed through to the + depends tree. Setting this to the same directory across multiple builds of the + depends tree can eliminate unnecessary building of packages. + + The path that this environment variable points to **must be a directory**, and + **NOT a symlink to a directory**. + +* _**SDK_PATH**_ + + Set the path where _extracted_ SDKs can be found. This is passed through to + the depends tree. Note that this should be set to the _parent_ directory of + the actual SDK (e.g. `SDK_PATH=$HOME/Downloads/macOS-SDKs` instead of + `$HOME/Downloads/macOS-SDKs/Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers`). + + The path that this environment variable points to **must be a directory**, and + **NOT a symlink to a directory**. + +* _**JOBS**_ + + Override the number of jobs to run simultaneously, you might want to do so on + a memory-limited machine. This may be passed to: + + - `guix` build commands as in `guix shell --cores="$JOBS"` + - `make` as in `make --jobs="$JOBS"` + - `cmake` as in `cmake --build build -j "$JOBS"` + - `xargs` as in `xargs -P"$JOBS"` + + See [here](#controlling-the-number-of-threads-used-by-guix-build-commands) for + more details. + + _(defaults to the value of `nproc` outside the container)_ + +* _**SOURCE_DATE_EPOCH**_ + + Override the reference UNIX timestamp used for bit-for-bit reproducibility, + the variable name conforms to [standard][r12e/source-date-epoch]. + + _(defaults to the output of `$(git log --format=%at -1)`)_ + +* _**V**_ + + If non-empty, will pass `V=1` to all `make` invocations, making `make` output + verbose. + + Note that any given value is ignored. The variable is only checked for + emptiness. More concretely, this means that `V=` (setting `V` to the empty + string) is interpreted the same way as not setting `V` at all, and that `V=0` + has the same effect as `V=1`. + +* _**SUBSTITUTE_URLS**_ + + A whitespace-delimited list of URLs from which to download pre-built packages. + A URL is only used if its signing key is authorized (refer to the [substitute + servers section](#option-1-building-with-substitutes) for more details). + +* _**ADDITIONAL_GUIX_COMMON_FLAGS**_ + + Additional flags to be passed to all `guix` commands. + +* _**ADDITIONAL_GUIX_TIMEMACHINE_FLAGS**_ + + Additional flags to be passed to `guix time-machine`. + +* _**ADDITIONAL_GUIX_ENVIRONMENT_FLAGS**_ + + Additional flags to be passed to the invocation of `guix shell` inside + `guix time-machine`. + +# Choosing your security model + +No matter how you installed Guix, you need to decide on your security model for +building packages with Guix. + +Guix allows us to achieve better binary security by using our CPU time to build +everything from scratch. However, it doesn't sacrifice user choice in pursuit of +this: users can decide whether or not to use **substitutes** (pre-built +packages). + +## Option 1: Building with substitutes + +### Step 1: Authorize the signing keys + +Depending on the installation procedure you followed, you may have already +authorized the Guix build farm key. In particular, the official shell installer +script asks you if you want the key installed, and the debian distribution +package authorized the key during installation. + +You can check the current list of authorized keys at `/etc/guix/acl`. + +At the time of writing, a `/etc/guix/acl` with just the Guix build farm key +authorized looks something like: + +```lisp +(acl + (entry + (public-key + (ecc + (curve Ed25519) + (q #8D156F295D24B0D9A86FA5741A840FF2D24F60F7B6C4134814AD55625971B394#) + ) + ) + (tag + (guix import) + ) + ) + ) +``` + +If you've determined that the official Guix build farm key hasn't been +authorized, and you would like to authorize it, run the following as root: + +``` +guix archive --authorize < /var/guix/profiles/per-user/root/current-guix/share/guix/ci.guix.gnu.org.pub +``` + +If +`/var/guix/profiles/per-user/root/current-guix/share/guix/ci.guix.gnu.org.pub` +doesn't exist, try: + +```sh +guix archive --authorize < /share/guix/ci.guix.gnu.org.pub +``` + +Where `` is likely: +- `/usr` if you installed from a distribution package +- `/usr/local` if you installed Guix from source and didn't supply any + prefix-modifying flags to Guix's `./configure` + +#### Removing authorized keys + +To remove previously authorized keys, simply edit `/etc/guix/acl` and remove the +`(entry (public-key ...))` entry. + +### Step 2: Specify the substitute servers + +Once its key is authorized, the official Guix build farm at +https://ci.guix.gnu.org is automatically used unless the `--no-substitutes` flag +is supplied. This default list of substitute servers is overridable both on a +`guix-daemon` level and when you invoke `guix` commands. See examples below for +the various ways of adding a substitute server after having [authorized +its signing key](#step-1-authorize-the-signing-keys). + +Change the **default list** of substitute servers by starting `guix-daemon` with +the `--substitute-urls` option (you will likely need to edit your init script): + +```sh +guix-daemon --substitute-urls='https://bordeaux.guix.gnu.org https://ci.guix.gnu.org' +``` + +Override the default list of substitute servers by passing the +`--substitute-urls` option for invocations of `guix` commands: + +```sh +guix --substitute-urls='https://bordeaux.guix.gnu.org https://ci.guix.gnu.org' +``` + +For scripts under `./contrib/guix`, set the `SUBSTITUTE_URLS` environment +variable: + +```sh +export SUBSTITUTE_URLS='https://bordeaux.guix.gnu.org https://ci.guix.gnu.org' +``` + +## Option 2: Disabling substitutes on an ad-hoc basis + +If you prefer not to use any substitutes, make sure to supply `--no-substitutes` +like in the following snippet. The first build will take a while, but the +resulting packages will be cached for future builds. + +For direct invocations of `guix`: +```sh +guix --no-substitutes +``` + +For the scripts under `./contrib/guix/`: +```sh +export ADDITIONAL_GUIX_COMMON_FLAGS='--no-substitutes' +``` + +## Option 3: Disabling substitutes by default + +`guix-daemon` accepts a `--no-substitutes` flag, which will make sure that, +unless otherwise overridden by a command line invocation, no substitutes will be +used. + +If you start `guix-daemon` using an init script, you can edit said script to +supply this flag. + +[b17e]: https://bootstrappable.org/ +[r12e/source-date-epoch]: https://reproducible-builds.org/docs/source-date-epoch/ +[env-vars-list]: #recognized-environment-variables diff --git a/bench-ci/guix/guix-attest b/bench-ci/guix/guix-attest new file mode 100755 index 000000000000..b0ef28dc3f92 --- /dev/null +++ b/bench-ci/guix/guix-attest @@ -0,0 +1,263 @@ +#!/usr/bin/env bash +export LC_ALL=C +set -e -o pipefail + +# Source the common prelude, which: +# 1. Checks if we're at the top directory of the Bitcoin Core repository +# 2. Defines a few common functions and variables +# +# shellcheck source=libexec/prelude.bash +source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash" + + +################### +## Sanity Checks ## +################### + +################ +# Required non-builtin commands should be invokable +################ + +check_tools cat env basename mkdir diff sort + +if [ -z "$NO_SIGN" ]; then + # make it possible to override the gpg binary + GPG=${GPG:-gpg} + + # $GPG can contain extra arguments passed to the binary + # so let's check only the existence of arg[0] + # shellcheck disable=SC2206 + GPG_ARRAY=($GPG) + check_tools "${GPG_ARRAY[0]}" +fi + +################ +# Required env vars should be non-empty +################ + +cmd_usage() { +cat < \\ + SIGNER=GPG_KEY_NAME[=SIGNER_NAME] \\ + [ NO_SIGN=1 ] + ./contrib/guix/guix-attest + +Example w/o overriding signing name: + + env GUIX_SIGS_REPO=/home/achow101/guix.sigs \\ + SIGNER=achow101 \\ + ./contrib/guix/guix-attest + +Example overriding signing name: + + env GUIX_SIGS_REPO=/home/dongcarl/guix.sigs \\ + SIGNER=0x96AB007F1A7ED999=dongcarl \\ + ./contrib/guix/guix-attest + +Example w/o signing, just creating SHA256SUMS: + + env GUIX_SIGS_REPO=/home/achow101/guix.sigs \\ + SIGNER=achow101 \\ + NO_SIGN=1 \\ + ./contrib/guix/guix-attest + +EOF +} + +if [ -z "$GUIX_SIGS_REPO" ] || [ -z "$SIGNER" ]; then + cmd_usage + exit 1 +fi + +################ +# GUIX_SIGS_REPO should exist as a directory +################ + +if [ ! -d "$GUIX_SIGS_REPO" ]; then +cat << EOF +ERR: The specified GUIX_SIGS_REPO is not an existent directory: + + '$GUIX_SIGS_REPO' + +Hint: Please clone the guix.sigs repository and point to it with the + GUIX_SIGS_REPO environment variable. + +EOF +cmd_usage +exit 1 +fi + +################ +# The key specified in SIGNER should be usable +################ + +IFS='=' read -r gpg_key_name signer_name <<< "$SIGNER" +if [ -z "${signer_name}" ]; then + signer_name="$gpg_key_name" +fi + +if [ -z "$NO_SIGN" ] && ! ${GPG} --dry-run --list-secret-keys "${gpg_key_name}" >/dev/null 2>&1; then + echo "ERR: GPG can't seem to find any key named '${gpg_key_name}'" + exit 1 +fi + +################ +# We should be able to find at least one output +################ + +echo "Looking for build output SHA256SUMS fragments in ${OUTDIR_BASE}" + +shopt -s nullglob +sha256sum_fragments=( "$OUTDIR_BASE"/*/SHA256SUMS.part ) # This expands to an array of directories... +shopt -u nullglob + +noncodesigned_fragments=() +codesigned_fragments=() + +if (( ${#sha256sum_fragments[@]} )); then + echo "Found build output SHA256SUMS fragments:" + for outdir in "${sha256sum_fragments[@]}"; do + echo " '$outdir'" + case "$outdir" in + "$OUTDIR_BASE"/*-codesigned/SHA256SUMS.part) + codesigned_fragments+=("$outdir") + ;; + *) + noncodesigned_fragments+=("$outdir") + ;; + esac + done + echo +else + echo "ERR: Could not find any build output SHA256SUMS fragments in ${OUTDIR_BASE}" + exit 1 +fi + +############## +## Attest ## +############## + +# Usage: out_name $outdir +# +# HOST: The output directory being attested +# +out_name() { + basename "$(dirname "$1")" +} + +shasum_already_exists() { +cat < "$temp_noncodesigned" + if [ -e noncodesigned.SHA256SUMS ]; then + # The SHA256SUMS already exists, make sure it's exactly what we + # expect, error out if not + if diff -u noncodesigned.SHA256SUMS "$temp_noncodesigned"; then + echo "A noncodesigned.SHA256SUMS file already exists for '${VERSION}' and is up-to-date." + else + shasum_already_exists noncodesigned.SHA256SUMS + exit 1 + fi + else + mv "$temp_noncodesigned" noncodesigned.SHA256SUMS + fi + else + echo "ERR: No noncodesigned outputs found for '${VERSION}', exiting..." + exit 1 + fi + + temp_all="$(mktemp)" + trap 'rm -rf -- "$temp_all"' EXIT + + if (( ${#codesigned_fragments[@]} )); then + # Note: all.SHA256SUMS attests to all of $sha256sum_fragments, but is + # not needed if there are no $codesigned_fragments + cat "${sha256sum_fragments[@]}" \ + | sort -u \ + | sort -k2 \ + | basenameify_SHA256SUMS \ + > "$temp_all" + if [ -e all.SHA256SUMS ]; then + # The SHA256SUMS already exists, make sure it's exactly what we + # expect, error out if not + if diff -u all.SHA256SUMS "$temp_all"; then + echo "An all.SHA256SUMS file already exists for '${VERSION}' and is up-to-date." + else + shasum_already_exists all.SHA256SUMS + exit 1 + fi + else + mv "$temp_all" all.SHA256SUMS + fi + else + # It is fine to have the codesigned outputs be missing (perhaps the + # detached codesigs have not been published yet), just print a log + # message instead of erroring out + echo "INFO: No codesigned outputs found for '${VERSION}', skipping..." + fi + + if [ -z "$NO_SIGN" ]; then + echo "Signing SHA256SUMS to produce SHA256SUMS.asc" + for i in *.SHA256SUMS; do + if [ ! -e "$i".asc ]; then + ${GPG} --detach-sign \ + --digest-algo sha256 \ + --local-user "$gpg_key_name" \ + --armor \ + --output "$i".asc "$i" + else + echo "Signature already there" + fi + done + else + echo "Not signing SHA256SUMS as \$NO_SIGN is not empty" + fi + echo "" +) diff --git a/bench-ci/guix/guix-build b/bench-ci/guix/guix-build new file mode 100755 index 000000000000..ee285bf322cf --- /dev/null +++ b/bench-ci/guix/guix-build @@ -0,0 +1,474 @@ +#!/usr/bin/env bash +export LC_ALL=C +set -e -o pipefail + +# Source the common prelude, which: +# 1. Checks if we're at the top directory of the Bitcoin Core repository +# 2. Defines a few common functions and variables +# +# shellcheck source=libexec/prelude.bash +source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash" + + +################### +## SANITY CHECKS ## +################### + +################ +# Required non-builtin commands should be invocable +################ + +check_tools cat mkdir make getent curl git guix + +################ +# GUIX_BUILD_OPTIONS should be empty +################ +# +# GUIX_BUILD_OPTIONS is an environment variable recognized by guix commands that +# can perform builds. This seems like what we want instead of +# ADDITIONAL_GUIX_COMMON_FLAGS, but the value of GUIX_BUILD_OPTIONS is actually +# _appended_ to normal command-line options. Meaning that they will take +# precedence over the command-specific ADDITIONAL_GUIX__FLAGS. +# +# This seems like a poor user experience. Thus we check for GUIX_BUILD_OPTIONS's +# existence here and direct users of this script to use our (more flexible) +# custom environment variables. +if [ -n "$GUIX_BUILD_OPTIONS" ]; then +cat << EOF +Error: Environment variable GUIX_BUILD_OPTIONS is not empty: + '$GUIX_BUILD_OPTIONS' + +Unfortunately this script is incompatible with GUIX_BUILD_OPTIONS, please unset +GUIX_BUILD_OPTIONS and use ADDITIONAL_GUIX_COMMON_FLAGS to set build options +across guix commands or ADDITIONAL_GUIX__FLAGS to set build options for a +specific guix command. + +See contrib/guix/README.md for more details. +EOF +exit 1 +fi + +################ +# The git worktree should not be dirty +################ + +if ! git diff-index --quiet HEAD -- && [ -z "$FORCE_DIRTY_WORKTREE" ]; then +cat << EOF +ERR: The current git worktree is dirty, which may lead to broken builds. + + Aborting... + +Hint: To make your git worktree clean, You may want to: + 1. Commit your changes, + 2. Stash your changes, or + 3. Set the 'FORCE_DIRTY_WORKTREE' environment variable if you insist on + using a dirty worktree +EOF +exit 1 +fi + +mkdir -p "$VERSION_BASE" + +################ +# SOURCE_DATE_EPOCH should not unintentionally be set +################ + +check_source_date_epoch + +################ +# Build directories should not exist +################ + +# Default to building for all supported HOSTs (overridable by environment) +# powerpc64le-linux-gnu currently disabled due non-determinism issues across build arches. +export HOSTS="${HOSTS:-x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu powerpc64-linux-gnu + x86_64-w64-mingw32 + x86_64-apple-darwin arm64-apple-darwin}" + +# Usage: distsrc_for_host HOST +# +# HOST: The current platform triple we're building for +# +distsrc_for_host() { + echo "${DISTSRC_BASE}/distsrc-${VERSION}-${1}" +} + +# Accumulate a list of build directories that already exist... +hosts_distsrc_exists="" +for host in $HOSTS; do + if [ -e "$(distsrc_for_host "$host")" ]; then + hosts_distsrc_exists+=" ${host}" + fi +done + +if [ -n "$hosts_distsrc_exists" ]; then +# ...so that we can print them out nicely in an error message +cat << EOF +ERR: Build directories for this commit already exist for the following platform + triples you're attempting to build, probably because of previous builds. + Please remove, or otherwise deal with them prior to starting another build. + + Aborting... + +Hint: To blow everything away, you may want to use: + + $ ./contrib/guix/guix-clean + +Specifically, this will remove all files without an entry in the index, +excluding the SDK directory, the depends download cache, the depends built +packages cache, the garbage collector roots for Guix environments, and the +output directory. +EOF +for host in $hosts_distsrc_exists; do + echo " ${host} '$(distsrc_for_host "$host")'" +done +exit 1 +else + mkdir -p "$DISTSRC_BASE" +fi + +################ +# When building for darwin, the macOS SDK should exist +################ + +for host in $HOSTS; do + case "$host" in + *darwin*) + OSX_SDK="$(make -C "${PWD}/depends" --no-print-directory HOST="$host" print-OSX_SDK | sed 's@^[^=]\+=@@g')" + if [ -e "$OSX_SDK" ]; then + echo "Found macOS SDK at '${OSX_SDK}', using..." + break + else + echo "macOS SDK does not exist at '${OSX_SDK}', please place the extracted, untarred SDK there to perform darwin builds, or define SDK_PATH environment variable. Exiting..." + exit 1 + fi + ;; + esac +done + +################ +# VERSION_BASE should have enough space +################ + +avail_KiB="$(df -Pk "$VERSION_BASE" | sed 1d | tr -s ' ' | cut -d' ' -f4)" +total_required_KiB=0 +for host in $HOSTS; do + case "$host" in + *darwin*) required_KiB=440000 ;; + *mingw*) required_KiB=7600000 ;; + *) required_KiB=6400000 ;; + esac + total_required_KiB=$((total_required_KiB+required_KiB)) +done + +if (( total_required_KiB > avail_KiB )); then + total_required_GiB=$((total_required_KiB / 1048576)) + avail_GiB=$((avail_KiB / 1048576)) + echo "Performing a Bitcoin Core Guix build for the selected HOSTS requires ${total_required_GiB} GiB, however, only ${avail_GiB} GiB is available. Please free up some disk space before performing the build." + exit 1 +fi + +################ +# Check that we can connect to the guix-daemon +################ + +cat << EOF +Checking that we can connect to the guix-daemon... + +Hint: If this hangs, you may want to try turning your guix-daemon off and on + again. + +EOF +if ! guix gc --list-failures > /dev/null; then +cat << EOF + +ERR: Failed to connect to the guix-daemon, please ensure that one is running and + reachable. +EOF +exit 1 +fi + +# Developer note: we could use `guix repl` for this check and run: +# +# (import (guix store)) (close-connection (open-connection)) +# +# However, the internal API is likely to change more than the CLI invocation + +################ +# Services database must have basic entries +################ + +if ! getent services http https ftp > /dev/null 2>&1; then +cat << EOF +ERR: Your system's C library cannot find service database entries for at least + one of the following services: http, https, ftp. + +Hint: Most likely, /etc/services does not exist yet (common for docker images + and minimal distros), or you don't have permissions to access it. + + If /etc/services does not exist yet, you may want to install the + appropriate package for your distro which provides it. + + On Debian/Ubuntu: netbase + On Arch Linux: iana-etc + + For more information, see: getent(1), services(5) + +EOF + +fi + +######### +# SETUP # +######### + +# Determine the maximum number of jobs to run simultaneously (overridable by +# environment) +JOBS="${JOBS:-$(nproc)}" + +# Usage: host_to_commonname HOST +# +# HOST: The current platform triple we're building for +# +host_to_commonname() { + case "$1" in + *darwin*) echo osx ;; + *mingw*) echo win ;; + *linux*) echo linux ;; + *) exit 1 ;; + esac +} + +# Determine the reference time used for determinism (overridable by environment) +SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git -c log.showSignature=false log --format=%at -1)}" + +# Precious directories are those which should not be cleaned between successive +# guix builds +depends_precious_dir_names='SOURCES_PATH BASE_CACHE SDK_PATH' +precious_dir_names="${depends_precious_dir_names} OUTDIR_BASE PROFILES_BASE" + +# Usage: contains IFS-SEPARATED-LIST ITEM +contains() { + for i in ${1}; do + if [ "$i" = "${2}" ]; then + return 0 # Found! + fi + done + return 1 +} + +# If the user explicitly specified a precious directory, create it so we +# can map it into the container +for precious_dir_name in $precious_dir_names; do + precious_dir_path="${!precious_dir_name}" + if [ -n "$precious_dir_path" ]; then + if [ ! -e "$precious_dir_path" ]; then + mkdir -p "$precious_dir_path" + elif [ -L "$precious_dir_path" ]; then + echo "ERR: ${precious_dir_name} cannot be a symbolic link" + exit 1 + elif [ ! -d "$precious_dir_path" ]; then + echo "ERR: ${precious_dir_name} must be a directory" + exit 1 + fi + fi +done + +mkdir -p "$VAR_BASE" + +# Record the _effective_ values of precious directories such that guix-clean can +# avoid clobbering them if appropriate. +# +# shellcheck disable=SC2046,SC2086 +{ + # Get depends precious dir definitions from depends + make -C "${PWD}/depends" \ + --no-print-directory \ + -- $(printf "print-%s\n" $depends_precious_dir_names) + + # Get remaining precious dir definitions from the environment + for precious_dir_name in $precious_dir_names; do + precious_dir_path="${!precious_dir_name}" + if ! contains "$depends_precious_dir_names" "$precious_dir_name"; then + echo "${precious_dir_name}=${precious_dir_path}" + fi + done +} > "${VAR_BASE}/precious_dirs" + +# Make sure an output directory exists for our builds +OUTDIR_BASE="${OUTDIR_BASE:-${VERSION_BASE}/output}" +mkdir -p "$OUTDIR_BASE" + +# Download the depends sources now as we won't have internet access in the build +# container +for host in $HOSTS; do + make -C "${PWD}/depends" -j"$JOBS" download-"$(host_to_commonname "$host")" ${V:+V=1} ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} +done + +# Usage: outdir_for_host HOST SUFFIX +# +# HOST: The current platform triple we're building for +# +outdir_for_host() { + echo "${OUTDIR_BASE}/${1}${2:+-${2}}" +} + +# Usage: profiledir_for_host HOST SUFFIX +# +# HOST: The current platform triple we're building for +# +profiledir_for_host() { + echo "${PROFILES_BASE}/${1}${2:+-${2}}" +} + + +######### +# BUILD # +######### + +# Function to be called when building for host ${1} and the user interrupts the +# build +int_trap() { +cat << EOF +** INT received while building ${1}, you may want to clean up the relevant + work directories (e.g. distsrc-*) before rebuilding + +Hint: To blow everything away, you may want to use: + + $ ./contrib/guix/guix-clean + +Specifically, this will remove all files without an entry in the index, +excluding the SDK directory, the depends download cache, the depends built +packages cache, the garbage collector roots for Guix environments, and the +output directory. +EOF +} + +# Deterministically build Bitcoin Core +# shellcheck disable=SC2153 +for host in $HOSTS; do + + # Display proper warning when the user interrupts the build + trap 'int_trap ${host}' INT + + ( + # Required for 'contrib/guix/manifest.scm' to output the right manifest + # for the particular $HOST we're building for + export HOST="$host" + + # shellcheck disable=SC2030 +cat << EOF +INFO: Building ${VERSION:?not set} for platform triple ${HOST:?not set}: + ...using reference timestamp: ${SOURCE_DATE_EPOCH:?not set} + ...running at most ${JOBS:?not set} jobs + ...from worktree directory: '${PWD}' + ...bind-mounted in container to: '/bitcoin' + ...in build directory: '$(distsrc_for_host "$HOST")' + ...bind-mounted in container to: '$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")' + ...outputting in: '$(outdir_for_host "$HOST")' + ...bind-mounted in container to: '$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST")' + ADDITIONAL FLAGS (if set) + ADDITIONAL_GUIX_COMMON_FLAGS: ${ADDITIONAL_GUIX_COMMON_FLAGS} + ADDITIONAL_GUIX_ENVIRONMENT_FLAGS: ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} + ADDITIONAL_GUIX_TIMEMACHINE_FLAGS: ${ADDITIONAL_GUIX_TIMEMACHINE_FLAGS} +EOF + + # Run the build script 'contrib/guix/libexec/build.sh' in the build + # container specified by 'contrib/guix/manifest.scm'. + # + # Explanation of `guix shell` flags: + # + # --container run command within an isolated container + # + # Running in an isolated container minimizes build-time differences + # between machines and improves reproducibility + # + # --pure unset existing environment variables + # + # Same rationale as --container + # + # --no-cwd do not share current working directory with an + # isolated container + # + # When --container is specified, the default behavior is to share + # the current working directory with the isolated container at the + # same exact path (e.g. mapping '/home/satoshi/bitcoin/' to + # '/home/satoshi/bitcoin/'). This means that the $PWD inside the + # container becomes a source of irreproducibility. --no-cwd disables + # this behaviour. + # + # --share=SPEC for containers, share writable host file system + # according to SPEC + # + # --share="$PWD"=/bitcoin + # + # maps our current working directory to /bitcoin + # inside the isolated container, which we later cd + # into. + # + # While we don't want to map our current working directory to the + # same exact path (as this introduces irreproducibility), we do want + # it to be at a _fixed_ path _somewhere_ inside the isolated + # container so that we have something to build. '/bitcoin' was + # chosen arbitrarily. + # + # ${SOURCES_PATH:+--share="$SOURCES_PATH"} + # + # make the downloaded depends sources path available + # inside the isolated container + # + # The isolated container has no network access as it's in a + # different network namespace from the main machine, so we have to + # make the downloaded depends sources available to it. The sources + # should have been downloaded prior to this invocation. + # + # --keep-failed keep build tree of failed builds + # + # When builds of the Guix environment itself (not Bitcoin Core) + # fail, it is useful for the build tree to be kept for debugging + # purposes. + # + # ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} + # + # fetch substitute from SUBSTITUTE_URLS if they are + # authorized + # + # Depending on the user's security model, it may be desirable to use + # substitutes (pre-built packages) from servers that the user trusts. + # Please read the README.md in the same directory as this file for + # more information. + # + # shellcheck disable=SC2086,SC2031 + time-machine shell --manifest="${PWD}/contrib/guix/manifest.scm" \ + --container \ + --pure \ + --no-cwd \ + --share="$PWD"=/bitcoin \ + --share="$DISTSRC_BASE"=/distsrc-base \ + --share="$OUTDIR_BASE"=/outdir-base \ + --expose="$(git rev-parse --git-common-dir)" \ + ${SOURCES_PATH:+--share="$SOURCES_PATH"} \ + ${BASE_CACHE:+--share="$BASE_CACHE"} \ + ${SDK_PATH:+--share="$SDK_PATH"} \ + --cores="$JOBS" \ + --keep-failed \ + --fallback \ + --link-profile \ + --root="$(profiledir_for_host "${HOST}")" \ + ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ + ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ + -- env HOST="$host" \ + DISTNAME="$DISTNAME" \ + JOBS="$JOBS" \ + SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:?unable to determine value}" \ + ${V:+V=1} \ + ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} \ + ${BASE_CACHE:+BASE_CACHE="$BASE_CACHE"} \ + ${SDK_PATH:+SDK_PATH="$SDK_PATH"} \ + DISTSRC="$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")" \ + OUTDIR="$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST")" \ + DIST_ARCHIVE_BASE=/outdir-base/dist-archive \ + bash -c "cd /bitcoin && bash contrib/guix/libexec/build.sh" + ) + +done diff --git a/bench-ci/guix/guix-clean b/bench-ci/guix/guix-clean new file mode 100755 index 000000000000..9af0a793cff7 --- /dev/null +++ b/bench-ci/guix/guix-clean @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +export LC_ALL=C +set -e -o pipefail + +# Source the common prelude, which: +# 1. Checks if we're at the top directory of the Bitcoin Core repository +# 2. Defines a few common functions and variables +# +# shellcheck source=libexec/prelude.bash +source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash" + + +################### +## Sanity Checks ## +################### + +################ +# Required non-builtin commands should be invokable +################ + +check_tools cat mkdir make git guix + + +############# +## Clean ## +############# + +# Usage: under_dir MAYBE_PARENT MAYBE_CHILD +# +# If MAYBE_CHILD is a subdirectory of MAYBE_PARENT, print the relative path +# from MAYBE_PARENT to MAYBE_CHILD. Otherwise, return 1 as the error code. +# +# NOTE: This does not perform any symlink-resolving or path canonicalization. +# +under_dir() { + local path_residue + path_residue="${2##"${1}"}" + if [ -z "$path_residue" ] || [ "$path_residue" = "$2" ]; then + return 1 + else + echo "$path_residue" + fi +} + +# Usage: dir_under_git_root MAYBE_CHILD +# +# If MAYBE_CHILD is under the current git repository and exists, print the +# relative path from the git repository's top-level directory to MAYBE_CHILD, +# otherwise, exit with an error code. +# +dir_under_git_root() { + local rv + rv="$(under_dir "$(git_root)" "$1")" + [ -n "$rv" ] && echo "$rv" +} + +shopt -s nullglob +found_precious_dirs_files=( "${version_base_prefix}"*/"${var_base_basename}/precious_dirs" ) # This expands to an array of directories... +shopt -u nullglob + +exclude_flags=() + +for precious_dirs_file in "${found_precious_dirs_files[@]}"; do + # Make sure the precious directories (e.g. SOURCES_PATH, BASE_CACHE, SDK_PATH) + # are excluded from git-clean + echo "Found precious_dirs file: '${precious_dirs_file}'" + + # Exclude the precious_dirs file itself + if dirs_file_exclude_fragment=$(dir_under_git_root "$(dirname "$precious_dirs_file")"); then + exclude_flags+=( --exclude="${dirs_file_exclude_fragment}/precious_dirs" ) + fi + + # Read each 'name=dir' pair from the precious_dirs file + while IFS='=' read -r name dir; do + # Add an exclusion flag if the precious directory is under the git root. + if under=$(dir_under_git_root "$dir"); then + echo "Avoiding ${name}: ${under}" + exclude_flags+=( --exclude="$under" ) + fi + done < "$precious_dirs_file" +done + +git clean -xdff "${exclude_flags[@]}" diff --git a/bench-ci/guix/guix-codesign b/bench-ci/guix/guix-codesign new file mode 100755 index 000000000000..ac7aae3a1802 --- /dev/null +++ b/bench-ci/guix/guix-codesign @@ -0,0 +1,384 @@ +#!/usr/bin/env bash +export LC_ALL=C +set -e -o pipefail + +# Source the common prelude, which: +# 1. Checks if we're at the top directory of the Bitcoin Core repository +# 2. Defines a few common functions and variables +# +# shellcheck source=libexec/prelude.bash +source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash" + + +################### +## SANITY CHECKS ## +################### + +################ +# Required non-builtin commands should be invocable +################ + +check_tools cat mkdir git guix + +################ +# Required env vars should be non-empty +################ + +cmd_usage() { + cat < \\ + ./contrib/guix/guix-codesign + +EOF +} + +if [ -z "$DETACHED_SIGS_REPO" ]; then + cmd_usage + exit 1 +fi + +################ +# GUIX_BUILD_OPTIONS should be empty +################ +# +# GUIX_BUILD_OPTIONS is an environment variable recognized by guix commands that +# can perform builds. This seems like what we want instead of +# ADDITIONAL_GUIX_COMMON_FLAGS, but the value of GUIX_BUILD_OPTIONS is actually +# _appended_ to normal command-line options. Meaning that they will take +# precedence over the command-specific ADDITIONAL_GUIX__FLAGS. +# +# This seems like a poor user experience. Thus we check for GUIX_BUILD_OPTIONS's +# existence here and direct users of this script to use our (more flexible) +# custom environment variables. +if [ -n "$GUIX_BUILD_OPTIONS" ]; then +cat << EOF +Error: Environment variable GUIX_BUILD_OPTIONS is not empty: + '$GUIX_BUILD_OPTIONS' + +Unfortunately this script is incompatible with GUIX_BUILD_OPTIONS, please unset +GUIX_BUILD_OPTIONS and use ADDITIONAL_GUIX_COMMON_FLAGS to set build options +across guix commands or ADDITIONAL_GUIX__FLAGS to set build options for a +specific guix command. + +See contrib/guix/README.md for more details. +EOF +exit 1 +fi + +################ +# SOURCE_DATE_EPOCH should not unintentionally be set +################ + +check_source_date_epoch + +################ +# The codesignature git worktree should not be dirty +################ + +if ! git -C "$DETACHED_SIGS_REPO" diff-index --quiet HEAD -- && [ -z "$FORCE_DIRTY_WORKTREE" ]; then + cat << EOF +ERR: The DETACHED CODESIGNATURE git worktree is dirty, which may lead to broken builds. + + Aborting... + +Hint: To make your git worktree clean, You may want to: + 1. Commit your changes, + 2. Stash your changes, or + 3. Set the 'FORCE_DIRTY_WORKTREE' environment variable if you insist on + using a dirty worktree +EOF + exit 1 +fi + +################ +# Build directories should not exist +################ + +# Default to building for all supported HOSTs (overridable by environment) +export HOSTS="${HOSTS:-x86_64-w64-mingw32 x86_64-apple-darwin arm64-apple-darwin}" + +# Usage: distsrc_for_host HOST +# +# HOST: The current platform triple we're building for +# +distsrc_for_host() { + echo "${DISTSRC_BASE}/distsrc-${VERSION}-${1}-codesigned" +} + +# Accumulate a list of build directories that already exist... +hosts_distsrc_exists="" +for host in $HOSTS; do + if [ -e "$(distsrc_for_host "$host")" ]; then + hosts_distsrc_exists+=" ${host}" + fi +done + +if [ -n "$hosts_distsrc_exists" ]; then +# ...so that we can print them out nicely in an error message +cat << EOF +ERR: Build directories for this commit already exist for the following platform + triples you're attempting to build, probably because of previous builds. + Please remove, or otherwise deal with them prior to starting another build. + + Aborting... + +Hint: To blow everything away, you may want to use: + + $ ./contrib/guix/guix-clean + +Specifically, this will remove all files without an entry in the index, +excluding the SDK directory, the depends download cache, the depends built +packages cache, the garbage collector roots for Guix environments, and the +output directory. +EOF +for host in $hosts_distsrc_exists; do + echo " ${host} '$(distsrc_for_host "$host")'" +done +exit 1 +else + mkdir -p "$DISTSRC_BASE" +fi + + +################ +# Codesigning tarballs SHOULD exist +################ + +# Usage: outdir_for_host HOST SUFFIX +# +# HOST: The current platform triple we're building for +# +outdir_for_host() { + echo "${OUTDIR_BASE}/${1}${2:+-${2}}" +} + + +codesigning_tarball_for_host() { + case "$1" in + *mingw*) + echo "$(outdir_for_host "$1")/${DISTNAME}-win64-codesigning.tar.gz" + ;; + *darwin*) + echo "$(outdir_for_host "$1")/${DISTNAME}-${1}-codesigning.tar.gz" + ;; + *) + exit 1 + ;; + esac +} + +# Accumulate a list of build directories that already exist... +hosts_codesigning_tarball_missing="" +for host in $HOSTS; do + if [ ! -e "$(codesigning_tarball_for_host "$host")" ]; then + hosts_codesigning_tarball_missing+=" ${host}" + fi +done + +if [ -n "$hosts_codesigning_tarball_missing" ]; then + # ...so that we can print them out nicely in an error message + cat << EOF +ERR: Codesigning tarballs do not exist +... + +EOF +for host in $hosts_codesigning_tarball_missing; do + echo " ${host} '$(codesigning_tarball_for_host "$host")'" +done +exit 1 +fi + +################ +# Check that we can connect to the guix-daemon +################ + +cat << EOF +Checking that we can connect to the guix-daemon... + +Hint: If this hangs, you may want to try turning your guix-daemon off and on + again. + +EOF +if ! guix gc --list-failures > /dev/null; then + cat << EOF + +ERR: Failed to connect to the guix-daemon, please ensure that one is running and + reachable. +EOF + exit 1 +fi + +# Developer note: we could use `guix repl` for this check and run: +# +# (import (guix store)) (close-connection (open-connection)) +# +# However, the internal API is likely to change more than the CLI invocation + + +######### +# SETUP # +######### + +# Determine the maximum number of jobs to run simultaneously (overridable by +# environment) +JOBS="${JOBS:-$(nproc)}" + +# Determine the reference time used for determinism (overridable by environment) +SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git -c log.showSignature=false log --format=%at -1)}" + +# Make sure an output directory exists for our builds +OUTDIR_BASE="${OUTDIR_BASE:-${VERSION_BASE}/output}" +mkdir -p "$OUTDIR_BASE" + +# Usage: profiledir_for_host HOST SUFFIX +# +# HOST: The current platform triple we're building for +# +profiledir_for_host() { + echo "${PROFILES_BASE}/${1}${2:+-${2}}" +} + +######### +# BUILD # +######### + +# Function to be called when codesigning for host ${1} and the user interrupts +# the codesign +int_trap() { +cat << EOF +** INT received while codesigning ${1}, you may want to clean up the relevant + work directories (e.g. distsrc-*) before recodesigning + +Hint: To blow everything away, you may want to use: + + $ ./contrib/guix/guix-clean + +Specifically, this will remove all files without an entry in the index, +excluding the SDK directory, the depends download cache, the depends built +packages cache, the garbage collector roots for Guix environments, and the +output directory. +EOF +} + +# Deterministically build Bitcoin Core +# shellcheck disable=SC2153 +for host in $HOSTS; do + + # Display proper warning when the user interrupts the build + trap 'int_trap ${host}' INT + + ( + # Required for 'contrib/guix/manifest.scm' to output the right manifest + # for the particular $HOST we're building for + export HOST="$host" + + # shellcheck disable=SC2030 +cat << EOF +INFO: Codesigning ${VERSION:?not set} for platform triple ${HOST:?not set}: + ...using reference timestamp: ${SOURCE_DATE_EPOCH:?not set} + ...from worktree directory: '${PWD}' + ...bind-mounted in container to: '/bitcoin' + ...in build directory: '$(distsrc_for_host "$HOST")' + ...bind-mounted in container to: '$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")' + ...outputting in: '$(outdir_for_host "$HOST" codesigned)' + ...bind-mounted in container to: '$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST" codesigned)' + ...using detached signatures in: '${DETACHED_SIGS_REPO:?not set}' + ...bind-mounted in container to: '/detached-sigs' +EOF + + + # Run the build script 'contrib/guix/libexec/build.sh' in the build + # container specified by 'contrib/guix/manifest.scm'. + # + # Explanation of `guix shell` flags: + # + # --container run command within an isolated container + # + # Running in an isolated container minimizes build-time differences + # between machines and improves reproducibility + # + # --pure unset existing environment variables + # + # Same rationale as --container + # + # --no-cwd do not share current working directory with an + # isolated container + # + # When --container is specified, the default behavior is to share + # the current working directory with the isolated container at the + # same exact path (e.g. mapping '/home/satoshi/bitcoin/' to + # '/home/satoshi/bitcoin/'). This means that the $PWD inside the + # container becomes a source of irreproducibility. --no-cwd disables + # this behaviour. + # + # --share=SPEC for containers, share writable host file system + # according to SPEC + # + # --share="$PWD"=/bitcoin + # + # maps our current working directory to /bitcoin + # inside the isolated container, which we later cd + # into. + # + # While we don't want to map our current working directory to the + # same exact path (as this introduces irreproducibility), we do want + # it to be at a _fixed_ path _somewhere_ inside the isolated + # container so that we have something to build. '/bitcoin' was + # chosen arbitrarily. + # + # ${SOURCES_PATH:+--share="$SOURCES_PATH"} + # + # make the downloaded depends sources path available + # inside the isolated container + # + # The isolated container has no network access as it's in a + # different network namespace from the main machine, so we have to + # make the downloaded depends sources available to it. The sources + # should have been downloaded prior to this invocation. + # + # ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} + # + # fetch substitute from SUBSTITUTE_URLS if they are + # authorized + # + # Depending on the user's security model, it may be desirable to use + # substitutes (pre-built packages) from servers that the user trusts. + # Please read the README.md in the same directory as this file for + # more information. + # + # shellcheck disable=SC2086,SC2031 + time-machine shell --manifest="${PWD}/contrib/guix/manifest.scm" \ + --container \ + --pure \ + --no-cwd \ + --share="$PWD"=/bitcoin \ + --share="$DISTSRC_BASE"=/distsrc-base \ + --share="$OUTDIR_BASE"=/outdir-base \ + --share="$DETACHED_SIGS_REPO"=/detached-sigs \ + --expose="$(git rev-parse --git-common-dir)" \ + --expose="$(git -C "$DETACHED_SIGS_REPO" rev-parse --git-common-dir)" \ + ${SOURCES_PATH:+--share="$SOURCES_PATH"} \ + --cores="$JOBS" \ + --keep-failed \ + --fallback \ + --link-profile \ + --root="$(profiledir_for_host "${HOST}" codesigned)" \ + ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ + ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ + -- env HOST="$host" \ + DISTNAME="$DISTNAME" \ + JOBS="$JOBS" \ + SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:?unable to determine value}" \ + ${V:+V=1} \ + ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} \ + DISTSRC="$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")" \ + OUTDIR="$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST" codesigned)" \ + DIST_ARCHIVE_BASE=/outdir-base/dist-archive \ + DETACHED_SIGS_REPO=/detached-sigs \ + CODESIGNING_TARBALL="$(OUTDIR_BASE=/outdir-base && codesigning_tarball_for_host "$HOST")" \ + bash -c "cd /bitcoin && bash contrib/guix/libexec/codesign.sh" + ) + +done diff --git a/bench-ci/guix/guix-verify b/bench-ci/guix/guix-verify new file mode 100755 index 000000000000..02ae022741ba --- /dev/null +++ b/bench-ci/guix/guix-verify @@ -0,0 +1,174 @@ +#!/usr/bin/env bash +export LC_ALL=C +set -e -o pipefail + +# Source the common prelude, which: +# 1. Checks if we're at the top directory of the Bitcoin Core repository +# 2. Defines a few common functions and variables +# +# shellcheck source=libexec/prelude.bash +source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash" + + +################### +## Sanity Checks ## +################### + +################ +# Required non-builtin commands should be invokable +################ + +check_tools cat diff gpg + +################ +# Required env vars should be non-empty +################ + +cmd_usage() { +cat < [ SIGNER= ] ./contrib/guix/guix-verify + +Example overriding signer's manifest to use as base + + env GUIX_SIGS_REPO=/home/dongcarl/guix.sigs SIGNER=achow101 ./contrib/guix/guix-verify + +EOF +} + +if [ -z "$GUIX_SIGS_REPO" ]; then + cmd_usage + exit 1 +fi + +################ +# GUIX_SIGS_REPO should exist as a directory +################ + +if [ ! -d "$GUIX_SIGS_REPO" ]; then +cat << EOF +ERR: The specified GUIX_SIGS_REPO is not an existent directory: + + '$GUIX_SIGS_REPO' + +Hint: Please clone the guix.sigs repository and point to it with the + GUIX_SIGS_REPO environment variable. + +EOF +cmd_usage +exit 1 +fi + +############## +## Verify ## +############## + +OUTSIGDIR_BASE="${GUIX_SIGS_REPO}/${VERSION}" +echo "Looking for signature directories in '${OUTSIGDIR_BASE}'" +echo "" + +# Usage: verify compare_manifest current_manifest +verify() { + local compare_manifest="$1" + local current_manifest="$2" + if ! gpg --quiet --batch --verify "$current_manifest".asc "$current_manifest" 1>&2; then + echo "ERR: Failed to verify GPG signature in '${current_manifest}'" + echo "" + echo "Hint: Either the signature is invalid or the public key is missing" + echo "" + failure=1 + elif ! diff --report-identical "$compare_manifest" "$current_manifest" 1>&2; then + echo "ERR: The SHA256SUMS attestation in these two directories differ:" + echo " '${compare_manifest}'" + echo " '${current_manifest}'" + echo "" + failure=1 + else + echo "Verified: '${current_manifest}'" + echo "" + fi +} + +shopt -s nullglob +all_noncodesigned=( "$OUTSIGDIR_BASE"/*/noncodesigned.SHA256SUMS ) +shopt -u nullglob + +echo "--------------------" +echo "" +if (( ${#all_noncodesigned[@]} )); then + compare_noncodesigned="${all_noncodesigned[0]}" + if [[ -n "$SIGNER" ]]; then + signer_noncodesigned="$OUTSIGDIR_BASE/$SIGNER/noncodesigned.SHA256SUMS" + if [[ -f "$signer_noncodesigned" ]]; then + echo "Using $SIGNER's manifest as the base to compare against" + compare_noncodesigned="$signer_noncodesigned" + else + echo "Unable to find $SIGNER's manifest, using the first one found" + fi + else + echo "No SIGNER provided, using the first manifest found" + fi + + for current_manifest in "${all_noncodesigned[@]}"; do + verify "$compare_noncodesigned" "$current_manifest" + done + + echo "DONE: Checking output signatures for noncodesigned.SHA256SUMS" + echo "" +else + echo "WARN: No signature directories with noncodesigned.SHA256SUMS found" + echo "" +fi + +shopt -s nullglob +all_all=( "$OUTSIGDIR_BASE"/*/all.SHA256SUMS ) +shopt -u nullglob + +echo "--------------------" +echo "" +if (( ${#all_all[@]} )); then + compare_all="${all_all[0]}" + if [[ -n "$SIGNER" ]]; then + signer_all="$OUTSIGDIR_BASE/$SIGNER/all.SHA256SUMS" + if [[ -f "$signer_all" ]]; then + echo "Using $SIGNER's manifest as the base to compare against" + compare_all="$signer_all" + else + echo "Unable to find $SIGNER's manifest, using the first one found" + fi + else + echo "No SIGNER provided, using the first manifest found" + fi + + for current_manifest in "${all_all[@]}"; do + verify "$compare_all" "$current_manifest" + done + + # Sanity check: there should be no entries that exist in + # noncodesigned.SHA256SUMS that doesn't exist in all.SHA256SUMS + if [[ "$(comm -23 <(sort "$compare_noncodesigned") <(sort "$compare_all") | wc -c)" -ne 0 ]]; then + echo "ERR: There are unique lines in noncodesigned.SHA256SUMS which" + echo " do not exist in all.SHA256SUMS, something went very wrong." + exit 1 + fi + + echo "DONE: Checking output signatures for all.SHA256SUMS" + echo "" +else + echo "WARN: No signature directories with all.SHA256SUMS found" + echo "" +fi + +echo "====================" +echo "" +if (( ${#all_noncodesigned[@]} + ${#all_all[@]} == 0 )); then + echo "ERR: Unable to perform any verifications as no signature directories" + echo " were found" + echo "" + exit 1 +fi + +if [ -n "$failure" ]; then + exit 1 +fi diff --git a/bench-ci/guix/libexec/build.sh b/bench-ci/guix/libexec/build.sh new file mode 100755 index 000000000000..16e12d563816 --- /dev/null +++ b/bench-ci/guix/libexec/build.sh @@ -0,0 +1,403 @@ +#!/usr/bin/env bash +# Copyright (c) 2019-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +export LC_ALL=C +set -e -o pipefail +export TZ=UTC + +# Although Guix _does_ set umask when building its own packages (in our case, +# this is all packages in manifest.scm), it does not set it for `guix +# shell`. It does make sense for at least `guix shell --container` +# to set umask, so if that change gets merged upstream and we bump the +# time-machine to a commit which includes the aforementioned change, we can +# remove this line. +# +# This line should be placed before any commands which creates files. +umask 0022 + +if [ -n "$V" ]; then + # Print both unexpanded (-v) and expanded (-x) forms of commands as they are + # read from this file. + set -vx + # Set VERBOSE for CMake-based builds + export VERBOSE="$V" +fi + +# Check that required environment variables are set +cat << EOF +Required environment variables as seen inside the container: + DIST_ARCHIVE_BASE: ${DIST_ARCHIVE_BASE:?not set} + DISTNAME: ${DISTNAME:?not set} + HOST: ${HOST:?not set} + SOURCE_DATE_EPOCH: ${SOURCE_DATE_EPOCH:?not set} + JOBS: ${JOBS:?not set} + DISTSRC: ${DISTSRC:?not set} + OUTDIR: ${OUTDIR:?not set} +EOF + +ACTUAL_OUTDIR="${OUTDIR}" +OUTDIR="${DISTSRC}/output" + +##################### +# Environment Setup # +##################### + +# The depends folder also serves as a base-prefix for depends packages for +# $HOSTs after successfully building. +BASEPREFIX="${PWD}/depends" + +# Given a package name and an output name, return the path of that output in our +# current guix environment +store_path() { + grep --extended-regexp "/[^-]{32}-${1}-[^-]+${2:+-${2}}" "${GUIX_ENVIRONMENT}/manifest" \ + | head --lines=1 \ + | sed --expression='s|\x29*$||' \ + --expression='s|^[[:space:]]*"||' \ + --expression='s|"[[:space:]]*$||' +} + + +# Set environment variables to point the NATIVE toolchain to the right +# includes/libs +NATIVE_GCC="$(store_path gcc-toolchain)" + +unset LIBRARY_PATH +unset CPATH +unset C_INCLUDE_PATH +unset CPLUS_INCLUDE_PATH +unset OBJC_INCLUDE_PATH +unset OBJCPLUS_INCLUDE_PATH + +# Set native toolchain +build_CC="${NATIVE_GCC}/bin/gcc -isystem ${NATIVE_GCC}/include" +build_CXX="${NATIVE_GCC}/bin/g++ -isystem ${NATIVE_GCC}/include/c++ -isystem ${NATIVE_GCC}/include" + +case "$HOST" in + *darwin*) export LIBRARY_PATH="${NATIVE_GCC}/lib" ;; # Required for native packages + *mingw*) export LIBRARY_PATH="${NATIVE_GCC}/lib" ;; + *) + NATIVE_GCC_STATIC="$(store_path gcc-toolchain static)" + export LIBRARY_PATH="${NATIVE_GCC}/lib:${NATIVE_GCC_STATIC}/lib" + ;; +esac + +# Set environment variables to point the CROSS toolchain to the right +# includes/libs for $HOST +case "$HOST" in + *mingw*) + # Determine output paths to use in CROSS_* environment variables + CROSS_GLIBC="$(store_path "mingw-w64-x86_64-winpthreads")" + CROSS_GCC="$(store_path "gcc-cross-${HOST}")" + CROSS_GCC_LIB_STORE="$(store_path "gcc-cross-${HOST}" lib)" + CROSS_GCC_LIBS=( "${CROSS_GCC_LIB_STORE}/lib/gcc/${HOST}"/* ) # This expands to an array of directories... + CROSS_GCC_LIB="${CROSS_GCC_LIBS[0]}" # ...we just want the first one (there should only be one) + + # The search path ordering is generally: + # 1. gcc-related search paths + # 2. libc-related search paths + # 2. kernel-header-related search paths (not applicable to mingw-w64 hosts) + export CROSS_C_INCLUDE_PATH="${CROSS_GCC_LIB}/include:${CROSS_GCC_LIB}/include-fixed:${CROSS_GLIBC}/include" + export CROSS_CPLUS_INCLUDE_PATH="${CROSS_GCC}/include/c++:${CROSS_GCC}/include/c++/${HOST}:${CROSS_GCC}/include/c++/backward:${CROSS_C_INCLUDE_PATH}" + export CROSS_LIBRARY_PATH="${CROSS_GCC_LIB_STORE}/lib:${CROSS_GCC_LIB}:${CROSS_GLIBC}/lib" + ;; + *darwin*) + # The CROSS toolchain for darwin uses the SDK and ignores environment variables. + # See depends/hosts/darwin.mk for more details. + ;; + *linux*) + CROSS_GLIBC="$(store_path "glibc-cross-${HOST}")" + CROSS_GLIBC_STATIC="$(store_path "glibc-cross-${HOST}" static)" + CROSS_KERNEL="$(store_path "linux-libre-headers-cross-${HOST}")" + CROSS_GCC="$(store_path "gcc-cross-${HOST}")" + CROSS_GCC_LIB_STORE="$(store_path "gcc-cross-${HOST}" lib)" + CROSS_GCC_LIBS=( "${CROSS_GCC_LIB_STORE}/lib/gcc/${HOST}"/* ) # This expands to an array of directories... + CROSS_GCC_LIB="${CROSS_GCC_LIBS[0]}" # ...we just want the first one (there should only be one) + + export CROSS_C_INCLUDE_PATH="${CROSS_GCC_LIB}/include:${CROSS_GCC_LIB}/include-fixed:${CROSS_GLIBC}/include:${CROSS_KERNEL}/include" + export CROSS_CPLUS_INCLUDE_PATH="${CROSS_GCC}/include/c++:${CROSS_GCC}/include/c++/${HOST}:${CROSS_GCC}/include/c++/backward:${CROSS_C_INCLUDE_PATH}" + export CROSS_LIBRARY_PATH="${CROSS_GCC_LIB_STORE}/lib:${CROSS_GCC_LIB}:${CROSS_GLIBC}/lib:${CROSS_GLIBC_STATIC}/lib" + ;; + *) + exit 1 ;; +esac + +# Sanity check CROSS_*_PATH directories +IFS=':' read -ra PATHS <<< "${CROSS_C_INCLUDE_PATH}:${CROSS_CPLUS_INCLUDE_PATH}:${CROSS_LIBRARY_PATH}" +for p in "${PATHS[@]}"; do + if [ -n "$p" ] && [ ! -d "$p" ]; then + echo "'$p' doesn't exist or isn't a directory... Aborting..." + exit 1 + fi +done + +# Disable Guix ld auto-rpath behavior +export GUIX_LD_WRAPPER_DISABLE_RPATH=yes + +# Make /usr/bin if it doesn't exist +[ -e /usr/bin ] || mkdir -p /usr/bin + +# Symlink env to a conventional path +[ -e /usr/bin/env ] || ln -s --no-dereference "$(command -v env)" /usr/bin/env + +# Determine the correct value for -Wl,--dynamic-linker for the current $HOST +case "$HOST" in + *linux*) + glibc_dynamic_linker=$( + case "$HOST" in + x86_64-linux-gnu) echo /lib64/ld-linux-x86-64.so.2 ;; + arm-linux-gnueabihf) echo /lib/ld-linux-armhf.so.3 ;; + aarch64-linux-gnu) echo /lib/ld-linux-aarch64.so.1 ;; + riscv64-linux-gnu) echo /lib/ld-linux-riscv64-lp64d.so.1 ;; + powerpc64-linux-gnu) echo /lib64/ld64.so.1;; + powerpc64le-linux-gnu) echo /lib64/ld64.so.2;; + *) exit 1 ;; + esac + ) + ;; +esac + +# Environment variables for determinism +export TAR_OPTIONS="--owner=0 --group=0 --numeric-owner --mtime='@${SOURCE_DATE_EPOCH}' --sort=name" +export TZ="UTC" + +#################### +# Depends Building # +#################### + +# Build the depends tree, overriding variables that assume multilib gcc +make -C depends --jobs="$JOBS" HOST="$HOST" \ + ${V:+V=1} \ + ${SOURCES_PATH+SOURCES_PATH="$SOURCES_PATH"} \ + ${BASE_CACHE+BASE_CACHE="$BASE_CACHE"} \ + ${SDK_PATH+SDK_PATH="$SDK_PATH"} \ + ${build_CC+build_CC="$build_CC"} \ + ${build_CXX+build_CXX="$build_CXX"} \ + x86_64_linux_CC=x86_64-linux-gnu-gcc \ + x86_64_linux_CXX=x86_64-linux-gnu-g++ \ + x86_64_linux_AR=x86_64-linux-gnu-gcc-ar \ + x86_64_linux_RANLIB=x86_64-linux-gnu-gcc-ranlib \ + x86_64_linux_NM=x86_64-linux-gnu-gcc-nm \ + x86_64_linux_STRIP=x86_64-linux-gnu-strip + +case "$HOST" in + *darwin*) + # Unset now that Qt is built + unset LIBRARY_PATH + ;; +esac + +########################### +# Source Tarball Building # +########################### + +GIT_ARCHIVE="${DIST_ARCHIVE_BASE}/${DISTNAME}.tar.gz" + +# Create the source tarball if not already there +if [ ! -e "$GIT_ARCHIVE" ]; then + mkdir -p "$(dirname "$GIT_ARCHIVE")" + git archive --prefix="${DISTNAME}/" --output="$GIT_ARCHIVE" HEAD +fi + +mkdir -p "$OUTDIR" + +########################### +# Binary Tarball Building # +########################### + +# CONFIGFLAGS +CONFIGFLAGS="-DREDUCE_EXPORTS=ON -DBUILD_BENCH=OFF -DBUILD_GUI_TESTS=OFF -DBUILD_FUZZ_BINARY=OFF -DCMAKE_SKIP_RPATH=TRUE" + +# CFLAGS +HOST_CFLAGS="-O2 -g" +HOST_CFLAGS+=$(find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;) +case "$HOST" in + *mingw*) HOST_CFLAGS+=" -fno-ident" ;; + *darwin*) unset HOST_CFLAGS ;; +esac + +# CXXFLAGS +HOST_CXXFLAGS="$HOST_CFLAGS" + +case "$HOST" in + arm-linux-gnueabihf) HOST_CXXFLAGS="${HOST_CXXFLAGS} -Wno-psabi" ;; +esac + +# LDFLAGS +case "$HOST" in + *linux*) HOST_LDFLAGS="-Wl,--as-needed -Wl,--dynamic-linker=$glibc_dynamic_linker -static-libstdc++ -Wl,-O2" ;; + *mingw*) HOST_LDFLAGS="-Wl,--no-insert-timestamp" ;; +esac + +mkdir -p "$DISTSRC" +( + cd "$DISTSRC" + + # Extract the source tarball + tar --strip-components=1 -xf "${GIT_ARCHIVE}" + + # Configure this DISTSRC for $HOST + # shellcheck disable=SC2086 + env CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" LDFLAGS="${HOST_LDFLAGS}" \ + cmake -S . -B build \ + --toolchain "${BASEPREFIX}/${HOST}/toolchain.cmake" \ + -DWITH_CCACHE=OFF \ + -Werror=dev \ + ${CONFIGFLAGS} + + # Build Bitcoin Core + cmake --build build -j "$JOBS" ${V:+--verbose} + + # Perform basic security checks on a series of executables. + cmake --build build -j 1 --target check-security ${V:+--verbose} + # Check that executables only contain allowed version symbols. + cmake --build build -j 1 --target check-symbols ${V:+--verbose} + + mkdir -p "$OUTDIR" + + # Make the os-specific installers + case "$HOST" in + *mingw*) + cmake --build build -j "$JOBS" -t deploy ${V:+--verbose} + mv build/bitcoin-win64-setup.exe "${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe" + ;; + esac + + # Setup the directory where our Bitcoin Core build for HOST will be + # installed. This directory will also later serve as the input for our + # binary tarballs. + INSTALLPATH="${PWD}/installed/${DISTNAME}" + mkdir -p "${INSTALLPATH}" + # Install built Bitcoin Core to $INSTALLPATH + case "$HOST" in + *darwin*) + # This workaround can be dropped for CMake >= 3.27. + # See the upstream commit 689616785f76acd844fd448c51c5b2a0711aafa2. + find build -name 'cmake_install.cmake' -exec sed -i 's| -u -r | |g' {} + + + cmake --install build --strip --prefix "${INSTALLPATH}" ${V:+--verbose} + ;; + *) + cmake --install build --prefix "${INSTALLPATH}" ${V:+--verbose} + ;; + esac + + ( + cd installed + + case "$HOST" in + *darwin*) ;; + *) + # Split binaries from their debug symbols + { + find "${DISTNAME}/bin" "${DISTNAME}/libexec" -type f -executable -print0 + } | xargs -0 -P"$JOBS" -I{} "${DISTSRC}/build/split-debug.sh" {} {} {}.dbg + ;; + esac + + case "$HOST" in + *mingw*) + cp "${DISTSRC}/doc/README_windows.txt" "${DISTNAME}/readme.txt" + ;; + *linux*) + cp "${DISTSRC}/README.md" "${DISTNAME}/" + ;; + esac + + # copy over the example bitcoin.conf file. if contrib/devtools/gen-bitcoin-conf.sh + # has not been run before buildling, this file will be a stub + cp "${DISTSRC}/share/examples/bitcoin.conf" "${DISTNAME}/" + + cp -r "${DISTSRC}/share/rpcauth" "${DISTNAME}/share/" + + # Deterministically produce {non-,}debug binary tarballs ready + # for release + case "$HOST" in + *mingw*) + find "${DISTNAME}" -not -name "*.dbg" -print0 \ + | xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}" + find "${DISTNAME}" -not -name "*.dbg" \ + | sort \ + | zip -X@ "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}-unsigned.zip" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}-unsigned.zip" && exit 1 ) + find "${DISTNAME}" -name "*.dbg" -print0 \ + | xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}" + find "${DISTNAME}" -name "*.dbg" \ + | sort \ + | zip -X@ "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}-debug.zip" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}-debug.zip" && exit 1 ) + ;; + *linux*) + find "${DISTNAME}" -not -name "*.dbg" -print0 \ + | sort --zero-terminated \ + | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ + | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" && exit 1 ) + find "${DISTNAME}" -name "*.dbg" -print0 \ + | sort --zero-terminated \ + | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ + | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}-debug.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}-debug.tar.gz" && exit 1 ) + ;; + *darwin*) + find "${DISTNAME}" -print0 \ + | sort --zero-terminated \ + | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ + | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}-unsigned.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}-unsigned.tar.gz" && exit 1 ) + ;; + esac + ) # $DISTSRC/installed + + # Finally make tarballs for codesigning + case "$HOST" in + *mingw*) + cp -rf --target-directory=. contrib/windeploy + ( + cd ./windeploy + mkdir -p unsigned + cp --target-directory=unsigned/ "${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe" + cp -r --target-directory=unsigned/ "${INSTALLPATH}" + find unsigned/ -name "*.dbg" -print0 \ + | xargs -0r rm + find . -print0 \ + | sort --zero-terminated \ + | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ + | gzip -9n > "${OUTDIR}/${DISTNAME}-win64-codesigning.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-win64-codesigning.tar.gz" && exit 1 ) + ) + ;; + *darwin*) + cmake --build build --target deploy ${V:+--verbose} + mv build/dist/bitcoin-macos-app.zip "${OUTDIR}/${DISTNAME}-${HOST}-unsigned.zip" + mkdir -p "unsigned-app-${HOST}" + cp --target-directory="unsigned-app-${HOST}" \ + contrib/macdeploy/detached-sig-create.sh + mv --target-directory="unsigned-app-${HOST}" build/dist + cp -r --target-directory="unsigned-app-${HOST}" "${INSTALLPATH}" + ( + cd "unsigned-app-${HOST}" + find . -print0 \ + | sort --zero-terminated \ + | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ + | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}-codesigning.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}-codesigning.tar.gz" && exit 1 ) + ) + ;; + esac +) # $DISTSRC + +rm -rf "$ACTUAL_OUTDIR" +mv --no-target-directory "$OUTDIR" "$ACTUAL_OUTDIR" \ + || ( rm -rf "$ACTUAL_OUTDIR" && exit 1 ) + +( + cd /outdir-base + { + echo "$GIT_ARCHIVE" + find "$ACTUAL_OUTDIR" -type f + } | xargs realpath --relative-base="$PWD" \ + | xargs sha256sum \ + | sort -k2 \ + | sponge "$ACTUAL_OUTDIR"/SHA256SUMS.part +) diff --git a/bench-ci/guix/libexec/codesign.sh b/bench-ci/guix/libexec/codesign.sh new file mode 100755 index 000000000000..fe86065350e9 --- /dev/null +++ b/bench-ci/guix/libexec/codesign.sh @@ -0,0 +1,153 @@ +#!/usr/bin/env bash +# Copyright (c) 2021-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +export LC_ALL=C +set -e -o pipefail + +# Environment variables for determinism +export TAR_OPTIONS="--owner=0 --group=0 --numeric-owner --mtime='@${SOURCE_DATE_EPOCH}' --sort=name" +export TZ=UTC + +# Although Guix _does_ set umask when building its own packages (in our case, +# this is all packages in manifest.scm), it does not set it for `guix +# shell`. It does make sense for at least `guix shell --container` +# to set umask, so if that change gets merged upstream and we bump the +# time-machine to a commit which includes the aforementioned change, we can +# remove this line. +# +# This line should be placed before any commands which creates files. +umask 0022 + +if [ -n "$V" ]; then + # Print both unexpanded (-v) and expanded (-x) forms of commands as they are + # read from this file. + set -vx + # Set VERBOSE for CMake-based builds + export VERBOSE="$V" +fi + +# Check that required environment variables are set +cat << EOF +Required environment variables as seen inside the container: + CODESIGNING_TARBALL: ${CODESIGNING_TARBALL:?not set} + DETACHED_SIGS_REPO: ${DETACHED_SIGS_REPO:?not set} + DIST_ARCHIVE_BASE: ${DIST_ARCHIVE_BASE:?not set} + DISTNAME: ${DISTNAME:?not set} + HOST: ${HOST:?not set} + SOURCE_DATE_EPOCH: ${SOURCE_DATE_EPOCH:?not set} + DISTSRC: ${DISTSRC:?not set} + OUTDIR: ${OUTDIR:?not set} +EOF + +ACTUAL_OUTDIR="${OUTDIR}" +OUTDIR="${DISTSRC}/output" + +git_head_version() { + local recent_tag + if recent_tag="$(git -C "$1" describe --exact-match HEAD 2> /dev/null)"; then + echo "${recent_tag#v}" + else + git -C "$1" rev-parse --short=12 HEAD + fi +} + +CODESIGNATURE_GIT_ARCHIVE="${DIST_ARCHIVE_BASE}/${DISTNAME}-codesignatures-$(git_head_version "$DETACHED_SIGS_REPO").tar.gz" + +# Create the codesignature tarball if not already there +if [ ! -e "$CODESIGNATURE_GIT_ARCHIVE" ]; then + mkdir -p "$(dirname "$CODESIGNATURE_GIT_ARCHIVE")" + git -C "$DETACHED_SIGS_REPO" archive --output="$CODESIGNATURE_GIT_ARCHIVE" HEAD +fi + +mkdir -p "$OUTDIR" + +mkdir -p "$DISTSRC" +( + cd "$DISTSRC" + + tar -xf "$CODESIGNING_TARBALL" + + mkdir -p codesignatures + tar -C codesignatures -xf "$CODESIGNATURE_GIT_ARCHIVE" + + case "$HOST" in + *mingw*) + # Apply detached codesignatures + WORKDIR=".tmp" + mkdir -p ${WORKDIR} + cp -r --target-directory="${WORKDIR}" "unsigned/${DISTNAME}" + find "${WORKDIR}/${DISTNAME}" -name "*.exe" -type f -exec rm {} \; + find unsigned/ -name "*.exe" -type f | while read -r bin + do + bin_base="$(realpath --relative-to=unsigned/ "${bin}")" + mkdir -p "${WORKDIR}/$(dirname "${bin_base}")" + osslsigncode attach-signature \ + -in "${bin}" \ + -out "${WORKDIR}/${bin_base/-unsigned}" \ + -CAfile "$GUIX_ENVIRONMENT/etc/ssl/certs/ca-certificates.crt" \ + -sigin codesignatures/win/"${bin_base}".pem + done + + # Move installer to outdir + cd "${WORKDIR}" + find . -name "*setup.exe" -print0 \ + | xargs -0r mv --target-directory="${OUTDIR}" + + # Make .zip from binaries + find "${DISTNAME}" -print0 \ + | xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}" + find "${DISTNAME}" \ + | sort \ + | zip -X@ "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}.zip" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}.zip" && exit 1 ) + ;; + *darwin*) + case "$HOST" in + arm64*) ARCH="arm64" ;; + x86_64*) ARCH="x86_64" ;; + esac + + # Apply detached codesignatures (in-place) + signapple apply dist/Bitcoin-Qt.app codesignatures/osx/"${HOST}"/dist/Bitcoin-Qt.app + find "${DISTNAME}" \( -wholename "*/bin/*" -o -wholename "*/libexec/*" \) -type f | while read -r bin + do + signapple apply "${bin}" "codesignatures/osx/${HOST}/${bin}.${ARCH}sign" + done + + # Make a .zip from dist/ + cd dist/ + find . -print0 \ + | xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}" + find . | sort \ + | zip -X@ "${OUTDIR}/${DISTNAME}-${HOST}.zip" + cd .. + + # Make a .tar.gz from bins + find "${DISTNAME}" -print0 \ + | sort --zero-terminated \ + | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ + | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" && exit 1 ) + ;; + *) + exit 1 + ;; + esac +) # $DISTSRC + +rm -rf "$ACTUAL_OUTDIR" +mv --no-target-directory "$OUTDIR" "$ACTUAL_OUTDIR" \ + || ( rm -rf "$ACTUAL_OUTDIR" && exit 1 ) + +( + cd /outdir-base + { + echo "$CODESIGNING_TARBALL" + echo "$CODESIGNATURE_GIT_ARCHIVE" + find "$ACTUAL_OUTDIR" -type f + } | xargs realpath --relative-base="$PWD" \ + | xargs sha256sum \ + | sort -k2 \ + | sponge "$ACTUAL_OUTDIR"/SHA256SUMS.part +) diff --git a/bench-ci/guix/libexec/prelude.bash b/bench-ci/guix/libexec/prelude.bash new file mode 100644 index 000000000000..b7c13cc91d8c --- /dev/null +++ b/bench-ci/guix/libexec/prelude.bash @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +export LC_ALL=C +set -e -o pipefail + +# shellcheck source=contrib/shell/realpath.bash +source contrib/shell/realpath.bash + +# shellcheck source=contrib/shell/git-utils.bash +source contrib/shell/git-utils.bash + +################ +# Required non-builtin commands should be invocable +################ + +check_tools() { + for cmd in "$@"; do + if ! command -v "$cmd" > /dev/null 2>&1; then + echo "ERR: This script requires that '$cmd' is installed and available in your \$PATH" + exit 1 + fi + done +} + +################ +# SOURCE_DATE_EPOCH should not unintentionally be set +################ + +check_source_date_epoch() { + if [ -n "$SOURCE_DATE_EPOCH" ] && [ -z "$FORCE_SOURCE_DATE_EPOCH" ]; then + cat << EOF +ERR: Environment variable SOURCE_DATE_EPOCH is set which may break reproducibility. + + Aborting... + +Hint: You may want to: + 1. Unset this variable: \`unset SOURCE_DATE_EPOCH\` before rebuilding + 2. Set the 'FORCE_SOURCE_DATE_EPOCH' environment variable if you insist on + using your own epoch +EOF + exit 1 + fi +} + +check_tools cat env readlink dirname basename git + +################ +# We should be at the top directory of the repository +################ + +same_dir() { + local resolved1 resolved2 + resolved1="$(bash_realpath "${1}")" + resolved2="$(bash_realpath "${2}")" + [ "$resolved1" = "$resolved2" ] +} + +if ! same_dir "${PWD}" "$(git_root)"; then +cat << EOF +ERR: This script must be invoked from the top level of the git repository + +Hint: This may look something like: + env FOO=BAR ./contrib/guix/guix- + +EOF +exit 1 +fi + +################ +# Execute "$@" in a pinned, possibly older version of Guix, for reproducibility +# across time. +time-machine() { + # shellcheck disable=SC2086 + guix time-machine --url=https://codeberg.org/guix/guix.git \ + --commit=5cb84f2013c5b1e48a7d0e617032266f1e6059e2 \ + --cores="$JOBS" \ + --keep-failed \ + --fallback \ + ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ + ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_TIMEMACHINE_FLAGS} \ + -- "$@" +} + + +################ +# Set common variables +################ + +VERSION="${FORCE_VERSION:-$(git_head_version)}" +DISTNAME="${DISTNAME:-bitcoin-${VERSION}}" + +version_base_prefix="${PWD}/guix-build-" +VERSION_BASE="${version_base_prefix}${VERSION}" # TOP + +DISTSRC_BASE="${DISTSRC_BASE:-${VERSION_BASE}}" + +OUTDIR_BASE="${OUTDIR_BASE:-${VERSION_BASE}/output}" + +var_base_basename="var" +VAR_BASE="${VAR_BASE:-${VERSION_BASE}/${var_base_basename}}" + +profiles_base_basename="profiles" +PROFILES_BASE="${PROFILES_BASE:-${VAR_BASE}/${profiles_base_basename}}" diff --git a/bench-ci/guix/manifest.scm b/bench-ci/guix/manifest.scm new file mode 100644 index 000000000000..59837e9647e9 --- /dev/null +++ b/bench-ci/guix/manifest.scm @@ -0,0 +1,574 @@ +(use-modules (gnu packages) + ((gnu packages bash) #:select (bash-minimal)) + (gnu packages bison) + ((gnu packages certs) #:select (nss-certs)) + ((gnu packages cmake) #:select (cmake-minimal)) + (gnu packages commencement) + (gnu packages compression) + (gnu packages cross-base) + (gnu packages gawk) + (gnu packages gcc) + ((gnu packages installers) #:select (nsis-x86_64)) + ((gnu packages linux) #:select (linux-libre-headers-6.1)) + (gnu packages llvm) + (gnu packages mingw) + (gnu packages ninja) + (gnu packages pkg-config) + ((gnu packages python) #:select (python-minimal)) + ((gnu packages python-build) #:select (python-poetry-core)) + ((gnu packages python-crypto) #:select (python-asn1crypto)) + ((gnu packages python-science) #:select (python-scikit-build-core)) + ((gnu packages python-xyz) #:select (python-pydantic-2)) + ((gnu packages tls) #:select (openssl)) + ((gnu packages version-control) #:select (git-minimal)) + (guix build-system cmake) + (guix build-system gnu) + (guix build-system python) + (guix build-system pyproject) + (guix build-system trivial) + (guix download) + (guix gexp) + (guix git-download) + ((guix licenses) #:prefix license:) + (guix packages) + ((guix utils) #:select (cc-for-target substitute-keyword-arguments))) + +(define-syntax-rule (search-our-patches file-name ...) + "Return the list of absolute file names corresponding to each +FILE-NAME found in ./patches relative to the current file." + (parameterize + ((%patch-path (list (string-append (dirname (current-filename)) "/patches")))) + (list (search-patch file-name) ...))) + +(define building-on (string-append "--build=" (list-ref (string-split (%current-system) #\-) 0) "-guix-linux-gnu")) + +(define (make-cross-toolchain target + base-gcc-for-libc + base-kernel-headers + base-libc + base-gcc) + "Create a cross-compilation toolchain package for TARGET" + (let* ((xbinutils (cross-binutils target)) + ;; 1. Build a cross-compiling gcc without targeting any libc, derived + ;; from BASE-GCC-FOR-LIBC + (xgcc-sans-libc (cross-gcc target + #:xgcc base-gcc-for-libc + #:xbinutils xbinutils)) + ;; 2. Build cross-compiled kernel headers with XGCC-SANS-LIBC, derived + ;; from BASE-KERNEL-HEADERS + (xkernel (cross-kernel-headers target + #:linux-headers base-kernel-headers + #:xgcc xgcc-sans-libc + #:xbinutils xbinutils)) + ;; 3. Build a cross-compiled libc with XGCC-SANS-LIBC and XKERNEL, + ;; derived from BASE-LIBC + (xlibc (cross-libc target + #:libc base-libc + #:xgcc xgcc-sans-libc + #:xbinutils xbinutils + #:xheaders xkernel)) + ;; 4. Build a cross-compiling gcc targeting XLIBC, derived from + ;; BASE-GCC + (xgcc (cross-gcc target + #:xgcc base-gcc + #:xbinutils xbinutils + #:libc xlibc))) + ;; Define a meta-package that propagates the resulting XBINUTILS, XLIBC, and + ;; XGCC + (package + (name (string-append target "-toolchain")) + (version (package-version xgcc)) + (source #f) + (build-system trivial-build-system) + (arguments '(#:builder (begin (mkdir %output) #t))) + (propagated-inputs + (list xbinutils + xlibc + xgcc + `(,xlibc "static") + `(,xgcc "lib"))) + (synopsis (string-append "Complete GCC tool chain for " target)) + (description (string-append "This package provides a complete GCC tool +chain for " target " development.")) + (home-page (package-home-page xgcc)) + (license (package-license xgcc))))) + +(define base-gcc gcc-13) ;; 13.3.0 + +(define base-linux-kernel-headers linux-libre-headers-6.1) + +(define* (make-bitcoin-cross-toolchain target + #:key + (base-gcc-for-libc linux-base-gcc) + (base-kernel-headers base-linux-kernel-headers) + (base-libc glibc-2.31) + (base-gcc linux-base-gcc)) + "Convenience wrapper around MAKE-CROSS-TOOLCHAIN with default values +desirable for building Bitcoin Core release binaries." + (make-cross-toolchain target + base-gcc-for-libc + base-kernel-headers + base-libc + base-gcc)) + +(define (gcc-mingw-patches gcc) + (package-with-extra-patches gcc + (search-our-patches "gcc-remap-guix-store.patch"))) + +(define (binutils-mingw-patches binutils) + (package-with-extra-patches binutils + (search-our-patches "binutils-unaligned-default.patch"))) + +(define (winpthreads-patches mingw-w64-x86_64-winpthreads) + (package-with-extra-patches mingw-w64-x86_64-winpthreads + (search-our-patches "winpthreads-remap-guix-store.patch"))) + +(define (make-mingw-pthreads-cross-toolchain target) + "Create a cross-compilation toolchain package for TARGET" + (let* ((xbinutils (binutils-mingw-patches (cross-binutils target))) + (machine (substring target 0 (string-index target #\-))) + (pthreads-xlibc (winpthreads-patches (make-mingw-w64 machine + #:xgcc (cross-gcc target #:xgcc (gcc-mingw-patches base-gcc)) + #:with-winpthreads? #t))) + (pthreads-xgcc (cross-gcc target + #:xgcc (gcc-mingw-patches mingw-w64-base-gcc) + #:xbinutils xbinutils + #:libc pthreads-xlibc))) + ;; Define a meta-package that propagates the resulting XBINUTILS, XLIBC, and + ;; XGCC + (package + (name (string-append target "-posix-toolchain")) + (version (package-version pthreads-xgcc)) + (source #f) + (build-system trivial-build-system) + (arguments '(#:builder (begin (mkdir %output) #t))) + (propagated-inputs + (list xbinutils + pthreads-xlibc + pthreads-xgcc + `(,pthreads-xgcc "lib"))) + (synopsis (string-append "Complete GCC tool chain for " target)) + (description (string-append "This package provides a complete GCC tool +chain for " target " development.")) + (home-page (package-home-page pthreads-xgcc)) + (license (package-license pthreads-xgcc))))) + +;; While LIEF is packaged in Guix, we maintain our own package, +;; to simplify building, and more easily apply updates. +;; Moreover, the Guix's package uses cmake, which caused build +;; failure; see https://github.com/bitcoin/bitcoin/pull/27296. +(define-public python-lief + (package + (name "python-lief") + (version "0.16.6") + (source (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/lief-project/LIEF") + (commit version))) + (file-name (git-file-name name version)) + (sha256 + (base32 + "1pq9nagrnkl1x943bqnpiyxmkd9vk99znfxiwqp6vf012b50bz2a")) + (patches (search-our-patches "lief-scikit-0-9.patch")))) + (build-system pyproject-build-system) + (native-inputs (list cmake-minimal + ninja + python-scikit-build-core + python-pydantic-2)) + (arguments + (list + #:tests? #f ;needs network + #:phases #~(modify-phases %standard-phases + (add-before 'build 'set-pythonpath + (lambda _ + (setenv "PYTHONPATH" + (string-append (string-append (getcwd) "/api/python/backend") + ":" (or (getenv "PYTHONPATH") ""))))) + (add-after 'set-pythonpath 'change-directory + (lambda _ + (chdir "api/python")))))) + (home-page "https://github.com/lief-project/LIEF") + (synopsis "Library to instrument executable formats") + (description + "@code{python-lief} is a cross platform library which can parse, modify +and abstract ELF, PE and MachO formats.") + (license license:asl2.0))) + +(define osslsigncode + (package + (name "osslsigncode") + (version "2.5") + (source (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/mtrojnar/osslsigncode") + (commit version))) + (sha256 + (base32 + "1j47vwq4caxfv0xw68kw5yh00qcpbd56d7rq6c483ma3y7s96yyz")))) + (build-system cmake-build-system) + (inputs (list openssl)) + (home-page "https://github.com/mtrojnar/osslsigncode") + (synopsis "Authenticode signing and timestamping tool") + (description "osslsigncode is a small tool that implements part of the +functionality of the Microsoft tool signtool.exe - more exactly the Authenticode +signing and timestamping. But osslsigncode is based on OpenSSL and cURL, and +thus should be able to compile on most platforms where these exist.") + (license license:gpl3+))) ; license is with openssl exception + +(define-public python-elfesteem + (let ((commit "2eb1e5384ff7a220fd1afacd4a0170acff54fe56")) + (package + (name "python-elfesteem") + (version (git-version "0.1" "1" commit)) + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/LRGH/elfesteem") + (commit commit))) + (file-name (git-file-name name commit)) + (sha256 + (base32 + "07x6p8clh11z8s1n2kdxrqwqm2almgc5qpkcr9ckb6y5ivjdr5r6")))) + (build-system python-build-system) + ;; There are no tests, but attempting to run python setup.py test leads to + ;; PYTHONPATH problems, just disable the test + (arguments '(#:tests? #f)) + (home-page "https://github.com/LRGH/elfesteem") + (synopsis "ELF/PE/Mach-O parsing library") + (description "elfesteem parses ELF, PE and Mach-O files.") + (license license:lgpl2.1)))) + +(define-public python-oscrypto + (package + (name "python-oscrypto") + (version "1.3.0") + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/wbond/oscrypto") + (commit version))) + (file-name (git-file-name name version)) + (sha256 + (base32 + "1v5wkmzcyiqy39db8j2dvkdrv2nlsc48556h73x4dzjwd6kg4q0a")) + (patches (search-our-patches "oscrypto-hard-code-openssl.patch")))) + (build-system python-build-system) + (native-search-paths + (list (search-path-specification + (variable "SSL_CERT_FILE") + (file-type 'regular) + (separator #f) ;single entry + (files '("etc/ssl/certs/ca-certificates.crt"))))) + + (propagated-inputs + (list python-asn1crypto openssl)) + (arguments + `(#:phases + (modify-phases %standard-phases + (add-after 'unpack 'hard-code-path-to-libscrypt + (lambda* (#:key inputs #:allow-other-keys) + (let ((openssl (assoc-ref inputs "openssl"))) + (substitute* "oscrypto/__init__.py" + (("@GUIX_OSCRYPTO_USE_OPENSSL@") + (string-append openssl "/lib/libcrypto.so" "," openssl "/lib/libssl.so"))) + #t))) + (add-after 'unpack 'disable-broken-tests + (lambda _ + ;; This test is broken as there is no keyboard interrupt. + (substitute* "tests/test_trust_list.py" + (("^(.*)class TrustListTests" line indent) + (string-append indent + "@unittest.skip(\"Disabled by Guix\")\n" + line))) + (substitute* "tests/test_tls.py" + (("^(.*)class TLSTests" line indent) + (string-append indent + "@unittest.skip(\"Disabled by Guix\")\n" + line))) + #t)) + (replace 'check + (lambda _ + (invoke "python" "run.py" "tests") + #t))))) + (home-page "https://github.com/wbond/oscrypto") + (synopsis "Compiler-free Python crypto library backed by the OS") + (description "oscrypto is a compilation-free, always up-to-date encryption library for Python.") + (license license:expat))) + +(define-public python-oscryptotests + (package (inherit python-oscrypto) + (name "python-oscryptotests") + (propagated-inputs + (list python-oscrypto)) + (arguments + `(#:tests? #f + #:phases + (modify-phases %standard-phases + (add-after 'unpack 'hard-code-path-to-libscrypt + (lambda* (#:key inputs #:allow-other-keys) + (chdir "tests") + #t))))))) + +(define-public python-certvalidator + (let ((commit "a145bf25eb75a9f014b3e7678826132efbba6213")) + (package + (name "python-certvalidator") + (version (git-version "0.1" "1" commit)) + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/achow101/certvalidator") + (commit commit))) + (file-name (git-file-name name commit)) + (sha256 + (base32 + "1qw2k7xis53179lpqdqyylbcmp76lj7sagp883wmxg5i7chhc96k")))) + (build-system python-build-system) + (propagated-inputs + (list python-asn1crypto + python-oscrypto + python-oscryptotests)) ;; certvalidator tests import oscryptotests + (arguments + `(#:phases + (modify-phases %standard-phases + (add-after 'unpack 'disable-broken-tests + (lambda _ + (substitute* "tests/test_certificate_validator.py" + (("^(.*)class CertificateValidatorTests" line indent) + (string-append indent + "@unittest.skip(\"Disabled by Guix\")\n" + line))) + (substitute* "tests/test_crl_client.py" + (("^(.*)def test_fetch_crl" line indent) + (string-append indent + "@unittest.skip(\"Disabled by Guix\")\n" + line))) + (substitute* "tests/test_ocsp_client.py" + (("^(.*)def test_fetch_ocsp" line indent) + (string-append indent + "@unittest.skip(\"Disabled by Guix\")\n" + line))) + (substitute* "tests/test_registry.py" + (("^(.*)def test_build_paths" line indent) + (string-append indent + "@unittest.skip(\"Disabled by Guix\")\n" + line))) + (substitute* "tests/test_validate.py" + (("^(.*)def test_revocation_mode_hard" line indent) + (string-append indent + "@unittest.skip(\"Disabled by Guix\")\n" + line))) + (substitute* "tests/test_validate.py" + (("^(.*)def test_revocation_mode_soft" line indent) + (string-append indent + "@unittest.skip(\"Disabled by Guix\")\n" + line))) + #t)) + (replace 'check + (lambda _ + (invoke "python" "run.py" "tests") + #t))))) + (home-page "https://github.com/wbond/certvalidator") + (synopsis "Python library for validating X.509 certificates and paths") + (description "certvalidator is a Python library for validating X.509 +certificates or paths. Supports various options, including: validation at a +specific moment in time, whitelisting and revocation checks.") + (license license:expat)))) + +(define-public python-signapple + (let ((commit "85bfcecc33d2773bc09bc318cec0614af2c8e287")) + (package + (name "python-signapple") + (version (git-version "0.2.0" "1" commit)) + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/achow101/signapple") + (commit commit))) + (file-name (git-file-name name commit)) + (sha256 + (base32 + "17yqjll8nw83q6dhgqhkl7w502z5vy9sln8m6mlx0f1c10isg8yg")))) + (build-system pyproject-build-system) + (propagated-inputs + (list python-asn1crypto + python-oscrypto + python-certvalidator + python-elfesteem)) + (native-inputs (list python-poetry-core)) + ;; There are no tests, but attempting to run python setup.py test leads to + ;; problems, just disable the test + (arguments '(#:tests? #f)) + (home-page "https://github.com/achow101/signapple") + (synopsis "Mach-O binary signature tool") + (description "signapple is a Python tool for creating, verifying, and +inspecting signatures in Mach-O binaries.") + (license license:expat)))) + +(define-public mingw-w64-base-gcc + (package + (inherit base-gcc) + (arguments + (substitute-keyword-arguments (package-arguments base-gcc) + ((#:configure-flags flags) + `(append ,flags + ;; https://gcc.gnu.org/install/configure.html + (list "--enable-threads=posix", + "--enable-default-ssp=yes", + "--disable-gcov", + building-on))))))) + +(define-public linux-base-gcc + (package + (inherit base-gcc) + (arguments + (substitute-keyword-arguments (package-arguments base-gcc) + ((#:configure-flags flags) + `(append ,flags + ;; https://gcc.gnu.org/install/configure.html + (list "--enable-initfini-array=yes", + "--enable-default-ssp=yes", + "--enable-default-pie=yes", + "--enable-standard-branch-protection=yes", + "--enable-cet=yes", + "--disable-gcov", + building-on))) + ((#:phases phases) + `(modify-phases ,phases + ;; Given a XGCC package, return a modified package that replace each instance of + ;; -rpath in the default system spec that's inserted by Guix with -rpath-link + (add-after 'pre-configure 'replace-rpath-with-rpath-link + (lambda _ + (substitute* (cons "gcc/config/rs6000/sysv4.h" + (find-files "gcc/config" + "^gnu-user.*\\.h$")) + (("-rpath=") "-rpath-link=")) + #t)))))))) + +(define-public glibc-2.31 + (let ((commit "7b27c450c34563a28e634cccb399cd415e71ebfe")) + (package + (inherit glibc) ;; 2.39 + (version "2.31") + (source (origin + (method git-fetch) + (uri (git-reference + (url "https://sourceware.org/git/glibc.git") + (commit commit))) + (file-name (git-file-name "glibc" commit)) + (sha256 + (base32 + "017qdpr5id7ddb4lpkzj2li1abvw916m3fc6n7nw28z4h5qbv2n0")) + (patches (search-our-patches "glibc-guix-prefix.patch" + "glibc-riscv-jumptarget.patch")))) + (arguments + (substitute-keyword-arguments (package-arguments glibc) + ((#:configure-flags flags) + `(append ,flags + ;; https://www.gnu.org/software/libc/manual/html_node/Configuring-and-compiling.html + (list "--enable-stack-protector=all", + "--enable-cet", + "--enable-bind-now", + "--disable-werror", + "--disable-timezone-tools", + "--disable-profile", + building-on))) + ((#:phases phases) + `(modify-phases ,phases + (add-before 'configure 'set-etc-rpc-installation-directory + (lambda* (#:key outputs #:allow-other-keys) + ;; Install the rpc data base file under `$out/etc/rpc'. + ;; Otherwise build will fail with "Permission denied." + ;; Can be removed when we are building 2.32 or later. + (let ((out (assoc-ref outputs "out"))) + (substitute* "sunrpc/Makefile" + (("^\\$\\(inst_sysconfdir\\)/rpc(.*)$" _ suffix) + (string-append out "/etc/rpc" suffix "\n")) + (("^install-others =.*$") + (string-append "install-others = " out "/etc/rpc\n"))))))))))))) + +;; The sponge tool from moreutils. +(define-public sponge + (package + (name "sponge") + (version "0.69") + (source (origin + (method url-fetch) + (uri (string-append + "https://git.joeyh.name/index.cgi/moreutils.git/snapshot/ + moreutils-" version ".tar.gz")) + (file-name (string-append "moreutils-" version ".tar.gz")) + (sha256 + (base32 + "1l859qnzccslvxlh5ghn863bkq2vgmqgnik6jr21b9kc6ljmsy8g")))) + (build-system gnu-build-system) + (arguments + (list #:phases + #~(modify-phases %standard-phases + (delete 'configure) + (replace 'install + (lambda* (#:key outputs #:allow-other-keys) + (let ((bin (string-append (assoc-ref outputs "out") "/bin"))) + (install-file "sponge" bin))))) + #:make-flags + #~(list "sponge" (string-append "CC=" #$(cc-for-target))))) + (home-page "https://joeyh.name/code/moreutils/") + (synopsis "Miscellaneous general-purpose command-line tools") + (description "Just sponge") + (license license:gpl2+))) + +(packages->manifest + (append + (list ;; The Basics + bash-minimal + which + coreutils-minimal + ;; File(system) inspection + grep + diffutils + findutils + ;; File transformation + patch + gawk + sed + sponge + ;; Compression and archiving + tar + gzip + xz + ;; Build tools + gcc-toolchain-13 + cmake-minimal + gnu-make + ninja + ;; Scripting + python-minimal ;; (3.10) + ;; Git + git-minimal + ;; Tests + python-lief) + (let ((target (getenv "HOST"))) + (cond ((string-suffix? "-mingw32" target) + (list zip + (make-mingw-pthreads-cross-toolchain "x86_64-w64-mingw32") + nsis-x86_64 + nss-certs + osslsigncode)) + ((string-contains target "-linux-") + (list bison + pkg-config + (list gcc-toolchain-13 "static") + (make-bitcoin-cross-toolchain target))) + ((string-contains target "darwin") + (list clang-toolchain-19 + lld-19 + (make-lld-wrapper lld-19 #:lld-as-ld? #t) + python-signapple + zip)) + (else '()))))) diff --git a/bench-ci/guix/patches/binutils-unaligned-default.patch b/bench-ci/guix/patches/binutils-unaligned-default.patch new file mode 100644 index 000000000000..d1bc71aee142 --- /dev/null +++ b/bench-ci/guix/patches/binutils-unaligned-default.patch @@ -0,0 +1,22 @@ +commit 6537181f59ed186a341db621812a6bc35e22eaf6 +Author: fanquake +Date: Wed Apr 10 12:15:52 2024 +0200 + + build: turn on -muse-unaligned-vector-move by default + + This allows us to avoid (more invasively) patching GCC, to avoid + unaligned instruction use. + +diff --git a/gas/config/tc-i386.c b/gas/config/tc-i386.c +index e0632681477..14a9653abdf 100644 +--- a/gas/config/tc-i386.c ++++ b/gas/config/tc-i386.c +@@ -801,7 +801,7 @@ static unsigned int no_cond_jump_promotion = 0; + static unsigned int sse2avx; + + /* Encode aligned vector move as unaligned vector move. */ +-static unsigned int use_unaligned_vector_move; ++static unsigned int use_unaligned_vector_move = 1; + + /* Encode scalar AVX instructions with specific vector length. */ + static enum diff --git a/bench-ci/guix/patches/gcc-remap-guix-store.patch b/bench-ci/guix/patches/gcc-remap-guix-store.patch new file mode 100644 index 000000000000..a8b41d485b04 --- /dev/null +++ b/bench-ci/guix/patches/gcc-remap-guix-store.patch @@ -0,0 +1,20 @@ +Without ffile-prefix-map, the debug symbols will contain paths for the +guix store which will include the hashes of each package. However, the +hash for the same package will differ when on different architectures. +In order to be reproducible regardless of the architecture used to build +the package, map all guix store prefixes to something fixed, e.g. /usr. + +--- a/libgcc/Makefile.in ++++ b/libgcc/Makefile.in +@@ -854,7 +854,7 @@ endif + # libgcc_eh.a, only LIB2ADDEH matters. If we do, only LIB2ADDEHSTATIC and + # LIB2ADDEHSHARED matter. (Usually all three are identical.) + +-c_flags := -fexceptions ++c_flags := -fexceptions $(shell find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;) + + ifeq ($(enable_shared),yes) + +-- +2.37.0 + diff --git a/bench-ci/guix/patches/glibc-2.42-guix-prefix.patch b/bench-ci/guix/patches/glibc-2.42-guix-prefix.patch new file mode 100644 index 000000000000..f2fc1b90f183 --- /dev/null +++ b/bench-ci/guix/patches/glibc-2.42-guix-prefix.patch @@ -0,0 +1,47 @@ +Without -ffile-prefix-map, the debug symbols will contain paths for the +guix store which will include the hashes of each package. However, the +hash for the same package will differ when on different architectures. +In order to be reproducible regardless of the architecture used to build +the package, map all guix store prefixes to something fixed, e.g. /usr. + +--- a/Makeconfig ++++ b/Makeconfig +@@ -1074,6 +1074,10 @@ CPPFLAGS-.o = $(pic-default) + CFLAGS-.o = $(filter %frame-pointer,$(+cflags)) $(pie-default) + CFLAGS-.o += $(call elide-fortify-source,.o,$(routines_no_fortify)) + CFLAGS-.o += $(call elide-fortify-source,_chk.o,$(routines_no_fortify)) ++ ++# Map Guix store paths to /usr ++CFLAGS-.o += `find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;` ++ + libtype.o := lib%.a + object-suffixes += .o + ifeq (yes,$(build-shared)) +diff --git a/iconv/Makefile b/iconv/Makefile +index afb3fb7bdb..5acee345e0 100644 +--- a/iconv/Makefile ++++ b/iconv/Makefile +@@ -65,6 +65,9 @@ CFLAGS-gconv_cache.c += -DGCONV_DIR='"$(gconvdir)"' + CFLAGS-gconv_conf.c += -DGCONV_PATH='"$(gconvdir)"' + CFLAGS-iconvconfig.c += -DGCONV_PATH='"$(gconvdir)"' -DGCONV_DIR='"$(gconvdir)"' + ++# Map Guix store paths to /usr ++CFLAGS-.c += `find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;` ++ + # Set libof-* for each routine. + cpp-srcs-left := $(iconv_prog-modules) $(iconvconfig-modules) + lib := iconvprogs +diff --git a/posix/Makefile b/posix/Makefile +index 3d368b91f6..d79d8fb648 100644 +--- a/posix/Makefile ++++ b/posix/Makefile +@@ -590,6 +590,9 @@ CFLAGS-execlp.os = -fomit-frame-pointer + CFLAGS-nanosleep.c += -fexceptions -fasynchronous-unwind-tables + CFLAGS-fork.c = $(libio-mtsafe) $(config-cflags-wno-ignored-attributes) + ++# Map Guix store paths to /usr ++CFLAGS-.c += `find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;` ++ + tstgetopt-ARGS = -a -b -cfoobar --required foobar --optional=bazbug \ + --none random --col --color --colour + diff --git a/bench-ci/guix/patches/glibc-guix-prefix.patch b/bench-ci/guix/patches/glibc-guix-prefix.patch new file mode 100644 index 000000000000..60e12ca52546 --- /dev/null +++ b/bench-ci/guix/patches/glibc-guix-prefix.patch @@ -0,0 +1,16 @@ +Without ffile-prefix-map, the debug symbols will contain paths for the +guix store which will include the hashes of each package. However, the +hash for the same package will differ when on different architectures. +In order to be reproducible regardless of the architecture used to build +the package, map all guix store prefixes to something fixed, e.g. /usr. + +--- a/Makeconfig ++++ b/Makeconfig +@@ -1007,6 +1007,7 @@ object-suffixes := + CPPFLAGS-.o = $(pic-default) + # libc.a must be compiled with -fPIE/-fpie for static PIE. + CFLAGS-.o = $(filter %frame-pointer,$(+cflags)) $(pie-default) ++CFLAGS-.o += `find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;` + libtype.o := lib%.a + object-suffixes += .o + ifeq (yes,$(build-shared)) diff --git a/bench-ci/guix/patches/glibc-riscv-jumptarget.patch b/bench-ci/guix/patches/glibc-riscv-jumptarget.patch new file mode 100644 index 000000000000..702959433d86 --- /dev/null +++ b/bench-ci/guix/patches/glibc-riscv-jumptarget.patch @@ -0,0 +1,57 @@ +commit 68389203832ab39dd0dbaabbc4059e7fff51c29b +Author: Fangrui Song +Date: Thu Oct 28 11:39:49 2021 -0700 + + riscv: Fix incorrect jal with HIDDEN_JUMPTARGET + + A non-local STV_DEFAULT defined symbol is by default preemptible in a + shared object. j/jal cannot target a preemptible symbol. On other + architectures, such a jump instruction either causes PLT [BZ #18822], or + if short-ranged, sometimes rejected by the linker (but not by GNU ld's + riscv port [ld PR/28509]). + + Use HIDDEN_JUMPTARGET to target a non-preemptible symbol instead. + + With this patch, ld.so and libc.so can be linked with LLD if source + files are compiled/assembled with -mno-relax/-Wa,-mno-relax. + + Acked-by: Palmer Dabbelt + Reviewed-by: Adhemerval Zanella + +Can be dropped when we are using glibc 2.35 or later. + +diff --git a/sysdeps/riscv/setjmp.S b/sysdeps/riscv/setjmp.S +index 0b92016b31..bec7ff80f4 100644 +--- a/sysdeps/riscv/setjmp.S ++++ b/sysdeps/riscv/setjmp.S +@@ -21,7 +21,7 @@ + + ENTRY (_setjmp) + li a1, 0 +- j __sigsetjmp ++ j HIDDEN_JUMPTARGET (__sigsetjmp) + END (_setjmp) + ENTRY (setjmp) + li a1, 1 +diff --git a/sysdeps/unix/sysv/linux/riscv/setcontext.S b/sysdeps/unix/sysv/linux/riscv/setcontext.S +index 9510518750..e44a68aad4 100644 +--- a/sysdeps/unix/sysv/linux/riscv/setcontext.S ++++ b/sysdeps/unix/sysv/linux/riscv/setcontext.S +@@ -95,6 +95,7 @@ LEAF (__setcontext) + 99: j __syscall_error + + END (__setcontext) ++libc_hidden_def (__setcontext) + weak_alias (__setcontext, setcontext) + + LEAF (__start_context) +@@ -108,7 +109,7 @@ LEAF (__start_context) + /* Invoke subsequent context if present, else exit(0). */ + mv a0, s2 + beqz s2, 1f +- jal __setcontext +-1: j exit ++ jal HIDDEN_JUMPTARGET (__setcontext) ++1: j HIDDEN_JUMPTARGET (exit) + + END (__start_context) diff --git a/bench-ci/guix/patches/lief-scikit-0-9.patch b/bench-ci/guix/patches/lief-scikit-0-9.patch new file mode 100644 index 000000000000..71e617834f07 --- /dev/null +++ b/bench-ci/guix/patches/lief-scikit-0-9.patch @@ -0,0 +1,21 @@ +Partially revert f23ced2f4ffc170d0a6f40ff4a1bee575e3447cf + +Restore compat with python-scikit-build-core 0.9.x +Can be dropped when using python-scikit-build-core >= 0.10.x + +--- a/api/python/backend/setup.py ++++ b/api/python/backend/setup.py +@@ -101,12 +101,12 @@ def _get_hooked_config(is_editable: bool) -> Optional[dict[str, Union[str, List[ + config_settings = { + "logging.level": "DEBUG", + "build-dir": config.build_dir, +- "build.targets": config.build.targets, + "install.strip": config.strip, + "backport.find-python": "0", + "wheel.py-api": config.build.py_api, + "cmake.source-dir": SRC_DIR.as_posix(), + "cmake.build-type": config.build.build_type, ++ "cmake.targets": config.build.targets, + "cmake.args": [ + *config.cmake_generator, + *config.get_cmake_args(is_editable), diff --git a/bench-ci/guix/patches/oscrypto-hard-code-openssl.patch b/bench-ci/guix/patches/oscrypto-hard-code-openssl.patch new file mode 100644 index 000000000000..32027f2d09af --- /dev/null +++ b/bench-ci/guix/patches/oscrypto-hard-code-openssl.patch @@ -0,0 +1,13 @@ +diff --git a/oscrypto/__init__.py b/oscrypto/__init__.py +index eb27313..371ab24 100644 +--- a/oscrypto/__init__.py ++++ b/oscrypto/__init__.py +@@ -302,3 +302,8 @@ def load_order(): + 'oscrypto._win.tls', + 'oscrypto.tls', + ] ++ ++ ++paths = '@GUIX_OSCRYPTO_USE_OPENSSL@'.split(',') ++assert len(paths) == 2, 'Value for OSCRYPTO_USE_OPENSSL env var must be two paths separated by a comma' ++use_openssl(*paths) diff --git a/bench-ci/guix/patches/winpthreads-remap-guix-store.patch b/bench-ci/guix/patches/winpthreads-remap-guix-store.patch new file mode 100644 index 000000000000..e1f1a6eba531 --- /dev/null +++ b/bench-ci/guix/patches/winpthreads-remap-guix-store.patch @@ -0,0 +1,17 @@ +Without ffile-prefix-map, the debug symbols will contain paths for the +guix store which will include the hashes of each package. However, the +hash for the same package will differ when on different architectures. +In order to be reproducible regardless of the architecture used to build +the package, map all guix store prefixes to something fixed, e.g. /usr. + +--- a/mingw-w64-libraries/winpthreads/Makefile.in ++++ b/mingw-w64-libraries/winpthreads/Makefile.in +@@ -478,7 +478,7 @@ top_build_prefix = @top_build_prefix@ + top_builddir = @top_builddir@ + top_srcdir = @top_srcdir@ + SUBDIRS = . tests +-AM_CFLAGS = -Wall -DWIN32_LEAN_AND_MEAN $(am__append_1) ++AM_CFLAGS = -Wall -DWIN32_LEAN_AND_MEAN $(am__append_1) $(shell find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;) + ACLOCAL_AMFLAGS = -I m4 + lib_LTLIBRARIES = libwinpthread.la + include_HEADERS = include/pthread.h include/sched.h include/semaphore.h include/pthread_unistd.h include/pthread_time.h include/pthread_compat.h include/pthread_signal.h diff --git a/bench-ci/guix/security-check.py b/bench-ci/guix/security-check.py new file mode 100755 index 000000000000..be2e0cfbe2af --- /dev/null +++ b/bench-ci/guix/security-check.py @@ -0,0 +1,293 @@ +#!/usr/bin/env python3 +# Copyright (c) 2015-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +''' +Perform basic security checks on a series of executables. +Exit status will be 0 if successful, and the program will be silent. +Otherwise the exit status will be 1 and it will log which executables failed which checks. + +Example usage: + + find ../path/to/guix/binaries -type f -executable | xargs python3 contrib/guix/security-check.py +''' +import re +import sys + +import lief + +def check_ELF_RELRO(binary) -> bool: + ''' + Check for read-only relocations. + GNU_RELRO program header must exist + Dynamic section must have BIND_NOW flag + ''' + have_gnu_relro = False + for segment in binary.segments: + # Note: not checking p_flags == PF_R: here as linkers set the permission differently + # This does not affect security: the permission flags of the GNU_RELRO program + # header are ignored, the PT_LOAD header determines the effective permissions. + # However, the dynamic linker need to write to this area so these are RW. + # Glibc itself takes care of mprotecting this area R after relocations are finished. + # See also https://marc.info/?l=binutils&m=1498883354122353 + if segment.type == lief.ELF.Segment.TYPE.GNU_RELRO: + have_gnu_relro = True + + have_bindnow = False + try: + flags = binary.get(lief.ELF.DynamicEntry.TAG.FLAGS) + if flags.has(lief.ELF.DynamicEntryFlags.FLAG.BIND_NOW): + have_bindnow = True + except Exception: + have_bindnow = False + + return have_gnu_relro and have_bindnow + +def check_ELF_CANARY(binary) -> bool: + ''' + Check for use of stack canary + ''' + return binary.has_symbol('__stack_chk_fail') + +def check_ELF_SEPARATE_CODE(binary): + ''' + Check that sections are appropriately separated in virtual memory, + based on their permissions. This checks for missing -Wl,-z,separate-code + and potentially other problems. + ''' + R = lief.ELF.Segment.FLAGS.R + W = lief.ELF.Segment.FLAGS.W + E = lief.ELF.Segment.FLAGS.X + EXPECTED_FLAGS = { + # Read + execute + '.init': R | E, + '.plt': R | E, + '.plt.got': R | E, + '.plt.sec': R | E, + '.text': R | E, + '.fini': R | E, + # Read-only data + '.interp': R, + '.note.gnu.property': R, + '.note.gnu.build-id': R, + '.note.ABI-tag': R, + '.gnu.hash': R, + '.dynsym': R, + '.dynstr': R, + '.gnu.version': R, + '.gnu.version_r': R, + '.rela.dyn': R, + '.rela.plt': R, + '.rodata': R, + '.eh_frame_hdr': R, + '.eh_frame': R, + '.qtmetadata': R, + '.gcc_except_table': R, + '.stapsdt.base': R, + # Writable data + '.init_array': R | W, + '.fini_array': R | W, + '.dynamic': R | W, + '.got': R | W, + '.data': R | W, + '.bss': R | W, + } + if binary.header.machine_type == lief.ELF.ARCH.PPC64: + # .plt is RW on ppc64 even with separate-code + EXPECTED_FLAGS['.plt'] = R | W + # For all LOAD program headers get mapping to the list of sections, + # and for each section, remember the flags of the associated program header. + flags_per_section = {} + for segment in binary.segments: + if segment.type == lief.ELF.Segment.TYPE.LOAD: + for section in segment.sections: + flags_per_section[section.name] = segment.flags + # Spot-check ELF LOAD program header flags per section + # If these sections exist, check them against the expected R/W/E flags + for (section, flags) in flags_per_section.items(): + if section in EXPECTED_FLAGS: + if int(EXPECTED_FLAGS[section]) != int(flags): + return False + return True + +def check_ELF_CONTROL_FLOW(binary) -> bool: + ''' + Check for control flow instrumentation + ''' + main = binary.get_function_address('main') + content = binary.get_content_from_virtual_address(main, 4, lief.Binary.VA_TYPES.AUTO) + + if content.tolist() == [243, 15, 30, 250]: # endbr64 + return True + return False + +def check_ELF_FORTIFY(binary) -> bool: + + # bitcoin wrapper does not currently contain any fortified functions + if '--monolithic' in binary.strings: + return True + + chk_funcs = set() + + for sym in binary.imported_symbols: + match = re.search(r'__[a-z]*_chk', sym.name) + if match: + chk_funcs.add(match.group(0)) + + # ignore stack-protector + chk_funcs.discard('__stack_chk') + + return len(chk_funcs) >= 1 + +def check_PE_DYNAMIC_BASE(binary) -> bool: + '''PIE: DllCharacteristics bit 0x40 signifies dynamicbase (ASLR)''' + return lief.PE.OptionalHeader.DLL_CHARACTERISTICS.DYNAMIC_BASE in binary.optional_header.dll_characteristics_lists + +# Must support high-entropy 64-bit address space layout randomization +# in addition to DYNAMIC_BASE to have secure ASLR. +def check_PE_HIGH_ENTROPY_VA(binary) -> bool: + '''PIE: DllCharacteristics bit 0x20 signifies high-entropy ASLR''' + return lief.PE.OptionalHeader.DLL_CHARACTERISTICS.HIGH_ENTROPY_VA in binary.optional_header.dll_characteristics_lists + +def check_PE_RELOC_SECTION(binary) -> bool: + '''Check for a reloc section. This is required for functional ASLR.''' + return binary.has_relocations + +def check_PE_CONTROL_FLOW(binary) -> bool: + ''' + Check for control flow instrumentation + ''' + main = binary.get_symbol('main').value + + section_addr = binary.section_from_rva(main).virtual_address + virtual_address = binary.optional_header.imagebase + section_addr + main + + content = binary.get_content_from_virtual_address(virtual_address, 4, lief.Binary.VA_TYPES.VA) + + if content.tolist() == [243, 15, 30, 250]: # endbr64 + return True + return False + +def check_PE_CANARY(binary) -> bool: + ''' + Check for use of stack canary + ''' + return binary.has_symbol('__stack_chk_fail') + +def check_MACHO_NOUNDEFS(binary) -> bool: + ''' + Check for no undefined references. + ''' + return binary.header.has(lief.MachO.Header.FLAGS.NOUNDEFS) + +def check_MACHO_FIXUP_CHAINS(binary) -> bool: + ''' + Check for use of chained fixups. + ''' + return binary.has_dyld_chained_fixups + +def check_MACHO_CANARY(binary) -> bool: + ''' + Check for use of stack canary + ''' + return binary.has_symbol('___stack_chk_fail') + +def check_PIE(binary) -> bool: + ''' + Check for position independent executable (PIE), + allowing for address space randomization. + ''' + return binary.is_pie + +def check_NX(binary) -> bool: + ''' + Check for no stack execution + ''' + + # binary.has_nx checks are only for the stack, but MachO binaries might + # have executable heaps. + if binary.format == lief.Binary.FORMATS.MACHO: + return binary.concrete.has_nx_stack and binary.concrete.has_nx_heap + else: + return binary.has_nx + +def check_MACHO_CONTROL_FLOW(binary) -> bool: + ''' + Check for control flow instrumentation + ''' + content = binary.get_content_from_virtual_address(binary.entrypoint, 4, lief.Binary.VA_TYPES.AUTO) + + if content.tolist() == [243, 15, 30, 250]: # endbr64 + return True + return False + +def check_MACHO_BRANCH_PROTECTION(binary) -> bool: + ''' + Check for branch protection instrumentation + ''' + content = binary.get_content_from_virtual_address(binary.entrypoint, 4, lief.Binary.VA_TYPES.AUTO) + + if content.tolist() == [95, 36, 3, 213]: # bti + return True + return False + +BASE_ELF = [ + ('FORTIFY', check_ELF_FORTIFY), + ('PIE', check_PIE), + ('NX', check_NX), + ('RELRO', check_ELF_RELRO), + ('CANARY', check_ELF_CANARY), + ('SEPARATE_CODE', check_ELF_SEPARATE_CODE), +] + +BASE_PE = [ + ('PIE', check_PIE), + ('DYNAMIC_BASE', check_PE_DYNAMIC_BASE), + ('HIGH_ENTROPY_VA', check_PE_HIGH_ENTROPY_VA), + ('NX', check_NX), + ('RELOC_SECTION', check_PE_RELOC_SECTION), + ('CONTROL_FLOW', check_PE_CONTROL_FLOW), + ('CANARY', check_PE_CANARY), +] + +BASE_MACHO = [ + ('NOUNDEFS', check_MACHO_NOUNDEFS), + ('CANARY', check_MACHO_CANARY), + ('FIXUP_CHAINS', check_MACHO_FIXUP_CHAINS), +] + +CHECKS = { + lief.Binary.FORMATS.ELF: { + lief.Header.ARCHITECTURES.X86_64: BASE_ELF + [('CONTROL_FLOW', check_ELF_CONTROL_FLOW)], + lief.Header.ARCHITECTURES.ARM: BASE_ELF, + lief.Header.ARCHITECTURES.ARM64: BASE_ELF, + lief.Header.ARCHITECTURES.PPC64: BASE_ELF, + lief.Header.ARCHITECTURES.RISCV: BASE_ELF, + }, + lief.Binary.FORMATS.PE: { + lief.Header.ARCHITECTURES.X86_64: BASE_PE, + }, + lief.Binary.FORMATS.MACHO: { + lief.Header.ARCHITECTURES.X86_64: BASE_MACHO + [('PIE', check_PIE), + ('NX', check_NX), + ('CONTROL_FLOW', check_MACHO_CONTROL_FLOW)], + lief.Header.ARCHITECTURES.ARM64: BASE_MACHO + [('BRANCH_PROTECTION', check_MACHO_BRANCH_PROTECTION)], + } +} + +if __name__ == '__main__': + retval: int = 0 + for filename in sys.argv[1:]: + binary = lief.parse(filename) + + etype = binary.format + arch = binary.abstract.header.architecture + + failed: list[str] = [] + for (name, func) in CHECKS[etype][arch]: + if not func(binary): + failed.append(name) + if failed: + print(f'{filename}: failed {" ".join(failed)}') + retval = 1 + sys.exit(retval) diff --git a/bench-ci/guix/symbol-check.py b/bench-ci/guix/symbol-check.py new file mode 100755 index 000000000000..464b33cf66fb --- /dev/null +++ b/bench-ci/guix/symbol-check.py @@ -0,0 +1,335 @@ +#!/usr/bin/env python3 +# Copyright (c) 2014 Wladimir J. van der Laan +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +''' +A script to check that release executables only contain certain symbols +and are only linked against allowed libraries. + +Example usage: + + find ../path/to/guix/binaries -type f -executable | xargs python3 contrib/guix/symbol-check.py +''' +import sys + +import lief + +# Debian 11 (Bullseye) EOL: 2026. https://wiki.debian.org/LTS +# +# - libgcc version 10.2.1 (https://packages.debian.org/bullseye/libgcc-s1) +# - libc version 2.31 (https://packages.debian.org/source/bullseye/glibc) +# +# Ubuntu 20.04 (Focal) EOL: 2030. https://wiki.ubuntu.com/ReleaseTeam +# +# - libgcc version 10.5.0 (https://packages.ubuntu.com/focal/libgcc1) +# - libc version 2.31 (https://packages.ubuntu.com/focal/libc6) +# +# CentOS Stream 9 EOL: 2027. https://www.centos.org/cl-vs-cs/#end-of-life +# +# - libgcc version 12.2.1 (https://mirror.stream.centos.org/9-stream/AppStream/x86_64/os/Packages/) +# - libc version 2.34 (https://mirror.stream.centos.org/9-stream/AppStream/x86_64/os/Packages/) +# +# See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html for more info. + +MAX_VERSIONS = { +'GCC': (7,0,0), +'GLIBC': { + lief.ELF.ARCH.X86_64: (2,31), + lief.ELF.ARCH.ARM: (2,31), + lief.ELF.ARCH.AARCH64:(2,31), + lief.ELF.ARCH.PPC64: (2,31), + lief.ELF.ARCH.RISCV: (2,31), +}, +'LIBATOMIC': (1,0), +'V': (0,5,0), # xkb (bitcoin-qt only) +} + +# Ignore symbols that are exported as part of every executable +IGNORE_EXPORTS = { +'environ', '_environ', '__environ', '_fini', '_init', 'stdin', +'stdout', 'stderr', +} + +# Expected linker-loader names can be found here: +# https://sourceware.org/glibc/wiki/ABIList?action=recall&rev=16 +ELF_INTERPRETER_NAMES: dict[lief.ELF.ARCH, dict[lief.Header.ENDIANNESS, str]] = { + lief.ELF.ARCH.X86_64: { + lief.Header.ENDIANNESS.LITTLE: "/lib64/ld-linux-x86-64.so.2", + }, + lief.ELF.ARCH.ARM: { + lief.Header.ENDIANNESS.LITTLE: "/lib/ld-linux-armhf.so.3", + }, + lief.ELF.ARCH.AARCH64: { + lief.Header.ENDIANNESS.LITTLE: "/lib/ld-linux-aarch64.so.1", + }, + lief.ELF.ARCH.PPC64: { + lief.Header.ENDIANNESS.BIG: "/lib64/ld64.so.1", + lief.Header.ENDIANNESS.LITTLE: "/lib64/ld64.so.2", + }, + lief.ELF.ARCH.RISCV: { + lief.Header.ENDIANNESS.LITTLE: "/lib/ld-linux-riscv64-lp64d.so.1", + }, +} + +ELF_ABIS: dict[lief.ELF.ARCH, dict[lief.Header.ENDIANNESS, list[int]]] = { + lief.ELF.ARCH.X86_64: { + lief.Header.ENDIANNESS.LITTLE: [3,2,0], + }, + lief.ELF.ARCH.ARM: { + lief.Header.ENDIANNESS.LITTLE: [3,2,0], + }, + lief.ELF.ARCH.AARCH64: { + lief.Header.ENDIANNESS.LITTLE: [3,7,0], + }, + lief.ELF.ARCH.PPC64: { + lief.Header.ENDIANNESS.LITTLE: [3,10,0], + lief.Header.ENDIANNESS.BIG: [3,2,0], + }, + lief.ELF.ARCH.RISCV: { + lief.Header.ENDIANNESS.LITTLE: [4,15,0], + }, +} + +# Allowed NEEDED libraries +ELF_ALLOWED_LIBRARIES = { +# bitcoind and bitcoin-qt +'libgcc_s.so.1', # GCC base support +'libc.so.6', # C library +'libpthread.so.0', # threading +'libm.so.6', # math library +'libatomic.so.1', +'ld-linux-x86-64.so.2', # 64-bit dynamic linker +'ld-linux.so.2', # 32-bit dynamic linker +'ld-linux-aarch64.so.1', # 64-bit ARM dynamic linker +'ld-linux-armhf.so.3', # 32-bit ARM dynamic linker +'ld64.so.1', # POWER64 ABIv1 dynamic linker +'ld64.so.2', # POWER64 ABIv2 dynamic linker +'ld-linux-riscv64-lp64d.so.1', # 64-bit RISC-V dynamic linker +# bitcoin-qt only +'libxcb.so.1', # part of X11 +'libxkbcommon.so.0', # keyboard keymapping +'libxkbcommon-x11.so.0', # keyboard keymapping +'libfontconfig.so.1', # font support +'libfreetype.so.6', # font parsing +'libdl.so.2', # programming interface to dynamic linker +'libxcb-icccm.so.4', +'libxcb-image.so.0', +'libxcb-shm.so.0', +'libxcb-keysyms.so.1', +'libxcb-randr.so.0', +'libxcb-render-util.so.0', +'libxcb-render.so.0', +'libxcb-shape.so.0', +'libxcb-sync.so.1', +'libxcb-xfixes.so.0', +'libxcb-xkb.so.1', +} + +MACHO_ALLOWED_LIBRARIES = { +# bitcoind and bitcoin-qt +'libc++.1.dylib', # C++ Standard Library +'libSystem.B.dylib', # libc, libm, libpthread, libinfo +# bitcoin-qt only +'AppKit', # user interface +'ApplicationServices', # common application tasks. +'Carbon', # deprecated c back-compat API +'ColorSync', +'CoreFoundation', # low level func, data types +'CoreGraphics', # 2D rendering +'CoreServices', # operating system services +'CoreText', # interface for laying out text and handling fonts. +'CoreVideo', # video processing +'Foundation', # base layer functionality for apps/frameworks +'ImageIO', # read and write image file formats. +'IOKit', # user-space access to hardware devices and drivers. +'IOSurface', # cross process image/drawing buffers +'libobjc.A.dylib', # Objective-C runtime library +'Metal', # 3D graphics +'QuartzCore', # animation +'Security', # access control and authentication +'UniformTypeIdentifiers', # collection of types that map to MIME and file types +} + +PE_ALLOWED_LIBRARIES = { +'ADVAPI32.dll', # legacy security & registry +'bcrypt.dll', # newer security and identity API +'IPHLPAPI.DLL', # IP helper API +'KERNEL32.dll', # win32 base APIs +'msvcrt.dll', # C standard library for MSVC +'SHELL32.dll', # shell API +'WS2_32.dll', # sockets +# bitcoin-qt only +'api-ms-win-core-synch-l1-2-0.dll', # Synchronization Primitives API +'api-ms-win-core-winrt-l1-1-0.dll', # Windows Runtime API +'api-ms-win-core-winrt-string-l1-1-0.dll', # WinRT String API +'AUTHZ.dll', # Windows Authorization Framework +'comdlg32.dll', # Common Dialog Box Library +'d3d11.dll', # Direct3D 11 API +'d3d12.dll', # Direct3D 12 API +'d3d9.dll', # Direct3D 9 API +'dwmapi.dll', # desktop window manager +'DWrite.dll', # DirectX Typography Services +'dxgi.dll', # DirectX Graphics Infrastructure +'GDI32.dll', # graphics device interface +'IMM32.dll', # input method editor +'NETAPI32.dll', # network management +'ole32.dll', # component object model +'OLEAUT32.dll', # OLE Automation API +'SHLWAPI.dll', # light weight shell API +'USER32.dll', # user interface +'USERENV.dll', # user management +'UxTheme.dll', # visual style +'VERSION.dll', # version checking +'WINMM.dll', # WinMM audio API +'WTSAPI32.dll', # Remote Desktop +'SETUPAPI.dll', # Windows Setup API +'SHCORE.dll', # Stream Handler Core +} + +def check_version(max_versions, version, arch) -> bool: + (lib, _, ver) = version.rpartition('_') + ver = tuple([int(x) for x in ver.split('.')]) + if not lib in max_versions: + return False + if isinstance(max_versions[lib], tuple): + return ver <= max_versions[lib] + else: + return ver <= max_versions[lib][arch] + +def check_imported_symbols(binary) -> bool: + ok: bool = True + + for symbol in binary.imported_symbols: + if not symbol.imported: + continue + + version = symbol.symbol_version if symbol.has_version else None + + if version: + aux_version = version.symbol_version_auxiliary.name if version.has_auxiliary_version else None + if aux_version and not check_version(MAX_VERSIONS, aux_version, binary.header.machine_type): + print(f'{filename}: symbol {symbol.name} from unsupported version {version}') + ok = False + return ok + +def check_exported_symbols(binary) -> bool: + ok: bool = True + + for symbol in binary.dynamic_symbols: + if not symbol.exported: + continue + name = symbol.name + if binary.header.machine_type == lief.ELF.ARCH.RISCV or name in IGNORE_EXPORTS: + continue + print(f'{filename}: export of symbol {name} not allowed!') + ok = False + return ok + +def check_RUNPATH(binary) -> bool: + assert binary.get(lief.ELF.DynamicEntry.TAG.RUNPATH) is None + assert binary.get(lief.ELF.DynamicEntry.TAG.RPATH) is None + return True + +def check_ELF_libraries(binary) -> bool: + ok: bool = True + for library in binary.libraries: + if library not in ELF_ALLOWED_LIBRARIES: + print(f'{filename}: {library} is not in ALLOWED_LIBRARIES!') + ok = False + return ok + +def check_MACHO_libraries(binary) -> bool: + ok: bool = True + for dylib in binary.libraries: + split = dylib.name.split('/') + if split[-1] not in MACHO_ALLOWED_LIBRARIES: + print(f'{split[-1]} is not in ALLOWED_LIBRARIES!') + ok = False + return ok + +def check_MACHO_min_os(binary) -> bool: + if binary.build_version.minos == [14,0,0]: + return True + return False + +def check_MACHO_sdk(binary) -> bool: + if binary.build_version.sdk == [14, 0, 0]: + return True + return False + +def check_MACHO_lld(binary) -> bool: + if binary.build_version.tools[0].version == [19, 1, 4]: + return True + return False + +def check_PE_libraries(binary) -> bool: + ok: bool = True + for dylib in binary.libraries: + if dylib not in PE_ALLOWED_LIBRARIES: + print(f'{dylib} is not in ALLOWED_LIBRARIES!') + ok = False + return ok + +def check_PE_subsystem_version(binary) -> bool: + major: int = binary.optional_header.major_subsystem_version + minor: int = binary.optional_header.minor_subsystem_version + if major == 6 and minor == 2: + return True + return False + +def check_PE_application_manifest(binary) -> bool: + if not binary.has_resources: + # No resources at all. + return False + + rm = binary.resources_manager + return rm.has_manifest + +def check_ELF_interpreter(binary) -> bool: + expected_interpreter = ELF_INTERPRETER_NAMES[binary.header.machine_type][binary.abstract.header.endianness] + + return binary.concrete.interpreter == expected_interpreter + +def check_ELF_ABI(binary) -> bool: + expected_abi = ELF_ABIS[binary.header.machine_type][binary.abstract.header.endianness] + note = binary.concrete.get(lief.ELF.Note.TYPE.GNU_ABI_TAG) + assert note.abi == lief.ELF.NoteAbi.ABI.LINUX + return note.version == expected_abi + +CHECKS = { +lief.Binary.FORMATS.ELF: [ + ('IMPORTED_SYMBOLS', check_imported_symbols), + ('EXPORTED_SYMBOLS', check_exported_symbols), + ('LIBRARY_DEPENDENCIES', check_ELF_libraries), + ('INTERPRETER_NAME', check_ELF_interpreter), + ('ABI', check_ELF_ABI), + ('RUNPATH', check_RUNPATH), +], +lief.Binary.FORMATS.MACHO: [ + ('DYNAMIC_LIBRARIES', check_MACHO_libraries), + ('MIN_OS', check_MACHO_min_os), + ('SDK', check_MACHO_sdk), + ('LLD', check_MACHO_lld), +], +lief.Binary.FORMATS.PE: [ + ('DYNAMIC_LIBRARIES', check_PE_libraries), + ('SUBSYSTEM_VERSION', check_PE_subsystem_version), + ('APPLICATION_MANIFEST', check_PE_application_manifest), +] +} + +if __name__ == '__main__': + retval: int = 0 + for filename in sys.argv[1:]: + binary = lief.parse(filename) + + etype = binary.format + + failed: list[str] = [] + for (name, func) in CHECKS[etype]: + if not func(binary): + failed.append(name) + if failed: + print(f'{filename}: failed {" ".join(failed)}') + retval = 1 + sys.exit(retval) From 17e6cc64650762fe8e4819c8f86f59ffc41d029d Mon Sep 17 00:00:00 2001 From: willcl-ark Date: Tue, 28 Oct 2025 22:23:37 +0000 Subject: [PATCH 06/51] modify bench-ci/guix params for benchmarking --- bench-ci/guix/guix-build | 4 ++-- bench-ci/guix/libexec/build.sh | 20 ++++++++++++++++---- bench-ci/guix/libexec/prelude.bash | 7 +++++++ 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/bench-ci/guix/guix-build b/bench-ci/guix/guix-build index ee285bf322cf..84d4f201b259 100755 --- a/bench-ci/guix/guix-build +++ b/bench-ci/guix/guix-build @@ -439,7 +439,7 @@ EOF # more information. # # shellcheck disable=SC2086,SC2031 - time-machine shell --manifest="${PWD}/contrib/guix/manifest.scm" \ + time-machine shell --manifest="${PWD}/bench-ci/guix/manifest.scm" \ --container \ --pure \ --no-cwd \ @@ -468,7 +468,7 @@ EOF DISTSRC="$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")" \ OUTDIR="$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST")" \ DIST_ARCHIVE_BASE=/outdir-base/dist-archive \ - bash -c "cd /bitcoin && bash contrib/guix/libexec/build.sh" + bash -c "cd /bitcoin && bash bench-ci/guix/libexec/build.sh" ) done diff --git a/bench-ci/guix/libexec/build.sh b/bench-ci/guix/libexec/build.sh index 16e12d563816..d12c795beaa4 100755 --- a/bench-ci/guix/libexec/build.sh +++ b/bench-ci/guix/libexec/build.sh @@ -72,6 +72,8 @@ unset OBJCPLUS_INCLUDE_PATH # Set native toolchain build_CC="${NATIVE_GCC}/bin/gcc -isystem ${NATIVE_GCC}/include" build_CXX="${NATIVE_GCC}/bin/g++ -isystem ${NATIVE_GCC}/include/c++ -isystem ${NATIVE_GCC}/include" +export C_INCLUDE_PATH="${NATIVE_GCC}/include" +export CPLUS_INCLUDE_PATH="${NATIVE_GCC}/include/c++:${NATIVE_GCC}/include" case "$HOST" in *darwin*) export LIBRARY_PATH="${NATIVE_GCC}/lib" ;; # Required for native packages @@ -178,7 +180,13 @@ make -C depends --jobs="$JOBS" HOST="$HOST" \ x86_64_linux_AR=x86_64-linux-gnu-gcc-ar \ x86_64_linux_RANLIB=x86_64-linux-gnu-gcc-ranlib \ x86_64_linux_NM=x86_64-linux-gnu-gcc-nm \ - x86_64_linux_STRIP=x86_64-linux-gnu-strip + x86_64_linux_STRIP=x86_64-linux-gnu-strip \ + NO_QT=1 \ + NO_QR=1 \ + NO_ZMQ=1 \ + NO_WALLET=1 \ + NO_BDB=1 \ + NO_USDT=1 case "$HOST" in *darwin*) @@ -208,6 +216,9 @@ mkdir -p "$OUTDIR" # CONFIGFLAGS CONFIGFLAGS="-DREDUCE_EXPORTS=ON -DBUILD_BENCH=OFF -DBUILD_GUI_TESTS=OFF -DBUILD_FUZZ_BINARY=OFF -DCMAKE_SKIP_RPATH=TRUE" +# BENCHCOINFLAGS +BENCHCOINFLAGS="-DBUILD_CLI=OFF -DBUILD_TESTS=OFF -DCMAKE_CXX_FLAGS=-fno-omit-frame-pointer" + # CFLAGS HOST_CFLAGS="-O2 -g" HOST_CFLAGS+=$(find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;) @@ -243,15 +254,16 @@ mkdir -p "$DISTSRC" --toolchain "${BASEPREFIX}/${HOST}/toolchain.cmake" \ -DWITH_CCACHE=OFF \ -Werror=dev \ - ${CONFIGFLAGS} + ${CONFIGFLAGS} \ + ${BENCHCOINFLAGS} # Build Bitcoin Core cmake --build build -j "$JOBS" ${V:+--verbose} # Perform basic security checks on a series of executables. - cmake --build build -j 1 --target check-security ${V:+--verbose} + # cmake --build build -j 1 --target check-security ${V:+--verbose} # Check that executables only contain allowed version symbols. - cmake --build build -j 1 --target check-symbols ${V:+--verbose} + # cmake --build build -j 1 --target check-symbols ${V:+--verbose} mkdir -p "$OUTDIR" diff --git a/bench-ci/guix/libexec/prelude.bash b/bench-ci/guix/libexec/prelude.bash index b7c13cc91d8c..b4e828013351 100644 --- a/bench-ci/guix/libexec/prelude.bash +++ b/bench-ci/guix/libexec/prelude.bash @@ -8,6 +8,13 @@ source contrib/shell/realpath.bash # shellcheck source=contrib/shell/git-utils.bash source contrib/shell/git-utils.bash +# Source guix profile from the runner home directory +GUIX_PROFILE=/home/github-runner/.config/guix/current +. "$GUIX_PROFILE/etc/profile" +echo "Using the following guix command:" +command -v guix +guix describe + ################ # Required non-builtin commands should be invocable ################ From 508a526185f48c9eccf5770e85afa819e70c37ce Mon Sep 17 00:00:00 2001 From: willcl-ark Date: Tue, 28 Oct 2025 22:24:58 +0000 Subject: [PATCH 07/51] bench: add benchmark ci workflows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: David Gumberg Co-authored-by: Lőrinc --- .github/workflows/benchmark.yml | 113 ++++++++++ .github/workflows/publish-results.yml | 298 ++++++++++++++++++++++++++ .github/workflows/sync_upstream.yml | 32 +++ bench-ci/build_binaries.sh | 52 +++++ bench-ci/parse_and_plot.py | 167 +++++++++++++++ bench-ci/run-assumeutxo-bench.sh | 154 +++++++++++++ bench-ci/run-benchmark.sh | 163 ++++++++++++++ 7 files changed, 979 insertions(+) create mode 100644 .github/workflows/benchmark.yml create mode 100644 .github/workflows/publish-results.yml create mode 100644 .github/workflows/sync_upstream.yml create mode 100755 bench-ci/build_binaries.sh create mode 100755 bench-ci/parse_and_plot.py create mode 100755 bench-ci/run-assumeutxo-bench.sh create mode 100755 bench-ci/run-benchmark.sh diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 000000000000..c8c1a992fa25 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,113 @@ +name: Benchmark +on: + pull_request: + branches: + - master +jobs: + build-binaries: + runs-on: [self-hosted, linux, x64] + env: + NIX_PATH: nixpkgs=channel:nixos-unstable + BASE_SHA: ${{ github.event.pull_request.base.sha }} + steps: + - name: Checkout repo + uses: actions/checkout@v4 + with: + fetch-depth: 1 + - name: Fetch base commit + run: | + echo "CHECKOUT_COMMIT=$(git rev-parse HEAD)" >> "$GITHUB_ENV" + git fetch --depth=1 origin ${{ github.event.pull_request.base.sha }} + - name: Setup ccache + run: | + mkdir -p /data/ccache + export CCACHE_DIR=/data/ccache + export CCACHE_MAXSIZE=50G + ccache -M 50G + ccache -s + - name: Build both binaries + env: + CCACHE_DIR: /data/ccache + run: | + mkdir -p ${{ runner.temp }}/binaries/base + mkdir -p ${{ runner.temp }}/binaries/head + nix-shell --command "just build-assumeutxo-binaries-guix $BASE_SHA $CHECKOUT_COMMIT" + cp binaries/base/bitcoind ${{ runner.temp }}/binaries/base/bitcoind + cp binaries/head/bitcoind ${{ runner.temp }}/binaries/head/bitcoind + - name: Upload binaries + uses: actions/upload-artifact@v4 + with: + name: bitcoind-binaries + path: ${{ runner.temp }}/binaries/ + assumeutxo: + needs: build-binaries + strategy: + matrix: + include: + - network: mainnet + name: mainnet-default + timeout: 600 + datadir_path: /data/pruned-840k + dbcache: 450 + - network: mainnet + name: mainnet-large + timeout: 600 + datadir_path: /data/pruned-840k + dbcache: 32000 + runs-on: [self-hosted, linux, x64] + timeout-minutes: ${{ matrix.timeout }} + env: + NIX_PATH: nixpkgs=channel:nixos-unstable + ORIGINAL_DATADIR: ${{ matrix.datadir_path }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + steps: + - name: Checkout repo + uses: actions/checkout@v4 + with: + fetch-depth: 1 + - name: Download binaries + uses: actions/download-artifact@v4 + with: + name: bitcoind-binaries + path: ${{ runner.temp }}/binaries + - name: Set binary permissions + run: | + chmod +x ${{ runner.temp }}/binaries/base/bitcoind + chmod +x ${{ runner.temp }}/binaries/head/bitcoind + - name: Fetch base commit + run: | + echo "CHECKOUT_COMMIT=$(git rev-parse HEAD)" >> "$GITHUB_ENV" + git fetch --depth=1 origin ${{ github.event.pull_request.base.sha }} + - name: Run AssumeUTXO ${{ matrix.network }} + env: + TMP_DATADIR: "${{ runner.temp }}/base_datadir" + BINARIES_DIR: "${{ runner.temp }}/binaries" + run: | + env + mkdir -p "$TMP_DATADIR" + nix-shell --command "just run-${{ matrix.network }}-ci $BASE_SHA $CHECKOUT_COMMIT $TMP_DATADIR $ORIGINAL_DATADIR ${{ runner.temp }}/results.json ${{ matrix.dbcache }} ${{ runner.temp }}/pngs $BINARIES_DIR" + - uses: actions/upload-artifact@v4 + with: + name: result-${{ matrix.name }} + path: "${{ runner.temp }}/results.json" + - uses: actions/upload-artifact@v4 + with: + name: pngs-${{ matrix.name }} + path: "${{ runner.temp }}/pngs/*.png" + - uses: actions/upload-artifact@v4 + with: + name: flamegraph-${{ matrix.name }} + path: "**/*-flamegraph.svg" + - name: Write GitHub and runner context files + env: + GITHUB_CONTEXT: ${{ toJSON(github) }} + RUNNER_CONTEXT: ${{ toJSON(runner) }} + run: | + mkdir contexts + echo "$GITHUB_CONTEXT" | nix-shell -p jq --command "jq 'del(.token)' > contexts/github.json" + echo "$RUNNER_CONTEXT" > contexts/runner.json + - name: Upload context metadata as artifact + uses: actions/upload-artifact@v4 + with: + name: run-metadata-${{ matrix.name }} + path: ./contexts/ diff --git a/.github/workflows/publish-results.yml b/.github/workflows/publish-results.yml new file mode 100644 index 000000000000..62076d85b5eb --- /dev/null +++ b/.github/workflows/publish-results.yml @@ -0,0 +1,298 @@ +name: Publish Results +on: + workflow_run: + workflows: ["Benchmark"] + types: [completed] +jobs: + build: + runs-on: ubuntu-latest + if: ${{ github.event.workflow_run.conclusion == 'success' }} + permissions: + actions: read + contents: write + checks: read + env: + NETWORKS: "mainnet-default,mainnet-large" + outputs: + speedups: ${{ steps.organize.outputs.speedups }} + pr-number: ${{ steps.organize.outputs.pr-number }} + steps: + - uses: actions/checkout@v4 + with: + ref: gh-pages + - name: Download artifacts + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh run download ${{ github.event.workflow_run.id }} --repo ${{ github.repository }} + + - name: Extract artifacts + run: | + for network in ${NETWORKS//,/ }; do + if [ -d "result-${network}" ]; then + mkdir -p "${network}-results" + mv "result-${network}/results.json" "${network}-results/" + fi + + if [ -d "flamegraph-${network}" ]; then + mkdir -p "${network}-flamegraph" + mv "flamegraph-${network}"/* "${network}-flamegraph/" + fi + + if [ -d "run-metadata-${network}" ]; then + mkdir -p "${network}-metadata" + mv "run-metadata-${network}"/* "${network}-metadata/" + fi + + if [ -d "pngs-${network}" ]; then + mkdir -p "${network}-plots" + mv "pngs-${network}"/*.png "${network}-plots/" + fi + done + - name: Organize results + id: organize + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const path = require('path'); + const networks = process.env.NETWORKS.split(','); + let prNumber = 'main'; + let runId; + + // First, extract metadata and get PR number + for (const network of networks) { + if (fs.existsSync(`${network}-metadata/github.json`)) { + const metadata = JSON.parse(fs.readFileSync(`${network}-metadata/github.json`, 'utf8')); + prNumber = metadata.event.pull_request?.number || prNumber; + runId = metadata.run_id; + } + } + + if (!runId) { + console.error('No valid metadata found for any network'); + process.exit(1); + } + + // Create directory structure + const resultDir = `results/pr-${prNumber}/${runId}`; + fs.mkdirSync(resultDir, { recursive: true }); + + // Now copy metadata files + for (const network of networks) { + if (fs.existsSync(`${network}-metadata/github.json`)) { + const metadataDir = `${resultDir}/${network}-metadata`; + fs.mkdirSync(metadataDir, { recursive: true }); + fs.copyFileSync(`${network}-metadata/github.json`, `${metadataDir}/github.json`); + } + } + + // Process each network's results + const combinedResults = { + results: [], + speedups: {} + }; + + for (const network of networks) { + if (fs.existsSync(`${network}-results`)) { + const networkResults = JSON.parse(fs.readFileSync(`${network}-results/results.json`, 'utf8')); + let baseMean, headMean; + + // Add network name to each result and collect means + networkResults.results.forEach(result => { + result.network = network; + combinedResults.results.push(result); + if (result.command.includes('base')) { + baseMean = result.mean; + } else if (result.command.includes('head')) { + headMean = result.mean; + } + }); + + // Calculate speedup if we have both measurements + if (baseMean && headMean) { + const speedup = baseMean > 0 ? ((baseMean - headMean) / baseMean * 100).toFixed(1) : 'N/A'; + combinedResults.speedups[network] = speedup; + } + + // Move flamegraphs + if (fs.existsSync(`${network}-flamegraph`)) { + fs.readdirSync(`${network}-flamegraph`).forEach(file => { + const sourceFile = `${network}-flamegraph/${file}`; + const targetFile = `${resultDir}/${network}-${file}`; + fs.copyFileSync(sourceFile, targetFile); + }); + } + + // Move plots + if (fs.existsSync(`${network}-plots`)) { + const targetPlotsDir = `${resultDir}/${network}-plots`; + fs.mkdirSync(targetPlotsDir, { recursive: true }); + fs.readdirSync(`${network}-plots`).forEach(plot => { + const sourcePlot = `${network}-plots/${plot}`; + const targetPlot = `${targetPlotsDir}/${plot}`; + fs.copyFileSync(sourcePlot, targetPlot); + }); + } + } + } + + // Write combined results + fs.writeFileSync(`${resultDir}/results.json`, JSON.stringify(combinedResults, null, 2)); + + // Create index.html for this run + const indexHtml = ` + + + Benchmark Results + + + +
+

Benchmark Results

+
+

PR #${prNumber} - Run ${runId}

+ ${networks.map(network => ` +
+

+ ${network} Results + ${combinedResults.speedups[network] ? + `(${combinedResults.speedups[network]}% speedup)` + : ''} +

+
+ ${combinedResults.results + .filter(result => result.network === network) + .map(result => { + const commitShortId = result.parameters.commit.slice(0, 8); + const flameGraphFile = `${network}-${result.parameters.commit}-flamegraph.svg`; + const flameGraphPath = `${resultDir}/${network}-${result.parameters.commit}-flamegraph.svg`; + + // Query PNG files dynamically + const plotDir = `${resultDir}/${network}-plots`; + const plots = fs.existsSync(plotDir) + ? fs.readdirSync(plotDir) + .map(plot => ` + + ${plot} + + `) + .join('') + : ''; + + return ` + + + + + + + + + + + + + + + + + + + +
CommandMean (s)Std DevUser (s)System (s)
+ ${result.command.replace( + /\((\w+)\)/, + (_, commit) => `(${commit.slice(0, 8)})` + )} + ${result.mean.toFixed(3)}${result.stddev?.toFixed(3) || 'N/A'}${result.user.toFixed(3)}${result.system.toFixed(3)}
+ ${fs.existsSync(flameGraphPath) ? ` + + ` : ''} + ${plots} + `; + }).join('')} +
+
+ `).join('')} +
+
+ + `; + + fs.writeFileSync(`${resultDir}/index.html`, indexHtml); + + // Update main index.html + const prs = fs.readdirSync('results') + .filter(dir => dir.startsWith('pr-')) + .map(dir => ({ + pr: dir.replace('pr-', ''), + runs: fs.readdirSync(`results/${dir}`) + })); + + const mainIndexHtml = ` + + + Bitcoin Benchmark Results + + + +
+

Bitcoin Benchmark Results

+
+

Available Results

+
    + ${prs.map(({pr, runs}) => ` +
  • PR #${pr} +
      + ${runs.map(run => ` +
    • Run ${run}
      • + `).join('')} +
    +
    • + `).join('')} +
+
+
+ + `; + + fs.writeFileSync('index.html', mainIndexHtml); + + // Set outputs for use in PR comment + const resultUrl = `https://${context.repo.owner}.github.io/${context.repo.name}/results/pr-${prNumber}/${runId}/index.html`; + const speedupString = Object.entries(combinedResults.speedups) + .map(([network, speedup]) => `${network}: ${speedup}%`) + .join(', '); + + core.setOutput('result-url', resultUrl); + core.setOutput('speedups', speedupString); + core.setOutput('pr-number', prNumber); + return { url: resultUrl, speedups: speedupString }; + - name: Upload Pages artifact + uses: actions/upload-pages-artifact@v3 + with: + path: results + - name: Commit and push to gh-pages + run: | + git config --global user.name "github-actions[bot]" + git config --global user.email "github-actions[bot]@users.noreply.github.com" + git add results/ index.html + git commit -m "Update benchmark results from run ${{ github.event.workflow_run.id }}" + git push origin gh-pages + comment-pr: + needs: build + runs-on: ubuntu-latest + permissions: + pull-requests: write + actions: read + steps: + - name: Comment on PR + if: ${{ needs.build.outputs.pr-number != 'main' }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh pr comment ${{ needs.build.outputs.pr-number }} \ + --repo ${{ github.repository }} \ + --body "📊 Benchmark results for this run (${{ github.event.workflow_run.id }}) will be available at: https://${{ github.repository_owner }}.github.io/${{ github.event.repository.name }}/results/pr-${{ needs.build.outputs.pr-number }}/${{ github.event.workflow_run.id }}/index.html after the github pages \"build and deployment\" action has completed. + 🚀 Speedups: ${{ needs.build.outputs.speedups }}" diff --git a/.github/workflows/sync_upstream.yml b/.github/workflows/sync_upstream.yml new file mode 100644 index 000000000000..11d9cfafc2e4 --- /dev/null +++ b/.github/workflows/sync_upstream.yml @@ -0,0 +1,32 @@ +name: Sync with Upstream +on: + schedule: + - cron: '0 3 * * *' # 03:00 UTC daily + workflow_dispatch: +permissions: + contents: write # Required for pushing to master +jobs: + sync: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + token: ${{ secrets.GITHUB_TOKEN }} + - name: Add upstream remote + run: | + git remote add upstream https://github.com/bitcoin/bitcoin.git + git remote -v + - name: Fetch upstream + run: git fetch upstream + - name: Configure Git + run: | + git config user.name github-actions + git config user.email github-actions@github.com + - name: Rebase onto upstream + run: | + git checkout master + git rebase upstream/master + - name: Push changes + run: git push --force-with-lease origin master diff --git a/bench-ci/build_binaries.sh b/bench-ci/build_binaries.sh new file mode 100755 index 000000000000..9a396a00659f --- /dev/null +++ b/bench-ci/build_binaries.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +set -euxo pipefail + +if [ $# -ne 2 ]; then + echo "Usage: $0 " + exit 1 +fi + +# Save current state of git +initial_ref=$(git symbolic-ref -q HEAD || git rev-parse HEAD) +if git symbolic-ref -q HEAD >/dev/null; then + initial_state="branch" + initial_branch=${initial_ref#refs/heads/} +else + initial_state="detached" +fi + +base_commit="$1" +head_commit="$2" + +mkdir -p binaries/base +mkdir -p binaries/head + +for build in "base:${base_commit}" "head:${head_commit}"; do + name="${build%%:*}" + commit="${build#*:}" + git checkout "$commit" + # Use environment variables if set, otherwise use defaults + HOSTS="${HOSTS:-x86_64-linux-gnu}" \ + SOURCES_PATH="${SOURCES_PATH:-/data/SOURCES_PATH}" \ + BASE_CACHE="${BASE_CACHE:-/data/BASE_CACHE}" \ + taskset -c 2-15 chrt -f 1 bench-ci/guix/guix-build + + # Truncate commit hash to 12 characters + short_commit=$(echo "$commit" | cut -c 1-12) + + # Extract the Guix output + tar -xzf "guix-build-${short_commit}/output/x86_64-linux-gnu/bitcoin-${short_commit}-x86_64-linux-gnu.tar.gz" + + # Copy the binary to our binaries directory + cp "bitcoin-${short_commit}/bin/bitcoind" "binaries/${name}/bitcoind" + + # Cleanup extracted files + rm -rf "bitcoin-${short_commit}" +done + +# Restore initial git state +if [ "$initial_state" = "branch" ]; then + git checkout "$initial_branch" +else + git checkout "$initial_ref" +fi diff --git a/bench-ci/parse_and_plot.py b/bench-ci/parse_and_plot.py new file mode 100755 index 000000000000..db577417b2ff --- /dev/null +++ b/bench-ci/parse_and_plot.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +import sys +import os +import re +import datetime +import matplotlib.pyplot as plt + + +def parse_updatetip_line(line): + match = re.match( + r'^([\d\-:TZ]+) UpdateTip: new best.+height=(\d+).+tx=(\d+).+cache=([\d.]+)MiB\((\d+)txo\)', + line + ) + if not match: + return None + iso_str, height_str, tx_str, cache_size_mb_str, cache_coins_count_str = match.groups() + parsed_datetime = datetime.datetime.strptime(iso_str, "%Y-%m-%dT%H:%M:%SZ") + return parsed_datetime, int(height_str), int(tx_str), float(cache_size_mb_str), int(cache_coins_count_str) + + +def parse_leveldb_compact_line(line): + match = re.match(r'^([\d\-:TZ]+) \[leveldb] Compacting.*files', line) + if not match: + return None + iso_str = match.groups()[0] + parsed_datetime = datetime.datetime.strptime(iso_str, "%Y-%m-%dT%H:%M:%SZ") + return parsed_datetime + + +def parse_leveldb_generated_table_line(line): + match = re.match(r'^([\d\-:TZ]+) \[leveldb] Generated table.*: (\d+) keys, (\d+) bytes', line) + if not match: + return None + iso_str, keys_count_str, bytes_count_str = match.groups() + parsed_datetime = datetime.datetime.strptime(iso_str, "%Y-%m-%dT%H:%M:%SZ") + return parsed_datetime, int(keys_count_str), int(bytes_count_str) + +def parse_validation_txadd_line(line): + match = re.match(r'^([\d\-:TZ]+) \[validation] TransactionAddedToMempool: txid=.+wtxid=.+', line) + if not match: + return None + iso_str = match.groups()[0] + parsed_datetime = datetime.datetime.strptime(iso_str, "%Y-%m-%dT%H:%M:%SZ") + return parsed_datetime + + +def parse_coindb_write_batch_line(line): + match = re.match(r'^([\d\-:TZ]+) \[coindb] Writing (partial|final) batch of ([\d.]+) MiB', line) + if not match: + return None + iso_str, is_partial_str, size_mb_str = match.groups() + parsed_datetime = datetime.datetime.strptime(iso_str, "%Y-%m-%dT%H:%M:%SZ") + return parsed_datetime, is_partial_str, float(size_mb_str) + + +def parse_coindb_commit_line(line): + match = re.match(r'^([\d\-:TZ]+) \[coindb] Committed (\d+) changed transaction outputs', line) + if not match: + return None + iso_str, txout_count_str = match.groups() + parsed_datetime = datetime.datetime.strptime(iso_str, "%Y-%m-%dT%H:%M:%SZ") + return parsed_datetime, int(txout_count_str) + +def parse_log_file(log_file): + with open(log_file, 'r', encoding='utf-8') as f: + update_tip_data = [] + leveldb_compact_data = [] + leveldb_gen_table_data = [] + validation_txadd_data = [] + coindb_write_batch_data = [] + coindb_commit_data = [] + + for line in f: + if result := parse_updatetip_line(line): + update_tip_data.append(result) + elif result := parse_leveldb_compact_line(line): + leveldb_compact_data.append(result) + elif result := parse_leveldb_generated_table_line(line): + leveldb_gen_table_data.append(result) + elif result := parse_validation_txadd_line(line): + validation_txadd_data.append(result) + elif result := parse_coindb_write_batch_line(line): + coindb_write_batch_data.append(result) + elif result := parse_coindb_commit_line(line): + coindb_commit_data.append(result) + + if not update_tip_data: + print("No UpdateTip entries found.") + sys.exit(0) + + assert all(update_tip_data[i][0] <= update_tip_data[i + 1][0] for i in + range(len(update_tip_data) - 1)), "UpdateTip entries are not sorted by time" + + return update_tip_data, leveldb_compact_data, leveldb_gen_table_data, validation_txadd_data, coindb_write_batch_data, coindb_commit_data + + +def generate_plot(x, y, x_label, y_label, title, output_file): + if not x or not y: + print(f"Skipping plot '{title}' as there is no data.") + return + + plt.figure(figsize=(30, 10)) + plt.plot(x, y) + plt.title(title, fontsize=20) + plt.xlabel(x_label, fontsize=16) + plt.ylabel(y_label, fontsize=16) + plt.grid(True) + plt.xticks(rotation=90, fontsize=12) + plt.yticks(fontsize=12) + plt.tight_layout() + plt.savefig(output_file) + plt.close() + print(f"Saved plot to {output_file}") + + +if __name__ == "__main__": + if len(sys.argv) != 3: + print(f"Usage: {sys.argv[0]} ") + sys.exit(1) + + log_file = sys.argv[1] + if not os.path.isfile(log_file): + print(f"File not found: {log_file}") + sys.exit(1) + + png_dir = sys.argv[2] + os.makedirs(png_dir, exist_ok=True) + + update_tip_data, leveldb_compact_data, leveldb_gen_table_data, validation_txadd_data, coindb_write_batch_data, coindb_commit_data = parse_log_file(log_file) + times, heights, tx_counts, cache_size, cache_count = zip(*update_tip_data) + float_minutes = [(t - times[0]).total_seconds() / 60 for t in times] + + generate_plot(float_minutes, heights, "Elapsed minutes", "Block Height", "Block Height vs Time", os.path.join(png_dir, "height_vs_time.png")) + generate_plot(heights, cache_size, "Block Height", "Cache Size (MiB)", "Cache Size vs Block Height", os.path.join(png_dir, "cache_vs_height.png")) + generate_plot(float_minutes, cache_size, "Elapsed minutes", "Cache Size (MiB)", "Cache Size vs Time", os.path.join(png_dir, "cache_vs_time.png")) + generate_plot(heights, tx_counts, "Block Height", "Transaction Count", "Transactions vs Block Height", os.path.join(png_dir, "tx_vs_height.png")) + generate_plot(times, cache_count, "Block Height", "Coins Cache Size", "Coins Cache Size vs Time", os.path.join(png_dir, "coins_cache_vs_time.png")) + + # LevelDB Compaction and Generated Tables + if leveldb_compact_data: + leveldb_compact_times = [(t - times[0]).total_seconds() / 60 for t in leveldb_compact_data] + leveldb_compact_y = [1 for _ in leveldb_compact_times] # dummy y axis to mark compactions + generate_plot(leveldb_compact_times, leveldb_compact_y, "Elapsed minutes", "LevelDB Compaction", "LevelDB Compaction Events vs Time", os.path.join(png_dir, "leveldb_compact_vs_time.png")) + if leveldb_gen_table_data: + leveldb_gen_table_times, leveldb_gen_table_keys, leveldb_gen_table_bytes = zip(*leveldb_gen_table_data) + leveldb_gen_table_float_minutes = [(t - times[0]).total_seconds() / 60 for t in leveldb_gen_table_times] + generate_plot(leveldb_gen_table_float_minutes, leveldb_gen_table_keys, "Elapsed minutes", "Number of keys", "LevelDB Keys Generated vs Time", os.path.join(png_dir, "leveldb_gen_keys_vs_time.png")) + generate_plot(leveldb_gen_table_float_minutes, leveldb_gen_table_bytes, "Elapsed minutes", "Number of bytes", "LevelDB Bytes Generated vs Time", os.path.join(png_dir, "leveldb_gen_bytes_vs_time.png")) + + # validation mempool add transaction lines + if validation_txadd_data: + validation_txadd_times = [(t - times[0]).total_seconds() / 60 for t in validation_txadd_data] + validation_txadd_y = [1 for _ in validation_txadd_times] # dummy y axis to mark transaction additions + generate_plot(validation_txadd_times, validation_txadd_y, "Elapsed minutes", "Transaction Additions", "Transaction Additions to Mempool vs Time", os.path.join(png_dir, "validation_txadd_vs_time.png")) + + # coindb write batch lines + if coindb_write_batch_data: + coindb_write_batch_times, is_partial_strs, sizes_mb = zip(*coindb_write_batch_data) + coindb_write_batch_float_minutes = [(t - times[0]).total_seconds() / 60 for t in coindb_write_batch_times] + generate_plot(coindb_write_batch_float_minutes, sizes_mb, "Elapsed minutes", "Batch Size MiB", "Coin Database Partial/Final Write Batch Size vs Time", os.path.join(png_dir, "coindb_write_batch_size_vs_time.png")) + if coindb_commit_data: + coindb_commit_times, txout_counts = zip(*coindb_commit_data) + coindb_commit_float_minutes = [(t - times[0]).total_seconds() / 60 for t in coindb_commit_times] + generate_plot(coindb_commit_float_minutes, txout_counts, "Elapsed minutes", "Transaction Output Count", "Coin Database Transaction Output Committed vs Time", os.path.join(png_dir, "coindb_commit_txout_vs_time.png")) + + + print("Plots saved!") \ No newline at end of file diff --git a/bench-ci/run-assumeutxo-bench.sh b/bench-ci/run-assumeutxo-bench.sh new file mode 100755 index 000000000000..a1ee910ed428 --- /dev/null +++ b/bench-ci/run-assumeutxo-bench.sh @@ -0,0 +1,154 @@ +#!/usr/bin/env bash + +set -euxo pipefail + +# Helper function to check and clean datadir +clean_datadir() { + set -euxo pipefail + + local TMP_DATADIR="$1" + + # Create the directory if it doesn't exist + mkdir -p "${TMP_DATADIR}" + + # If we're in CI, clean without confirmation + if [ -n "${CI:-}" ]; then + rm -Rf "${TMP_DATADIR:?}"/* + else + read -rp "Are you sure you want to delete everything in ${TMP_DATADIR}? [y/N] " response + if [[ "$response" =~ ^[Yy]$ ]]; then + rm -Rf "${TMP_DATADIR:?}"/* + else + echo "Aborting..." + exit 1 + fi + fi +} + +# Helper function to clear logs +clean_logs() { + set -euxo pipefail + + local TMP_DATADIR="$1" + local logfile="${TMP_DATADIR}/debug.log" + + echo "Checking for ${logfile}" + if [ -e "${logfile}" ]; then + echo "Removing ${logfile}" + rm "${logfile}" + fi +} + +# Execute CMD before each set of timing runs. +setup_assumeutxo_snapshot_run() { + set -euxo pipefail + + local TMP_DATADIR="$1" + local commit="$2" + clean_datadir "${TMP_DATADIR}" +} + +# Execute CMD before each timing run. +prepare_assumeutxo_snapshot_run() { + set -euxo pipefail + + local TMP_DATADIR="$1" + local UTXO_PATH="$2" + local CONNECT_ADDRESS="$3" + local CHAIN="$4" + local DBCACHE="$5" + local commit="$6" + local BINARIES_DIR="$7" + + # Run the actual preparation steps + clean_datadir "${TMP_DATADIR}" + # Use the pre-built binaries from BINARIES_DIR + "${BINARIES_DIR}/${commit}/bitcoind" --help + taskset -c 0-15 "${BINARIES_DIR}/${commit}/bitcoind" -datadir="${TMP_DATADIR}" -connect="${CONNECT_ADDRESS}" -daemon=0 -chain="${CHAIN}" -stopatheight=1 -printtoconsole=0 + taskset -c 0-15 "${BINARIES_DIR}/${commit}/bitcoind" -datadir="${TMP_DATADIR}" -connect="${CONNECT_ADDRESS}" -daemon=0 -chain="${CHAIN}" -dbcache="${DBCACHE}" -pausebackgroundsync=1 -loadutxosnapshot="${UTXO_PATH}" -printtoconsole=0 || true + clean_logs "${TMP_DATADIR}" +} + +# Executed after each timing run +conclude_assumeutxo_snapshot_run() { + set -euxo pipefail + + local commit="$1" + local TMP_DATADIR="$2" + local PNG_DIR="$3" + + # Search in subdirs e.g. $datadir/signet + debug_log=$(find "${TMP_DATADIR}" -name debug.log -print -quit) + if [ -n "${debug_log}" ]; then + echo "Generating plots from ${debug_log}" + if [ -x "bench-ci/parse_and_plot.py" ]; then + bench-ci/parse_and_plot.py "${debug_log}" "${PNG_DIR}" + else + ls -al "bench-ci/" + echo "parse_and_plot.py not found or not executable, skipping plot generation" + fi + else + ls -al "${TMP_DATADIR}/" + echo "debug.log not found, skipping plot generation" + fi + + # Move flamegraph if exists + if [ -e flamegraph.svg ]; then + mv flamegraph.svg "${commit}"-flamegraph.svg + fi +} + +# Execute CMD after the completion of all benchmarking runs for each individual +# command to be benchmarked. +cleanup_assumeutxo_snapshot_run() { + set -euxo pipefail + + local TMP_DATADIR="$1" + + # Clean up the datadir + clean_datadir "${TMP_DATADIR}" +} + +run_benchmark() { + local base_commit="$1" + local head_commit="$2" + local TMP_DATADIR="$3" + local UTXO_PATH="$4" + local results_file="$5" + local png_dir="$6" + local chain="$7" + local stop_at_height="$8" + local connect_address="$9" + local dbcache="${10}" + local BINARIES_DIR="${11}" + + # Export functions so they can be used by hyperfine + export -f setup_assumeutxo_snapshot_run + export -f prepare_assumeutxo_snapshot_run + export -f conclude_assumeutxo_snapshot_run + export -f cleanup_assumeutxo_snapshot_run + export -f clean_datadir + export -f clean_logs + + # Run hyperfine + hyperfine \ + --shell=bash \ + --setup "setup_assumeutxo_snapshot_run ${TMP_DATADIR} {commit}" \ + --prepare "prepare_assumeutxo_snapshot_run ${TMP_DATADIR} ${UTXO_PATH} ${connect_address} ${chain} ${dbcache} {commit} ${BINARIES_DIR}" \ + --conclude "conclude_assumeutxo_snapshot_run {commit} ${TMP_DATADIR} ${png_dir}" \ + --cleanup "cleanup_assumeutxo_snapshot_run ${TMP_DATADIR}" \ + --runs 1 \ + --export-json "${results_file}" \ + --command-name "base (${base_commit})" \ + --command-name "head (${head_commit})" \ + "taskset -c 1 flamegraph --palette bitcoin --title 'bitcoind assumeutxo IBD@{commit}' -c 'record -F 101 --call-graph fp' -- taskset -c 2-15 ${BINARIES_DIR}/{commit}/bitcoind -datadir=${TMP_DATADIR} -connect=${connect_address} -daemon=0 -chain=${chain} -stopatheight=${stop_at_height} -dbcache=${dbcache} -printtoconsole=0 -debug=coindb -debug=leveldb -debug=bench -debug=validation" \ + -L commit "base,head" +} + +# Main execution +if [ "$#" -ne 11 ]; then + echo "Usage: $0 base_commit head_commit TMP_DATADIR UTXO_PATH results_dir png_dir chain stop_at_height connect_address dbcache BINARIES_DIR" + exit 1 +fi + +run_benchmark "$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8" "$9" "${10}" "${11}" diff --git a/bench-ci/run-benchmark.sh b/bench-ci/run-benchmark.sh new file mode 100755 index 000000000000..dc190a5fa207 --- /dev/null +++ b/bench-ci/run-benchmark.sh @@ -0,0 +1,163 @@ +#!/usr/bin/env bash + +set -euxo pipefail + +# Helper function to check and clean datadir +clean_datadir() { + set -euxo pipefail + + local TMP_DATADIR="$1" + + # Create the directory if it doesn't exist + mkdir -p "${TMP_DATADIR}" + + # If we're in CI, clean without confirmation + if [ -n "${CI:-}" ]; then + rm -Rf "${TMP_DATADIR:?}"/* + else + read -rp "Are you sure you want to delete everything in ${TMP_DATADIR}? [y/N] " response + if [[ "$response" =~ ^[Yy]$ ]]; then + rm -Rf "${TMP_DATADIR:?}"/* + else + echo "Aborting..." + exit 1 + fi + fi +} + +# Helper function to clear logs +clean_logs() { + set -euxo pipefail + + local TMP_DATADIR="$1" + local logfile="${TMP_DATADIR}/debug.log" + + echo "Checking for ${logfile}" + if [ -e "${logfile}" ]; then + echo "Removing ${logfile}" + rm "${logfile}" + fi +} + +# Execute CMD before each set of timing runs. +setup_run() { + set -euxo pipefail + + local TMP_DATADIR="$1" + local commit="$2" + clean_datadir "${TMP_DATADIR}" +} + +# Execute CMD before each timing run. +prepare_run() { + set -euxo pipefail + + local TMP_DATADIR="$1" + local ORIGINAL_DATADIR="$2" + + # Run the actual preparation steps + clean_datadir "${TMP_DATADIR}" + # Don't copy hidden files so use * + taskset -c 0-15 cp -r "$ORIGINAL_DATADIR"/* "$TMP_DATADIR" + clean_logs "${TMP_DATADIR}" +} + +# Executed after each timing run +conclude_run() { + set -euxo pipefail + + local commit="$1" + local TMP_DATADIR="$2" + local PNG_DIR="$3" + + # Search in subdirs e.g. $datadir/signet + debug_log=$(find "${TMP_DATADIR}" -name debug.log -print -quit) + if [ -n "${debug_log}" ]; then + echo "Generating plots from ${debug_log}" + if [ -x "bench-ci/parse_and_plot.py" ]; then + bench-ci/parse_and_plot.py "${debug_log}" "${PNG_DIR}" + else + ls -al "bench-ci/" + echo "parse_and_plot.py not found or not executable, skipping plot generation" + fi + else + ls -al "${TMP_DATADIR}/" + echo "debug.log not found, skipping plot generation" + fi + + # Move flamegraph if exists + if [ -e flamegraph.svg ]; then + mv flamegraph.svg "${commit}"-flamegraph.svg + fi +} + +# Execute CMD after the completion of all benchmarking runs for each individual +# command to be benchmarked. +cleanup_run() { + set -euxo pipefail + + local TMP_DATADIR="$1" + + # Clean up the datadir + clean_datadir "${TMP_DATADIR}" +} + +run_benchmark() { + local base_commit="$1" + local head_commit="$2" + local TMP_DATADIR="$3" + local ORIGINAL_DATADIR="$4" + local results_file="$5" + local png_dir="$6" + local chain="$7" + local stop_at_height="$8" + local connect_address="$9" + local dbcache="${10}" + local BINARIES_DIR="${11}" + + # Export functions so they can be used by hyperfine + export -f setup_run + export -f prepare_run + export -f conclude_run + export -f cleanup_run + export -f clean_datadir + export -f clean_logs + + # Debug: Print all variables being used + echo "=== Debug Information ===" + echo "TMP_DATADIR: ${TMP_DATADIR}" + echo "ORIGINAL_DATADIR: ${ORIGINAL_DATADIR}" + echo "BINARIES_DIR: ${BINARIES_DIR}" + echo "base_commit: ${base_commit}" + echo "head_commit: ${head_commit}" + echo "results_file: ${results_file}" + echo "png_dir: ${png_dir}" + echo "chain: ${chain}" + echo "stop_at_height: ${stop_at_height}" + echo "connect_address: ${connect_address}" + echo "dbcache: ${dbcache}" + echo "\n" + + # Run hyperfine + hyperfine \ + --shell=bash \ + --setup "setup_run ${TMP_DATADIR} {commit}" \ + --prepare "prepare_run ${TMP_DATADIR} ${ORIGINAL_DATADIR}" \ + --conclude "conclude_run {commit} ${TMP_DATADIR} ${png_dir}" \ + --cleanup "cleanup_run ${TMP_DATADIR}" \ + --runs 2 \ + --export-json "${results_file}" \ + --show-output \ + --command-name "base (${base_commit})" \ + --command-name "head (${head_commit})" \ + "taskset -c 1 flamegraph --palette bitcoin --title 'bitcoind IBD@{commit}' -c 'record -F 101 --call-graph fp' -- taskset -c 2-15 chrt -r 1 ${BINARIES_DIR}/{commit}/bitcoind -datadir=${TMP_DATADIR} -connect=${connect_address} -daemon=0 -prune=10000 -chain=${chain} -stopatheight=${stop_at_height} -dbcache=${dbcache} -printtoconsole=0 -debug=coindb -debug=leveldb -debug=bench -debug=validation" \ + -L commit "base,head" +} + +# Main execution +if [ "$#" -ne 11 ]; then + echo "Usage: $0 base_commit head_commit TMP_DATADIR ORIGINAL_DATADIR results_dir png_dir chain stop_at_height connect_address dbcache BINARIES_DIR" + exit 1 +fi + +run_benchmark "$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8" "$9" "${10}" "${11}" From 0cc13911d8fed4a2407e32259e5ec3d579b060c1 Mon Sep 17 00:00:00 2001 From: willcl-ark Date: Tue, 28 Oct 2025 22:37:54 +0000 Subject: [PATCH 08/51] add justfile --- justfile | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 justfile diff --git a/justfile b/justfile new file mode 100644 index 000000000000..0ecd3e990f77 --- /dev/null +++ b/justfile @@ -0,0 +1,36 @@ +set shell := ["bash", "-uc"] + +os := os() + +default: + just --list + +# Build base and head binaries for CI +[group('ci')] +build-assumeutxo-binaries-guix base_commit head_commit: + #!/usr/bin/env bash + set -euxo pipefail + unset SOURCE_DATE_EPOCH # needed to run on NixOS + ./bench-ci/build_binaries.sh {{ base_commit }} {{ head_commit }} + +# Run mainnet benchmark workflow for large cache +[group('ci')] +run-mainnet-ci base_commit head_commit TMP_DATADIR ORIGINAL_DATADIR results_file dbcache png_dir binaries_dir: + #!/usr/bin/env bash + set -euxo pipefail + unset SOURCE_DATE_EPOCH # needed to run on NixOS + ./bench-ci/run-benchmark.sh {{ base_commit }} {{ head_commit }} {{ TMP_DATADIR }} {{ ORIGINAL_DATADIR }} {{ results_file }} {{ png_dir }} main 855000 "148.251.128.115:33333" {{ dbcache }} {{ binaries_dir }} + +# Cherry-pick commits from a bitcoin core PR onto this branch +[group('git')] +pick-pr pr_number: + #!/usr/bin/env bash + set -euxo pipefail + + if ! git remote get-url upstream 2>/dev/null | grep -q "bitcoin/bitcoin"; then + echo "Error: 'upstream' remote not found or doesn't point to bitcoin/bitcoin" + echo "Please add it with: `git remote add upstream https://github.com/bitcoin/bitcoin.git`" + exit 1 + fi + + git fetch upstream pull/{{ pr_number }}/head:bench-{{ pr_number }} && git cherry-pick $(git rev-list --reverse bench-{{ pr_number }} --not upstream/master) From be3cb01da766aa943a56fa194da2ea9e64d05e04 Mon Sep 17 00:00:00 2001 From: willcl-ark Date: Wed, 29 Oct 2025 00:04:54 +0000 Subject: [PATCH 09/51] guix: build static Essentially picked (into bench-ci/guix) from https://github.com/bitcoin/bitcoin/pull/25573 Co-authored-by: fanquake --- bench-ci/guix/libexec/build.sh | 3 ++- bench-ci/guix/libexec/prelude.bash | 5 +++++ bench-ci/guix/manifest.scm | 36 ++++++++++++++++++++++++++++++ bench-ci/guix/security-check.py | 4 ++++ bench-ci/guix/symbol-check.py | 11 +++++---- 5 files changed, 54 insertions(+), 5 deletions(-) diff --git a/bench-ci/guix/libexec/build.sh b/bench-ci/guix/libexec/build.sh index d12c795beaa4..87ed1996cf38 100755 --- a/bench-ci/guix/libexec/build.sh +++ b/bench-ci/guix/libexec/build.sh @@ -144,10 +144,10 @@ export GUIX_LD_WRAPPER_DISABLE_RPATH=yes # Determine the correct value for -Wl,--dynamic-linker for the current $HOST case "$HOST" in + x86_64-linux-gnu) ;; *linux*) glibc_dynamic_linker=$( case "$HOST" in - x86_64-linux-gnu) echo /lib64/ld-linux-x86-64.so.2 ;; arm-linux-gnueabihf) echo /lib/ld-linux-armhf.so.3 ;; aarch64-linux-gnu) echo /lib/ld-linux-aarch64.so.1 ;; riscv64-linux-gnu) echo /lib/ld-linux-riscv64-lp64d.so.1 ;; @@ -236,6 +236,7 @@ esac # LDFLAGS case "$HOST" in + x86_64-linux-gnu) HOST_LDFLAGS=" -static-pie -static-libgcc -Wl,-O2" ;; *linux*) HOST_LDFLAGS="-Wl,--as-needed -Wl,--dynamic-linker=$glibc_dynamic_linker -static-libstdc++ -Wl,-O2" ;; *mingw*) HOST_LDFLAGS="-Wl,--no-insert-timestamp" ;; esac diff --git a/bench-ci/guix/libexec/prelude.bash b/bench-ci/guix/libexec/prelude.bash index b4e828013351..5756e856b240 100644 --- a/bench-ci/guix/libexec/prelude.bash +++ b/bench-ci/guix/libexec/prelude.bash @@ -13,6 +13,11 @@ GUIX_PROFILE=/home/github-runner/.config/guix/current . "$GUIX_PROFILE/etc/profile" echo "Using the following guix command:" command -v guix +echo "Guix command symlink points to:" +readlink -f "$(command -v guix)" +echo "Current Guix profile:" +echo "$GUIX_PROFILE" +echo "Profile generation info:" guix describe ################ diff --git a/bench-ci/guix/manifest.scm b/bench-ci/guix/manifest.scm index 59837e9647e9..f89eccc85360 100644 --- a/bench-ci/guix/manifest.scm +++ b/bench-ci/guix/manifest.scm @@ -438,6 +438,7 @@ inspecting signatures in Mach-O binaries.") "--enable-standard-branch-protection=yes", "--enable-cet=yes", "--disable-gcov", + "--disable-libsanitizer", building-on))) ((#:phases phases) `(modify-phases ,phases @@ -493,6 +494,37 @@ inspecting signatures in Mach-O binaries.") (("^install-others =.*$") (string-append "install-others = " out "/etc/rpc\n"))))))))))))) +(define-public glibc-2.42 + (let ((commit "71874f167aa5bb1538ff7e394beaacee28ebe65f")) + (package + (inherit glibc) ;; 2.39 + (version "2.42") + (source (origin + (method git-fetch) + (uri (git-reference + (url "https://sourceware.org/git/glibc.git") + (commit commit))) + (file-name (git-file-name "glibc" commit)) + (sha256 + (base32 + "1pfbk907fkbavg7grbvb5zlhd3y47f8jj3d2v1s5w7xjnn0ypigq")) + (patches (search-our-patches "glibc-2.42-guix-prefix.patch")))) + (arguments + (substitute-keyword-arguments (package-arguments glibc) + ((#:configure-flags flags) + `(append ,flags + ;; https://www.gnu.org/software/libc/manual/html_node/Configuring-and-compiling.html + (list "--enable-stack-protector=all", + "--enable-bind-now", + "--enable-fortify-source", + "--enable-cet=yes", + "--enable-nscd=no", + "--enable-static-nss=yes", + "--disable-timezone-tools", + "--disable-profile", + "--disable-werror", + building-on)))))))) + ;; The sponge tool from moreutils. (define-public sponge (package @@ -560,6 +592,10 @@ inspecting signatures in Mach-O binaries.") nsis-x86_64 nss-certs osslsigncode)) + ((string-contains target "x86_64-linux-") + (list (list gcc-toolchain-13 "static") + (make-bitcoin-cross-toolchain target + #:base-libc glibc-2.42))) ((string-contains target "-linux-") (list bison pkg-config diff --git a/bench-ci/guix/security-check.py b/bench-ci/guix/security-check.py index be2e0cfbe2af..ac943e33aabd 100755 --- a/bench-ci/guix/security-check.py +++ b/bench-ci/guix/security-check.py @@ -122,6 +122,10 @@ def check_ELF_CONTROL_FLOW(binary) -> bool: return False def check_ELF_FORTIFY(binary) -> bool: + # no imported fortified funcs if we are fully static + # check could be changed to include all symbols + if binary.header.machine_type == lief.ELF.ARCH.X86_64: + return True # bitcoin wrapper does not currently contain any fortified functions if '--monolithic' in binary.strings: diff --git a/bench-ci/guix/symbol-check.py b/bench-ci/guix/symbol-check.py index 464b33cf66fb..3d7a654c8589 100755 --- a/bench-ci/guix/symbol-check.py +++ b/bench-ci/guix/symbol-check.py @@ -34,7 +34,7 @@ MAX_VERSIONS = { 'GCC': (7,0,0), 'GLIBC': { - lief.ELF.ARCH.X86_64: (2,31), + lief.ELF.ARCH.X86_64: (0,0), lief.ELF.ARCH.ARM: (2,31), lief.ELF.ARCH.AARCH64:(2,31), lief.ELF.ARCH.PPC64: (2,31), @@ -47,14 +47,14 @@ # Ignore symbols that are exported as part of every executable IGNORE_EXPORTS = { 'environ', '_environ', '__environ', '_fini', '_init', 'stdin', -'stdout', 'stderr', +'stdout', 'stderr', '__libc_single_threaded', } # Expected linker-loader names can be found here: # https://sourceware.org/glibc/wiki/ABIList?action=recall&rev=16 ELF_INTERPRETER_NAMES: dict[lief.ELF.ARCH, dict[lief.Header.ENDIANNESS, str]] = { lief.ELF.ARCH.X86_64: { - lief.Header.ENDIANNESS.LITTLE: "/lib64/ld-linux-x86-64.so.2", + lief.Header.ENDIANNESS.LITTLE: "", }, lief.ELF.ARCH.ARM: { lief.Header.ENDIANNESS.LITTLE: "/lib/ld-linux-armhf.so.3", @@ -98,7 +98,6 @@ 'libpthread.so.0', # threading 'libm.so.6', # math library 'libatomic.so.1', -'ld-linux-x86-64.so.2', # 64-bit dynamic linker 'ld-linux.so.2', # 32-bit dynamic linker 'ld-linux-aarch64.so.1', # 64-bit ARM dynamic linker 'ld-linux-armhf.so.3', # 32-bit ARM dynamic linker @@ -232,6 +231,10 @@ def check_RUNPATH(binary) -> bool: def check_ELF_libraries(binary) -> bool: ok: bool = True + + if binary.header.machine_type == lief.ELF.ARCH.X86_64: + return len(binary.libraries) == 0 + for library in binary.libraries: if library not in ELF_ALLOWED_LIBRARIES: print(f'{filename}: {library} is not in ALLOWED_LIBRARIES!') From 7306412ff27e28eaf7f4663b012924ae1125088b Mon Sep 17 00:00:00 2001 From: willcl-ark Date: Mon, 10 Feb 2025 12:21:14 +0000 Subject: [PATCH 10/51] doc: add benchcoin docs --- .github/README.md | 1 + doc/benchcoin.md | 127 ++++++++++++ doc/flamegraph.svg | 491 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 619 insertions(+) create mode 120000 .github/README.md create mode 100644 doc/benchcoin.md create mode 100644 doc/flamegraph.svg diff --git a/.github/README.md b/.github/README.md new file mode 120000 index 000000000000..e5c578ba74b5 --- /dev/null +++ b/.github/README.md @@ -0,0 +1 @@ +../doc/benchcoin.md \ No newline at end of file diff --git a/doc/benchcoin.md b/doc/benchcoin.md new file mode 100644 index 000000000000..0b4159256c95 --- /dev/null +++ b/doc/benchcoin.md @@ -0,0 +1,127 @@ +# benchcoin + +A Bitcoin Core benchmarking fork + +This repository is a fork of Bitcoin Core that performs automated IBD benchmarking. +It allows you to measure and compare the performance impact of certain types of changes to Bitcoin Core's codebase on a longer-running IBD benchmark, in a (pretty) reproducible fashion. + +## Features + +- Automated IBD benchmarking on pull requests +- Multiple configurations: + - Mainnet with default cache + - Mainnet with large cache +- Performance visualizations including: + - Flamegraphs for CPU profiling + - Time series plots of various metrics + - Compare `base` (bitcoin/bitcoin:master) and `head` (PR) + +## Example Flamegraph + +Below is an example flamegraph showing CPU utilization during IBD: + +![Example Flamegraph](../doc/flamegraph.svg) + +## How to use it + +1. Open a Pull Request against **this repo** +2. Wait for the bot to comment on your PR after it's finished. + +See the [Contributing](#contributing) section for more details. + +## How it works + +When you open a pull request against this repository: + +1. The CI workflow automatically builds both the base and PR versions of bitcoind +2. Runs IBD benchmarks +3. Records performance metrics and creates various visualizations +4. Posts results as a comment on your PR + +The benchmarks test three configurations: +- Mainnet-default: with default (450 MB) dbcache + - From a pruned datadir @ height 840,000 to height 855,000 +- Mainnet-large: with 32000 MB dbcache + - From a pruned datadir @ height 840,000 to height 855,000 + +## Benchmark Outputs + +For each benchmark run, you'll get a github pages page with: + +- Timing comparisons between base and PR versions +- CPU flamegraphs showing where time is spent +- Time series plots showing: + - Block height vs time + - Cache size vs block height + - Cache size vs time + - Transaction count vs block height + - Coins cache size vs time + - LevelDB metrics + - Memory pool metrics + +## Local Development (WIP) + +To run benchmarks locally (WIP, and Linux-only due to [shell.nix](../shell.nix) limitations): + +1. Make sure you have [Nix package manager](https://nixos.org/download/) installed + +2. Setup the Nix development environment: +```bash +nix-shell +``` + +3. Run a local benchmark: +```bash +just run-signet +``` + +This will: +- Create a temporary directory for testing +- Build both base and PR versions +- Download the required UTXO snapshot if needed +- Run the benchmark +- Generate performance visualizations + +## Technical Details + +The benchmarking system uses: +- [Hyperfine](https://github.com/sharkdp/hyperfine) for benchmark timing +- [Flamegraph](https://github.com/willcl-ark/flamegraph) for CPU profiling +- [matplotlib](https://matplotlib.org/) for metric visualization +- [GitHub Actions](https://github.com/features/actions) for CI automation + +The system copies over a pruned datadir to speed up IBD to a more interesting height (840k). + +### Runner & seed + +The CI runner is self-hosted on a Hetzner AX52 running at the bitcoin-dev-tools organsation level. +It is running NixOS using configuration found in this repo: [nix-github-runner](https://github.com/bitcoin-dev-tools/nix-github-runner) for easier deployment and reproducibility. + +The runner host has 16 cores, with one used for system, one for `flamegraph` (i.e. `perf record`) and 14 dedicated to the Bitcoin Core node under test. + +The benchmarking peer on the runner is served blocks over the (real) "internet" (it may be LAN as it's within a single Hetzner region) via a single peer to exercise full IBD codepaths. This naturally may introduce some variance, but it was deemed preferable to running another bitcoin core on the same machine. + +This seed peer is another Hetzner VPS in the same region, and its configuration can be found here: [nix-seed-node](https://github.com/bitcoin-dev-tools/nix-seed-node) + +## Contributing + +### Benchmark an existing bitcoin/bitcoin PR + +This requires `just` be installed. If you don't have `just` installed you can run the commands in the [justfile](../justfile) manually. + +1. Fork this repository (or bitcoin/bitcoin and add this as a remote) +2. Create a new branch from benchcoin/master +3. Run: `just pick-pr ` to cherry-pick commits from the PR +4. Push the branch +5. Open a pull request **against this repo. NOT bitcoin/bitcoin** + +### Benchmark standalone/new changes + +1. Fork this repository (or bitcoin/bitcoin and add this as a remote) +2. Make your changes to Bitcoin Core +3. Open a pull request **against this repo. NOT bitcoin/bitcoin** +4. Wait for benchmark results to be posted on your PR here + +## License + +This project is licensed under the same terms as Bitcoin Core - see the [COPYING](../COPYING) file for details. diff --git a/doc/flamegraph.svg b/doc/flamegraph.svg new file mode 100644 index 000000000000..77f05068edd1 --- /dev/null +++ b/doc/flamegraph.svg @@ -0,0 +1,491 @@ +bitcoind assumeutxo IBD@head Reset ZoomSearch [unknown] (930,216,305 samples, 0.03%)libc.so.6::__GI___libc_open (1,277,437,934 samples, 0.04%)[unknown] (1,277,437,934 samples, 0.04%)[unknown] (1,121,698,471 samples, 0.03%)[unknown] (1,121,698,471 samples, 0.03%)[unknown] (1,121,698,471 samples, 0.03%)[unknown] (808,723,138 samples, 0.02%)[unknown] (705,370,773 samples, 0.02%)[unknown] (654,247,113 samples, 0.02%)[unknown] (601,840,190 samples, 0.02%)[unknown] (412,286,776 samples, 0.01%)libc.so.6::__lll_lock_wait_private (3,169,140,832 samples, 0.09%)[unknown] (3,068,852,192 samples, 0.09%)[unknown] (2,912,247,498 samples, 0.08%)[unknown] (2,859,869,350 samples, 0.08%)[unknown] (2,547,374,665 samples, 0.07%)[unknown] (2,442,338,234 samples, 0.07%)[unknown] (2,018,530,007 samples, 0.06%)[unknown] (1,768,059,272 samples, 0.05%)[unknown] (1,360,516,543 samples, 0.04%)[unknown] (941,780,033 samples, 0.03%)[unknown] (732,126,125 samples, 0.02%)[unknown] (367,091,733 samples, 0.01%)libc.so.6::__lll_lock_wake_private (53,149,822,463 samples, 1.49%)l..[unknown] (52,891,684,033 samples, 1.49%)[..[unknown] (51,489,363,011 samples, 1.45%)[..[unknown] (51,020,482,662 samples, 1.43%)[..[unknown] (46,915,115,303 samples, 1.32%)[unknown] (45,255,852,290 samples, 1.27%)[unknown] (38,150,418,340 samples, 1.07%)[unknown] (35,292,486,865 samples, 0.99%)[unknown] (7,892,404,247 samples, 0.22%)[unknown] (3,327,749,547 samples, 0.09%)[unknown] (1,188,855,625 samples, 0.03%)[unknown] (566,758,595 samples, 0.02%)libc.so.6::_int_free_create_chunk (628,326,946 samples, 0.02%)libc.so.6::_int_free_merge_chunk (358,656,602 samples, 0.01%)libc.so.6::_int_malloc (74,559,659,927 samples, 2.10%)li..[unknown] (721,620,417 samples, 0.02%)[unknown] (610,988,583 samples, 0.02%)[unknown] (610,988,583 samples, 0.02%)[unknown] (610,988,583 samples, 0.02%)[unknown] (559,250,914 samples, 0.02%)[unknown] (559,250,914 samples, 0.02%)libc.so.6::alloc_perturb (425,154,213 samples, 0.01%)libc.so.6::malloc (24,700,554,078 samples, 0.69%)libc.so.6::malloc_consolidate (735,996,757 samples, 0.02%)libc.so.6::unlink_chunk.isra.0 (6,120,352,373 samples, 0.17%)[unknown] (167,607,884,597 samples, 4.71%)[unknown]libstdc++.so.6.0.32::virtual thunk to std::__cxx11::basic_ostringstream<char, std::char_traits<char>, std::allocator<char> >::~basic_ostringstream (417,178,495 samples, 0.01%)[unknown] (417,178,495 samples, 0.01%)libc.so.6::_IO_default_xsputn (371,898,668 samples, 0.01%)libc.so.6::_IO_do_write@@GLIBC_2.2.5 (415,186,042 samples, 0.01%)libc.so.6::_IO_file_xsputn@@GLIBC_2.2.5 (52,841,892,362 samples, 1.49%)l..libc.so.6::_IO_fwrite (157,971,658,633 samples, 4.44%)libc.so...[[ext4]] (1,657,432,113 samples, 0.05%)[unknown] (573,069,492 samples, 0.02%)[[ext4]] (2,536,153,731 samples, 0.07%)[[ext4]] (10,537,322,599 samples, 0.30%)[unknown] (7,422,408,080 samples, 0.21%)[unknown] (6,329,696,449 samples, 0.18%)[unknown] (5,353,636,150 samples, 0.15%)[unknown] (5,041,980,997 samples, 0.14%)[unknown] (3,383,888,214 samples, 0.10%)[unknown] (1,348,486,405 samples, 0.04%)[unknown] (477,579,410 samples, 0.01%)[unknown] (424,961,857 samples, 0.01%)[[ext4]] (48,707,811,335 samples, 1.37%)[..[unknown] (37,296,429,178 samples, 1.05%)[unknown] (35,118,068,672 samples, 0.99%)[unknown] (29,610,843,695 samples, 0.83%)[unknown] (24,208,827,110 samples, 0.68%)[unknown] (17,096,181,771 samples, 0.48%)[unknown] (6,112,761,166 samples, 0.17%)[unknown] (1,344,893,459 samples, 0.04%)[unknown] (458,831,632 samples, 0.01%)[[ext4]] (365,017,200 samples, 0.01%)[[ext4]] (518,180,627 samples, 0.01%)[[ext4]] (466,259,788 samples, 0.01%)[[ext4]] (673,383,386 samples, 0.02%)[[ext4]] (59,764,846,104 samples, 1.68%)[..[unknown] (58,060,722,922 samples, 1.63%)[..[unknown] (7,950,480,723 samples, 0.22%)[unknown] (5,540,377,500 samples, 0.16%)[unknown] (865,590,582 samples, 0.02%)[unknown] (813,212,612 samples, 0.02%)[unknown] (813,212,612 samples, 0.02%)[unknown] (813,212,612 samples, 0.02%)[unknown] (711,368,524 samples, 0.02%)libc.so.6::__GI___libc_write (70,786,161,691 samples, 1.99%)li..[unknown] (70,568,950,557 samples, 1.98%)[u..[unknown] (69,379,113,892 samples, 1.95%)[u..[unknown] (68,772,280,665 samples, 1.93%)[u..[unknown] (66,697,097,059 samples, 1.88%)[u..[unknown] (3,800,961,354 samples, 0.11%)[unknown] (780,895,718 samples, 0.02%)libc.so.6::__memmove_avx512_unaligned_erms (15,769,232,267 samples, 0.44%)libc.so.6::__mempcpy@plt (4,938,637,189 samples, 0.14%)libc.so.6::__send (1,149,037,952 samples, 0.03%)[unknown] (1,149,037,952 samples, 0.03%)[unknown] (1,149,037,952 samples, 0.03%)[unknown] (1,149,037,952 samples, 0.03%)[unknown] (1,096,533,096 samples, 0.03%)[unknown] (1,096,533,096 samples, 0.03%)[unknown] (1,096,533,096 samples, 0.03%)[unknown] (1,094,640,456 samples, 0.03%)[unknown] (943,771,904 samples, 0.03%)[unknown] (626,496,659 samples, 0.02%)[unknown] (522,399,654 samples, 0.01%)[unknown] (469,549,544 samples, 0.01%)[unknown] (469,549,544 samples, 0.01%)[unknown] (366,321,373 samples, 0.01%)libc.so.6::_int_free (16,918,597,179 samples, 0.48%)libc.so.6::_int_free_merge_chunk (716,678,677 samples, 0.02%)libc.so.6::_int_malloc (1,269,524,481 samples, 0.04%)libc.so.6::cfree@GLIBC_2.2.5 (4,352,992,616 samples, 0.12%)libc.so.6::malloc (8,032,159,513 samples, 0.23%)libc.so.6::malloc_consolidate (39,479,511,598 samples, 1.11%)[unknown] (401,333,554 samples, 0.01%)[unknown] (401,333,554 samples, 0.01%)[unknown] (401,333,554 samples, 0.01%)[unknown] (401,333,554 samples, 0.01%)[unknown] (401,333,554 samples, 0.01%)[unknown] (401,333,554 samples, 0.01%)libc.so.6::new_do_write (469,906,341 samples, 0.01%)libc.so.6::read (459,442,054 samples, 0.01%)[unknown] (459,442,054 samples, 0.01%)[unknown] (360,200,514 samples, 0.01%)[unknown] (360,200,514 samples, 0.01%)[unknown] (360,200,514 samples, 0.01%)[unknown] (360,200,514 samples, 0.01%)libc.so.6::sysmalloc (469,717,952 samples, 0.01%)[unknown] (469,717,952 samples, 0.01%)[unknown] (415,893,983 samples, 0.01%)[unknown] (366,135,265 samples, 0.01%)[unknown] (366,135,265 samples, 0.01%)libc.so.6::unlink_chunk.isra.0 (2,862,604,776 samples, 0.08%)bitcoind::CBlockIndex::GetAncestor (412,360,660 samples, 0.01%)bitcoind::CCoinsViewCache::AccessCoin (421,783,849 samples, 0.01%)bitcoind::SipHashUint256Extra (6,150,872,313 samples, 0.17%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::_M_erase (100,736,697,557 samples, 2.83%)bitc..bitcoind::SipHashUint256Extra (1,991,693,392 samples, 0.06%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::_M_find_before_node (90,084,545,787 samples, 2.53%)bit..bitcoind::SipHashUint256Extra (71,251,854,599 samples, 2.00%)bi..bitcoind::SipHashUint256Extra (26,794,756,611 samples, 0.75%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::_M_insert_unique_node (46,369,997,648 samples, 1.30%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::_M_rehash (18,471,505,609 samples, 0.52%)libc.so.6::__memset_avx512_unaligned_erms (632,105,655 samples, 0.02%)[unknown] (579,371,219 samples, 0.02%)[unknown] (474,387,191 samples, 0.01%)[unknown] (421,585,797 samples, 0.01%)[unknown] (421,585,797 samples, 0.01%)[unknown] (368,759,434 samples, 0.01%)[unknown] (368,759,434 samples, 0.01%)[unknown] (368,759,434 samples, 0.01%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::erase (1,518,987,687 samples, 0.04%)bitcoind::SipHashUint256Extra (625,645,482 samples, 0.02%)bitcoind::SipHashUint256Extra (6,692,957,315 samples, 0.19%)[unknown] (1,036,177,296 samples, 0.03%)[unknown] (928,879,608 samples, 0.03%)[unknown] (877,183,919 samples, 0.02%)[unknown] (719,026,447 samples, 0.02%)[unknown] (666,701,067 samples, 0.02%)[unknown] (626,005,752 samples, 0.02%)[unknown] (364,282,815 samples, 0.01%)[unknown] (364,282,815 samples, 0.01%)[unknown] (364,282,815 samples, 0.01%)[unknown] (364,282,815 samples, 0.01%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::find (133,163,328,034 samples, 3.75%)bitcoi..bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::_M_find_before_node (119,438,100,972 samples, 3.36%)bitco..bitcoind::SipHashUint256Extra (986,497,657 samples, 0.03%)bitcoind::std::__detail::_Hash_node<std::pair<COutPoint const, CCoinsCacheEntry>, false>* std::__detail::_Hashtable_alloc<PoolAllocator<std::__detail::_Hash_node<std::pair<COutPoint const, CCoinsCacheEntry>, false>, 144ul, 8ul> >::_M_allocate_node<std::piecewise_construct_t const&, std::tuple<COutPoint const&>, std::tuple<> > (5,414,052,109 samples, 0.15%)libc.so.6::cfree@GLIBC_2.2.5 (4,527,272,747 samples, 0.13%)bitcoind::CCoinsViewCache::BatchWrite (408,297,908,928 samples, 11.48%)bitcoind::CCoinsViewCac..bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::clear (4,431,167,402 samples, 0.12%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::find (676,133,697 samples, 0.02%)bitcoind::CCoinsViewCache::Flush (414,604,793,420 samples, 11.66%)bitcoind::CCoinsViewCach..bitcoind::CTxMemPool::removeConflicts (1,307,189,422 samples, 0.04%)bitcoind::std::_Rb_tree<COutPoint const*, std::pair<COutPoint const* const, CTransaction const*>, std::_Select1st<std::pair<COutPoint const* const, CTransaction const*> >, DereferencingComparator<COutPoint const*>, std::allocator<std::pair<COutPoint const* const, CTransaction const*> > >::find (940,298,479 samples, 0.03%)bitcoind::SipHashUint256 (1,301,282,993 samples, 0.04%)bitcoind::std::_Rb_tree<uint256, std::pair<uint256 const, long>, std::_Select1st<std::pair<uint256 const, long> >, std::less<uint256>, std::allocator<std::pair<uint256 const, long> > >::_M_erase (1,201,625,005 samples, 0.03%)bitcoind::CTxMemPool::removeForBlock (17,028,655,239 samples, 0.48%)bitcoind::std::_Rb_tree<uint256, std::pair<uint256 const, long>, std::_Select1st<std::pair<uint256 const, long> >, std::less<uint256>, std::allocator<std::pair<uint256 const, long> > >::erase (12,855,923,134 samples, 0.36%)bitcoind::std::_Rb_tree<uint256, std::pair<uint256 const, long>, std::_Select1st<std::pair<uint256 const, long> >, std::less<uint256>, std::allocator<std::pair<uint256 const, long> > >::equal_range (2,508,971,022 samples, 0.07%)[unknown] (3,441,479,431 samples, 0.10%)[unknown] (3,089,709,936 samples, 0.09%)[unknown] (2,820,174,820 samples, 0.08%)[unknown] (2,720,356,939 samples, 0.08%)[unknown] (2,720,356,939 samples, 0.08%)[unknown] (2,557,087,196 samples, 0.07%)[unknown] (2,356,775,337 samples, 0.07%)[unknown] (1,672,816,080 samples, 0.05%)[unknown] (1,100,674,926 samples, 0.03%)[unknown] (787,217,059 samples, 0.02%)[unknown] (574,492,426 samples, 0.02%)bitcoind::SipHashUint256Extra (359,543,734 samples, 0.01%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::_M_find_before_node (10,977,516,042 samples, 0.31%)bitcoind::SipHashUint256Extra (3,562,058,963 samples, 0.10%)bitcoind::SipHashUint256Extra (1,836,963,585 samples, 0.05%)bitcoind::SipHashUint256Extra (6,867,820,925 samples, 0.19%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::_M_insert_unique_node (16,890,522,357 samples, 0.48%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::_M_rehash (12,768,158,119 samples, 0.36%)bitcoind::std::__detail::_Hash_node<std::pair<COutPoint const, CCoinsCacheEntry>, false>* std::__detail::_Hashtable_alloc<PoolAllocator<std::__detail::_Hash_node<std::pair<COutPoint const, CCoinsCacheEntry>, false>, 144ul, 8ul> >::_M_allocate_node<std::piecewise_construct_t const&, std::tuple<COutPoint const&>, std::tuple<> > (6,083,575,685 samples, 0.17%)[unknown] (2,667,289,880 samples, 0.08%)[unknown] (2,453,773,220 samples, 0.07%)[unknown] (2,293,236,868 samples, 0.06%)[unknown] (2,189,852,142 samples, 0.06%)[unknown] (1,978,814,058 samples, 0.06%)[unknown] (1,713,021,112 samples, 0.05%)[unknown] (1,360,558,892 samples, 0.04%)[unknown] (1,099,770,850 samples, 0.03%)[unknown] (785,095,967 samples, 0.02%)[unknown] (468,560,942 samples, 0.01%)[unknown] (366,515,283 samples, 0.01%)bitcoind::CCoinsViewCache::AddCoin (67,517,205,631 samples, 1.90%)bi..bitcoind::AddCoins (83,151,504,659 samples, 2.34%)bit..bitcoind::std::__detail::_Hash_node<std::pair<COutPoint const, CCoinsCacheEntry>, false>* std::__detail::_Hashtable_alloc<PoolAllocator<std::__detail::_Hash_node<std::pair<COutPoint const, CCoinsCacheEntry>, false>, 144ul, 8ul> >::_M_allocate_node<std::piecewise_construct_t const&, std::tuple<COutPoint const&>, std::tuple<> > (368,308,911 samples, 0.01%)bitcoind::CBlockIndex::GetAncestor (780,828,411 samples, 0.02%)bitcoind::SipHashUint256Extra (6,967,127,022 samples, 0.20%)bitcoind::CCoinsViewCache::FetchCoin (11,631,656,359 samples, 0.33%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::_M_find_before_node (2,762,447,333 samples, 0.08%)bitcoind::CCoinsViewCache::AccessCoin (13,718,933,582 samples, 0.39%)bitcoind::CCoinsViewCache::AddCoin (935,848,977 samples, 0.03%)bitcoind::CCoinsViewCache::HaveInputs (363,967,847 samples, 0.01%)bitcoind::CCoinsViewCache::SpendCoin (775,446,488 samples, 0.02%)bitcoind::CTransaction::GetValueOut (571,129,594 samples, 0.02%)bitcoind::SipHashUint256Extra (6,132,196,838 samples, 0.17%)bitcoind::CCoinsViewCache::FetchCoin (22,771,955,106 samples, 0.64%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::_M_find_before_node (8,387,260,917 samples, 0.24%)bitcoind::SipHashUint256Extra (672,360,582 samples, 0.02%)bitcoind::CCoinsViewCache::AccessCoin (27,541,380,041 samples, 0.77%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::_M_find_before_node (840,128,595 samples, 0.02%)bitcoind::CCoinsViewCache::FetchCoin (9,862,576,991 samples, 0.28%)bitcoind::CCoinsViewCache::FetchCoin (723,258,358 samples, 0.02%)bitcoind::CCoinsViewBacked::GetCoin (1,001,559,892 samples, 0.03%)bitcoind::leveldb::LookupKey::LookupKey (468,932,422 samples, 0.01%)bitcoind::leveldb::(anonymous namespace)::BytewiseComparatorImpl::Compare (477,889,771 samples, 0.01%)bitcoind::leveldb::(anonymous namespace)::BytewiseComparatorImpl::Compare (2,464,437,114 samples, 0.07%)bitcoind::leveldb::FindFile (12,889,348,897 samples, 0.36%)bitcoind::leveldb::InternalKeyComparator::Compare (8,952,657,039 samples, 0.25%)libc.so.6::__memcmp_evex_movbe (3,658,168,717 samples, 0.10%)bitcoind::leveldb::InternalKeyComparator::Compare (468,603,758 samples, 0.01%)bitcoind::leveldb::(anonymous namespace)::ShardedLRUCache::Lookup (2,481,353,143 samples, 0.07%)[unknown] (470,703,247 samples, 0.01%)[unknown] (419,110,322 samples, 0.01%)[unknown] (367,081,554 samples, 0.01%)[unknown] (367,081,554 samples, 0.01%)bitcoind::std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_mutate (723,558,367 samples, 0.02%)libc.so.6::__memmove_avx512_unaligned_erms (682,634,544 samples, 0.02%)bitcoind::leveldb::Block::Iter::ParseNextKey (6,607,693,428 samples, 0.19%)libc.so.6::malloc (468,621,157 samples, 0.01%)bitcoind::leveldb::(anonymous namespace)::BytewiseComparatorImpl::Compare (3,736,764,421 samples, 0.11%)bitcoind::leveldb::InternalKeyComparator::Compare (22,752,758,306 samples, 0.64%)libc.so.6::__memcmp_evex_movbe (16,502,022,326 samples, 0.46%)bitcoind::leveldb::Block::Iter::Seek (81,753,854,146 samples, 2.30%)bit..libc.so.6::__memmove_avx512_unaligned_erms (624,754,079 samples, 0.02%)bitcoind::leveldb::Block::Iter::~Iter (1,202,042,453 samples, 0.03%)bitcoind::leveldb::Iterator::~Iterator (886,809,043 samples, 0.02%)bitcoind::leveldb::DeleteBlock (418,661,180 samples, 0.01%)bitcoind::leveldb::Block::NewIterator (1,830,741,267 samples, 0.05%)bitcoind::leveldb::BlockHandle::DecodeFrom (1,350,133,609 samples, 0.04%)bitcoind::leveldb::FilterBlockReader::KeyMayMatch (3,241,956,535 samples, 0.09%)bitcoind::leveldb::InternalFilterPolicy::KeyMayMatch (470,469,134 samples, 0.01%)bitcoind::leveldb::(anonymous namespace)::BloomFilterPolicy::KeyMayMatch (470,469,134 samples, 0.01%)bitcoind::leveldb::InternalKeyComparator::Compare (2,930,394,374 samples, 0.08%)bitcoind::leveldb::SaveValue (885,107,264 samples, 0.02%)bitcoind::leveldb::(anonymous namespace)::ShardedLRUCache::Lookup (1,152,034,360 samples, 0.03%)bitcoind::leveldb::Hash (363,890,191 samples, 0.01%)bitcoind::leveldb::Block::NewIterator (1,259,229,813 samples, 0.04%)bitcoind::leveldb::BlockHandle::DecodeFrom (1,156,612,863 samples, 0.03%)bitcoind::leveldb::GetVarint64 (416,693,035 samples, 0.01%)bitcoind::leveldb::Iterator::RegisterCleanup (363,166,691 samples, 0.01%)[unknown] (2,314,123,053 samples, 0.07%)[unknown] (2,156,687,800 samples, 0.06%)[unknown] (2,051,108,413 samples, 0.06%)[unknown] (1,945,393,833 samples, 0.05%)[unknown] (1,894,650,811 samples, 0.05%)[unknown] (1,894,650,811 samples, 0.05%)[unknown] (1,794,842,453 samples, 0.05%)[unknown] (1,315,291,384 samples, 0.04%)[unknown] (733,842,157 samples, 0.02%)[unknown] (421,059,647 samples, 0.01%)[unknown] (367,252,654 samples, 0.01%)bitcoind::crc32c::ExtendSse42 (56,521,776,403 samples, 1.59%)b..bitcoind::leveldb::ReadBlock (62,722,682,079 samples, 1.76%)b..libc.so.6::__GI___pthread_mutex_unlock_usercnt (978,769,336 samples, 0.03%)libc.so.6::cfree@GLIBC_2.2.5 (571,745,263 samples, 0.02%)bitcoind::leveldb::Table::BlockReader (93,027,689,265 samples, 2.62%)bit..libc.so.6::__memmove_avx512_unaligned_erms (525,280,305 samples, 0.01%)bitcoind::leveldb::Table::InternalGet (191,009,481,478 samples, 5.37%)bitcoind::..bitcoind::leveldb::(anonymous namespace)::ShardedLRUCache::Lookup (2,456,558,609 samples, 0.07%)bitcoind::leveldb::Hash (674,476,478 samples, 0.02%)libc.so.6::__GI___pthread_mutex_unlock_usercnt (949,827,762 samples, 0.03%)libc.so.6::__memcmp_evex_movbe (672,469,665 samples, 0.02%)libc.so.6::pthread_mutex_lock@@GLIBC_2.2.5 (770,697,666 samples, 0.02%)bitcoind::leveldb::TableCache::FindTable (5,889,647,371 samples, 0.17%)bitcoind::leveldb::TableCache::Get (199,229,141,358 samples, 5.60%)bitcoind::..bitcoind::leveldb::Version::Get (200,226,855,069 samples, 5.63%)bitcoind::..libc.so.6::__GI___pthread_mutex_unlock_usercnt (733,288,816 samples, 0.02%)bitcoind::leveldb::Version::ForEachOverlapping (215,208,197,899 samples, 6.05%)bitcoind::l..libc.so.6::__memcmp_evex_movbe (359,285,284 samples, 0.01%)bitcoind::leveldb::Version::Get (216,049,507,027 samples, 6.08%)bitcoind::l..bitcoind::leveldb::DBImpl::Get (217,672,929,621 samples, 6.12%)bitcoind::l..libc.so.6::__GI___pthread_mutex_unlock_usercnt (1,861,877,233 samples, 0.05%)bitcoind::CDBWrapper::ReadImpl[abi:cxx11] (221,752,252,623 samples, 6.24%)bitcoind::CD..libc.so.6::pthread_mutex_lock@@GLIBC_2.2.5 (1,748,433,964 samples, 0.05%)bitcoind::DecompressAmount (1,005,313,570 samples, 0.03%)bitcoind::void ScriptCompression::Unser<DataStream> (2,769,444,330 samples, 0.08%)bitcoind::void std::vector<std::byte, zero_after_free_allocator<std::byte> >::_M_range_insert<std::byte const*> (7,911,029,894 samples, 0.22%)libc.so.6::__memmove_avx512_unaligned_erms (416,410,569 samples, 0.01%)bitcoind::CCoinsViewDB::GetCoin (247,131,705,346 samples, 6.95%)bitcoind::CCo..bitcoind::CCoinsViewBacked::GetCoin (251,714,610,750 samples, 7.08%)bitcoind::CCo..bitcoind::CCoinsViewErrorCatcher::GetCoin (257,960,090,912 samples, 7.26%)bitcoind::CCoi..bitcoind::CCoinsViewDB::GetCoin (5,789,812,101 samples, 0.16%)bitcoind::SipHashUint256Extra (686,778,601 samples, 0.02%)[unknown] (1,028,820,936 samples, 0.03%)[unknown] (974,950,139 samples, 0.03%)[unknown] (867,196,862 samples, 0.02%)[unknown] (710,030,298 samples, 0.02%)[unknown] (710,030,298 samples, 0.02%)[unknown] (600,430,034 samples, 0.02%)[unknown] (489,234,171 samples, 0.01%)[unknown] (434,975,120 samples, 0.01%)[unknown] (434,975,120 samples, 0.01%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::_M_find_before_node (29,304,700,539 samples, 0.82%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::_M_insert_unique_node (21,307,639,964 samples, 0.60%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::_M_rehash (20,780,827,998 samples, 0.58%)libc.so.6::__memset_avx512_unaligned_erms (579,451,231 samples, 0.02%)[unknown] (579,451,231 samples, 0.02%)[unknown] (526,649,228 samples, 0.01%)[unknown] (526,649,228 samples, 0.01%)[unknown] (526,649,228 samples, 0.01%)[unknown] (473,772,435 samples, 0.01%)[unknown] (420,996,348 samples, 0.01%)[unknown] (368,735,591 samples, 0.01%)[unknown] (368,735,591 samples, 0.01%)bitcoind::std::__detail::_Hash_node<std::pair<COutPoint const, CCoinsCacheEntry>, false>* std::__detail::_Hashtable_alloc<PoolAllocator<std::__detail::_Hash_node<std::pair<COutPoint const, CCoinsCacheEntry>, false>, 144ul, 8ul> >::_M_allocate_node<std::piecewise_construct_t const&, std::tuple<COutPoint const&>, std::tuple<> > (4,934,629,385 samples, 0.14%)[unknown] (421,130,280 samples, 0.01%)[unknown] (368,737,467 samples, 0.01%)[unknown] (368,737,467 samples, 0.01%)bitcoind::CCoinsViewCache::FetchCoin (327,425,895,563 samples, 9.21%)bitcoind::CCoinsVi..bitcoind::CCoinsViewErrorCatcher::GetCoin (601,145,923 samples, 0.02%)bitcoind::CCoinsViewCache::GetCoin (349,247,006,292 samples, 9.82%)bitcoind::CCoinsView..bitcoind::SipHashUint256Extra (17,454,209,723 samples, 0.49%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::_M_find_before_node (22,697,810,020 samples, 0.64%)bitcoind::SipHashUint256Extra (4,124,049,750 samples, 0.12%)bitcoind::SipHashUint256Extra (4,306,133,540 samples, 0.12%)bitcoind::SipHashUint256Extra (7,085,914,542 samples, 0.20%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::_M_insert_unique_node (19,180,887,889 samples, 0.54%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::_M_rehash (12,199,005,039 samples, 0.34%)libc.so.6::__memset_avx512_unaligned_erms (574,777,734 samples, 0.02%)bitcoind::std::__detail::_Hash_node<std::pair<COutPoint const, CCoinsCacheEntry>, false>* std::__detail::_Hashtable_alloc<PoolAllocator<std::__detail::_Hash_node<std::pair<COutPoint const, CCoinsCacheEntry>, false>, 144ul, 8ul> >::_M_allocate_node<std::piecewise_construct_t const&, std::tuple<COutPoint const&>, std::tuple<> > (7,865,255,678 samples, 0.22%)[unknown] (1,969,736,150 samples, 0.06%)[unknown] (1,916,111,977 samples, 0.05%)[unknown] (1,812,200,695 samples, 0.05%)[unknown] (1,812,200,695 samples, 0.05%)[unknown] (1,812,200,695 samples, 0.05%)[unknown] (1,496,076,465 samples, 0.04%)[unknown] (1,234,917,855 samples, 0.03%)[unknown] (921,179,131 samples, 0.03%)[unknown] (658,036,512 samples, 0.02%)[unknown] (507,636,670 samples, 0.01%)bitcoind::CCoinsViewCache::FetchCoin (439,862,693,437 samples, 12.37%)bitcoind::CCoinsViewCache..bitcoind::CCoinsViewCache::GetCoin (567,408,453 samples, 0.02%)bitcoind::SipHashUint256Extra (11,079,411,759 samples, 0.31%)bitcoind::CCoinsViewCache::HaveInputs (468,021,622,384 samples, 13.16%)bitcoind::CCoinsViewCache::..bitcoind::Consensus::CheckTxInputs (525,550,058,887 samples, 14.78%)bitcoind::Consensus::CheckTxInp..bitcoind::CTransaction::GetValueOut (8,116,827,965 samples, 0.23%)bitcoind::EvaluateSequenceLocks (13,084,419,728 samples, 0.37%)bitcoind::CBlockIndex::GetMedianTimePast (12,762,378,539 samples, 0.36%)bitcoind::void std::__introsort_loop<long*, long, __gnu_cxx::__ops::_Iter_less_iter> (1,776,177,595 samples, 0.05%)bitcoind::SipHashUint256Extra (3,528,590,848 samples, 0.10%)bitcoind::CCoinsViewCache::FetchCoin (9,099,104,563 samples, 0.26%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::_M_find_before_node (3,448,133,251 samples, 0.10%)bitcoind::SipHashUint256Extra (373,550,141 samples, 0.01%)bitcoind::CCoinsViewCache::AccessCoin (10,147,664,939 samples, 0.29%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::_M_find_before_node (368,697,772 samples, 0.01%)bitcoind::CScript::GetSigOpCount (1,181,105,155 samples, 0.03%)bitcoind::CScript::IsPayToScriptHash (361,942,649 samples, 0.01%)bitcoind::CScript::IsPushOnly (1,550,137,517 samples, 0.04%)bitcoind::CScript::IsWitnessProgram (14,154,912,421 samples, 0.40%)bitcoind::GetScriptOp (1,727,592,712 samples, 0.05%)bitcoind::CScript::GetSigOpCount (1,617,517,251 samples, 0.05%)bitcoind::GetScriptOp (834,793,526 samples, 0.02%)bitcoind::WitnessSigOps (3,120,635,596 samples, 0.09%)bitcoind::CountWitnessSigOps (25,211,941,345 samples, 0.71%)bitcoind::CScript::GetSigOpCount (21,895,087,837 samples, 0.62%)bitcoind::GetScriptOp (11,871,223,047 samples, 0.33%)bitcoind::GetLegacySigOpCount (26,548,006,408 samples, 0.75%)bitcoind::GetScriptOp (1,822,747,918 samples, 0.05%)bitcoind::SipHashUint256Extra (1,613,835,917 samples, 0.05%)bitcoind::CCoinsViewCache::FetchCoin (6,631,397,326 samples, 0.19%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::_M_find_before_node (2,817,298,340 samples, 0.08%)bitcoind::CCoinsViewCache::AccessCoin (7,316,792,317 samples, 0.21%)bitcoind::CCoinsViewCache::FetchCoin (363,943,746 samples, 0.01%)bitcoind::CScript::GetSigOpCount (1,160,904,417 samples, 0.03%)bitcoind::GetScriptOp (688,273,084 samples, 0.02%)bitcoind::GetScriptOp (2,964,048,193 samples, 0.08%)bitcoind::CScript::GetSigOpCount (5,643,658,755 samples, 0.16%)bitcoind::CScript::IsPayToScriptHash (581,631,871 samples, 0.02%)bitcoind::GetP2SHSigOpCount (15,633,133,461 samples, 0.44%)bitcoind::GetTransactionSigOpCost (84,183,784,739 samples, 2.37%)bit..libstdc++.so.6.0.32::operator delete (405,410,027 samples, 0.01%)bitcoind::SequenceLocks (1,661,951,664 samples, 0.05%)bitcoind::CalculateSequenceLocks (1,453,270,225 samples, 0.04%)bitcoind::SipHashUint256Extra (937,441,713 samples, 0.03%)bitcoind::CCoinsViewCache::FetchCoin (2,049,216,208 samples, 0.06%)bitcoind::SipHashUint256Extra (1,345,870,966 samples, 0.04%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::_M_erase (363,086,362 samples, 0.01%)bitcoind::CCoinsViewCache::SpendCoin (20,676,663,595 samples, 0.58%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::erase (2,777,265,349 samples, 0.08%)bitcoind::SipHashUint256Extra (1,428,091,877 samples, 0.04%)bitcoind::UpdateCoins (24,385,621,354 samples, 0.69%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::erase (473,710,256 samples, 0.01%)bitcoind::AutoFile::write (1,453,030,200 samples, 0.04%)bitcoind::CSHA256::Write (1,519,729,645 samples, 0.04%)bitcoind::CompressAmount (618,711,609 samples, 0.02%)bitcoind::CompressScript (985,913,050 samples, 0.03%)[[ext4]] (404,866,263 samples, 0.01%)bitcoind::node::BlockManager::FindUndoPos (561,722,604 samples, 0.02%)bitcoind::FlatFileSeq::Allocate (509,374,850 samples, 0.01%)libc.so.6::posix_fallocate (509,374,850 samples, 0.01%)[unknown] (509,374,850 samples, 0.01%)[unknown] (509,374,850 samples, 0.01%)[unknown] (509,374,850 samples, 0.01%)[unknown] (457,299,763 samples, 0.01%)bitcoind::AutoFile::write (10,042,610,399 samples, 0.28%)bitcoind::CSHA256::Write (19,844,383,315 samples, 0.56%)bitcoind::sha256_x86_shani::Transform (3,151,148,807 samples, 0.09%)bitcoind::CompressAmount (1,773,668,392 samples, 0.05%)bitcoind::CompressScript (4,638,408,540 samples, 0.13%)bitcoind::prevector<33u, unsigned char, unsigned int, int>::resize (3,040,914,869 samples, 0.09%)bitcoind::CompressAmount (831,251,028 samples, 0.02%)bitcoind::prevector<33u, unsigned char, unsigned int, int>::resize (1,914,945,145 samples, 0.05%)bitcoind::void VectorFormatter<DefaultFormatter>::Ser<SizeComputer, std::vector<CTxUndo, std::allocator<CTxUndo> > > (7,020,871,233 samples, 0.20%)bitcoind::CompressScript (2,957,454,406 samples, 0.08%)bitcoind::AutoFile::write (4,887,544,250 samples, 0.14%)bitcoind::void WriteVarInt<AutoFile, (VarIntMode)0, unsigned int> (5,868,765,238 samples, 0.17%)bitcoind::CSHA256::Write (8,012,816,481 samples, 0.23%)bitcoind::sha256_x86_shani::Transform (938,301,513 samples, 0.03%)bitcoind::void WriteVarInt<HashWriter, (VarIntMode)0, unsigned int> (12,386,753,309 samples, 0.35%)libc.so.6::__memmove_avx512_unaligned_erms (941,007,723 samples, 0.03%)libc.so.6::_IO_fwrite (1,409,554,078 samples, 0.04%)bitcoind::node::BlockManager::UndoWriteToDisk (74,178,487,109 samples, 2.09%)bi..libc.so.6::__memmove_avx512_unaligned_erms (3,806,477,393 samples, 0.11%)bitcoind::CompressAmount (730,340,863 samples, 0.02%)bitcoind::void VectorFormatter<DefaultFormatter>::Ser<SizeComputer, std::vector<CTxUndo, std::allocator<CTxUndo> > > (9,108,229,147 samples, 0.26%)bitcoind::CompressScript (3,027,453,269 samples, 0.09%)bitcoind::prevector<33u, unsigned char, unsigned int, int>::resize (2,034,465,890 samples, 0.06%)bitcoind::void WriteVarInt<AutoFile, (VarIntMode)0, unsigned int> (367,022,852 samples, 0.01%)bitcoind::void WriteVarInt<HashWriter, (VarIntMode)0, unsigned int> (521,478,522 samples, 0.01%)bitcoind::node::BlockManager::WriteUndoDataForBlock (89,569,504,650 samples, 2.52%)bit..bitcoind::std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_dispose (583,464,405 samples, 0.02%)libc.so.6::malloc (1,716,514,762 samples, 0.05%)bitcoind::Chainstate::ConnectBlock (855,466,273,851 samples, 24.06%)bitcoind::Chainstate::ConnectBlockbitcoind::std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_dispose (418,091,278 samples, 0.01%)libc.so.6::cfree@GLIBC_2.2.5 (1,144,767,280 samples, 0.03%)bitcoind::Chainstate::ConnectTip (1,291,793,481,748 samples, 36.33%)bitcoind::Chainstate::ConnectTiplibstdc++.so.6.0.32::operator delete (627,918,999 samples, 0.02%)bitcoind::Chainstate::ActivateBestChainStep (1,291,995,942,063 samples, 36.34%)bitcoind::Chainstate::ActivateBestChainStepbitcoind::Chainstate::ActivateBestChain (1,292,515,820,515 samples, 36.35%)bitcoind::Chainstate::ActivateBestChainbitcoind::IsFinalTx (467,637,167 samples, 0.01%)bitcoind::void SerializeTransaction<ParamsStream<SizeComputer&, TransactionSerParams>, CTransaction> (25,890,452,766 samples, 0.73%)bitcoind::void WriteCompactSize<ParamsStream<SizeComputer&, TransactionSerParams> > (6,846,762,723 samples, 0.19%)bitcoind::ContextualCheckBlock (27,706,291,261 samples, 0.78%)bitcoind::void WriteCompactSize<ParamsStream<SizeComputer&, TransactionSerParams> > (1,089,597,648 samples, 0.03%)[[ext4]] (5,576,796,020 samples, 0.16%)[unknown] (3,815,201,507 samples, 0.11%)[unknown] (2,345,433,446 samples, 0.07%)[unknown] (520,783,293 samples, 0.01%)[[ext4]] (7,954,320,588 samples, 0.22%)[unknown] (1,662,013,865 samples, 0.05%)[unknown] (1,269,240,468 samples, 0.04%)[unknown] (705,348,263 samples, 0.02%)[unknown] (455,918,938 samples, 0.01%)[[nvme]] (807,919,787 samples, 0.02%)[[nvme]] (807,919,787 samples, 0.02%)[unknown] (807,919,787 samples, 0.02%)[unknown] (807,919,787 samples, 0.02%)[unknown] (807,919,787 samples, 0.02%)[unknown] (547,049,759 samples, 0.02%)[unknown] (496,243,932 samples, 0.01%)[unknown] (448,114,949 samples, 0.01%)[[ext4]] (12,810,206,632 samples, 0.36%)[unknown] (3,316,731,307 samples, 0.09%)[unknown] (2,036,481,321 samples, 0.06%)[unknown] (1,478,602,939 samples, 0.04%)[unknown] (1,322,893,322 samples, 0.04%)[unknown] (1,227,580,922 samples, 0.03%)[[ext4]] (13,121,603,080 samples, 0.37%)[[ext4]] (13,121,603,080 samples, 0.37%)bitcoind::FlatFileSeq::Flush (13,525,287,477 samples, 0.38%)libc.so.6::fdatasync (13,525,287,477 samples, 0.38%)[unknown] (13,525,287,477 samples, 0.38%)[unknown] (13,525,287,477 samples, 0.38%)[unknown] (13,525,287,477 samples, 0.38%)[[ext4]] (13,525,287,477 samples, 0.38%)[unknown] (13,525,287,477 samples, 0.38%)[unknown] (13,525,287,477 samples, 0.38%)[unknown] (13,525,287,477 samples, 0.38%)[unknown] (13,525,287,477 samples, 0.38%)[unknown] (403,684,397 samples, 0.01%)[unknown] (403,684,397 samples, 0.01%)[unknown] (403,684,397 samples, 0.01%)[[ext4]] (619,895,319 samples, 0.02%)[unknown] (483,644,425 samples, 0.01%)[[ext4]] (981,510,072 samples, 0.03%)[[ext4]] (1,916,459,846 samples, 0.05%)[unknown] (398,097,615 samples, 0.01%)[[ext4]] (1,967,105,500 samples, 0.06%)[[ext4]] (1,967,105,500 samples, 0.06%)bitcoind::node::BlockManager::FindNextBlockPos (16,065,033,072 samples, 0.45%)bitcoind::node::BlockManager::FlushBlockFile (15,700,018,553 samples, 0.44%)bitcoind::node::BlockManager::FlushUndoFile (2,174,731,076 samples, 0.06%)bitcoind::FlatFileSeq::Flush (2,174,731,076 samples, 0.06%)libc.so.6::fdatasync (2,174,731,076 samples, 0.06%)[unknown] (2,174,731,076 samples, 0.06%)[unknown] (2,174,731,076 samples, 0.06%)[unknown] (2,174,731,076 samples, 0.06%)[[ext4]] (2,174,731,076 samples, 0.06%)[unknown] (2,174,731,076 samples, 0.06%)[unknown] (2,174,731,076 samples, 0.06%)[unknown] (2,174,731,076 samples, 0.06%)[unknown] (2,119,891,081 samples, 0.06%)bitcoind::AutoFile::write (5,548,941,818 samples, 0.16%)libc.so.6::__GI___fstatat64 (365,833,677 samples, 0.01%)bitcoind::node::BlockManager::OpenBlockFile (470,569,767 samples, 0.01%)bitcoind::AutoFile::write (61,167,375,809 samples, 1.72%)b..[unknown] (598,511,547 samples, 0.02%)[unknown] (457,806,853 samples, 0.01%)[unknown] (457,806,853 samples, 0.01%)[unknown] (457,806,853 samples, 0.01%)[unknown] (409,659,414 samples, 0.01%)[unknown] (357,939,661 samples, 0.01%)bitcoind::AutoFile::write (4,647,493,060 samples, 0.13%)bitcoind::void WriteCompactSize<ParamsStream<AutoFile&, TransactionSerParams> > (6,592,272,733 samples, 0.19%)libc.so.6::_IO_fwrite (910,505,012 samples, 0.03%)bitcoind::void SerializeMany<ParamsStream<AutoFile&, TransactionSerParams>, CBlockHeader, std::vector<std::shared_ptr<CTransaction const>, std::allocator<std::shared_ptr<CTransaction const> > > > (82,131,751,453 samples, 2.31%)bit..libc.so.6::_IO_fwrite (8,527,040,897 samples, 0.24%)bitcoind::void SerializeMany<ParamsStream<SizeComputer&, TransactionSerParams>, CBlockHeader, std::vector<std::shared_ptr<CTransaction const>, std::allocator<std::shared_ptr<CTransaction const> > > > (15,937,770,258 samples, 0.45%)bitcoind::void WriteCompactSize<ParamsStream<SizeComputer&, TransactionSerParams> > (3,905,423,573 samples, 0.11%)bitcoind::void WriteCompactSize<ParamsStream<AutoFile&, TransactionSerParams> > (571,858,007 samples, 0.02%)bitcoind::node::BlockManager::WriteBlockToDisk (106,357,642,754 samples, 2.99%)bitc..bitcoind::void WriteCompactSize<ParamsStream<SizeComputer&, TransactionSerParams> > (1,696,751,451 samples, 0.05%)bitcoind::void SerializeMany<ParamsStream<SizeComputer&, TransactionSerParams>, CBlockHeader, std::vector<std::shared_ptr<CTransaction const>, std::allocator<std::shared_ptr<CTransaction const> > > > (17,168,001,989 samples, 0.48%)bitcoind::void WriteCompactSize<ParamsStream<SizeComputer&, TransactionSerParams> > (4,825,716,114 samples, 0.14%)bitcoind::node::BlockManager::SaveBlockToDisk (141,154,624,112 samples, 3.97%)bitcoi..bitcoind::void WriteCompactSize<ParamsStream<SizeComputer&, TransactionSerParams> > (1,511,914,109 samples, 0.04%)bitcoind::ChainstateManager::AcceptBlock (169,805,644,100 samples, 4.78%)bitcoind..bitcoind::void SerializeTransaction<ParamsStream<SizeComputer&, TransactionSerParams>, CTransaction> (419,732,705 samples, 0.01%)bitcoind::CScript::GetSigOpCount (1,244,733,942 samples, 0.04%)bitcoind::memcmp@plt (416,583,431 samples, 0.01%)bitcoind::std::_Rb_tree<COutPoint, COutPoint, std::_Identity<COutPoint>, std::less<COutPoint>, std::allocator<COutPoint> >::_M_erase (1,490,186,398 samples, 0.04%)bitcoind::std::pair<std::_Rb_tree_iterator<COutPoint>, bool> std::_Rb_tree<COutPoint, COutPoint, std::_Identity<COutPoint>, std::less<COutPoint>, std::allocator<COutPoint> >::_M_insert_unique<COutPoint const&> (4,247,810,353 samples, 0.12%)bitcoind::void WriteCompactSize<ParamsStream<SizeComputer&, TransactionSerParams> > (781,801,877 samples, 0.02%)libc.so.6::__memcmp_evex_movbe (6,070,441,149 samples, 0.17%)libc.so.6::cfree@GLIBC_2.2.5 (421,482,290 samples, 0.01%)libstdc++.so.6.0.32::operator delete (614,232,991 samples, 0.02%)bitcoind::CheckTransaction (25,650,523,240 samples, 0.72%)libstdc++.so.6.0.32::std::_Rb_tree_insert_and_rebalance (2,281,327,330 samples, 0.06%)bitcoind::CScript::GetSigOpCount (19,161,186,078 samples, 0.54%)bitcoind::GetScriptOp (8,992,060,021 samples, 0.25%)bitcoind::GetLegacySigOpCount (22,614,517,690 samples, 0.64%)bitcoind::GetScriptOp (1,176,069,512 samples, 0.03%)bitcoind::std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_dispose (1,349,955,285 samples, 0.04%)bitcoind::void SerializeTransaction<ParamsStream<SizeComputer&, TransactionSerParams>, CTransaction> (6,676,130,736 samples, 0.19%)bitcoind::void WriteCompactSize<ParamsStream<SizeComputer&, TransactionSerParams> > (1,329,967,416 samples, 0.04%)bitcoind::CheckBlock (60,550,319,748 samples, 1.70%)b..bitcoind::void WriteCompactSize<ParamsStream<SizeComputer&, TransactionSerParams> > (572,803,846 samples, 0.02%)bitcoind::ChainstateManager::ProcessNewBlock (1,523,688,403,640 samples, 42.85%)bitcoind::ChainstateManager::ProcessNewBlockbitcoind::sha256d64_x86_shani::Transform_2way (15,194,464,935 samples, 0.43%)bitcoind::BlockMerkleRoot (16,172,687,252 samples, 0.45%)bitcoind::ComputeMerkleRoot (15,499,928,925 samples, 0.44%)bitcoind::SHA256D64 (15,246,405,066 samples, 0.43%)bitcoind::CheckMerkleRoot (16,532,547,442 samples, 0.46%)libc.so.6::__memset_avx512_unaligned_erms (359,860,190 samples, 0.01%)bitcoind::sha256d64_x86_shani::Transform_2way (12,972,294,835 samples, 0.36%)bitcoind::SHA256D64 (13,025,009,373 samples, 0.37%)bitcoind::IsBlockMutated (30,129,022,002 samples, 0.85%)bitcoind::CheckWitnessMalleation (13,596,474,560 samples, 0.38%)bitcoind::BlockWitnessMerkleRoot (13,596,474,560 samples, 0.38%)bitcoind::ComputeMerkleRoot (13,077,728,889 samples, 0.37%)bitcoind::void (anonymous namespace)::PeerManagerImpl::MakeAndPushMessage<std::vector<CInv, std::allocator<CInv> >&> (406,479,193 samples, 0.01%)bitcoind::CConnman::PushMessage (406,479,193 samples, 0.01%)bitcoind::std::vector<unsigned char, std::allocator<unsigned char> >::_M_default_append (367,056,757 samples, 0.01%)bitcoind::unsigned long ReadCompactSize<ParamsStream<DataStream&, TransactionSerParams> > (622,762,372 samples, 0.02%)bitcoind::CTransaction::ComputeHasWitness (1,387,667,716 samples, 0.04%)bitcoind::CSHA256::Write (17,955,645,390 samples, 0.51%)bitcoind::sha256_x86_shani::Transform (11,932,913,194 samples, 0.34%)bitcoind::memcpy@plt (418,918,061 samples, 0.01%)bitcoind::sha256_x86_shani::Transform (3,306,980,273 samples, 0.09%)bitcoind::CSHA256::Finalize (22,917,960,073 samples, 0.64%)libc.so.6::__memmove_avx512_unaligned_erms (668,127,949 samples, 0.02%)bitcoind::CSHA256::Write (3,163,584,691 samples, 0.09%)bitcoind::CSHA256::Write (33,313,763,000 samples, 0.94%)bitcoind::sha256_x86_shani::Transform (14,194,928,537 samples, 0.40%)bitcoind::sha256_x86_shani::Transform (767,994,599 samples, 0.02%)bitcoind::CSHA256::Write (5,341,265,376 samples, 0.15%)bitcoind::void WriteCompactSize<ParamsStream<HashWriter&, TransactionSerParams> > (7,984,745,468 samples, 0.22%)bitcoind::void SerializeTransaction<ParamsStream<HashWriter&, TransactionSerParams>, CTransaction> (50,933,406,220 samples, 1.43%)b..libc.so.6::__memmove_avx512_unaligned_erms (5,183,727,187 samples, 0.15%)bitcoind::void WriteCompactSize<ParamsStream<HashWriter&, TransactionSerParams> > (1,613,593,834 samples, 0.05%)bitcoind::CTransaction::ComputeHash (80,845,793,271 samples, 2.27%)bit..bitcoind::CSHA256::Write (23,348,148,278 samples, 0.66%)bitcoind::sha256_x86_shani::Transform (11,595,812,714 samples, 0.33%)bitcoind::CSHA256::Finalize (24,335,325,870 samples, 0.68%)bitcoind::CSHA256::Write (2,288,432,816 samples, 0.06%)bitcoind::CSHA256::Write (64,681,112,465 samples, 1.82%)bi..bitcoind::sha256_x86_shani::Transform (33,677,349,718 samples, 0.95%)bitcoind::sha256_x86_shani::Transform (622,627,277 samples, 0.02%)bitcoind::CSHA256::Write (11,395,509,513 samples, 0.32%)bitcoind::sha256_x86_shani::Transform (523,186,685 samples, 0.01%)bitcoind::void WriteCompactSize<ParamsStream<HashWriter&, TransactionSerParams> > (17,046,149,334 samples, 0.48%)libc.so.6::__memmove_avx512_unaligned_erms (2,169,704,353 samples, 0.06%)bitcoind::void SerializeTransaction<ParamsStream<HashWriter&, TransactionSerParams>, CTransaction> (92,366,151,212 samples, 2.60%)bit..libc.so.6::__memmove_avx512_unaligned_erms (5,813,350,330 samples, 0.16%)bitcoind::void WriteCompactSize<ParamsStream<HashWriter&, TransactionSerParams> > (2,181,533,875 samples, 0.06%)bitcoind::CTransaction::ComputeWitnessHash (122,098,239,092 samples, 3.43%)bitco..bitcoind::CTransaction::CTransaction (213,407,475,563 samples, 6.00%)bitcoind::C..bitcoind::CTransaction::ComputeHasWitness (420,121,661 samples, 0.01%)bitcoind::CTransaction::ComputeHash (409,129,353 samples, 0.01%)bitcoind::DataStream::read (3,530,026,319 samples, 0.10%)bitcoind::operator new (628,502,415 samples, 0.02%)bitcoind::std::vector<unsigned char, std::allocator<unsigned char> >::_M_default_append (6,309,702,925 samples, 0.18%)bitcoind::unsigned long ReadCompactSize<ParamsStream<DataStream&, TransactionSerParams> > (4,193,618,734 samples, 0.12%)bitcoind::void Unserialize<ParamsStream<DataStream&, TransactionSerParams>, 28u, unsigned char> (727,834,750 samples, 0.02%)bitcoind::unsigned long ReadCompactSize<ParamsStream<DataStream&, TransactionSerParams> > (3,066,020,716 samples, 0.09%)bitcoind::void Unserialize<ParamsStream<DataStream&, TransactionSerParams>, 28u, unsigned char> (3,105,560,893 samples, 0.09%)bitcoind::unsigned long ReadCompactSize<ParamsStream<DataStream&, TransactionSerParams> > (1,455,846,726 samples, 0.04%)libc.so.6::__memmove_avx512_unaligned_erms (465,834,593 samples, 0.01%)bitcoind::void VectorFormatter<DefaultFormatter>::Unser<ParamsStream<DataStream&, TransactionSerParams>, std::vector<CTxIn, std::allocator<CTxIn> > > (15,677,650,112 samples, 0.44%)bitcoind::unsigned long ReadCompactSize<ParamsStream<DataStream&, TransactionSerParams> > (1,078,029,303 samples, 0.03%)bitcoind::void Unserialize<ParamsStream<DataStream&, TransactionSerParams>, 28u, unsigned char> (6,171,925,860 samples, 0.17%)bitcoind::unsigned long ReadCompactSize<ParamsStream<DataStream&, TransactionSerParams> > (982,486,879 samples, 0.03%)libc.so.6::__memmove_avx512_unaligned_erms (1,458,516,290 samples, 0.04%)bitcoind::void VectorFormatter<DefaultFormatter>::Unser<ParamsStream<DataStream&, TransactionSerParams>, std::vector<CTxOut, std::allocator<CTxOut> > > (13,963,877,725 samples, 0.39%)libc.so.6::__memmove_avx512_unaligned_erms (1,048,169,614 samples, 0.03%)libc.so.6::__memset_avx512_unaligned (1,046,482,105 samples, 0.03%)libc.so.6::__memset_avx512_unaligned_erms (1,963,080,141 samples, 0.06%)libc.so.6::malloc (3,025,102,825 samples, 0.09%)libstdc++.so.6.0.32::malloc@plt (1,462,651,744 samples, 0.04%)bitcoind::void Unserialize<ParamsStream<DataStream&, TransactionSerParams>, CTransaction> (283,898,001,379 samples, 7.98%)bitcoind::void U..libstdc++.so.6.0.32::operator new (2,257,486,798 samples, 0.06%)bitcoind::void VectorFormatter<DefaultFormatter>::Unser<ParamsStream<DataStream&, TransactionSerParams>, std::vector<CTxIn, std::allocator<CTxIn> > > (1,090,260,916 samples, 0.03%)libc.so.6::__memmove_avx512_unaligned_erms (2,753,503,546 samples, 0.08%)libc.so.6::malloc (1,149,716,024 samples, 0.03%)bitcoind::void ParamsWrapper<TransactionSerParams, CBlock>::Unserialize<DataStream> (291,189,121,636 samples, 8.19%)bitcoind::void P..bitcoind::void VectorFormatter<DefaultFormatter>::Unser<ParamsStream<DataStream&, TransactionSerParams>, std::vector<std::shared_ptr<CTransaction const>, std::allocator<std::shared_ptr<CTransaction const> > > > (291,134,772,004 samples, 8.19%)bitcoind::void V..libstdc++.so.6.0.32::operator new (518,854,210 samples, 0.01%)libc.so.6::__memset_avx512_unaligned_erms (3,588,859,593 samples, 0.10%)bitcoind::CConnman::ThreadMessageHandler (1,852,055,734,561 samples, 52.09%)bitcoind::CConnman::ThreadMessageHandlerlibstdc++.so.6.0.32::execute_native_thread_routine (1,852,107,180,016 samples, 52.09%)libstdc++.so.6.0.32::execute_native_thread_routinebitcoind::std::thread::_State_impl<std::thread::_Invoker<std::tuple<void (*)(std::basic_string_view<char, std::char_traits<char> >, std::function<void ()>), char const*, CConnman::Start(CScheduler&, CConnman::Options const&)::{lambda()#5}> > >::_M_run (1,852,107,180,016 samples, 52.09%)bitcoind::std::thread::_State_impl<std::thread::_Invoker<std::tuple<void (*)(std::basic_string_view<char, std::char..bitcoind::util::TraceThread (1,852,107,180,016 samples, 52.09%)bitcoind::util::TraceThreadlibstdc++.so.6.0.32::std::__cxx11::basic_stringbuf<char, std::char_traits<char>, std::allocator<char> >::overflow (397,900,679 samples, 0.01%)b-msghand (2,401,934,372,954 samples, 67.55%)b-msghand[[igc]] (638,737,826 samples, 0.02%)[unknown] (492,740,386 samples, 0.01%)[unknown] (492,740,386 samples, 0.01%)[unknown] (492,740,386 samples, 0.01%)libc.so.6::__libc_recv (23,769,090,268 samples, 0.67%)[unknown] (23,681,676,959 samples, 0.67%)[unknown] (23,585,908,630 samples, 0.66%)[unknown] (23,544,049,599 samples, 0.66%)[unknown] (23,499,819,825 samples, 0.66%)[unknown] (23,453,162,931 samples, 0.66%)[unknown] (23,205,326,716 samples, 0.65%)[unknown] (23,046,242,743 samples, 0.65%)[unknown] (23,000,657,790 samples, 0.65%)[unknown] (22,592,454,604 samples, 0.64%)[unknown] (21,715,983,496 samples, 0.61%)[unknown] (20,537,782,242 samples, 0.58%)[unknown] (19,311,079,312 samples, 0.54%)[unknown] (6,108,735,942 samples, 0.17%)[unknown] (1,360,583,546 samples, 0.04%)bitcoind::std::vector<std::byte, zero_after_free_allocator<std::byte> >::_M_fill_insert (16,619,401,507 samples, 0.47%)bitcoind::V2Transport::GetReceivedMessage (16,718,730,797 samples, 0.47%)[[igc]] (507,437,414 samples, 0.01%)[unknown] (412,781,498 samples, 0.01%)bitcoind::ChaCha20::Crypt (134,944,431,601 samples, 3.80%)bitcoi..bitcoind::ChaCha20Aligned::Crypt (134,944,431,601 samples, 3.80%)bitcoi..[unknown] (955,536,462 samples, 0.03%)[unknown] (955,536,462 samples, 0.03%)[unknown] (906,440,192 samples, 0.03%)[unknown] (861,463,927 samples, 0.02%)[unknown] (760,654,093 samples, 0.02%)[unknown] (658,510,836 samples, 0.02%)bitcoind::BIP324Cipher::Decrypt (196,638,059,936 samples, 5.53%)bitcoind::..bitcoind::FSChaCha20Poly1305::Decrypt (196,638,059,936 samples, 5.53%)bitcoind::..bitcoind::AEADChaCha20Poly1305::Decrypt (196,638,059,936 samples, 5.53%)bitcoind::..bitcoind::poly1305_donna::poly1305_update (61,693,628,335 samples, 1.74%)b..bitcoind::poly1305_donna::poly1305_blocks (61,693,628,335 samples, 1.74%)b..[unknown] (655,063,915 samples, 0.02%)[unknown] (607,270,235 samples, 0.02%)[unknown] (525,964,847 samples, 0.01%)[unknown] (525,964,847 samples, 0.01%)[unknown] (470,111,416 samples, 0.01%)[unknown] (470,107,658 samples, 0.01%)bitcoind::V2Transport::ProcessReceivedPacketBytes (198,460,164,481 samples, 5.58%)bitcoind::..libc.so.6::__memset_avx512_unaligned_erms (1,781,860,401 samples, 0.05%)bitcoind::V2Transport::ReceivedBytes (203,432,631,557 samples, 5.72%)bitcoind::..libc.so.6::__memmove_avx512_unaligned_erms (4,655,332,308 samples, 0.13%)libc.so.6::__memmove_avx512_unaligned_erms (10,715,799,436 samples, 0.30%)bitcoind::CNode::ReceiveMsgBytes (231,225,287,054 samples, 6.50%)bitcoind::CN..bitcoind::CConnman::SocketHandlerConnected (231,463,366,433 samples, 6.51%)bitcoind::CC..libc.so.6::__poll (3,830,838,327 samples, 0.11%)[unknown] (3,830,838,327 samples, 0.11%)[unknown] (3,782,920,191 samples, 0.11%)[unknown] (3,725,807,764 samples, 0.10%)[unknown] (3,522,157,004 samples, 0.10%)[unknown] (3,150,768,515 samples, 0.09%)[unknown] (2,627,277,437 samples, 0.07%)[unknown] (2,338,467,135 samples, 0.07%)[unknown] (2,037,878,870 samples, 0.06%)[unknown] (1,480,962,324 samples, 0.04%)[unknown] (688,242,613 samples, 0.02%)bitcoind::CConnman::SocketHandler (236,436,484,949 samples, 6.65%)bitcoind::CCo..b-net (260,905,688,952 samples, 7.34%)b-netlibstdc++.so.6.0.32::execute_native_thread_routine (236,875,778,634 samples, 6.66%)libstdc++.so...bitcoind::std::thread::_State_impl<std::thread::_Invoker<std::tuple<void (*)(std::basic_string_view<char, std::char_traits<char> >, std::function<void ()>), char const*, CConnman::Start(CScheduler&, CConnman::Options const&)::{lambda()#1}> > >::_M_run (236,875,778,634 samples, 6.66%)bitcoind::std..bitcoind::util::TraceThread (236,875,778,634 samples, 6.66%)bitcoind::uti..bitcoind::CConnman::ThreadSocketHandler (236,875,778,634 samples, 6.66%)bitcoind::CCo..libc.so.6::_int_free_create_chunk (982,572,444 samples, 0.03%)libc.so.6::_int_free_merge_chunk (797,147,451 samples, 0.02%)[unknown] (2,170,942,655 samples, 0.06%)libc.so.6::__futex_abstimed_wait_common (459,293,920 samples, 0.01%)[unknown] (459,293,920 samples, 0.01%)[unknown] (405,487,988 samples, 0.01%)[unknown] (405,482,438 samples, 0.01%)[unknown] (405,482,438 samples, 0.01%)[unknown] (356,784,451 samples, 0.01%)libc.so.6::__lll_lock_wait_private (57,276,007,979 samples, 1.61%)l..[unknown] (54,932,210,267 samples, 1.54%)[..[unknown] (52,306,124,993 samples, 1.47%)[..[unknown] (51,843,804,338 samples, 1.46%)[..[unknown] (49,115,074,635 samples, 1.38%)[..[unknown] (47,020,328,627 samples, 1.32%)[unknown] (41,124,744,672 samples, 1.16%)[unknown] (38,571,784,780 samples, 1.08%)[unknown] (36,085,617,902 samples, 1.01%)[unknown] (32,172,048,607 samples, 0.90%)[unknown] (24,296,172,973 samples, 0.68%)[unknown] (14,033,556,774 samples, 0.39%)[unknown] (7,508,395,799 samples, 0.21%)[unknown] (3,295,574,070 samples, 0.09%)[unknown] (1,590,496,727 samples, 0.04%)[unknown] (1,002,849,637 samples, 0.03%)[unknown] (414,545,859 samples, 0.01%)libc.so.6::__lll_lock_wake_private (11,041,124,764 samples, 0.31%)[unknown] (10,991,162,572 samples, 0.31%)[unknown] (9,603,504,474 samples, 0.27%)[unknown] (9,459,439,012 samples, 0.27%)[unknown] (7,207,430,735 samples, 0.20%)[unknown] (5,830,933,319 samples, 0.16%)[unknown] (1,889,493,619 samples, 0.05%)[unknown] (394,342,984 samples, 0.01%)libc.so.6::_int_free (67,830,842,133 samples, 1.91%)li..libc.so.6::_int_free_merge_chunk (832,998,780 samples, 0.02%)libc.so.6::cfree@GLIBC_2.2.5 (2,087,601,863 samples, 0.06%)libc.so.6::malloc_consolidate (3,954,686,383 samples, 0.11%)libc.so.6::unlink_chunk.isra.0 (497,585,449 samples, 0.01%)bitcoind::CRollingBloomFilter::insert (356,229,732 samples, 0.01%)[unknown] (444,029,098 samples, 0.01%)[unknown] (397,328,353 samples, 0.01%)[unknown] (397,328,353 samples, 0.01%)[unknown] (397,328,353 samples, 0.01%)[unknown] (397,328,353 samples, 0.01%)bitcoind::CRollingBloomFilter::insert (165,056,371,702 samples, 4.64%)bitcoind..bitcoind::MurmurHash3 (79,485,956,130 samples, 2.24%)bit..[unknown] (508,285,343 samples, 0.01%)[unknown] (450,228,615 samples, 0.01%)[unknown] (404,433,625 samples, 0.01%)[unknown] (404,433,625 samples, 0.01%)[unknown] (404,433,625 samples, 0.01%)[unknown] (404,433,625 samples, 0.01%)bitcoind::MurmurHash3 (5,783,718,949 samples, 0.16%)bitcoind::TxOrphanage::EraseForBlock (4,219,830,042 samples, 0.12%)bitcoind::std::_Rb_tree<COutPoint, std::pair<COutPoint const, std::set<std::_Rb_tree_iterator<std::pair<transaction_identifier<true> const, TxOrphanage::OrphanTx> >, TxOrphanage::IteratorComparator, std::allocator<std::_Rb_tree_iterator<std::pair<transaction_identifier<true> const, TxOrphanage::OrphanTx> > > > >, std::_Select1st<std::pair<COutPoint const, std::set<std::_Rb_tree_iterator<std::pair<transaction_identifier<true> const, TxOrphanage::OrphanTx> >, TxOrphanage::IteratorComparator, std::allocator<std::_Rb_tree_iterator<std::pair<transaction_identifier<true> const, TxOrphanage::OrphanTx> > > > > >, std::less<COutPoint>, std::allocator<std::pair<COutPoint const, std::set<std::_Rb_tree_iterator<std::pair<transaction_identifier<true> const, TxOrphanage::OrphanTx> >, TxOrphanage::IteratorComparator, std::allocator<std::_Rb_tree_iterator<std::pair<transaction_identifier<true> const, TxOrphanage::OrphanTx> > > > > > >::find (834,275,777 samples, 0.02%)bitcoind::node::TxDownloadManagerImpl::BlockConnected (176,131,189,628 samples, 4.95%)bitcoind:..bitcoind::TxRequestTracker::ForgetTxHash (789,439,865 samples, 0.02%)bitcoind::std::_Function_handler<void (), ValidationSignals::BlockConnected(ChainstateRole, std::shared_ptr<CBlock const> const&, CBlockIndex const*)::{lambda()#2}>::_M_invoke (177,028,683,872 samples, 4.98%)bitcoind:..bitcoind::std::_Sp_counted_ptr_inplace<CTransaction const, std::allocator<void>, (__gnu_cxx::_Lock_policy)2>::_M_dispose (18,094,676,466 samples, 0.51%)libc.so.6::cfree@GLIBC_2.2.5 (20,756,908,966 samples, 0.58%)bitcoind::std::_Sp_counted_ptr_inplace<CBlock, std::allocator<void>, (__gnu_cxx::_Lock_policy)2>::_M_dispose (48,020,349,476 samples, 1.35%)b..libstdc++.so.6.0.32::operator delete (7,010,168,745 samples, 0.20%)bitcoind::std::_Sp_counted_ptr_inplace<CTransaction const, std::allocator<void>, (__gnu_cxx::_Lock_policy)2>::_M_dispose (595,474,492 samples, 0.02%)libc.so.6::cfree@GLIBC_2.2.5 (1,199,917,863 samples, 0.03%)bitcoind::std::_Function_handler<void (), ValidationSignals::BlockConnected(ChainstateRole, std::shared_ptr<CBlock const> const&, CBlockIndex const*)::{lambda()#2}>::_M_manager (50,738,017,178 samples, 1.43%)b..bitcoind::std::_Sp_counted_base<(__gnu_cxx::_Lock_policy)2>::_M_release (50,738,017,178 samples, 1.43%)b..libstdc++.so.6.0.32::operator delete (678,866,047 samples, 0.02%)bitcoind::CBlockPolicyEstimator::processBlock (2,721,087,031 samples, 0.08%)bitcoind::TxConfirmStats::UpdateMovingAverages (2,530,304,686 samples, 0.07%)bitcoind::std::_Function_handler<void (), ValidationSignals::MempoolTransactionsRemovedForBlock(std::vector<RemovedMempoolTransactionInfo, std::allocator<RemovedMempoolTransactionInfo> > const&, unsigned int)::{lambda()#2}>::_M_invoke (2,804,941,944 samples, 0.08%)bitcoind::SerialTaskRunner::ProcessQueue (230,828,220,555 samples, 6.49%)bitcoind::Se..bitcoind::CScheduler::serviceQueue (231,341,597,555 samples, 6.51%)bitcoind::CS..bitcoind::std::_Function_handler<void (), Repeat(CScheduler&, std::function<void ()>, std::chrono::duration<long, std::ratio<1l, 1000l> >)::{lambda()#1}>::_M_invoke (386,989,959 samples, 0.01%)bitcoind::Repeat (386,989,959 samples, 0.01%)bitcoind::CSHA512::Finalize (386,989,959 samples, 0.01%)b-scheduler (378,036,629,725 samples, 10.63%)b-schedulerlibstdc++.so.6.0.32::execute_native_thread_routine (231,550,611,141 samples, 6.51%)libstdc++.so..bitcoind::std::thread::_State_impl<std::thread::_Invoker<std::tuple<void (*)(std::basic_string_view<char, std::char_traits<char> >, std::function<void ()>), char const*, AppInitMain(node::NodeContext&, interfaces::BlockAndHeaderTipInfo*)::{lambda()#1}> > >::_M_run (231,550,611,141 samples, 6.51%)bitcoind::st..bitcoind::util::TraceThread (231,550,611,141 samples, 6.51%)bitcoind::ut..[unknown] (1,418,945,425 samples, 0.04%)[unknown] (1,418,945,425 samples, 0.04%)[unknown] (1,418,945,425 samples, 0.04%)[unknown] (1,418,945,425 samples, 0.04%)[unknown] (1,418,945,425 samples, 0.04%)[unknown] (1,418,945,425 samples, 0.04%)[unknown] (1,418,945,425 samples, 0.04%)[unknown] (1,368,549,335 samples, 0.04%)[unknown] (1,263,446,697 samples, 0.04%)[unknown] (1,105,228,005 samples, 0.03%)[unknown] (684,110,353 samples, 0.02%)[unknown] (1,463,102,999 samples, 0.04%)libc.so.6::_int_malloc (1,478,820,457 samples, 0.04%)[unknown] (1,323,329,878 samples, 0.04%)[unknown] (1,219,148,488 samples, 0.03%)[unknown] (1,167,736,581 samples, 0.03%)[unknown] (1,167,736,581 samples, 0.03%)[unknown] (1,115,451,061 samples, 0.03%)[unknown] (1,014,330,812 samples, 0.03%)[unknown] (911,337,057 samples, 0.03%)[unknown] (714,835,817 samples, 0.02%)[unknown] (456,457,319 samples, 0.01%)[unknown] (3,458,133,839 samples, 0.10%)bitcoind::CDBWrapper::~CDBWrapper (1,160,687,762 samples, 0.03%)bitcoind::leveldb::DBImpl::~DBImpl (1,160,687,762 samples, 0.03%)bitcoind::leveldb::DBImpl::~DBImpl (1,160,687,762 samples, 0.03%)bitcoind::leveldb::TableCache::~TableCache (1,160,687,762 samples, 0.03%)bitcoind::leveldb::(anonymous namespace)::ShardedLRUCache::~ShardedLRUCache (1,160,687,762 samples, 0.03%)bitcoind::leveldb::(anonymous namespace)::LRUCache::~LRUCache (1,160,687,762 samples, 0.03%)bitcoind::leveldb::DeleteEntry (1,160,687,762 samples, 0.03%)libc.so.6::__munmap (1,160,687,762 samples, 0.03%)[unknown] (1,160,687,762 samples, 0.03%)[unknown] (1,160,687,762 samples, 0.03%)[unknown] (1,160,687,762 samples, 0.03%)[unknown] (1,160,687,762 samples, 0.03%)[unknown] (1,160,687,762 samples, 0.03%)[unknown] (1,160,687,762 samples, 0.03%)[unknown] (1,160,687,762 samples, 0.03%)[unknown] (1,160,687,762 samples, 0.03%)[unknown] (1,160,687,762 samples, 0.03%)[unknown] (580,697,270 samples, 0.02%)bitcoind::leveldb::PutVarint32 (363,737,260 samples, 0.01%)bitcoind::leveldb::PutLengthPrefixedSlice (571,217,019 samples, 0.02%)bitcoind::leveldb::WriteBatch::Delete (2,702,574,018 samples, 0.08%)bitcoind::leveldb::WriteBatchInternal::SetCount (1,715,286,573 samples, 0.05%)bitcoind::leveldb::WriteBatchInternal::SetCount (1,453,616,163 samples, 0.04%)bitcoind::CDBBatch::EraseImpl (5,090,452,967 samples, 0.14%)bitcoind::leveldb::PutVarint32 (1,872,876,736 samples, 0.05%)bitcoind::leveldb::PutLengthPrefixedSlice (2,343,591,543 samples, 0.07%)bitcoind::leveldb::PutVarint32 (572,117,605 samples, 0.02%)bitcoind::leveldb::PutVarint32 (567,491,257 samples, 0.02%)bitcoind::leveldb::PutLengthPrefixedSlice (938,977,738 samples, 0.03%)bitcoind::leveldb::WriteBatchInternal::Count (619,405,896 samples, 0.02%)bitcoind::leveldb::WriteBatch::Put (2,689,024,451 samples, 0.08%)bitcoind::CDBBatch::WriteImpl (10,634,135,335 samples, 0.30%)bitcoind::leveldb::GetLengthPrefixedSlice (463,225,027 samples, 0.01%)bitcoind::leveldb::GetLengthPrefixedSlice (6,489,010,398 samples, 0.18%)bitcoind::leveldb::GetVarint32 (3,004,905,545 samples, 0.08%)bitcoind::leveldb::GetVarint32 (1,160,323,181 samples, 0.03%)bitcoind::leveldb::Arena::AllocateAligned (406,996,319 samples, 0.01%)bitcoind::leveldb::(anonymous namespace)::BytewiseComparatorImpl::Compare (717,398,174 samples, 0.02%)bitcoind::leveldb::MemTable::KeyComparator::operator (5,108,835,410 samples, 0.14%)bitcoind::leveldb::InternalKeyComparator::Compare (3,324,232,989 samples, 0.09%)bitcoind::leveldb::InternalKeyComparator::Compare (4,244,823,969 samples, 0.12%)bitcoind::leveldb::(anonymous namespace)::BytewiseComparatorImpl::Compare (3,179,677,931 samples, 0.09%)bitcoind::leveldb::(anonymous namespace)::BytewiseComparatorImpl::Compare (33,796,395,298 samples, 0.95%)bitcoind::memcmp@plt (943,665,852 samples, 0.03%)bitcoind::leveldb::SkipList<char const*, leveldb::MemTable::KeyComparator>::Insert (185,524,871,422 samples, 5.22%)bitcoind:..bitcoind::leveldb::SkipList<char const*, leveldb::MemTable::KeyComparator>::FindGreaterOrEqual (178,286,921,652 samples, 5.01%)bitcoind:..bitcoind::leveldb::MemTable::KeyComparator::operator (98,574,957,808 samples, 2.77%)bitc..bitcoind::leveldb::InternalKeyComparator::Compare (75,114,665,063 samples, 2.11%)bi..libc.so.6::__memcmp_evex_movbe (8,323,863,446 samples, 0.23%)bitcoind::leveldb::MemTable::Add (188,893,844,275 samples, 5.31%)bitcoind::..bitcoind::leveldb::VarintLength (766,638,876 samples, 0.02%)bitcoind::leveldb::WriteBatchInternal::InsertInto (199,306,778,687 samples, 5.61%)bitcoind::..bitcoind::leveldb::WriteBatch::Iterate (198,740,714,232 samples, 5.59%)bitcoind::..bitcoind::crc32c::ExtendSse42 (471,197,509 samples, 0.01%)[[ext4]] (679,093,773 samples, 0.02%)[unknown] (522,409,669 samples, 0.01%)[[ext4]] (1,096,838,426 samples, 0.03%)[[ext4]] (1,722,362,275 samples, 0.05%)[unknown] (625,523,849 samples, 0.02%)[unknown] (574,147,567 samples, 0.02%)[unknown] (469,028,477 samples, 0.01%)[unknown] (469,028,477 samples, 0.01%)[unknown] (365,648,781 samples, 0.01%)[[ext4]] (4,389,086,262 samples, 0.12%)[unknown] (2,561,710,219 samples, 0.07%)[unknown] (2,561,710,219 samples, 0.07%)[unknown] (2,352,117,097 samples, 0.07%)[unknown] (1,880,182,821 samples, 0.05%)[unknown] (1,308,734,829 samples, 0.04%)[unknown] (523,736,031 samples, 0.01%)[[ext4]] (5,069,490,473 samples, 0.14%)[unknown] (5,069,490,473 samples, 0.14%)[unknown] (575,311,800 samples, 0.02%)[unknown] (470,084,210 samples, 0.01%)libc.so.6::__GI___libc_write (5,174,401,795 samples, 0.15%)[unknown] (5,174,401,795 samples, 0.15%)[unknown] (5,174,401,795 samples, 0.15%)[unknown] (5,174,401,795 samples, 0.15%)[unknown] (5,174,401,795 samples, 0.15%)bitcoind::CDBWrapper::WriteBatch (205,215,727,495 samples, 5.77%)bitcoind::C..bitcoind::leveldb::DBImpl::Write (205,215,727,495 samples, 5.77%)bitcoind::l..bitcoind::leveldb::log::Writer::AddRecord (5,908,948,808 samples, 0.17%)bitcoind::leveldb::log::Writer::EmitPhysicalRecord (5,908,948,808 samples, 0.17%)bitcoind::CompressScript (1,030,024,630 samples, 0.03%)bitcoind::prevector<33u, unsigned char, unsigned int, int>::resize (459,767,226 samples, 0.01%)bitcoind::void WriteVarInt<DataStream, (VarIntMode)0, unsigned int> (11,377,276,951 samples, 0.32%)bitcoind::void std::vector<std::byte, zero_after_free_allocator<std::byte> >::_M_range_insert<std::byte const*> (8,938,854,890 samples, 0.25%)bitcoind::CCoinsViewDB::BatchWrite (244,230,597,449 samples, 6.87%)bitcoind::CCo..bitcoind::void std::vector<std::byte, zero_after_free_allocator<std::byte> >::_M_range_insert<std::byte const*> (6,482,431,215 samples, 0.18%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::clear (13,761,064,935 samples, 0.39%)bitcoind::void std::vector<std::byte, zero_after_free_allocator<std::byte> >::_M_range_insert<std::byte const*> (364,557,178 samples, 0.01%)bitcoind::CCoinsViewCache::Flush (264,031,161,045 samples, 7.43%)bitcoind::CCoi..libc.so.6::cfree@GLIBC_2.2.5 (5,262,867,110 samples, 0.15%)bitcoind::Chainstate::ForceFlushStateToDisk (264,186,830,154 samples, 7.43%)bitcoind::Chai..bitcoind::Chainstate::FlushStateToDisk (264,186,830,154 samples, 7.43%)bitcoind::Chai..libc.so.6::__libc_start_call_main (265,453,083,455 samples, 7.47%)libc.so.6::__l..bitcoind::main (265,453,083,455 samples, 7.47%)bitcoind::mainbitcoind::Shutdown (265,453,083,455 samples, 7.47%)bitcoind::Shut..libc.so.6::_int_free (2,825,988,487 samples, 0.08%)libc.so.6::malloc_consolidate (2,950,349,980 samples, 0.08%)b-shutoff (278,389,331,208 samples, 7.83%)b-shutofflibc.so.6::unlink_chunk.isra.0 (3,181,018,445 samples, 0.09%)libc.so.6::_int_malloc (620,560,935 samples, 0.02%)[unknown] (518,649,070 samples, 0.01%)[unknown] (466,591,536 samples, 0.01%)[unknown] (466,591,536 samples, 0.01%)[unknown] (466,591,536 samples, 0.01%)[unknown] (415,625,450 samples, 0.01%)[unknown] (415,625,450 samples, 0.01%)[unknown] (363,215,208 samples, 0.01%)[unknown] (1,501,827,638 samples, 0.04%)bitcoind::leveldb::BlockBuilder::Add (581,064,351 samples, 0.02%)bitcoind::leveldb::TableBuilder::Add (1,003,488,869 samples, 0.03%)bitcoind::leveldb::DBImpl::WriteLevel0Table (1,214,913,728 samples, 0.03%)bitcoind::leveldb::BuildTable (1,214,913,728 samples, 0.03%)bitcoind::leveldb::WriteBatchInternal::InsertInto (2,528,384,688 samples, 0.07%)bitcoind::leveldb::WriteBatch::Iterate (2,528,384,688 samples, 0.07%)bitcoind::leveldb::MemTable::Add (2,422,985,691 samples, 0.07%)bitcoind::leveldb::SkipList<char const*, leveldb::MemTable::KeyComparator>::Insert (2,422,985,691 samples, 0.07%)bitcoind::leveldb::SkipList<char const*, leveldb::MemTable::KeyComparator>::FindGreaterOrEqual (2,318,036,540 samples, 0.07%)bitcoind::leveldb::MemTable::KeyComparator::operator (1,429,299,251 samples, 0.04%)bitcoind::leveldb::InternalKeyComparator::Compare (910,982,229 samples, 0.03%)bitcoind::CDBWrapper::CDBWrapper (5,007,147,537 samples, 0.14%)bitcoind::leveldb::DB::Open (5,007,147,537 samples, 0.14%)bitcoind::leveldb::DBImpl::Recover (4,954,666,055 samples, 0.14%)bitcoind::leveldb::DBImpl::RecoverLogFile (4,954,666,055 samples, 0.14%)libc.so.6::__memmove_avx512_unaligned_erms (1,000,227,273 samples, 0.03%)[unknown] (1,000,227,273 samples, 0.03%)[unknown] (1,000,227,273 samples, 0.03%)[unknown] (947,397,460 samples, 0.03%)[unknown] (947,397,460 samples, 0.03%)[unknown] (947,397,460 samples, 0.03%)[unknown] (841,684,608 samples, 0.02%)[unknown] (841,684,608 samples, 0.02%)[unknown] (841,684,608 samples, 0.02%)[unknown] (788,837,171 samples, 0.02%)bitcoind::node::BlockManager::GetAllBlockIndices (356,174,463 samples, 0.01%)bitcoind::base_uint<256u>::operator/= (4,353,340,184 samples, 0.12%)bitcoind::base_uint<256u>::operator>>=(unsigned int) (1,651,178,228 samples, 0.05%)bitcoind::GetBlockProof (4,611,529,418 samples, 0.13%)bitcoind::CSHA256::Finalize (469,172,416 samples, 0.01%)bitcoind::CSHA256::Write (416,395,152 samples, 0.01%)bitcoind::CBlockHeader::GetHash (889,433,319 samples, 0.03%)bitcoind::CSHA256::Write (420,260,903 samples, 0.01%)bitcoind::CheckProofOfWorkImpl (628,054,325 samples, 0.02%)bitcoind::arith_uint256::SetCompact (474,577,125 samples, 0.01%)bitcoind::leveldb::(anonymous namespace)::MergingIterator::Next (359,545,524 samples, 0.01%)bitcoind::leveldb::(anonymous namespace)::DBIter::FindNextUserEntry (615,884,661 samples, 0.02%)bitcoind::std::_Hashtable<uint256, std::pair<uint256 const, CBlockIndex>, std::allocator<std::pair<uint256 const, CBlockIndex> >, std::__detail::_Select1st, std::equal_to<uint256>, BlockHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true> >::_M_rehash (412,349,637 samples, 0.01%)bitcoind::node::BlockManager::InsertBlockIndex (926,266,820 samples, 0.03%)bitcoind::std::_Hashtable<uint256, std::pair<uint256 const, CBlockIndex>, std::allocator<std::pair<uint256 const, CBlockIndex> >, std::__detail::_Select1st, std::equal_to<uint256>, BlockHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true> >::_M_insert_unique_node (621,546,429 samples, 0.02%)bitcoind::kernel::BlockTreeDB::LoadBlockIndexGuts (4,371,914,721 samples, 0.12%)bitcoind::node::BlockManager::GetAllBlockIndices (360,699,633 samples, 0.01%)bitcoind::void std::__introsort_loop<__gnu_cxx::__normal_iterator<CBlockIndex**, std::vector<CBlockIndex*, std::allocator<CBlockIndex*> > >, long, __gnu_cxx::__ops::_Iter_comp_iter<node::CBlockIndexHeightOnlyComparator> > (487,613,426 samples, 0.01%)bitcoind::node::BlockManager::LoadBlockIndexDB (10,756,421,448 samples, 0.30%)bitcoind::node::BlockManager::LoadBlockIndex (10,397,563,911 samples, 0.29%)libc.so.6::__libc_start_call_main (17,915,410,780 samples, 0.50%)bitcoind::main (17,915,410,780 samples, 0.50%)bitcoind::AppInitMain (17,915,410,780 samples, 0.50%)bitcoind::InitAndLoadChainstate (17,915,410,780 samples, 0.50%)bitcoind::node::LoadChainstate (17,915,410,780 samples, 0.50%)bitcoind::node::CompleteChainstateInitialization (17,915,410,780 samples, 0.50%)bitcoind::ChainstateManager::LoadBlockIndex (12,499,349,673 samples, 0.35%)bitcoind::void std::__introsort_loop<__gnu_cxx::__normal_iterator<CBlockIndex**, std::vector<CBlockIndex*, std::allocator<CBlockIndex*> > >, long, __gnu_cxx::__ops::_Iter_comp_iter<node::CBlockIndexHeightOnlyComparator> > (711,414,524 samples, 0.02%)bitcoind::void std::__introsort_loop<__gnu_cxx::__normal_iterator<CBlockIndex**, std::vector<CBlockIndex*, std::allocator<CBlockIndex*> > >, long, __gnu_cxx::__ops::_Iter_comp_iter<node::CBlockIndexHeightOnlyComparator> > (401,238,745 samples, 0.01%)libc.so.6::_int_free (620,663,041 samples, 0.02%)bitcoind::leveldb::(anonymous namespace)::BytewiseComparatorImpl::Compare (868,925,227 samples, 0.02%)bitcoind::leveldb::(anonymous namespace)::MergingIterator::Valid (404,648,282 samples, 0.01%)bitcoind::leveldb::(anonymous namespace)::MergingIterator::value (764,967,422 samples, 0.02%)bitcoind::leveldb::Compaction::ShouldStopBefore (811,517,390 samples, 0.02%)bitcoind::leveldb::TableCache::Evict (924,140,736 samples, 0.03%)bitcoind::leveldb::(anonymous namespace)::ShardedLRUCache::Erase (924,140,736 samples, 0.03%)bitcoind::leveldb::(anonymous namespace)::LRUCache::FinishErase (924,140,736 samples, 0.03%)bitcoind::leveldb::(anonymous namespace)::LRUCache::Unref (924,140,736 samples, 0.03%)bitcoind::leveldb::DeleteEntry (924,140,736 samples, 0.03%)libc.so.6::__munmap (924,140,736 samples, 0.03%)[unknown] (924,140,736 samples, 0.03%)[unknown] (924,140,736 samples, 0.03%)[unknown] (924,140,736 samples, 0.03%)[unknown] (924,140,736 samples, 0.03%)[unknown] (924,140,736 samples, 0.03%)[unknown] (924,140,736 samples, 0.03%)[unknown] (924,140,736 samples, 0.03%)[unknown] (924,140,736 samples, 0.03%)[unknown] (924,140,736 samples, 0.03%)[unknown] (622,330,840 samples, 0.02%)[[jbd2]] (579,680,739 samples, 0.02%)bitcoind::leveldb::DBImpl::DeleteObsoleteFiles (5,386,961,700 samples, 0.15%)libc.so.6::__unlink (4,462,820,964 samples, 0.13%)[unknown] (4,462,820,964 samples, 0.13%)[unknown] (4,462,820,964 samples, 0.13%)[unknown] (4,462,820,964 samples, 0.13%)[unknown] (4,462,820,964 samples, 0.13%)[unknown] (4,462,820,964 samples, 0.13%)[[ext4]] (4,462,820,964 samples, 0.13%)[unknown] (4,413,928,808 samples, 0.12%)[unknown] (4,413,928,808 samples, 0.12%)[unknown] (3,629,480,214 samples, 0.10%)[unknown] (2,527,606,876 samples, 0.07%)[unknown] (1,289,801,972 samples, 0.04%)[unknown] (411,890,158 samples, 0.01%)bitcoind::leveldb::(anonymous namespace)::BytewiseComparatorImpl::Compare (1,451,370,022 samples, 0.04%)bitcoind::leveldb::(anonymous namespace)::BytewiseComparatorImpl::Compare (2,100,345,679 samples, 0.06%)bitcoind::leveldb::(anonymous namespace)::MergingIterator::FindSmallest (7,036,670,089 samples, 0.20%)bitcoind::leveldb::InternalKeyComparator::Compare (5,331,785,618 samples, 0.15%)libc.so.6::__memcmp_evex_movbe (467,739,292 samples, 0.01%)bitcoind::leveldb::Block::Iter::ParseNextKey (1,597,295,639 samples, 0.04%)bitcoind::leveldb::Block::Iter::key (719,412,755 samples, 0.02%)bitcoind::leveldb::(anonymous namespace)::TwoLevelIterator::Next (3,651,719,685 samples, 0.10%)[unknown] (775,514,001 samples, 0.02%)[unknown] (775,514,001 samples, 0.02%)[unknown] (775,514,001 samples, 0.02%)[unknown] (775,514,001 samples, 0.02%)[unknown] (723,468,265 samples, 0.02%)[unknown] (671,854,971 samples, 0.02%)[unknown] (620,745,631 samples, 0.02%)[unknown] (467,020,775 samples, 0.01%)bitcoind::leveldb::ReadBlock (5,036,746,240 samples, 0.14%)bitcoind::crc32c::ExtendSse42 (4,003,982,142 samples, 0.11%)bitcoind::leveldb::(anonymous namespace)::TwoLevelIterator::InitDataBlock (6,011,618,239 samples, 0.17%)bitcoind::leveldb::Table::BlockReader (5,654,181,527 samples, 0.16%)bitcoind::leveldb::(anonymous namespace)::TwoLevelIterator::SkipEmptyDataBlocksForward (6,370,810,843 samples, 0.18%)bitcoind::leveldb::Block::Iter::Valid (514,863,214 samples, 0.01%)bitcoind::leveldb::(anonymous namespace)::TwoLevelIterator::Next (11,878,686,935 samples, 0.33%)[unknown] (357,525,803 samples, 0.01%)bitcoind::leveldb::ReadBlock (1,021,671,534 samples, 0.03%)bitcoind::crc32c::ExtendSse42 (664,145,731 samples, 0.02%)bitcoind::leveldb::(anonymous namespace)::TwoLevelIterator::InitDataBlock (1,177,165,099 samples, 0.03%)bitcoind::leveldb::Table::BlockReader (1,073,054,446 samples, 0.03%)bitcoind::leveldb::(anonymous namespace)::TwoLevelIterator::SkipEmptyDataBlocksForward (3,944,657,665 samples, 0.11%)bitcoind::leveldb::(anonymous namespace)::TwoLevelIterator::Valid (468,344,432 samples, 0.01%)bitcoind::leveldb::Block::Iter::Valid (360,443,695 samples, 0.01%)bitcoind::leveldb::(anonymous namespace)::MergingIterator::Next (25,840,019,062 samples, 0.73%)bitcoind::leveldb::InternalKeyComparator::Compare (877,755,927 samples, 0.02%)bitcoind::leveldb::(anonymous namespace)::MergingIterator::value (460,966,118 samples, 0.01%)bitcoind::leveldb::(anonymous namespace)::TwoLevelIterator::value (1,168,083,499 samples, 0.03%)bitcoind::leveldb::(anonymous namespace)::BytewiseComparatorImpl::Compare (1,027,703,208 samples, 0.03%)bitcoind::leveldb::Compaction::IsBaseLevelForKey (3,331,453,084 samples, 0.09%)libc.so.6::__memcmp_evex_movbe (1,380,364,868 samples, 0.04%)bitcoind::leveldb::(anonymous namespace)::BytewiseComparatorImpl::Compare (1,428,625,135 samples, 0.04%)bitcoind::leveldb::Compaction::ShouldStopBefore (5,019,787,360 samples, 0.14%)bitcoind::leveldb::InternalKeyComparator::Compare (3,376,359,370 samples, 0.09%)libc.so.6::__memcmp_evex_movbe (1,229,056,330 samples, 0.03%)bitcoind::leveldb::DBImpl::DeleteObsoleteFiles (947,024,277 samples, 0.03%)libc.so.6::__unlink (947,024,277 samples, 0.03%)[unknown] (947,024,277 samples, 0.03%)[unknown] (947,024,277 samples, 0.03%)[unknown] (947,024,277 samples, 0.03%)[unknown] (947,024,277 samples, 0.03%)[unknown] (947,024,277 samples, 0.03%)[[ext4]] (947,024,277 samples, 0.03%)[unknown] (947,024,277 samples, 0.03%)[unknown] (947,024,277 samples, 0.03%)[unknown] (891,341,341 samples, 0.03%)[unknown] (632,138,490 samples, 0.02%)[unknown] (416,723,130 samples, 0.01%)bitcoind::leveldb::MemTableIterator::key (1,087,232,643 samples, 0.03%)bitcoind::leveldb::(anonymous namespace)::BytewiseComparatorImpl::Compare (7,618,678,897 samples, 0.21%)bitcoind::leveldb::(anonymous namespace)::BytewiseComparatorImpl::Compare (3,626,299,706 samples, 0.10%)bitcoind::leveldb::InternalKeyComparator::Compare (5,542,426,443 samples, 0.16%)bitcoind::leveldb::PutVarint32 (983,352,417 samples, 0.03%)bitcoind::leveldb::EncodeVarint32 (516,604,326 samples, 0.01%)bitcoind::leveldb::BlockBuilder::Add (15,702,002,539 samples, 0.44%)bitcoind::leveldb::FilterBlockBuilder::AddKey (412,090,761 samples, 0.01%)bitcoind::leveldb::(anonymous namespace)::BloomFilterPolicy::CreateFilter (2,066,056,339 samples, 0.06%)bitcoind::leveldb::Hash (665,922,831 samples, 0.02%)bitcoind::leveldb::FilterBlockBuilder::StartBlock (2,377,127,267 samples, 0.07%)bitcoind::leveldb::FilterBlockBuilder::GenerateFilter (2,377,127,267 samples, 0.07%)bitcoind::leveldb::InternalKeyComparator::Compare (774,664,618 samples, 0.02%)[[ext4]] (567,830,671 samples, 0.02%)[[ext4]] (929,740,986 samples, 0.03%)[unknown] (361,910,315 samples, 0.01%)[[ext4]] (3,216,007,087 samples, 0.09%)[unknown] (2,077,722,358 samples, 0.06%)[unknown] (2,025,638,088 samples, 0.06%)[unknown] (1,766,421,841 samples, 0.05%)[unknown] (1,349,297,830 samples, 0.04%)[unknown] (985,540,031 samples, 0.03%)[[ext4]] (4,245,378,964 samples, 0.12%)[unknown] (4,245,378,964 samples, 0.12%)[unknown] (821,567,389 samples, 0.02%)bitcoind::leveldb::TableBuilder::Flush (5,177,109,910 samples, 0.15%)libc.so.6::__GI___libc_write (4,762,579,653 samples, 0.13%)[unknown] (4,762,579,653 samples, 0.13%)[unknown] (4,762,579,653 samples, 0.13%)[unknown] (4,762,579,653 samples, 0.13%)[unknown] (4,607,316,631 samples, 0.13%)libc.so.6::__memcmp_evex_movbe (2,327,620,616 samples, 0.07%)bitcoind::leveldb::TableBuilder::Add (29,098,360,859 samples, 0.82%)libc.so.6::__memmove_avx512_unaligned_erms (880,376,005 samples, 0.02%)[[ext4]] (576,641,035 samples, 0.02%)[unknown] (419,245,830 samples, 0.01%)[[ext4]] (681,686,302 samples, 0.02%)[[ext4]] (886,736,982 samples, 0.02%)[[ext4]] (886,736,982 samples, 0.02%)[[ext4]] (886,736,982 samples, 0.02%)bitcoind::leveldb::BuildTable (31,594,879,610 samples, 0.89%)libc.so.6::fdatasync (990,237,376 samples, 0.03%)[unknown] (990,237,376 samples, 0.03%)[unknown] (990,237,376 samples, 0.03%)[unknown] (990,237,376 samples, 0.03%)[[ext4]] (990,237,376 samples, 0.03%)[unknown] (990,237,376 samples, 0.03%)[unknown] (990,237,376 samples, 0.03%)[unknown] (990,237,376 samples, 0.03%)[unknown] (990,237,376 samples, 0.03%)bitcoind::leveldb::DBImpl::CompactMemTable (32,644,397,020 samples, 0.92%)bitcoind::leveldb::DBImpl::WriteLevel0Table (31,697,372,743 samples, 0.89%)[[ext4]] (360,420,776 samples, 0.01%)bitcoind::leveldb::TableBuilder::Finish (565,702,739 samples, 0.02%)bitcoind::leveldb::TableBuilder::WriteRawBlock (411,712,919 samples, 0.01%)libc.so.6::__GI___libc_write (411,712,919 samples, 0.01%)[unknown] (411,712,919 samples, 0.01%)[unknown] (411,712,919 samples, 0.01%)[unknown] (411,712,919 samples, 0.01%)[unknown] (411,712,919 samples, 0.01%)[[ext4]] (411,712,919 samples, 0.01%)[unknown] (411,712,919 samples, 0.01%)[[ext4]] (2,407,378,967 samples, 0.07%)[unknown] (1,896,402,811 samples, 0.05%)[unknown] (1,223,588,483 samples, 0.03%)[unknown] (359,102,837 samples, 0.01%)[[ext4]] (3,282,391,421 samples, 0.09%)[unknown] (669,408,205 samples, 0.02%)[[nvme]] (410,427,902 samples, 0.01%)[[nvme]] (410,427,902 samples, 0.01%)[unknown] (410,427,902 samples, 0.01%)[unknown] (410,427,902 samples, 0.01%)[[ext4]] (5,846,551,102 samples, 0.16%)[unknown] (1,539,251,741 samples, 0.04%)[unknown] (1,332,958,992 samples, 0.04%)[unknown] (1,230,554,197 samples, 0.03%)[unknown] (1,230,554,197 samples, 0.03%)[unknown] (1,230,554,197 samples, 0.03%)[unknown] (410,593,098 samples, 0.01%)[[ext4]] (5,999,990,575 samples, 0.17%)[[ext4]] (5,999,990,575 samples, 0.17%)bitcoind::leveldb::DBImpl::FinishCompactionOutputFile (7,646,287,561 samples, 0.22%)libc.so.6::fdatasync (6,926,959,748 samples, 0.19%)[unknown] (6,926,959,748 samples, 0.19%)[unknown] (6,926,959,748 samples, 0.19%)[unknown] (6,926,959,748 samples, 0.19%)[[ext4]] (6,926,959,748 samples, 0.19%)[unknown] (6,926,959,748 samples, 0.19%)[unknown] (6,926,959,748 samples, 0.19%)[unknown] (6,926,959,748 samples, 0.19%)[unknown] (6,720,904,548 samples, 0.19%)[unknown] (618,546,651 samples, 0.02%)[unknown] (618,541,816 samples, 0.02%)[unknown] (618,541,816 samples, 0.02%)[unknown] (618,541,816 samples, 0.02%)[unknown] (513,838,124 samples, 0.01%)[unknown] (411,261,494 samples, 0.01%)bitcoind::leveldb::InternalKeyComparator::Compare (1,176,215,358 samples, 0.03%)bitcoind::leveldb::(anonymous namespace)::BytewiseComparatorImpl::Compare (1,597,987,748 samples, 0.04%)bitcoind::leveldb::EncodeVarint32 (667,083,479 samples, 0.02%)bitcoind::leveldb::(anonymous namespace)::BytewiseComparatorImpl::Compare (3,023,925,193 samples, 0.09%)bitcoind::leveldb::InternalKeyComparator::Compare (5,485,200,607 samples, 0.15%)libc.so.6::__memcmp_evex_movbe (768,462,744 samples, 0.02%)bitcoind::leveldb::BlockBuilder::Add (19,355,464,658 samples, 0.54%)bitcoind::leveldb::PutVarint32 (3,963,072,776 samples, 0.11%)bitcoind::leveldb::EncodeVarint32 (2,006,933,285 samples, 0.06%)bitcoind::leveldb::FilterBlockBuilder::AddKey (1,861,448,821 samples, 0.05%)bitcoind::leveldb::(anonymous namespace)::BloomFilterPolicy::CreateFilter (13,758,298,035 samples, 0.39%)bitcoind::leveldb::Hash (5,062,387,301 samples, 0.14%)bitcoind::leveldb::InternalFilterPolicy::CreateFilter (408,507,196 samples, 0.01%)bitcoind::std::vector<leveldb::Slice, std::allocator<leveldb::Slice> >::_M_default_append (1,029,970,476 samples, 0.03%)bitcoind::leveldb::FilterBlockBuilder::GenerateFilter (16,416,940,319 samples, 0.46%)bitcoind::leveldb::FilterBlockBuilder::StartBlock (16,468,035,714 samples, 0.46%)bitcoind::leveldb::(anonymous namespace)::BytewiseComparatorImpl::Compare (1,040,462,681 samples, 0.03%)bitcoind::leveldb::InternalKeyComparator::Compare (2,329,875,977 samples, 0.07%)bitcoind::leveldb::InternalKeyComparator::FindShortestSeparator (358,735,789 samples, 0.01%)bitcoind::leveldb::PutVarint32 (610,801,466 samples, 0.02%)bitcoind::crc32c::ExtendSse42 (874,382,210 samples, 0.02%)bitcoind::leveldb::TableBuilder::WriteBlock (1,806,524,733 samples, 0.05%)bitcoind::leveldb::TableBuilder::WriteRawBlock (1,390,163,236 samples, 0.04%)libc.so.6::__memmove_avx512_unaligned_erms (413,005,584 samples, 0.01%)[[ext4]] (720,896,427 samples, 0.02%)[[ext4]] (2,836,852,977 samples, 0.08%)[unknown] (1,356,279,497 samples, 0.04%)[[ext4]] (3,876,087,820 samples, 0.11%)[unknown] (634,447,162 samples, 0.02%)[[ext4]] (6,595,884,839 samples, 0.19%)[unknown] (2,409,927,037 samples, 0.07%)[unknown] (2,152,146,763 samples, 0.06%)[unknown] (1,946,544,284 samples, 0.05%)[unknown] (1,691,057,617 samples, 0.05%)[unknown] (1,332,315,567 samples, 0.04%)[unknown] (618,194,201 samples, 0.02%)[unknown] (411,783,313 samples, 0.01%)[[ext4]] (21,402,165,352 samples, 0.60%)[unknown] (13,825,328,165 samples, 0.39%)[unknown] (12,948,506,018 samples, 0.36%)[unknown] (10,591,496,268 samples, 0.30%)[unknown] (8,635,293,060 samples, 0.24%)[unknown] (5,512,816,463 samples, 0.16%)[unknown] (1,755,230,935 samples, 0.05%)[unknown] (358,610,982 samples, 0.01%)[[ext4]] (26,848,872,865 samples, 0.76%)[unknown] (26,183,441,807 samples, 0.74%)[unknown] (3,805,768,350 samples, 0.11%)[unknown] (2,522,380,066 samples, 0.07%)libc.so.6::__GI___libc_write (29,870,807,469 samples, 0.84%)[unknown] (29,663,737,328 samples, 0.83%)[unknown] (29,456,391,053 samples, 0.83%)[unknown] (29,306,607,963 samples, 0.82%)[unknown] (28,793,621,717 samples, 0.81%)[unknown] (869,287,921 samples, 0.02%)bitcoind::leveldb::TableBuilder::Flush (32,039,566,359 samples, 0.90%)bitcoind::leveldb::TableBuilder::status (2,416,608,293 samples, 0.07%)bitcoind::memcpy@plt (1,533,086,169 samples, 0.04%)libc.so.6::__memcmp_evex_movbe (11,663,095,994 samples, 0.33%)libc.so.6::__memmove_avx512_unaligned_erms (6,084,682,703 samples, 0.17%)bitcoind::leveldb::TableBuilder::Add (101,316,031,082 samples, 2.85%)bitc..bitcoind::leveldb::TableBuilder::NumEntries (460,667,349 samples, 0.01%)libc.so.6::__memcmp_evex_movbe (359,824,779 samples, 0.01%)bitcoind::leveldb::DBImpl::DoCompactionWork (188,768,693,249 samples, 5.31%)bitcoind:..libc.so.6::__memmove_avx512_unaligned_erms (972,425,560 samples, 0.03%)bitcoind::leveldb::TableBuilder::NumEntries (767,314,029 samples, 0.02%)bitcoind::leveldb::DBImpl::BackgroundCompaction (198,697,568,504 samples, 5.59%)bitcoind::..libc.so.6::__memmove_avx512_unaligned_erms (569,144,596 samples, 0.02%)bitcoind::leveldb::DBImpl::DeleteObsoleteFiles (591,819,871 samples, 0.02%)libc.so.6::__unlink (591,819,871 samples, 0.02%)[unknown] (591,819,871 samples, 0.02%)[unknown] (591,819,871 samples, 0.02%)[unknown] (591,819,871 samples, 0.02%)[unknown] (591,819,871 samples, 0.02%)[unknown] (591,819,871 samples, 0.02%)[[ext4]] (591,819,871 samples, 0.02%)[unknown] (591,819,871 samples, 0.02%)[unknown] (591,819,871 samples, 0.02%)[unknown] (479,954,726 samples, 0.01%)[unknown] (428,868,095 samples, 0.01%)bitcoind::leveldb::(anonymous namespace)::BytewiseComparatorImpl::Compare (4,760,717,074 samples, 0.13%)bitcoind::leveldb::(anonymous namespace)::BytewiseComparatorImpl::Compare (1,338,954,347 samples, 0.04%)bitcoind::leveldb::InternalKeyComparator::Compare (2,113,914,207 samples, 0.06%)bitcoind::leveldb::BlockBuilder::Add (8,483,080,141 samples, 0.24%)bitcoind::leveldb::PutVarint32 (468,110,226 samples, 0.01%)bitcoind::leveldb::(anonymous namespace)::BloomFilterPolicy::CreateFilter (981,556,026 samples, 0.03%)bitcoind::leveldb::Hash (364,078,664 samples, 0.01%)bitcoind::leveldb::FilterBlockBuilder::StartBlock (1,085,605,353 samples, 0.03%)bitcoind::leveldb::FilterBlockBuilder::GenerateFilter (1,085,605,353 samples, 0.03%)[[ext4]] (363,216,075 samples, 0.01%)[[ext4]] (414,434,148 samples, 0.01%)[[ext4]] (622,462,403 samples, 0.02%)[[ext4]] (2,219,690,360 samples, 0.06%)[unknown] (1,545,848,943 samples, 0.04%)[unknown] (1,545,848,943 samples, 0.04%)[unknown] (1,344,249,592 samples, 0.04%)[unknown] (1,034,709,836 samples, 0.03%)[unknown] (463,122,475 samples, 0.01%)[[ext4]] (2,730,864,687 samples, 0.08%)[unknown] (2,627,509,960 samples, 0.07%)bitcoind::leveldb::TableBuilder::Flush (2,941,422,377 samples, 0.08%)libc.so.6::__GI___libc_write (2,889,358,538 samples, 0.08%)[unknown] (2,889,358,538 samples, 0.08%)[unknown] (2,837,160,085 samples, 0.08%)[unknown] (2,837,160,085 samples, 0.08%)[unknown] (2,837,160,085 samples, 0.08%)libc.so.6::__memcmp_evex_movbe (870,026,684 samples, 0.02%)bitcoind::leveldb::TableBuilder::Add (14,671,945,001 samples, 0.41%)libc.so.6::__memmove_avx512_unaligned_erms (516,334,186 samples, 0.01%)[[ext4]] (366,285,823 samples, 0.01%)bitcoind::leveldb::BuildTable (15,764,968,843 samples, 0.44%)libc.so.6::fdatasync (522,804,809 samples, 0.01%)[unknown] (522,804,809 samples, 0.01%)[unknown] (522,804,809 samples, 0.01%)[unknown] (522,804,809 samples, 0.01%)[[ext4]] (522,804,809 samples, 0.01%)[unknown] (522,804,809 samples, 0.01%)[unknown] (522,804,809 samples, 0.01%)[unknown] (522,804,809 samples, 0.01%)[unknown] (522,804,809 samples, 0.01%)[[ext4]] (470,717,222 samples, 0.01%)[[ext4]] (470,717,222 samples, 0.01%)[[ext4]] (470,717,222 samples, 0.01%)libstdc++.so.6.0.32::execute_native_thread_routine (215,158,735,915 samples, 6.05%)libstdc++.s..bitcoind::leveldb::(anonymous namespace)::PosixEnv::BackgroundThreadEntryPoint (215,158,735,915 samples, 6.05%)bitcoind::l..bitcoind::leveldb::DBImpl::BackgroundCall (215,158,735,915 samples, 6.05%)bitcoind::l..bitcoind::leveldb::DBImpl::CompactMemTable (16,461,167,411 samples, 0.46%)bitcoind::leveldb::DBImpl::WriteLevel0Table (15,869,347,540 samples, 0.45%)bitcoind (236,278,709,104 samples, 6.65%)bitcoindall (3,555,551,407,309 samples, 100%) From ea7616b7499e19d809b5cf5374adea872f3a8d2a Mon Sep 17 00:00:00 2001 From: willcl-ark Date: Thu, 13 Feb 2025 12:57:21 +0000 Subject: [PATCH 11/51] remove legacy assumeutxo bench This is still available in the testing repo: https://github.com/bitcoin-dev-tools/benchcoin-testing --- bench-ci/run-assumeutxo-bench.sh | 154 ------------------------------- 1 file changed, 154 deletions(-) delete mode 100755 bench-ci/run-assumeutxo-bench.sh diff --git a/bench-ci/run-assumeutxo-bench.sh b/bench-ci/run-assumeutxo-bench.sh deleted file mode 100755 index a1ee910ed428..000000000000 --- a/bench-ci/run-assumeutxo-bench.sh +++ /dev/null @@ -1,154 +0,0 @@ -#!/usr/bin/env bash - -set -euxo pipefail - -# Helper function to check and clean datadir -clean_datadir() { - set -euxo pipefail - - local TMP_DATADIR="$1" - - # Create the directory if it doesn't exist - mkdir -p "${TMP_DATADIR}" - - # If we're in CI, clean without confirmation - if [ -n "${CI:-}" ]; then - rm -Rf "${TMP_DATADIR:?}"/* - else - read -rp "Are you sure you want to delete everything in ${TMP_DATADIR}? [y/N] " response - if [[ "$response" =~ ^[Yy]$ ]]; then - rm -Rf "${TMP_DATADIR:?}"/* - else - echo "Aborting..." - exit 1 - fi - fi -} - -# Helper function to clear logs -clean_logs() { - set -euxo pipefail - - local TMP_DATADIR="$1" - local logfile="${TMP_DATADIR}/debug.log" - - echo "Checking for ${logfile}" - if [ -e "${logfile}" ]; then - echo "Removing ${logfile}" - rm "${logfile}" - fi -} - -# Execute CMD before each set of timing runs. -setup_assumeutxo_snapshot_run() { - set -euxo pipefail - - local TMP_DATADIR="$1" - local commit="$2" - clean_datadir "${TMP_DATADIR}" -} - -# Execute CMD before each timing run. -prepare_assumeutxo_snapshot_run() { - set -euxo pipefail - - local TMP_DATADIR="$1" - local UTXO_PATH="$2" - local CONNECT_ADDRESS="$3" - local CHAIN="$4" - local DBCACHE="$5" - local commit="$6" - local BINARIES_DIR="$7" - - # Run the actual preparation steps - clean_datadir "${TMP_DATADIR}" - # Use the pre-built binaries from BINARIES_DIR - "${BINARIES_DIR}/${commit}/bitcoind" --help - taskset -c 0-15 "${BINARIES_DIR}/${commit}/bitcoind" -datadir="${TMP_DATADIR}" -connect="${CONNECT_ADDRESS}" -daemon=0 -chain="${CHAIN}" -stopatheight=1 -printtoconsole=0 - taskset -c 0-15 "${BINARIES_DIR}/${commit}/bitcoind" -datadir="${TMP_DATADIR}" -connect="${CONNECT_ADDRESS}" -daemon=0 -chain="${CHAIN}" -dbcache="${DBCACHE}" -pausebackgroundsync=1 -loadutxosnapshot="${UTXO_PATH}" -printtoconsole=0 || true - clean_logs "${TMP_DATADIR}" -} - -# Executed after each timing run -conclude_assumeutxo_snapshot_run() { - set -euxo pipefail - - local commit="$1" - local TMP_DATADIR="$2" - local PNG_DIR="$3" - - # Search in subdirs e.g. $datadir/signet - debug_log=$(find "${TMP_DATADIR}" -name debug.log -print -quit) - if [ -n "${debug_log}" ]; then - echo "Generating plots from ${debug_log}" - if [ -x "bench-ci/parse_and_plot.py" ]; then - bench-ci/parse_and_plot.py "${debug_log}" "${PNG_DIR}" - else - ls -al "bench-ci/" - echo "parse_and_plot.py not found or not executable, skipping plot generation" - fi - else - ls -al "${TMP_DATADIR}/" - echo "debug.log not found, skipping plot generation" - fi - - # Move flamegraph if exists - if [ -e flamegraph.svg ]; then - mv flamegraph.svg "${commit}"-flamegraph.svg - fi -} - -# Execute CMD after the completion of all benchmarking runs for each individual -# command to be benchmarked. -cleanup_assumeutxo_snapshot_run() { - set -euxo pipefail - - local TMP_DATADIR="$1" - - # Clean up the datadir - clean_datadir "${TMP_DATADIR}" -} - -run_benchmark() { - local base_commit="$1" - local head_commit="$2" - local TMP_DATADIR="$3" - local UTXO_PATH="$4" - local results_file="$5" - local png_dir="$6" - local chain="$7" - local stop_at_height="$8" - local connect_address="$9" - local dbcache="${10}" - local BINARIES_DIR="${11}" - - # Export functions so they can be used by hyperfine - export -f setup_assumeutxo_snapshot_run - export -f prepare_assumeutxo_snapshot_run - export -f conclude_assumeutxo_snapshot_run - export -f cleanup_assumeutxo_snapshot_run - export -f clean_datadir - export -f clean_logs - - # Run hyperfine - hyperfine \ - --shell=bash \ - --setup "setup_assumeutxo_snapshot_run ${TMP_DATADIR} {commit}" \ - --prepare "prepare_assumeutxo_snapshot_run ${TMP_DATADIR} ${UTXO_PATH} ${connect_address} ${chain} ${dbcache} {commit} ${BINARIES_DIR}" \ - --conclude "conclude_assumeutxo_snapshot_run {commit} ${TMP_DATADIR} ${png_dir}" \ - --cleanup "cleanup_assumeutxo_snapshot_run ${TMP_DATADIR}" \ - --runs 1 \ - --export-json "${results_file}" \ - --command-name "base (${base_commit})" \ - --command-name "head (${head_commit})" \ - "taskset -c 1 flamegraph --palette bitcoin --title 'bitcoind assumeutxo IBD@{commit}' -c 'record -F 101 --call-graph fp' -- taskset -c 2-15 ${BINARIES_DIR}/{commit}/bitcoind -datadir=${TMP_DATADIR} -connect=${connect_address} -daemon=0 -chain=${chain} -stopatheight=${stop_at_height} -dbcache=${dbcache} -printtoconsole=0 -debug=coindb -debug=leveldb -debug=bench -debug=validation" \ - -L commit "base,head" -} - -# Main execution -if [ "$#" -ne 11 ]; then - echo "Usage: $0 base_commit head_commit TMP_DATADIR UTXO_PATH results_dir png_dir chain stop_at_height connect_address dbcache BINARIES_DIR" - exit 1 -fi - -run_benchmark "$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8" "$9" "${10}" "${11}" From 0401fa88d9c62b498616b76c0fe70b59decc0df8 Mon Sep 17 00:00:00 2001 From: willcl-ark Date: Thu, 13 Feb 2025 13:06:42 +0000 Subject: [PATCH 12/51] use *instrumented for flame runs --- .github/workflows/benchmark.yml | 4 +- bench-ci/run-benchmark-instrumented.sh | 163 +++++++++++++++++++++++++ bench-ci/run-benchmark.sh | 29 +---- justfile | 10 +- 4 files changed, 177 insertions(+), 29 deletions(-) create mode 100755 bench-ci/run-benchmark-instrumented.sh diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index c8c1a992fa25..fa10b537a931 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -39,7 +39,7 @@ jobs: with: name: bitcoind-binaries path: ${{ runner.temp }}/binaries/ - assumeutxo: + instrumented: needs: build-binaries strategy: matrix: @@ -85,7 +85,7 @@ jobs: run: | env mkdir -p "$TMP_DATADIR" - nix-shell --command "just run-${{ matrix.network }}-ci $BASE_SHA $CHECKOUT_COMMIT $TMP_DATADIR $ORIGINAL_DATADIR ${{ runner.temp }}/results.json ${{ matrix.dbcache }} ${{ runner.temp }}/pngs $BINARIES_DIR" + nix-shell --command "just run-${{ matrix.network }}-ci-instrumented $BASE_SHA $CHECKOUT_COMMIT $TMP_DATADIR $ORIGINAL_DATADIR ${{ runner.temp }}/results.json ${{ matrix.dbcache }} ${{ runner.temp }}/pngs $BINARIES_DIR" - uses: actions/upload-artifact@v4 with: name: result-${{ matrix.name }} diff --git a/bench-ci/run-benchmark-instrumented.sh b/bench-ci/run-benchmark-instrumented.sh new file mode 100755 index 000000000000..db7f5be6ead1 --- /dev/null +++ b/bench-ci/run-benchmark-instrumented.sh @@ -0,0 +1,163 @@ +#!/usr/bin/env bash + +set -euxo pipefail + +# Helper function to check and clean datadir +clean_datadir() { + set -euxo pipefail + + local TMP_DATADIR="$1" + + # Create the directory if it doesn't exist + mkdir -p "${TMP_DATADIR}" + + # If we're in CI, clean without confirmation + if [ -n "${CI:-}" ]; then + rm -Rf "${TMP_DATADIR:?}"/* + else + read -rp "Are you sure you want to delete everything in ${TMP_DATADIR}? [y/N] " response + if [[ "$response" =~ ^[Yy]$ ]]; then + rm -Rf "${TMP_DATADIR:?}"/* + else + echo "Aborting..." + exit 1 + fi + fi +} + +# Helper function to clear logs +clean_logs() { + set -euxo pipefail + + local TMP_DATADIR="$1" + local logfile="${TMP_DATADIR}/debug.log" + + echo "Checking for ${logfile}" + if [ -e "${logfile}" ]; then + echo "Removing ${logfile}" + rm "${logfile}" + fi +} + +# Execute CMD before each set of timing runs. +setup_run() { + set -euxo pipefail + + local TMP_DATADIR="$1" + local commit="$2" + clean_datadir "${TMP_DATADIR}" +} + +# Execute CMD before each timing run. +prepare_run() { + set -euxo pipefail + + local TMP_DATADIR="$1" + local ORIGINAL_DATADIR="$2" + + # Run the actual preparation steps + clean_datadir "${TMP_DATADIR}" + # Don't copy hidden files so use * + taskset -c 0-15 cp -r "$ORIGINAL_DATADIR"/* "$TMP_DATADIR" + clean_logs "${TMP_DATADIR}" +} + +# Executed after each timing run +conclude_run() { + set -euxo pipefail + + local commit="$1" + local TMP_DATADIR="$2" + local PNG_DIR="$3" + + # Search in subdirs e.g. $datadir/signet + debug_log=$(find "${TMP_DATADIR}" -name debug.log -print -quit) + if [ -n "${debug_log}" ]; then + echo "Generating plots from ${debug_log}" + if [ -x "bench-ci/parse_and_plot.py" ]; then + bench-ci/parse_and_plot.py "${debug_log}" "${PNG_DIR}" + else + ls -al "bench-ci/" + echo "parse_and_plot.py not found or not executable, skipping plot generation" + fi + else + ls -al "${TMP_DATADIR}/" + echo "debug.log not found, skipping plot generation" + fi + + # Move flamegraph if exists + if [ -e flamegraph.svg ]; then + mv flamegraph.svg "${commit}"-flamegraph.svg + fi +} + +# Execute CMD after the completion of all benchmarking runs for each individual +# command to be benchmarked. +cleanup_run() { + set -euxo pipefail + + local TMP_DATADIR="$1" + + # Clean up the datadir + clean_datadir "${TMP_DATADIR}" +} + +run_benchmark() { + local base_commit="$1" + local head_commit="$2" + local TMP_DATADIR="$3" + local ORIGINAL_DATADIR="$4" + local results_file="$5" + local png_dir="$6" + local chain="$7" + local stop_at_height="$8" + local connect_address="$9" + local dbcache="${10}" + local BINARIES_DIR="${11}" + + # Export functions so they can be used by hyperfine + export -f setup_run + export -f prepare_run + export -f conclude_run + export -f cleanup_run + export -f clean_datadir + export -f clean_logs + + # Debug: Print all variables being used + echo "=== Debug Information ===" + echo "TMP_DATADIR: ${TMP_DATADIR}" + echo "ORIGINAL_DATADIR: ${ORIGINAL_DATADIR}" + echo "BINARIES_DIR: ${BINARIES_DIR}" + echo "base_commit: ${base_commit}" + echo "head_commit: ${head_commit}" + echo "results_file: ${results_file}" + echo "png_dir: ${png_dir}" + echo "chain: ${chain}" + echo "stop_at_height: ${stop_at_height}" + echo "connect_address: ${connect_address}" + echo "dbcache: ${dbcache}" + echo "\n" + + # Run hyperfine + hyperfine \ + --shell=bash \ + --setup "setup_run ${TMP_DATADIR} {commit}" \ + --prepare "prepare_run ${TMP_DATADIR} ${ORIGINAL_DATADIR}" \ + --conclude "conclude_run {commit} ${TMP_DATADIR} ${png_dir}" \ + --cleanup "cleanup_run ${TMP_DATADIR}" \ + --runs 1 \ + --export-json "${results_file}" \ + --show-output \ + --command-name "base (${base_commit})" \ + --command-name "head (${head_commit})" \ + "taskset -c 1 flamegraph --palette bitcoin --title 'bitcoind IBD@{commit}' -c 'record -F 101 --call-graph fp' -- taskset -c 2-15 chrt -r 1 ${BINARIES_DIR}/{commit}/bitcoind -datadir=${TMP_DATADIR} -connect=${connect_address} -daemon=0 -prune=10000 -chain=${chain} -stopatheight=${stop_at_height} -dbcache=${dbcache} -printtoconsole=0 -debug=coindb -debug=leveldb -debug=bench -debug=validation" \ + -L commit "base,head" +} + +# Main execution +if [ "$#" -ne 11 ]; then + echo "Usage: $0 base_commit head_commit TMP_DATADIR ORIGINAL_DATADIR results_dir png_dir chain stop_at_height connect_address dbcache BINARIES_DIR" + exit 1 +fi + +run_benchmark "$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8" "$9" "${10}" "${11}" diff --git a/bench-ci/run-benchmark.sh b/bench-ci/run-benchmark.sh index dc190a5fa207..ca77427bb31f 100755 --- a/bench-ci/run-benchmark.sh +++ b/bench-ci/run-benchmark.sh @@ -65,30 +65,7 @@ prepare_run() { # Executed after each timing run conclude_run() { set -euxo pipefail - - local commit="$1" - local TMP_DATADIR="$2" - local PNG_DIR="$3" - - # Search in subdirs e.g. $datadir/signet - debug_log=$(find "${TMP_DATADIR}" -name debug.log -print -quit) - if [ -n "${debug_log}" ]; then - echo "Generating plots from ${debug_log}" - if [ -x "bench-ci/parse_and_plot.py" ]; then - bench-ci/parse_and_plot.py "${debug_log}" "${PNG_DIR}" - else - ls -al "bench-ci/" - echo "parse_and_plot.py not found or not executable, skipping plot generation" - fi - else - ls -al "${TMP_DATADIR}/" - echo "debug.log not found, skipping plot generation" - fi - - # Move flamegraph if exists - if [ -e flamegraph.svg ]; then - mv flamegraph.svg "${commit}"-flamegraph.svg - fi + return 0 } # Execute CMD after the completion of all benchmarking runs for each individual @@ -145,12 +122,12 @@ run_benchmark() { --prepare "prepare_run ${TMP_DATADIR} ${ORIGINAL_DATADIR}" \ --conclude "conclude_run {commit} ${TMP_DATADIR} ${png_dir}" \ --cleanup "cleanup_run ${TMP_DATADIR}" \ - --runs 2 \ + --runs 3 \ --export-json "${results_file}" \ --show-output \ --command-name "base (${base_commit})" \ --command-name "head (${head_commit})" \ - "taskset -c 1 flamegraph --palette bitcoin --title 'bitcoind IBD@{commit}' -c 'record -F 101 --call-graph fp' -- taskset -c 2-15 chrt -r 1 ${BINARIES_DIR}/{commit}/bitcoind -datadir=${TMP_DATADIR} -connect=${connect_address} -daemon=0 -prune=10000 -chain=${chain} -stopatheight=${stop_at_height} -dbcache=${dbcache} -printtoconsole=0 -debug=coindb -debug=leveldb -debug=bench -debug=validation" \ + "taskset -c 2-15 chrt -o 1 ${BINARIES_DIR}/{commit}/bitcoind -datadir=${TMP_DATADIR} -connect=${connect_address} -daemon=0 -prune=10000 -chain=${chain} -stopatheight=${stop_at_height} -dbcache=${dbcache} -printtoconsole=0" \ -L commit "base,head" } diff --git a/justfile b/justfile index 0ecd3e990f77..e87616602ce7 100644 --- a/justfile +++ b/justfile @@ -13,7 +13,7 @@ build-assumeutxo-binaries-guix base_commit head_commit: unset SOURCE_DATE_EPOCH # needed to run on NixOS ./bench-ci/build_binaries.sh {{ base_commit }} {{ head_commit }} -# Run mainnet benchmark workflow for large cache +# Run uninstrumented benchmarks on mainnet [group('ci')] run-mainnet-ci base_commit head_commit TMP_DATADIR ORIGINAL_DATADIR results_file dbcache png_dir binaries_dir: #!/usr/bin/env bash @@ -21,6 +21,14 @@ run-mainnet-ci base_commit head_commit TMP_DATADIR ORIGINAL_DATADIR results_file unset SOURCE_DATE_EPOCH # needed to run on NixOS ./bench-ci/run-benchmark.sh {{ base_commit }} {{ head_commit }} {{ TMP_DATADIR }} {{ ORIGINAL_DATADIR }} {{ results_file }} {{ png_dir }} main 855000 "148.251.128.115:33333" {{ dbcache }} {{ binaries_dir }} +# Run instrumented benchmarks on mainnet +[group('ci')] +run-mainnet-ci-instrumented base_commit head_commit TMP_DATADIR ORIGINAL_DATADIR results_file dbcache png_dir binaries_dir: + #!/usr/bin/env bash + set -euxo pipefail + unset SOURCE_DATE_EPOCH # needed to run on NixOS + ./bench-ci/run-benchmark-instrumented.sh {{ base_commit }} {{ head_commit }} {{ TMP_DATADIR }} {{ ORIGINAL_DATADIR }} {{ results_file }} {{ png_dir }} main 855000 "148.251.128.115:33333" {{ dbcache }} {{ binaries_dir }} + # Cherry-pick commits from a bitcoin core PR onto this branch [group('git')] pick-pr pr_number: From 52c52bb66d715837a1c1ad514d192e487f9ea575 Mon Sep 17 00:00:00 2001 From: willcl-ark Date: Thu, 13 Feb 2025 13:19:50 +0000 Subject: [PATCH 13/51] add uninstrumented run --- .github/workflows/benchmark.yml | 64 +++++++++++++++++++++++++++++++++ bench-ci/run-benchmark.sh | 27 ++++++-------- justfile | 4 +-- 3 files changed, 77 insertions(+), 18 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index fa10b537a931..cedbd3b562e6 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -39,6 +39,70 @@ jobs: with: name: bitcoind-binaries path: ${{ runner.temp }}/binaries/ + uninstrumented: + needs: build-binaries + strategy: + matrix: + include: + - network: mainnet + name: mainnet-default + timeout: 600 + datadir_path: /data/pruned-840k + dbcache: 450 + - network: mainnet + name: mainnet-large + timeout: 600 + datadir_path: /data/pruned-840k + dbcache: 32000 + runs-on: [self-hosted, linux, x64] + timeout-minutes: ${{ matrix.timeout }} + env: + NIX_PATH: nixpkgs=channel:nixos-unstable + ORIGINAL_DATADIR: ${{ matrix.datadir_path }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + steps: + - name: Checkout repo + uses: actions/checkout@v4 + with: + fetch-depth: 1 + - name: Download binaries + uses: actions/download-artifact@v4 + with: + name: bitcoind-binaries + path: ${{ runner.temp }}/binaries + - name: Set binary permissions + run: | + chmod +x ${{ runner.temp }}/binaries/base/bitcoind + chmod +x ${{ runner.temp }}/binaries/head/bitcoind + - name: Fetch base commit + run: | + echo "CHECKOUT_COMMIT=$(git rev-parse HEAD)" >> "$GITHUB_ENV" + git fetch --depth=1 origin ${{ github.event.pull_request.base.sha }} + - name: Run AssumeUTXO ${{ matrix.network }} + env: + TMP_DATADIR: "${{ runner.temp }}/base_datadir" + BINARIES_DIR: "${{ runner.temp }}/binaries" + run: | + env + mkdir -p "$TMP_DATADIR" + nix-shell --command "just run-${{ matrix.network }}-ci $BASE_SHA $CHECKOUT_COMMIT $TMP_DATADIR $ORIGINAL_DATADIR ${{ runner.temp }}/results.json ${{ matrix.dbcache }} $BINARIES_DIR" + - uses: actions/upload-artifact@v4 + with: + name: result-${{ matrix.name }} + path: "${{ runner.temp }}/results.json" + - name: Write GitHub and runner context files + env: + GITHUB_CONTEXT: ${{ toJSON(github) }} + RUNNER_CONTEXT: ${{ toJSON(runner) }} + run: | + mkdir contexts + echo "$GITHUB_CONTEXT" | nix-shell -p jq --command "jq 'del(.token)' > contexts/github.json" + echo "$RUNNER_CONTEXT" > contexts/runner.json + - name: Upload context metadata as artifact + uses: actions/upload-artifact@v4 + with: + name: run-metadata-${{ matrix.name }} + path: ./contexts/ instrumented: needs: build-binaries strategy: diff --git a/bench-ci/run-benchmark.sh b/bench-ci/run-benchmark.sh index ca77427bb31f..d0e3ba60e1f1 100755 --- a/bench-ci/run-benchmark.sh +++ b/bench-ci/run-benchmark.sh @@ -44,7 +44,6 @@ setup_run() { set -euxo pipefail local TMP_DATADIR="$1" - local commit="$2" clean_datadir "${TMP_DATADIR}" } @@ -85,17 +84,15 @@ run_benchmark() { local TMP_DATADIR="$3" local ORIGINAL_DATADIR="$4" local results_file="$5" - local png_dir="$6" - local chain="$7" - local stop_at_height="$8" - local connect_address="$9" - local dbcache="${10}" - local BINARIES_DIR="${11}" + local chain="$6" + local stop_at_height="$7" + local connect_address="$8" + local dbcache="${9}" + local BINARIES_DIR="${10}" # Export functions so they can be used by hyperfine export -f setup_run export -f prepare_run - export -f conclude_run export -f cleanup_run export -f clean_datadir export -f clean_logs @@ -108,33 +105,31 @@ run_benchmark() { echo "base_commit: ${base_commit}" echo "head_commit: ${head_commit}" echo "results_file: ${results_file}" - echo "png_dir: ${png_dir}" echo "chain: ${chain}" echo "stop_at_height: ${stop_at_height}" echo "connect_address: ${connect_address}" echo "dbcache: ${dbcache}" - echo "\n" + printf \n # Run hyperfine hyperfine \ --shell=bash \ - --setup "setup_run ${TMP_DATADIR} {commit}" \ + --setup "setup_run ${TMP_DATADIR}" \ --prepare "prepare_run ${TMP_DATADIR} ${ORIGINAL_DATADIR}" \ - --conclude "conclude_run {commit} ${TMP_DATADIR} ${png_dir}" \ --cleanup "cleanup_run ${TMP_DATADIR}" \ --runs 3 \ --export-json "${results_file}" \ --show-output \ --command-name "base (${base_commit})" \ --command-name "head (${head_commit})" \ - "taskset -c 2-15 chrt -o 1 ${BINARIES_DIR}/{commit}/bitcoind -datadir=${TMP_DATADIR} -connect=${connect_address} -daemon=0 -prune=10000 -chain=${chain} -stopatheight=${stop_at_height} -dbcache=${dbcache} -printtoconsole=0" \ + "taskset -c 2-15 chrt -o 0 ${BINARIES_DIR}/{commit}/bitcoind -datadir=${TMP_DATADIR} -connect=${connect_address} -daemon=0 -prune=10000 -chain=${chain} -stopatheight=${stop_at_height} -dbcache=${dbcache} -printtoconsole=0" \ -L commit "base,head" } # Main execution -if [ "$#" -ne 11 ]; then - echo "Usage: $0 base_commit head_commit TMP_DATADIR ORIGINAL_DATADIR results_dir png_dir chain stop_at_height connect_address dbcache BINARIES_DIR" +if [ "$#" -ne 10 ]; then + echo "Usage: $0 base_commit head_commit TMP_DATADIR ORIGINAL_DATADIR results_dir chain stop_at_height connect_address dbcache BINARIES_DIR" exit 1 fi -run_benchmark "$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8" "$9" "${10}" "${11}" +run_benchmark "$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8" "${9}" "${10}" diff --git a/justfile b/justfile index e87616602ce7..86dbbab91115 100644 --- a/justfile +++ b/justfile @@ -15,11 +15,11 @@ build-assumeutxo-binaries-guix base_commit head_commit: # Run uninstrumented benchmarks on mainnet [group('ci')] -run-mainnet-ci base_commit head_commit TMP_DATADIR ORIGINAL_DATADIR results_file dbcache png_dir binaries_dir: +run-mainnet-ci base_commit head_commit TMP_DATADIR ORIGINAL_DATADIR results_file dbcache binaries_dir: #!/usr/bin/env bash set -euxo pipefail unset SOURCE_DATE_EPOCH # needed to run on NixOS - ./bench-ci/run-benchmark.sh {{ base_commit }} {{ head_commit }} {{ TMP_DATADIR }} {{ ORIGINAL_DATADIR }} {{ results_file }} {{ png_dir }} main 855000 "148.251.128.115:33333" {{ dbcache }} {{ binaries_dir }} + ./bench-ci/run-benchmark.sh {{ base_commit }} {{ head_commit }} {{ TMP_DATADIR }} {{ ORIGINAL_DATADIR }} {{ results_file }} main 855000 "148.251.128.115:33333" {{ dbcache }} {{ binaries_dir }} # Run instrumented benchmarks on mainnet [group('ci')] From 67676e7b126ed339977083a7a64abe6ce0c2402f Mon Sep 17 00:00:00 2001 From: willcl-ark Date: Fri, 14 Feb 2025 09:12:52 +0000 Subject: [PATCH 14/51] include instrumentation in name to avoid conflicts --- .github/workflows/benchmark.yml | 8 ++++---- .github/workflows/publish-results.yml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index cedbd3b562e6..5a4e1a4efe8c 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -45,12 +45,12 @@ jobs: matrix: include: - network: mainnet - name: mainnet-default + name: mainnet-default-uninstrumented timeout: 600 datadir_path: /data/pruned-840k dbcache: 450 - network: mainnet - name: mainnet-large + name: mainnet-large-uninstrumented timeout: 600 datadir_path: /data/pruned-840k dbcache: 32000 @@ -109,12 +109,12 @@ jobs: matrix: include: - network: mainnet - name: mainnet-default + name: mainnet-default-instrumented timeout: 600 datadir_path: /data/pruned-840k dbcache: 450 - network: mainnet - name: mainnet-large + name: mainnet-large-instrumented timeout: 600 datadir_path: /data/pruned-840k dbcache: 32000 diff --git a/.github/workflows/publish-results.yml b/.github/workflows/publish-results.yml index 62076d85b5eb..5cf97732430e 100644 --- a/.github/workflows/publish-results.yml +++ b/.github/workflows/publish-results.yml @@ -12,7 +12,7 @@ jobs: contents: write checks: read env: - NETWORKS: "mainnet-default,mainnet-large" + NETWORKS: "mainnet-default-instrumented,mainnet-large-instrumented,mainnet-default-uninstrumented,mainnet-large-uninstrumented" outputs: speedups: ${{ steps.organize.outputs.speedups }} pr-number: ${{ steps.organize.outputs.pr-number }} From 39fda0df6afa3eec9f32bfec3620c52e2112a870 Mon Sep 17 00:00:00 2001 From: willcl-ark Date: Fri, 14 Feb 2025 15:53:52 +0000 Subject: [PATCH 15/51] allow failing source guix profile --- bench-ci/guix/libexec/prelude.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench-ci/guix/libexec/prelude.bash b/bench-ci/guix/libexec/prelude.bash index 5756e856b240..0a145479b6a7 100644 --- a/bench-ci/guix/libexec/prelude.bash +++ b/bench-ci/guix/libexec/prelude.bash @@ -10,7 +10,7 @@ source contrib/shell/git-utils.bash # Source guix profile from the runner home directory GUIX_PROFILE=/home/github-runner/.config/guix/current -. "$GUIX_PROFILE/etc/profile" +. "$GUIX_PROFILE/etc/profile" || true echo "Using the following guix command:" command -v guix echo "Guix command symlink points to:" From d4f546a9a805081fb1a54fcfdf343a29824c84c6 Mon Sep 17 00:00:00 2001 From: willcl-ark Date: Tue, 11 Mar 2025 10:01:22 +0000 Subject: [PATCH 16/51] use github guix mirror (faster) --- bench-ci/guix/libexec/prelude.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench-ci/guix/libexec/prelude.bash b/bench-ci/guix/libexec/prelude.bash index 0a145479b6a7..3cf568279f70 100644 --- a/bench-ci/guix/libexec/prelude.bash +++ b/bench-ci/guix/libexec/prelude.bash @@ -82,7 +82,7 @@ fi # across time. time-machine() { # shellcheck disable=SC2086 - guix time-machine --url=https://codeberg.org/guix/guix.git \ + guix time-machine --url=https://github.com/fanquake/guix.git \ --commit=5cb84f2013c5b1e48a7d0e617032266f1e6059e2 \ --cores="$JOBS" \ --keep-failed \ From 9e41eabbbe9ec97943bb8bfd814da21e8a2fb21c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C5=91rinc?= Date: Thu, 13 Mar 2025 13:41:36 +0100 Subject: [PATCH 17/51] Ignore speedup of instrumented runs --- .github/workflows/publish-results.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/publish-results.yml b/.github/workflows/publish-results.yml index 5cf97732430e..2a66dad88d59 100644 --- a/.github/workflows/publish-results.yml +++ b/.github/workflows/publish-results.yml @@ -262,6 +262,7 @@ jobs: // Set outputs for use in PR comment const resultUrl = `https://${context.repo.owner}.github.io/${context.repo.name}/results/pr-${prNumber}/${runId}/index.html`; const speedupString = Object.entries(combinedResults.speedups) + .filter(([network]) => network.includes('uninstrumented')) .map(([network, speedup]) => `${network}: ${speedup}%`) .join(', '); From 201b2946c5c54fca1aca449ffd2d8ea3ad671b25 Mon Sep 17 00:00:00 2001 From: willcl-ark Date: Mon, 17 Mar 2025 17:19:01 +0000 Subject: [PATCH 18/51] remove nightly upstream sync this just creates needless rebasing. Remove it. --- .github/workflows/sync_upstream.yml | 32 ----------------------------- 1 file changed, 32 deletions(-) delete mode 100644 .github/workflows/sync_upstream.yml diff --git a/.github/workflows/sync_upstream.yml b/.github/workflows/sync_upstream.yml deleted file mode 100644 index 11d9cfafc2e4..000000000000 --- a/.github/workflows/sync_upstream.yml +++ /dev/null @@ -1,32 +0,0 @@ -name: Sync with Upstream -on: - schedule: - - cron: '0 3 * * *' # 03:00 UTC daily - workflow_dispatch: -permissions: - contents: write # Required for pushing to master -jobs: - sync: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - token: ${{ secrets.GITHUB_TOKEN }} - - name: Add upstream remote - run: | - git remote add upstream https://github.com/bitcoin/bitcoin.git - git remote -v - - name: Fetch upstream - run: git fetch upstream - - name: Configure Git - run: | - git config user.name github-actions - git config user.email github-actions@github.com - - name: Rebase onto upstream - run: | - git checkout master - git rebase upstream/master - - name: Push changes - run: git push --force-with-lease origin master From ec3c842885a00cec88136287f34a7f31eff55fe9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C5=91rinc?= Date: Wed, 19 Mar 2025 14:08:59 +0100 Subject: [PATCH 19/51] Add commit id to the plots to make sure they're not overwritten --- .github/workflows/publish-results.yml | 3 ++- bench-ci/parse_and_plot.py | 33 +++++++++++++------------- bench-ci/run-benchmark-instrumented.sh | 2 +- 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/.github/workflows/publish-results.yml b/.github/workflows/publish-results.yml index 2a66dad88d59..2fa320db2072 100644 --- a/.github/workflows/publish-results.yml +++ b/.github/workflows/publish-results.yml @@ -46,7 +46,7 @@ jobs: if [ -d "pngs-${network}" ]; then mkdir -p "${network}-plots" - mv "pngs-${network}"/*.png "${network}-plots/" + mv "pngs-${network}"/* "${network}-plots/" fi done - name: Organize results @@ -172,6 +172,7 @@ jobs: const plotDir = `${resultDir}/${network}-plots`; const plots = fs.existsSync(plotDir) ? fs.readdirSync(plotDir) + .filter(plot => plot.startsWith(`${result.parameters.commit}-`)) .map(plot => ` ${plot} diff --git a/bench-ci/parse_and_plot.py b/bench-ci/parse_and_plot.py index db577417b2ff..4ff4d970a4f9 100755 --- a/bench-ci/parse_and_plot.py +++ b/bench-ci/parse_and_plot.py @@ -114,54 +114,55 @@ def generate_plot(x, y, x_label, y_label, title, output_file): if __name__ == "__main__": - if len(sys.argv) != 3: - print(f"Usage: {sys.argv[0]} ") + if len(sys.argv) != 4: + print(f"Usage: {sys.argv[0]} ") sys.exit(1) - log_file = sys.argv[1] + commit = sys.argv[1] + + log_file = sys.argv[2] if not os.path.isfile(log_file): print(f"File not found: {log_file}") sys.exit(1) - png_dir = sys.argv[2] + png_dir = sys.argv[3] os.makedirs(png_dir, exist_ok=True) update_tip_data, leveldb_compact_data, leveldb_gen_table_data, validation_txadd_data, coindb_write_batch_data, coindb_commit_data = parse_log_file(log_file) times, heights, tx_counts, cache_size, cache_count = zip(*update_tip_data) float_minutes = [(t - times[0]).total_seconds() / 60 for t in times] - generate_plot(float_minutes, heights, "Elapsed minutes", "Block Height", "Block Height vs Time", os.path.join(png_dir, "height_vs_time.png")) - generate_plot(heights, cache_size, "Block Height", "Cache Size (MiB)", "Cache Size vs Block Height", os.path.join(png_dir, "cache_vs_height.png")) - generate_plot(float_minutes, cache_size, "Elapsed minutes", "Cache Size (MiB)", "Cache Size vs Time", os.path.join(png_dir, "cache_vs_time.png")) - generate_plot(heights, tx_counts, "Block Height", "Transaction Count", "Transactions vs Block Height", os.path.join(png_dir, "tx_vs_height.png")) - generate_plot(times, cache_count, "Block Height", "Coins Cache Size", "Coins Cache Size vs Time", os.path.join(png_dir, "coins_cache_vs_time.png")) + generate_plot(float_minutes, heights, "Elapsed minutes", "Block Height", "Block Height vs Time", os.path.join(png_dir, f"{commit}-height_vs_time.png")) + generate_plot(heights, cache_size, "Block Height", "Cache Size (MiB)", "Cache Size vs Block Height", os.path.join(png_dir, f"{commit}-cache_vs_height.png")) + generate_plot(float_minutes, cache_size, "Elapsed minutes", "Cache Size (MiB)", "Cache Size vs Time", os.path.join(png_dir, f"{commit}-cache_vs_time.png")) + generate_plot(heights, tx_counts, "Block Height", "Transaction Count", "Transactions vs Block Height", os.path.join(png_dir, f"{commit}-tx_vs_height.png")) + generate_plot(times, cache_count, "Block Height", "Coins Cache Size", "Coins Cache Size vs Time", os.path.join(png_dir, f"{commit}-coins_cache_vs_time.png")) # LevelDB Compaction and Generated Tables if leveldb_compact_data: leveldb_compact_times = [(t - times[0]).total_seconds() / 60 for t in leveldb_compact_data] leveldb_compact_y = [1 for _ in leveldb_compact_times] # dummy y axis to mark compactions - generate_plot(leveldb_compact_times, leveldb_compact_y, "Elapsed minutes", "LevelDB Compaction", "LevelDB Compaction Events vs Time", os.path.join(png_dir, "leveldb_compact_vs_time.png")) + generate_plot(leveldb_compact_times, leveldb_compact_y, "Elapsed minutes", "LevelDB Compaction", "LevelDB Compaction Events vs Time", os.path.join(png_dir, f"{commit}-leveldb_compact_vs_time.png")) if leveldb_gen_table_data: leveldb_gen_table_times, leveldb_gen_table_keys, leveldb_gen_table_bytes = zip(*leveldb_gen_table_data) leveldb_gen_table_float_minutes = [(t - times[0]).total_seconds() / 60 for t in leveldb_gen_table_times] - generate_plot(leveldb_gen_table_float_minutes, leveldb_gen_table_keys, "Elapsed minutes", "Number of keys", "LevelDB Keys Generated vs Time", os.path.join(png_dir, "leveldb_gen_keys_vs_time.png")) - generate_plot(leveldb_gen_table_float_minutes, leveldb_gen_table_bytes, "Elapsed minutes", "Number of bytes", "LevelDB Bytes Generated vs Time", os.path.join(png_dir, "leveldb_gen_bytes_vs_time.png")) + generate_plot(leveldb_gen_table_float_minutes, leveldb_gen_table_keys, "Elapsed minutes", "Number of keys", "LevelDB Keys Generated vs Time", os.path.join(png_dir, f"{commit}-leveldb_gen_keys_vs_time.png")) + generate_plot(leveldb_gen_table_float_minutes, leveldb_gen_table_bytes, "Elapsed minutes", "Number of bytes", "LevelDB Bytes Generated vs Time", os.path.join(png_dir, f"{commit}-leveldb_gen_bytes_vs_time.png")) # validation mempool add transaction lines if validation_txadd_data: validation_txadd_times = [(t - times[0]).total_seconds() / 60 for t in validation_txadd_data] validation_txadd_y = [1 for _ in validation_txadd_times] # dummy y axis to mark transaction additions - generate_plot(validation_txadd_times, validation_txadd_y, "Elapsed minutes", "Transaction Additions", "Transaction Additions to Mempool vs Time", os.path.join(png_dir, "validation_txadd_vs_time.png")) + generate_plot(validation_txadd_times, validation_txadd_y, "Elapsed minutes", "Transaction Additions", "Transaction Additions to Mempool vs Time", os.path.join(png_dir, f"{commit}-validation_txadd_vs_time.png")) # coindb write batch lines if coindb_write_batch_data: coindb_write_batch_times, is_partial_strs, sizes_mb = zip(*coindb_write_batch_data) coindb_write_batch_float_minutes = [(t - times[0]).total_seconds() / 60 for t in coindb_write_batch_times] - generate_plot(coindb_write_batch_float_minutes, sizes_mb, "Elapsed minutes", "Batch Size MiB", "Coin Database Partial/Final Write Batch Size vs Time", os.path.join(png_dir, "coindb_write_batch_size_vs_time.png")) + generate_plot(coindb_write_batch_float_minutes, sizes_mb, "Elapsed minutes", "Batch Size MiB", "Coin Database Partial/Final Write Batch Size vs Time", os.path.join(png_dir, f"{commit}-coindb_write_batch_size_vs_time.png")) if coindb_commit_data: coindb_commit_times, txout_counts = zip(*coindb_commit_data) coindb_commit_float_minutes = [(t - times[0]).total_seconds() / 60 for t in coindb_commit_times] - generate_plot(coindb_commit_float_minutes, txout_counts, "Elapsed minutes", "Transaction Output Count", "Coin Database Transaction Output Committed vs Time", os.path.join(png_dir, "coindb_commit_txout_vs_time.png")) - + generate_plot(coindb_commit_float_minutes, txout_counts, "Elapsed minutes", "Transaction Output Count", "Coin Database Transaction Output Committed vs Time", os.path.join(png_dir, f"{commit}-coindb_commit_txout_vs_time.png")) print("Plots saved!") \ No newline at end of file diff --git a/bench-ci/run-benchmark-instrumented.sh b/bench-ci/run-benchmark-instrumented.sh index db7f5be6ead1..5c8a0481f4f4 100755 --- a/bench-ci/run-benchmark-instrumented.sh +++ b/bench-ci/run-benchmark-instrumented.sh @@ -75,7 +75,7 @@ conclude_run() { if [ -n "${debug_log}" ]; then echo "Generating plots from ${debug_log}" if [ -x "bench-ci/parse_and_plot.py" ]; then - bench-ci/parse_and_plot.py "${debug_log}" "${PNG_DIR}" + bench-ci/parse_and_plot.py "${commit}" "${debug_log}" "${PNG_DIR}" else ls -al "bench-ci/" echo "parse_and_plot.py not found or not executable, skipping plot generation" From e602abbbd7b012e7c8fa032a0230e1c95ca03f03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C5=91rinc?= Date: Mon, 7 Apr 2025 13:10:38 +0200 Subject: [PATCH 20/51] Plot coins_cache_vs_height instead of coins_cache_vs_time --- bench-ci/parse_and_plot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench-ci/parse_and_plot.py b/bench-ci/parse_and_plot.py index 4ff4d970a4f9..68a5c0fcb384 100755 --- a/bench-ci/parse_and_plot.py +++ b/bench-ci/parse_and_plot.py @@ -136,7 +136,7 @@ def generate_plot(x, y, x_label, y_label, title, output_file): generate_plot(heights, cache_size, "Block Height", "Cache Size (MiB)", "Cache Size vs Block Height", os.path.join(png_dir, f"{commit}-cache_vs_height.png")) generate_plot(float_minutes, cache_size, "Elapsed minutes", "Cache Size (MiB)", "Cache Size vs Time", os.path.join(png_dir, f"{commit}-cache_vs_time.png")) generate_plot(heights, tx_counts, "Block Height", "Transaction Count", "Transactions vs Block Height", os.path.join(png_dir, f"{commit}-tx_vs_height.png")) - generate_plot(times, cache_count, "Block Height", "Coins Cache Size", "Coins Cache Size vs Time", os.path.join(png_dir, f"{commit}-coins_cache_vs_time.png")) + generate_plot(heights, cache_count, "Block Height", "Coins Cache Size", "Coins Cache Size vs Height", os.path.join(png_dir, f"{commit}-coins_cache_vs_height.png")) # LevelDB Compaction and Generated Tables if leveldb_compact_data: From 315bc1f4c499ca3e25b9b76b76fb2de4a260c71e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C5=91rinc?= Date: Mon, 7 Apr 2025 13:29:26 +0200 Subject: [PATCH 21/51] Add vertical lines for major protocol upgrades if this is a height-based plot --- bench-ci/parse_and_plot.py | 89 +++++++++++++++++++++++++++++++++++--- 1 file changed, 84 insertions(+), 5 deletions(-) diff --git a/bench-ci/parse_and_plot.py b/bench-ci/parse_and_plot.py index 68a5c0fcb384..2a8a112cc4cf 100755 --- a/bench-ci/parse_and_plot.py +++ b/bench-ci/parse_and_plot.py @@ -4,6 +4,7 @@ import re import datetime import matplotlib.pyplot as plt +from collections import OrderedDict def parse_updatetip_line(line): @@ -35,6 +36,7 @@ def parse_leveldb_generated_table_line(line): parsed_datetime = datetime.datetime.strptime(iso_str, "%Y-%m-%dT%H:%M:%SZ") return parsed_datetime, int(keys_count_str), int(bytes_count_str) + def parse_validation_txadd_line(line): match = re.match(r'^([\d\-:TZ]+) \[validation] TransactionAddedToMempool: txid=.+wtxid=.+', line) if not match: @@ -61,6 +63,7 @@ def parse_coindb_commit_line(line): parsed_datetime = datetime.datetime.strptime(iso_str, "%Y-%m-%dT%H:%M:%SZ") return parsed_datetime, int(txout_count_str) + def parse_log_file(log_file): with open(log_file, 'r', encoding='utf-8') as f: update_tip_data = [] @@ -94,7 +97,7 @@ def parse_log_file(log_file): return update_tip_data, leveldb_compact_data, leveldb_gen_table_data, validation_txadd_data, coindb_write_batch_data, coindb_commit_data -def generate_plot(x, y, x_label, y_label, title, output_file): +def generate_plot(x, y, x_label, y_label, title, output_file, is_height_based=False): if not x or not y: print(f"Skipping plot '{title}' as there is no data.") return @@ -105,6 +108,82 @@ def generate_plot(x, y, x_label, y_label, title, output_file): plt.xlabel(x_label, fontsize=16) plt.ylabel(y_label, fontsize=16) plt.grid(True) + + # Make sure the x-axis covers the full data range + min_x, max_x = min(x), max(x) + plt.xlim(min_x, max_x) + + # Add vertical lines for major protocol upgrades if this is a height-based plot + if is_height_based: + # Define all notable heights from the chainparams file + fork_heights = OrderedDict([ + ('BIP34', 227931), # Block v2, coinbase includes height + ('BIP66', 363725), # Strict DER signatures + ('BIP65', 388381), # OP_CHECKLOCKTIMEVERIFY + ('CSV', 419328), # BIP68, 112, 113 - OP_CHECKSEQUENCEVERIFY + ('Segwit', 481824), # BIP141, 143, 144, 145 - Segregated Witness + ('Taproot', 709632), # BIP341, 342 - Schnorr signatures & Taproot + ('Halving 1', 210000), # First halving + ('Halving 2', 420000), # Second halving + ('Halving 3', 630000), # Third halving + ('Halving 4', 840000), # Fourth halving + ]) + + # Colors for the different types of events + fork_colors = { + 'BIP34': 'blue', + 'BIP66': 'blue', + 'BIP65': 'blue', + 'CSV': 'blue', + 'Segwit': 'green', + 'Taproot': 'red', + 'Halving 1': 'purple', + 'Halving 2': 'purple', + 'Halving 3': 'purple', + 'Halving 4': 'purple', + } + + # Line styles for different types of events + fork_styles = { + 'BIP34': '--', + 'BIP66': '--', + 'BIP65': '--', + 'CSV': '--', + 'Segwit': '--', + 'Taproot': '--', + 'Halving 1': ':', + 'Halving 2': ':', + 'Halving 3': ':', + 'Halving 4': ':', + } + + max_y = max(y) + + # Position text labels at different heights to avoid overlap + text_positions = {} + position_increment = max_y * 0.05 + current_position = max_y * 0.9 + + # Add lines for forks that are in range + for fork_name, height in fork_heights.items(): + if min_x <= height <= max_x: + plt.axvline(x=height, color=fork_colors[fork_name], + linestyle=fork_styles[fork_name]) + + # Avoid label overlaps by staggering vertical positions + if height in text_positions: + # If this x position already has a label, adjust position + text_positions[height] -= position_increment + else: + text_positions[height] = current_position + current_position -= position_increment + if current_position < max_y * 0.1: + current_position = max_y * 0.9 # Reset if we're too low + + plt.text(height, text_positions[height], f'{fork_name} ({height})', + rotation=90, verticalalignment='top', + color=fork_colors[fork_name]) + plt.xticks(rotation=90, fontsize=12) plt.yticks(fontsize=12) plt.tight_layout() @@ -133,10 +212,10 @@ def generate_plot(x, y, x_label, y_label, title, output_file): float_minutes = [(t - times[0]).total_seconds() / 60 for t in times] generate_plot(float_minutes, heights, "Elapsed minutes", "Block Height", "Block Height vs Time", os.path.join(png_dir, f"{commit}-height_vs_time.png")) - generate_plot(heights, cache_size, "Block Height", "Cache Size (MiB)", "Cache Size vs Block Height", os.path.join(png_dir, f"{commit}-cache_vs_height.png")) + generate_plot(heights, cache_size, "Block Height", "Cache Size (MiB)", "Cache Size vs Block Height", os.path.join(png_dir, f"{commit}-cache_vs_height.png"), is_height_based=True) generate_plot(float_minutes, cache_size, "Elapsed minutes", "Cache Size (MiB)", "Cache Size vs Time", os.path.join(png_dir, f"{commit}-cache_vs_time.png")) - generate_plot(heights, tx_counts, "Block Height", "Transaction Count", "Transactions vs Block Height", os.path.join(png_dir, f"{commit}-tx_vs_height.png")) - generate_plot(heights, cache_count, "Block Height", "Coins Cache Size", "Coins Cache Size vs Height", os.path.join(png_dir, f"{commit}-coins_cache_vs_height.png")) + generate_plot(heights, tx_counts, "Block Height", "Transaction Count", "Transactions vs Block Height", os.path.join(png_dir, f"{commit}-tx_vs_height.png"), is_height_based=True) + generate_plot(heights, cache_count, "Block Height", "Coins Cache Size", "Coins Cache Size vs Height", os.path.join(png_dir, f"{commit}-coins_cache_vs_height.png"), is_height_based=True) # LevelDB Compaction and Generated Tables if leveldb_compact_data: @@ -165,4 +244,4 @@ def generate_plot(x, y, x_label, y_label, title, output_file): coindb_commit_float_minutes = [(t - times[0]).total_seconds() / 60 for t in coindb_commit_times] generate_plot(coindb_commit_float_minutes, txout_counts, "Elapsed minutes", "Transaction Output Count", "Coin Database Transaction Output Committed vs Time", os.path.join(png_dir, f"{commit}-coindb_commit_txout_vs_time.png")) - print("Plots saved!") \ No newline at end of file + print("Plots saved!") From 84bb746c5151fcb8f97370daa7b746bf7a7357a6 Mon Sep 17 00:00:00 2001 From: will Date: Fri, 5 Dec 2025 09:42:48 +0000 Subject: [PATCH 22/51] drop page cache before each run --- bench-ci/run-benchmark-instrumented.sh | 2 ++ bench-ci/run-benchmark.sh | 2 ++ 2 files changed, 4 insertions(+) diff --git a/bench-ci/run-benchmark-instrumented.sh b/bench-ci/run-benchmark-instrumented.sh index 5c8a0481f4f4..29d9b51875cb 100755 --- a/bench-ci/run-benchmark-instrumented.sh +++ b/bench-ci/run-benchmark-instrumented.sh @@ -59,6 +59,8 @@ prepare_run() { clean_datadir "${TMP_DATADIR}" # Don't copy hidden files so use * taskset -c 0-15 cp -r "$ORIGINAL_DATADIR"/* "$TMP_DATADIR" + # Clear page caches + sync && echo 3 | sudo tee /proc/sys/vm/drop_caches > /dev/null clean_logs "${TMP_DATADIR}" } diff --git a/bench-ci/run-benchmark.sh b/bench-ci/run-benchmark.sh index d0e3ba60e1f1..d008dc0fa799 100755 --- a/bench-ci/run-benchmark.sh +++ b/bench-ci/run-benchmark.sh @@ -58,6 +58,8 @@ prepare_run() { clean_datadir "${TMP_DATADIR}" # Don't copy hidden files so use * taskset -c 0-15 cp -r "$ORIGINAL_DATADIR"/* "$TMP_DATADIR" + # Clear page caches + sync && echo 3 | sudo tee /proc/sys/vm/drop_caches > /dev/null clean_logs "${TMP_DATADIR}" } From 8340346ea890b913829a39b3bc257c255d204931 Mon Sep 17 00:00:00 2001 From: will Date: Fri, 5 Dec 2025 09:49:30 +0000 Subject: [PATCH 23/51] refactor shared functions --- bench-ci/prelude.sh | 78 +++++++++++++++++++++++ bench-ci/run-benchmark-instrumented.sh | 86 ++------------------------ bench-ci/run-benchmark.sh | 83 ++----------------------- 3 files changed, 89 insertions(+), 158 deletions(-) create mode 100644 bench-ci/prelude.sh diff --git a/bench-ci/prelude.sh b/bench-ci/prelude.sh new file mode 100644 index 000000000000..4424f8ab25b0 --- /dev/null +++ b/bench-ci/prelude.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# Shared functions + +set -euxo pipefail + +clean_datadir() { + set -euxo pipefail + + local TMP_DATADIR="$1" + + mkdir -p "${TMP_DATADIR}" + + # If we're in CI, clean without confirmation + if [ -n "${CI:-}" ]; then + rm -Rf "${TMP_DATADIR:?}"/* + else + read -rp "Are you sure you want to delete everything in ${TMP_DATADIR}? [y/N] " response + if [[ "$response" =~ ^[Yy]$ ]]; then + rm -Rf "${TMP_DATADIR:?}"/* + else + echo "Aborting..." + exit 1 + fi + fi +} + +clean_logs() { + set -euxo pipefail + + local TMP_DATADIR="$1" + local logfile="${TMP_DATADIR}/debug.log" + + echo "Checking for ${logfile}" + if [ -e "${logfile}" ]; then + echo "Removing ${logfile}" + rm "${logfile}" + fi +} + +# Executes once before each *set* of timing runs. +setup_run() { + set -euxo pipefail + + local TMP_DATADIR="$1" + clean_datadir "${TMP_DATADIR}" +} + +# Executes before each timing run. +prepare_run() { + set -euxo pipefail + + local TMP_DATADIR="$1" + local ORIGINAL_DATADIR="$2" + + clean_datadir "${TMP_DATADIR}" + # Don't copy hidden files so use * + taskset -c 0-15 cp -r "$ORIGINAL_DATADIR"/* "$TMP_DATADIR" + # Clear page caches + sync && echo 3 | sudo tee /proc/sys/vm/drop_caches > /dev/null + clean_logs "${TMP_DATADIR}" +} + +# Executes after the completion of all benchmarking runs for each individual +# command to be benchmarked. +cleanup_run() { + set -euxo pipefail + local TMP_DATADIR="$1" + clean_datadir "${TMP_DATADIR}" +} + +# Export all shared functions for use by hyperfine subshells +export_shared_functions() { + export -f clean_datadir + export -f clean_logs + export -f setup_run + export -f prepare_run + export -f cleanup_run +} diff --git a/bench-ci/run-benchmark-instrumented.sh b/bench-ci/run-benchmark-instrumented.sh index 29d9b51875cb..6cb307bf395b 100755 --- a/bench-ci/run-benchmark-instrumented.sh +++ b/bench-ci/run-benchmark-instrumented.sh @@ -2,69 +2,10 @@ set -euxo pipefail -# Helper function to check and clean datadir -clean_datadir() { - set -euxo pipefail - - local TMP_DATADIR="$1" - - # Create the directory if it doesn't exist - mkdir -p "${TMP_DATADIR}" - - # If we're in CI, clean without confirmation - if [ -n "${CI:-}" ]; then - rm -Rf "${TMP_DATADIR:?}"/* - else - read -rp "Are you sure you want to delete everything in ${TMP_DATADIR}? [y/N] " response - if [[ "$response" =~ ^[Yy]$ ]]; then - rm -Rf "${TMP_DATADIR:?}"/* - else - echo "Aborting..." - exit 1 - fi - fi -} - -# Helper function to clear logs -clean_logs() { - set -euxo pipefail - - local TMP_DATADIR="$1" - local logfile="${TMP_DATADIR}/debug.log" - - echo "Checking for ${logfile}" - if [ -e "${logfile}" ]; then - echo "Removing ${logfile}" - rm "${logfile}" - fi -} +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/prelude.sh" -# Execute CMD before each set of timing runs. -setup_run() { - set -euxo pipefail - - local TMP_DATADIR="$1" - local commit="$2" - clean_datadir "${TMP_DATADIR}" -} - -# Execute CMD before each timing run. -prepare_run() { - set -euxo pipefail - - local TMP_DATADIR="$1" - local ORIGINAL_DATADIR="$2" - - # Run the actual preparation steps - clean_datadir "${TMP_DATADIR}" - # Don't copy hidden files so use * - taskset -c 0-15 cp -r "$ORIGINAL_DATADIR"/* "$TMP_DATADIR" - # Clear page caches - sync && echo 3 | sudo tee /proc/sys/vm/drop_caches > /dev/null - clean_logs "${TMP_DATADIR}" -} - -# Executed after each timing run +# Executed after each timing run - generates plots and handles flamegraph conclude_run() { set -euxo pipefail @@ -93,17 +34,6 @@ conclude_run() { fi } -# Execute CMD after the completion of all benchmarking runs for each individual -# command to be benchmarked. -cleanup_run() { - set -euxo pipefail - - local TMP_DATADIR="$1" - - # Clean up the datadir - clean_datadir "${TMP_DATADIR}" -} - run_benchmark() { local base_commit="$1" local head_commit="$2" @@ -118,12 +48,8 @@ run_benchmark() { local BINARIES_DIR="${11}" # Export functions so they can be used by hyperfine - export -f setup_run - export -f prepare_run + export_shared_functions export -f conclude_run - export -f cleanup_run - export -f clean_datadir - export -f clean_logs # Debug: Print all variables being used echo "=== Debug Information ===" @@ -138,12 +64,12 @@ run_benchmark() { echo "stop_at_height: ${stop_at_height}" echo "connect_address: ${connect_address}" echo "dbcache: ${dbcache}" - echo "\n" + printf '\n' # Run hyperfine hyperfine \ --shell=bash \ - --setup "setup_run ${TMP_DATADIR} {commit}" \ + --setup "setup_run ${TMP_DATADIR}" \ --prepare "prepare_run ${TMP_DATADIR} ${ORIGINAL_DATADIR}" \ --conclude "conclude_run {commit} ${TMP_DATADIR} ${png_dir}" \ --cleanup "cleanup_run ${TMP_DATADIR}" \ diff --git a/bench-ci/run-benchmark.sh b/bench-ci/run-benchmark.sh index d008dc0fa799..bce7857919ac 100755 --- a/bench-ci/run-benchmark.sh +++ b/bench-ci/run-benchmark.sh @@ -2,84 +2,15 @@ set -euxo pipefail -# Helper function to check and clean datadir -clean_datadir() { - set -euxo pipefail - - local TMP_DATADIR="$1" - - # Create the directory if it doesn't exist - mkdir -p "${TMP_DATADIR}" - - # If we're in CI, clean without confirmation - if [ -n "${CI:-}" ]; then - rm -Rf "${TMP_DATADIR:?}"/* - else - read -rp "Are you sure you want to delete everything in ${TMP_DATADIR}? [y/N] " response - if [[ "$response" =~ ^[Yy]$ ]]; then - rm -Rf "${TMP_DATADIR:?}"/* - else - echo "Aborting..." - exit 1 - fi - fi -} - -# Helper function to clear logs -clean_logs() { - set -euxo pipefail - - local TMP_DATADIR="$1" - local logfile="${TMP_DATADIR}/debug.log" - - echo "Checking for ${logfile}" - if [ -e "${logfile}" ]; then - echo "Removing ${logfile}" - rm "${logfile}" - fi -} - -# Execute CMD before each set of timing runs. -setup_run() { - set -euxo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/prelude.sh" - local TMP_DATADIR="$1" - clean_datadir "${TMP_DATADIR}" -} - -# Execute CMD before each timing run. -prepare_run() { - set -euxo pipefail - - local TMP_DATADIR="$1" - local ORIGINAL_DATADIR="$2" - - # Run the actual preparation steps - clean_datadir "${TMP_DATADIR}" - # Don't copy hidden files so use * - taskset -c 0-15 cp -r "$ORIGINAL_DATADIR"/* "$TMP_DATADIR" - # Clear page caches - sync && echo 3 | sudo tee /proc/sys/vm/drop_caches > /dev/null - clean_logs "${TMP_DATADIR}" -} - -# Executed after each timing run +# Executed after each timing run (no-op for uninstrumented) conclude_run() { set -euxo pipefail return 0 } -# Execute CMD after the completion of all benchmarking runs for each individual -# command to be benchmarked. -cleanup_run() { - set -euxo pipefail - - local TMP_DATADIR="$1" - - # Clean up the datadir - clean_datadir "${TMP_DATADIR}" -} - run_benchmark() { local base_commit="$1" local head_commit="$2" @@ -93,11 +24,7 @@ run_benchmark() { local BINARIES_DIR="${10}" # Export functions so they can be used by hyperfine - export -f setup_run - export -f prepare_run - export -f cleanup_run - export -f clean_datadir - export -f clean_logs + export_shared_functions # Debug: Print all variables being used echo "=== Debug Information ===" @@ -111,7 +38,7 @@ run_benchmark() { echo "stop_at_height: ${stop_at_height}" echo "connect_address: ${connect_address}" echo "dbcache: ${dbcache}" - printf \n + printf '\n' # Run hyperfine hyperfine \ From f571372385ec0eca0cea50b6c1a900a54ff3cd9e Mon Sep 17 00:00:00 2001 From: will Date: Fri, 5 Dec 2025 20:49:26 +0000 Subject: [PATCH 24/51] fixup! drop page cache before each run --- bench-ci/prelude.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench-ci/prelude.sh b/bench-ci/prelude.sh index 4424f8ab25b0..98a5232b40fb 100644 --- a/bench-ci/prelude.sh +++ b/bench-ci/prelude.sh @@ -56,7 +56,7 @@ prepare_run() { # Don't copy hidden files so use * taskset -c 0-15 cp -r "$ORIGINAL_DATADIR"/* "$TMP_DATADIR" # Clear page caches - sync && echo 3 | sudo tee /proc/sys/vm/drop_caches > /dev/null + /run/wrappers/bin/drop-caches clean_logs "${TMP_DATADIR}" } From fa11663f28b80829cd01976255b6061ea1dc0577 Mon Sep 17 00:00:00 2001 From: will Date: Fri, 5 Dec 2025 21:56:24 +0000 Subject: [PATCH 25/51] use SCHED_OTHER in instrumented run --- bench-ci/run-benchmark-instrumented.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench-ci/run-benchmark-instrumented.sh b/bench-ci/run-benchmark-instrumented.sh index 6cb307bf395b..1f4be0b0b2dd 100755 --- a/bench-ci/run-benchmark-instrumented.sh +++ b/bench-ci/run-benchmark-instrumented.sh @@ -78,7 +78,7 @@ run_benchmark() { --show-output \ --command-name "base (${base_commit})" \ --command-name "head (${head_commit})" \ - "taskset -c 1 flamegraph --palette bitcoin --title 'bitcoind IBD@{commit}' -c 'record -F 101 --call-graph fp' -- taskset -c 2-15 chrt -r 1 ${BINARIES_DIR}/{commit}/bitcoind -datadir=${TMP_DATADIR} -connect=${connect_address} -daemon=0 -prune=10000 -chain=${chain} -stopatheight=${stop_at_height} -dbcache=${dbcache} -printtoconsole=0 -debug=coindb -debug=leveldb -debug=bench -debug=validation" \ + "taskset -c 1 flamegraph --palette bitcoin --title 'bitcoind IBD@{commit}' -c 'record -F 101 --call-graph fp' -- taskset -c 2-15 chrt -o 0 ${BINARIES_DIR}/{commit}/bitcoind -datadir=${TMP_DATADIR} -connect=${connect_address} -daemon=0 -prune=10000 -chain=${chain} -stopatheight=${stop_at_height} -dbcache=${dbcache} -printtoconsole=0 -debug=coindb -debug=leveldb -debug=bench -debug=validation" \ -L commit "base,head" } From 48e26de9aac7f17ccd83abfe633c62c77c1f5200 Mon Sep 17 00:00:00 2001 From: will Date: Sat, 6 Dec 2025 12:03:40 +0000 Subject: [PATCH 26/51] test clearer results chart --- .github/workflows/publish-results.yml | 163 ++++++++++++++++---------- 1 file changed, 102 insertions(+), 61 deletions(-) diff --git a/.github/workflows/publish-results.yml b/.github/workflows/publish-results.yml index 2fa320db2072..11d0327192c5 100644 --- a/.github/workflows/publish-results.yml +++ b/.github/workflows/publish-results.yml @@ -141,6 +141,14 @@ jobs: fs.writeFileSync(`${resultDir}/results.json`, JSON.stringify(combinedResults, null, 2)); // Create index.html for this run + // Sort results by network then by command type (base first) + const sortedResults = combinedResults.results.sort((a, b) => { + if (a.network !== b.network) return a.network.localeCompare(b.network); + const aIsBase = a.command.includes('base'); + const bIsBase = b.command.includes('base'); + return bIsBase - aIsBase; // base first + }); + const indexHtml = ` @@ -152,70 +160,103 @@ jobs:

Benchmark Results

PR #${prNumber} - Run ${runId}

- ${networks.map(network => ` -
-

- ${network} Results - ${combinedResults.speedups[network] ? - `(${combinedResults.speedups[network]}% speedup)` - : ''} -

-
- ${combinedResults.results - .filter(result => result.network === network) - .map(result => { - const commitShortId = result.parameters.commit.slice(0, 8); - const flameGraphFile = `${network}-${result.parameters.commit}-flamegraph.svg`; - const flameGraphPath = `${resultDir}/${network}-${result.parameters.commit}-flamegraph.svg`; - // Query PNG files dynamically - const plotDir = `${resultDir}/${network}-plots`; - const plots = fs.existsSync(plotDir) - ? fs.readdirSync(plotDir) - .filter(plot => plot.startsWith(`${result.parameters.commit}-`)) - .map(plot => ` - - ${plot} - - `) - .join('') - : ''; + +

Run Data

+
+ + + + + + + + + + + + + ${sortedResults.map(result => ` + + + + + + + + + `).join('')} + +
NetworkCommandMean (s)Std DevUser (s)System (s)
${result.network} + ${result.command.replace( + /\((\w+)\)/, + (_, commit) => `(${commit.slice(0, 8)})` + )} + ${result.mean.toFixed(3)}${result.stddev?.toFixed(3) || 'N/A'}${result.user.toFixed(3)}${result.system.toFixed(3)}
+
+ + +

Speedup Summary

+
+ + + + + + + + + ${Object.entries(combinedResults.speedups).map(([network, speedup]) => ` + + + + + `).join('')} + +
NetworkSpeedup (%)
${network}${speedup}%
+
+ + + ${networks.filter(network => network.includes('instrumented')).map(network => { + const networkResults = combinedResults.results.filter(r => r.network === network); + const graphsHtml = networkResults.map(result => { + const flameGraphFile = `${network}-${result.parameters.commit}-flamegraph.svg`; + const flameGraphPath = `${resultDir}/${network}-${result.parameters.commit}-flamegraph.svg`; + + const plotDir = `${resultDir}/${network}-plots`; + const plots = fs.existsSync(plotDir) + ? fs.readdirSync(plotDir) + .filter(plot => plot.startsWith(`${result.parameters.commit}-`)) + .map(plot => ` + + ${plot} + + `) + .join('') + : ''; + + if (!fs.existsSync(flameGraphPath) && !plots) return ''; + + return ` +
+

${result.command.replace(/\((\w+)\)/, (_, commit) => `(${commit.slice(0, 8)})`)}

+ ${fs.existsSync(flameGraphPath) ? ` + + ` : ''} + ${plots} +
+ `; + }).join(''); + + if (!graphsHtml.trim()) return ''; - return ` - - - - - - - - - - - - - - - - - - - -
CommandMean (s)Std DevUser (s)System (s)
- ${result.command.replace( - /\((\w+)\)/, - (_, commit) => `(${commit.slice(0, 8)})` - )} - ${result.mean.toFixed(3)}${result.stddev?.toFixed(3) || 'N/A'}${result.user.toFixed(3)}${result.system.toFixed(3)}
- ${fs.existsSync(flameGraphPath) ? ` - - ` : ''} - ${plots} - `; - }).join('')} + return ` +
+

${network} Graphs

+ ${graphsHtml}
-
- `).join('')} + `; + }).join('')}
From bfae96436060bff5dbd050e7ccbfa317acdec74a Mon Sep 17 00:00:00 2001 From: willcl-ark Date: Thu, 4 Dec 2025 11:06:06 +0000 Subject: [PATCH 27/51] bump to nixos 25.11 --- shell.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shell.nix b/shell.nix index b1914a75d99a..35a9fdbf49c2 100644 --- a/shell.nix +++ b/shell.nix @@ -1,6 +1,6 @@ # Copyright 0xB10C, willcl-ark { pkgs ? import - (fetchTarball "https://github.com/nixos/nixpkgs/archive/nixos-24.11.tar.gz") + (fetchTarball "https://github.com/nixos/nixpkgs/archive/nixos-25.11.tar.gz") { }, }: let inherit (pkgs.lib) optionals strings; From 191ffc5c57f0d90d583828d78f491e432acca88a Mon Sep 17 00:00:00 2001 From: will Date: Mon, 8 Dec 2025 09:57:45 +0000 Subject: [PATCH 28/51] add nix flake --- flake.lock | 27 +++++++++ flake.nix | 167 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 194 insertions(+) create mode 100644 flake.lock create mode 100644 flake.nix diff --git a/flake.lock b/flake.lock new file mode 100644 index 000000000000..fc1308c520fa --- /dev/null +++ b/flake.lock @@ -0,0 +1,27 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1764983851, + "narHash": "sha256-y7RPKl/jJ/KAP/VKLMghMgXTlvNIJMHKskl8/Uuar7o=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "d9bc5c7dceb30d8d6fafa10aeb6aa8a48c218454", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-25.11", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 000000000000..f769290ebca5 --- /dev/null +++ b/flake.nix @@ -0,0 +1,167 @@ +{ + description = "bitcoind for benchmarking"; + + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.11"; + + outputs = + { self, nixpkgs }: + let + systems = [ + "x86_64-linux" + "aarch64-darwin" + ]; + + forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f system); + + pkgsFor = system: import nixpkgs { inherit system; }; + + mkBitcoinCore = + system: + let + pkgs = pkgsFor system; + inherit (pkgs) lib; + + pname = "bitcoin-core"; + version = self.shortRev or "dirty"; + + CFlags = toString [ + "-O2" + "-g" + ]; + CXXFlags = "${CFlags} -fno-omit-frame-pointer"; + + nativeBuildInputs = with pkgs; [ + cmake + ninja + pkg-config + python3 + ]; + + buildInputs = with pkgs; [ + boost188.dev + libevent.dev + ]; + + cmakeFlags = [ + "-DBUILD_BENCH=OFF" + "-DBUILD_BITCOIN_BIN=OFF" + "-DBUILD_CLI=OFF" + "-DBUILD_DAEMON=ON" + "-DBUILD_FUZZ_BINARY=OFF" + "-DBUILD_GUI_TESTS=OFF" + "-DBUILD_TESTS=OFF" + "-DBUILD_TX=OFF" + "-DBUILD_UTIL=OFF" + "-DBUILD_WALLET_TOOL=OFF" + "-DCMAKE_BUILD_TYPE=RelWithDebInfo" + "-DCMAKE_SKIP_RPATH=ON" + "-DENABLE_EXTERNAL_SIGNER=OFF" + "-DENABLE_IPC=OFF" + "-DENABLE_WALLET=OFF" + "-DREDUCE_EXPORTS=ON" + "-DWITH_ZMQ=OFF" + ]; + in + pkgs.stdenv.mkDerivation { + inherit + pname + version + nativeBuildInputs + buildInputs + cmakeFlags + ; + + preConfigure = '' + cmakeFlagsArray+=( + "-DAPPEND_CFLAGS=${CFlags}" + "-DAPPEND_CXXFLAGS=${CXXFlags}" + "-DAPPEND_LDFLAGS=-Wl,--as-needed -Wl,-O2" + ) + ''; + + src = builtins.path { + path = ./.; + name = "source"; + }; + + env = { + CMAKE_GENERATOR = "Ninja"; + LC_ALL = "C"; + LIBRARY_PATH = ""; + CPATH = ""; + C_INCLUDE_PATH = ""; + CPLUS_INCLUDE_PATH = ""; + OBJC_INCLUDE_PATH = ""; + OBJCPLUS_INCLUDE_PATH = ""; + }; + + dontStrip = true; + + meta = { + description = "bitcoind for benchmarking"; + homepage = "https://bitcoincore.org/"; + license = lib.licenses.mit; + }; + }; + in + { + packages = forAllSystems (system: { + default = mkBitcoinCore system; + }); + + formatter = forAllSystems (system: (pkgsFor system).nixfmt-tree); + + devShells = forAllSystems ( + system: + let + pkgs = pkgsFor system; + inherit (pkgs) stdenv; + + # Override the default cargo-flamegraph with a custom fork including bitcoin highlighting + cargo-flamegraph = pkgs.rustPlatform.buildRustPackage rec { + pname = "flamegraph"; + version = "bitcoin-core"; + + src = pkgs.fetchFromGitHub { + owner = "willcl-ark"; + repo = "flamegraph"; + rev = "bitcoin-core"; + sha256 = "sha256-tQbr3MYfAiOxeT12V9au5KQK5X5JeGuV6p8GR/Sgen4="; + }; + + doCheck = false; + cargoHash = "sha256-QWPqTyTFSZNJNayNqLmsQSu0rX26XBKfdLROZ9tRjrg="; + + nativeBuildInputs = pkgs.lib.optionals stdenv.hostPlatform.isLinux [ pkgs.makeWrapper ]; + buildInputs = pkgs.lib.optionals stdenv.hostPlatform.isDarwin [ + pkgs.darwin.apple_sdk.frameworks.Security + ]; + + postFixup = pkgs.lib.optionalString stdenv.hostPlatform.isLinux '' + wrapProgram $out/bin/cargo-flamegraph \ + --set-default PERF ${pkgs.perf}/bin/perf + wrapProgram $out/bin/flamegraph \ + --set-default PERF ${pkgs.perf}/bin/perf + ''; + }; + in + { + default = pkgs.mkShell { + buildInputs = [ + # Benchmarking + pkgs.cargo-flamegraph + pkgs.flamegraph + pkgs.hyperfine + pkgs.jq + pkgs.perf + pkgs.perf-tools + pkgs.util-linux + + # Binary patching + pkgs.patchelf + ]; + }; + } + ); + }; +} From 541d1db215eac764e82c0cbacc8d5bfea1ce2c06 Mon Sep 17 00:00:00 2001 From: will Date: Mon, 8 Dec 2025 10:17:48 +0000 Subject: [PATCH 29/51] switch to nix build --- .github/workflows/benchmark.yml | 42 +- bench-ci/build_binaries.sh | 20 +- bench-ci/guix/INSTALL.md | 814 ------------------ bench-ci/guix/README.md | 430 --------- bench-ci/guix/guix-attest | 263 ------ bench-ci/guix/guix-build | 474 ---------- bench-ci/guix/guix-clean | 83 -- bench-ci/guix/guix-codesign | 384 --------- bench-ci/guix/guix-verify | 174 ---- bench-ci/guix/libexec/build.sh | 416 --------- bench-ci/guix/libexec/codesign.sh | 153 ---- bench-ci/guix/libexec/prelude.bash | 114 --- bench-ci/guix/manifest.scm | 610 ------------- .../patches/binutils-unaligned-default.patch | 22 - .../guix/patches/gcc-remap-guix-store.patch | 20 - .../guix/patches/glibc-2.42-guix-prefix.patch | 47 - bench-ci/guix/patches/glibc-guix-prefix.patch | 16 - .../guix/patches/glibc-riscv-jumptarget.patch | 57 -- bench-ci/guix/patches/lief-scikit-0-9.patch | 21 - .../patches/oscrypto-hard-code-openssl.patch | 13 - .../winpthreads-remap-guix-store.patch | 17 - bench-ci/guix/security-check.py | 297 ------- bench-ci/guix/symbol-check.py | 338 -------- flake.nix | 5 +- justfile | 3 +- shell.nix | 101 --- 26 files changed, 25 insertions(+), 4909 deletions(-) delete mode 100644 bench-ci/guix/INSTALL.md delete mode 100644 bench-ci/guix/README.md delete mode 100755 bench-ci/guix/guix-attest delete mode 100755 bench-ci/guix/guix-build delete mode 100755 bench-ci/guix/guix-clean delete mode 100755 bench-ci/guix/guix-codesign delete mode 100755 bench-ci/guix/guix-verify delete mode 100755 bench-ci/guix/libexec/build.sh delete mode 100755 bench-ci/guix/libexec/codesign.sh delete mode 100644 bench-ci/guix/libexec/prelude.bash delete mode 100644 bench-ci/guix/manifest.scm delete mode 100644 bench-ci/guix/patches/binutils-unaligned-default.patch delete mode 100644 bench-ci/guix/patches/gcc-remap-guix-store.patch delete mode 100644 bench-ci/guix/patches/glibc-2.42-guix-prefix.patch delete mode 100644 bench-ci/guix/patches/glibc-guix-prefix.patch delete mode 100644 bench-ci/guix/patches/glibc-riscv-jumptarget.patch delete mode 100644 bench-ci/guix/patches/lief-scikit-0-9.patch delete mode 100644 bench-ci/guix/patches/oscrypto-hard-code-openssl.patch delete mode 100644 bench-ci/guix/patches/winpthreads-remap-guix-store.patch delete mode 100755 bench-ci/guix/security-check.py delete mode 100755 bench-ci/guix/symbol-check.py delete mode 100644 shell.nix diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 5a4e1a4efe8c..65e667e88fe1 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -7,7 +7,6 @@ jobs: build-binaries: runs-on: [self-hosted, linux, x64] env: - NIX_PATH: nixpkgs=channel:nixos-unstable BASE_SHA: ${{ github.event.pull_request.base.sha }} steps: - name: Checkout repo @@ -18,22 +17,17 @@ jobs: run: | echo "CHECKOUT_COMMIT=$(git rev-parse HEAD)" >> "$GITHUB_ENV" git fetch --depth=1 origin ${{ github.event.pull_request.base.sha }} - - name: Setup ccache - run: | - mkdir -p /data/ccache - export CCACHE_DIR=/data/ccache - export CCACHE_MAXSIZE=50G - ccache -M 50G - ccache -s - name: Build both binaries env: - CCACHE_DIR: /data/ccache + CCACHE_DIR: /nix/var/cache/ccache run: | mkdir -p ${{ runner.temp }}/binaries/base mkdir -p ${{ runner.temp }}/binaries/head - nix-shell --command "just build-assumeutxo-binaries-guix $BASE_SHA $CHECKOUT_COMMIT" - cp binaries/base/bitcoind ${{ runner.temp }}/binaries/base/bitcoind - cp binaries/head/bitcoind ${{ runner.temp }}/binaries/head/bitcoind + nix develop --command bash -c ' + just build-binaries $BASE_SHA $CHECKOUT_COMMIT + cp binaries/base/bitcoind ${{ runner.temp }}/binaries/base/bitcoind + cp binaries/head/bitcoind ${{ runner.temp }}/binaries/head/bitcoind + ' - name: Upload binaries uses: actions/upload-artifact@v4 with: @@ -57,7 +51,6 @@ jobs: runs-on: [self-hosted, linux, x64] timeout-minutes: ${{ matrix.timeout }} env: - NIX_PATH: nixpkgs=channel:nixos-unstable ORIGINAL_DATADIR: ${{ matrix.datadir_path }} BASE_SHA: ${{ github.event.pull_request.base.sha }} steps: @@ -83,9 +76,8 @@ jobs: TMP_DATADIR: "${{ runner.temp }}/base_datadir" BINARIES_DIR: "${{ runner.temp }}/binaries" run: | - env mkdir -p "$TMP_DATADIR" - nix-shell --command "just run-${{ matrix.network }}-ci $BASE_SHA $CHECKOUT_COMMIT $TMP_DATADIR $ORIGINAL_DATADIR ${{ runner.temp }}/results.json ${{ matrix.dbcache }} $BINARIES_DIR" + nix develop --command just run-${{ matrix.network }}-ci $BASE_SHA $CHECKOUT_COMMIT $TMP_DATADIR $ORIGINAL_DATADIR ${{ runner.temp }}/results.json ${{ matrix.dbcache }} $BINARIES_DIR - uses: actions/upload-artifact@v4 with: name: result-${{ matrix.name }} @@ -95,9 +87,11 @@ jobs: GITHUB_CONTEXT: ${{ toJSON(github) }} RUNNER_CONTEXT: ${{ toJSON(runner) }} run: | - mkdir contexts - echo "$GITHUB_CONTEXT" | nix-shell -p jq --command "jq 'del(.token)' > contexts/github.json" - echo "$RUNNER_CONTEXT" > contexts/runner.json + mkdir -p contexts + nix develop --command bash -c ' + echo "$GITHUB_CONTEXT" | jq "del(.token)" > contexts/github.json + echo "$RUNNER_CONTEXT" > contexts/runner.json + ' - name: Upload context metadata as artifact uses: actions/upload-artifact@v4 with: @@ -121,7 +115,6 @@ jobs: runs-on: [self-hosted, linux, x64] timeout-minutes: ${{ matrix.timeout }} env: - NIX_PATH: nixpkgs=channel:nixos-unstable ORIGINAL_DATADIR: ${{ matrix.datadir_path }} BASE_SHA: ${{ github.event.pull_request.base.sha }} steps: @@ -147,9 +140,8 @@ jobs: TMP_DATADIR: "${{ runner.temp }}/base_datadir" BINARIES_DIR: "${{ runner.temp }}/binaries" run: | - env mkdir -p "$TMP_DATADIR" - nix-shell --command "just run-${{ matrix.network }}-ci-instrumented $BASE_SHA $CHECKOUT_COMMIT $TMP_DATADIR $ORIGINAL_DATADIR ${{ runner.temp }}/results.json ${{ matrix.dbcache }} ${{ runner.temp }}/pngs $BINARIES_DIR" + nix develop --command just run-${{ matrix.network }}-ci-instrumented $BASE_SHA $CHECKOUT_COMMIT $TMP_DATADIR $ORIGINAL_DATADIR ${{ runner.temp }}/results.json ${{ matrix.dbcache }} ${{ runner.temp }}/pngs $BINARIES_DIR - uses: actions/upload-artifact@v4 with: name: result-${{ matrix.name }} @@ -167,9 +159,11 @@ jobs: GITHUB_CONTEXT: ${{ toJSON(github) }} RUNNER_CONTEXT: ${{ toJSON(runner) }} run: | - mkdir contexts - echo "$GITHUB_CONTEXT" | nix-shell -p jq --command "jq 'del(.token)' > contexts/github.json" - echo "$RUNNER_CONTEXT" > contexts/runner.json + mkdir -p contexts + nix develop --command bash -c ' + echo "$GITHUB_CONTEXT" | jq "del(.token)" > contexts/github.json + echo "$RUNNER_CONTEXT" > contexts/runner.json + ' - name: Upload context metadata as artifact uses: actions/upload-artifact@v4 with: diff --git a/bench-ci/build_binaries.sh b/bench-ci/build_binaries.sh index 9a396a00659f..242456250d8d 100755 --- a/bench-ci/build_binaries.sh +++ b/bench-ci/build_binaries.sh @@ -25,23 +25,9 @@ for build in "base:${base_commit}" "head:${head_commit}"; do name="${build%%:*}" commit="${build#*:}" git checkout "$commit" - # Use environment variables if set, otherwise use defaults - HOSTS="${HOSTS:-x86_64-linux-gnu}" \ - SOURCES_PATH="${SOURCES_PATH:-/data/SOURCES_PATH}" \ - BASE_CACHE="${BASE_CACHE:-/data/BASE_CACHE}" \ - taskset -c 2-15 chrt -f 1 bench-ci/guix/guix-build - - # Truncate commit hash to 12 characters - short_commit=$(echo "$commit" | cut -c 1-12) - - # Extract the Guix output - tar -xzf "guix-build-${short_commit}/output/x86_64-linux-gnu/bitcoin-${short_commit}-x86_64-linux-gnu.tar.gz" - - # Copy the binary to our binaries directory - cp "bitcoin-${short_commit}/bin/bitcoind" "binaries/${name}/bitcoind" - - # Cleanup extracted files - rm -rf "bitcoin-${short_commit}" + taskset -c 2-15 chrt -f 1 nix build -L + cp "./result/bin/bitcoind" "./binaries/${name}/bitcoind" + rm -rf "./result" done # Restore initial git state diff --git a/bench-ci/guix/INSTALL.md b/bench-ci/guix/INSTALL.md deleted file mode 100644 index f9a79f66349c..000000000000 --- a/bench-ci/guix/INSTALL.md +++ /dev/null @@ -1,814 +0,0 @@ -# Guix Installation and Setup - -This only needs to be done once per machine. If you have already completed the -installation and setup, please proceed to [perform a build](./README.md). - -Otherwise, you may choose from one of the following options to install Guix: - -1. Using the official **shell installer script** [⤓ skip to section][install-script] - - Maintained by Guix developers - - Easiest (automatically performs *most* setup) - - Works on nearly all Linux distributions - - Only installs latest release - - Binary installation only, requires high level of trust - - Note: The script needs to be run as root, so it should be inspected before it's run -2. Using the official **binary tarball** [⤓ skip to section][install-bin-tarball] - - Maintained by Guix developers - - Normal difficulty (full manual setup required) - - Works on nearly all Linux distributions - - Installs any release - - Binary installation only, requires high level of trust -3. Using fanquake's **container image** [↗︎ external instructions][install-fanquake-container] - - Maintained by fanquake - - Easy (automatically performs *some* setup) - - Works wherever container images work (Docker/Podman) - - Installs any release - - Binary installation only, requires high level of trust -4. Using a **distribution-maintained package** [⤓ skip to section][install-distro-pkg] - - Maintained by distribution's Guix package maintainer - - Normal difficulty (manual setup required) - - Works only on distributions with Guix packaged, see: https://repology.org/project/guix/versions - - Installs a release decided on by package maintainer - - Source or binary installation depending on the distribution -5. Building **from source** [⤓ skip to section][install-source] - - Maintained by you - - Hard, but rewarding - - Can be made to work on most Linux distributions - - Installs any commit (more granular) - - Source installation, requires lower level of trust - -## Options 1 and 2: Using the official shell installer script or binary tarball - -The installation instructions for both the official shell installer script and -the binary tarballs can be found in the GNU Guix Manual's [Binary Installation -section](https://guix.gnu.org/manual/en/html_node/Binary-Installation.html). - -Note that running through the binary tarball installation steps is largely -equivalent to manually performing what the shell installer script does. - -Note that at the time of writing (July 5th, 2021), the shell installer script -automatically creates an `/etc/profile.d` entry which the binary tarball -installation instructions do not ask you to create. However, you will likely -need this entry for better desktop integration. Please see [this -section](#add-an-etcprofiled-entry) for instructions on how to add a -`/etc/profile.d/guix.sh` entry. - -Regardless of which installation option you chose, the changes to -`/etc/profile.d` will not take effect until the next shell or desktop session, -so you should log out and log back in. - -## Option 3: Using fanquake's container image - -Please refer to fanquake's instructions -[here](https://github.com/fanquake/core-review/tree/master/guix). - -## Option 4: Using a distribution-maintained package - -Note that this section is based on the distro packaging situation at the time of -writing (July 2021). Guix is expected to be more widely packaged over time. For -an up-to-date view on Guix's package status/version across distros, please see: -https://repology.org/project/guix/versions - -### Debian / Ubuntu - -Guix is available as a distribution package in [Debian -](https://packages.debian.org/search?keywords=guix) and [Ubuntu -](https://packages.ubuntu.com/search?keywords=guix). - -To install: -```sh -sudo apt install guix -``` - -### Arch Linux - -Guix is available in the AUR as -[`guix`](https://aur.archlinux.org/packages/guix/), please follow the -installation instructions in the Arch Linux Wiki ([live -link](https://wiki.archlinux.org/index.php/Guix#AUR_Package_Installation), -[2021/03/30 -permalink](https://wiki.archlinux.org/index.php?title=Guix&oldid=637559#AUR_Package_Installation)) -to install Guix. - -At the time of writing (2021/03/30), the `check` phase will fail if the path to -guix's build directory is longer than 36 characters due to an anachronistic -character limit on the shebang line. Since the `check` phase happens after the -`build` phase, which may take quite a long time, it is recommended that users -either: - -1. Skip the `check` phase - - For `makepkg`: `makepkg --nocheck ...` - - For `yay`: `yay --mflags="--nocheck" ...` - - For `paru`: `paru --nocheck ...` -2. Or, check their build directory's length beforehand - - For those building with `makepkg`: `pwd | wc -c` - -## Option 5: Building from source - -Building Guix from source is a rather involved process but a rewarding one for -those looking to minimize trust and maximize customizability (e.g. building a -particular commit of Guix). Previous experience with using autotools-style build -systems to build packages from source will be helpful. *hic sunt dracones.* - -I strongly urge you to at least skim through the entire section once before you -start issuing commands, as it will save you a lot of unnecessary pain and -anguish. - -### Installing common build tools - -There are a few basic build tools that are required for most things we'll build, -so let's install them now: - -Text transformation/i18n: -- `autopoint` (sometimes packaged in `gettext`) -- `help2man` -- `po4a` -- `texinfo` - -Build system tools: -- `g++` w/ C++11 support -- `libtool` -- `autoconf` -- `automake` -- `pkg-config` (sometimes packaged as `pkgconf`) -- `make` -- `cmake` - -Miscellaneous: -- `git` -- `gnupg` -- `python3` - -### Building and Installing Guix's dependencies - -In order to build Guix itself from source, we need to first make sure that the -necessary dependencies are installed and discoverable. The most up-to-date list -of Guix's dependencies is kept in the ["Requirements" -section](https://guix.gnu.org/manual/en/html_node/Requirements.html) of the Guix -Reference Manual. - -Depending on your distribution, most or all of these dependencies may already be -packaged and installable without manually building and installing. - -For reference, the graphic below outlines Guix v1.3.0's dependency graph: - -![bootstrap map](https://user-images.githubusercontent.com/6399679/125064185-a9a59880-e0b0-11eb-82c1-9b8e5dc9950d.png) - -If you do not care about building each dependency from source, and Guix is -already packaged for your distribution, you can easily install only the build -dependencies of Guix. For example, to enable deb-src and install the Guix build -dependencies on Ubuntu/Debian: - -```sh -sed -i 's|# deb-src|deb-src|g' /etc/apt/sources.list -apt update -apt-get build-dep -y guix -``` - -If this succeeded, you can likely skip to section -["Building and Installing Guix itself"](#building-and-installing-guix-itself). - -#### Guile - -###### Corner case: Multiple versions of Guile on one system - -It is recommended to only install the required version of Guile, so that build systems do -not get confused about which Guile to use. - -However, if you insist on having more versions of Guile installed on -your system, then you need to **consistently** specify -`GUILE_EFFECTIVE_VERSION=3.0` to all -`./configure` invocations for Guix and its dependencies. - -##### Installing Guile - -If your distribution splits packages into `-dev`-suffixed and -non-`-dev`-suffixed sub-packages (as is the case for Debian-derived -distributions), please make sure to install both. For example, to install Guile -v3.0 on Debian/Ubuntu: - -```sh -apt install guile-3.0 guile-3.0-dev -``` - -#### Mixing distribution packages and source-built packages - -At the time of writing, most distributions have _some_ of Guix's dependencies -packaged, but not all. This means that you may want to install the distribution -package for some dependencies, and manually build-from-source for others. - -Distribution packages usually install to `/usr`, which is different from the -default `./configure` prefix of source-built packages: `/usr/local`. - -This means that if you mix-and-match distribution packages and source-built -packages and do not specify exactly `--prefix=/usr` to `./configure` for -source-built packages, you will need to augment the `GUILE_LOAD_PATH` and -`GUILE_LOAD_COMPILED_PATH` environment variables so that Guile will look -under the right prefix and find your source-built packages. - -For example, if you are using Guile v3.0, and have Guile packages in the -`/usr/local` prefix, either add the following lines to your `.profile` or -`.bash_profile` so that the environment variable is properly set for all future -shell logins, or paste the lines into a POSIX-style shell to temporarily modify -the environment variables of your current shell session. - -```sh -# Help Guile v3.0.x find packages in /usr/local -export GUILE_LOAD_PATH="/usr/local/share/guile/site/3.0${GUILE_LOAD_PATH:+:}$GUILE_LOAD_PATH" -export GUILE_LOAD_COMPILED_PATH="/usr/local/lib/guile/3.0/site-ccache${GUILE_LOAD_COMPILED_PATH:+:}$GUILE_COMPILED_LOAD_PATH" -``` - -Note that these environment variables are used to check for packages during -`./configure`, so they should be set as soon as possible should you want to use -a prefix other than `/usr`. - -#### Building and installing source-built packages - -***IMPORTANT**: A few dependencies have non-obvious quirks/errata which are -documented in the sub-sections immediately below. Please read these sections -before proceeding to build and install these packages.* - -Although you should always refer to the README or INSTALL files for the most -accurate information, most of these dependencies use autoconf-style build -systems (check if there's a `configure.ac` file), and will likely do the right -thing with the following: - -Clone the repository and check out the latest release: -```sh -git clone /.git -cd -git tag -l # check for the latest release -git checkout -``` - -For autoconf-based build systems (if `./autogen.sh` or `configure.ac` exists at -the root of the repository): - -```sh -./autogen.sh || autoreconf -vfi -./configure --prefix= -make -sudo make install -``` - -For CMake-based build systems (if `CMakeLists.txt` exists at the root of the -repository): - -```sh -mkdir build && cd build -cmake .. -DCMAKE_INSTALL_PREFIX= -sudo cmake --build . --target install -``` - -If you choose not to specify exactly `--prefix=/usr` to `./configure`, please -make sure you've carefully read the [previous section] on mixing distribution -packages and source-built packages. - -##### Binding packages require `-dev`-suffixed packages - -Relevant for: -- Everyone - -When building bindings, the `-dev`-suffixed version of the original package -needs to be installed. For example, building `Guile-zlib` on Debian-derived -distributions requires that `zlib1g-dev` is installed. - -When using bindings, the `-dev`-suffixed version of the original package still -needs to be installed. This is particularly problematic when distribution -packages are mispackaged like `guile-sqlite3` is in Ubuntu Focal such that -installing `guile-sqlite3` does not automatically install `libsqlite3-dev` as a -dependency. - -Below is a list of relevant Guile bindings and their corresponding `-dev` -packages in Debian at the time of writing. - -| Guile binding package | -dev Debian package | -|-----------------------|---------------------| -| guile-gcrypt | libgcrypt-dev | -| guile-git | libgit2-dev | -| guile-gnutls | (none) | -| guile-json | (none) | -| guile-lzlib | liblz-dev | -| guile-ssh | libssh-dev | -| guile-sqlite3 | libsqlite3-dev | -| guile-zlib | zlib1g-dev | - -##### `guile-git` actually depends on `libgit2 >= 1.1` - -Relevant for: -- Those building `guile-git` from source against `libgit2 < 1.1` -- Those installing `guile-git` from their distribution where `guile-git` is - built against `libgit2 < 1.1` - -As of v0.5.2, `guile-git` claims to only require `libgit2 >= 0.28.0`, however, -it actually requires `libgit2 >= 1.1`, otherwise, it will be confused by a -reference of `origin/keyring`: instead of interpreting the reference as "the -'keyring' branch of the 'origin' remote", the reference is interpreted as "the -branch literally named 'origin/keyring'" - -This is especially notable because Ubuntu Focal packages `libgit2 v0.28.4`, and -`guile-git` is built against it. - -Should you be in this situation, you need to build both `libgit2 v1.1.x` and -`guile-git` from source. - -Source: https://logs.guix.gnu.org/guix/2020-11-12.log#232527 - -### Building and Installing Guix itself - -Start by cloning Guix: - -``` -git clone https://codeberg.org/guix/guix.git -cd guix -``` - -You will likely want to build the latest release. -At the time of writing (November 2023), the latest release was `v1.4.0`. - -``` -git branch -a -l 'origin/version-*' # check for the latest release -git checkout -``` - -Bootstrap the build system: -``` -./bootstrap -``` - -Configure with the recommended `--localstatedir` flag: -``` -./configure --localstatedir=/var -``` - -Note: If you intend to hack on Guix in the future, you will need to supply the -same `--localstatedir=` flag for all future Guix `./configure` invocations. See -the last paragraph of this -[section](https://guix.gnu.org/manual/en/html_node/Requirements.html) for more -details. - -Build Guix (this will take a while): -``` -make -j$(nproc) -``` - -Install Guix: - -``` -sudo make install -``` - -### Post-"build from source" Setup - -#### Creating and starting a `guix-daemon-original` service with a fixed `argv[0]` - -At this point, guix will be installed to `${bindir}`, which is likely -`/usr/local/bin` if you did not override directory variables at -`./configure`-time. More information on standard Automake directory variables -can be found -[here](https://www.gnu.org/software/automake/manual/html_node/Standard-Directory-Variables.html). - -However, the Guix init scripts and service configurations for Upstart, systemd, -SysV, and OpenRC are installed (in `${libdir}`) to launch -`${localstatedir}/guix/profiles/per-user/root/current-guix/bin/guix-daemon`, -which does not yet exist, and will only exist after [`root` performs their first -`guix pull`](#guix-pull-as-root). - -We need to create a `-original` version of these init scripts that's pointed to -the binaries we just built and `make install`'ed in `${bindir}` (normally, -`/usr/local/bin`). - -Example for `systemd`, run as `root`: - -```sh -# Create guix-daemon-original.service by modifying guix-daemon.service -libdir=# set according to your PREFIX (default is /usr/local/lib) -bindir="$(dirname $(command -v guix-daemon))" -sed -E -e "s|/\S*/guix/profiles/per-user/root/current-guix/bin/guix-daemon|${bindir}/guix-daemon|" "${libdir}"/systemd/system/guix-daemon.service > /etc/systemd/system/guix-daemon-original.service -chmod 664 /etc/systemd/system/guix-daemon-original.service - -# Make systemd recognize the new service -systemctl daemon-reload - -# Make sure that the non-working guix-daemon.service is stopped and disabled -systemctl stop guix-daemon -systemctl disable guix-daemon - -# Make sure that the working guix-daemon-original.service is started and enabled -systemctl enable guix-daemon-original -systemctl start guix-daemon-original -``` - -#### Creating `guix-daemon` users / groups - -Please see the [relevant -section](https://guix.gnu.org/manual/en/html_node/Build-Environment-Setup.html) -in the Guix Reference Manual for more details. - -## Optional setup - -At this point, you are set up to [use Guix to build Bitcoin -Core](./README.md#usage). However, if you want to polish your setup a bit and -make it "what Guix intended", then read the next few subsections. - -### Add an `/etc/profile.d` entry - -This section definitely does not apply to you if you installed Guix using: -1. The shell installer script -2. fanquake's container image -3. Debian's `guix` package - -#### Background - -Although Guix knows how to update itself and its packages, it does so in a -non-invasive way (it does not modify `/usr/local/bin/guix`). - -Instead, it does the following: - -- After a `guix pull`, it updates - `/var/guix/profiles/per-user/$USER/current-guix`, and creates a symlink - targeting this directory at `$HOME/.config/guix/current` - -- After a `guix install`, it updates - `/var/guix/profiles/per-user/$USER/guix-profile`, and creates a symlink - targeting this directory at `$HOME/.guix-profile` - -Therefore, in order for these operations to affect your shell/desktop sessions -(and for the principle of least astonishment to hold), their corresponding -directories have to be added to well-known environment variables like `$PATH`, -`$INFOPATH`, `$XDG_DATA_DIRS`, etc. - -In other words, if `$HOME/.config/guix/current/bin` does not exist in your -`$PATH`, a `guix pull` will have no effect on what `guix` you are using. Same -goes for `$HOME/.guix-profile/bin`, `guix install`, and installed packages. - -Helpfully, after a `guix pull` or `guix install`, a message will be printed like -so: - -``` -hint: Consider setting the necessary environment variables by running: - - GUIX_PROFILE="$HOME/.guix-profile" - . "$GUIX_PROFILE/etc/profile" - -Alternately, see `guix package --search-paths -p "$HOME/.guix-profile"'. -``` - -However, this is somewhat tedious to do for both `guix pull` and `guix install` -for each user on the system that wants to properly use `guix`. I recommend that -you add an entry to `/etc/profile.d` instead. This is done by default -when installing the Debian package later than 1.2.0-4 and when using the shell -script installer. - -#### Instructions - -Create `/etc/profile.d/guix.sh` with the following content: -```sh -# _GUIX_PROFILE: `guix pull` profile -_GUIX_PROFILE="$HOME/.config/guix/current" -if [ -L $_GUIX_PROFILE ]; then - export PATH="$_GUIX_PROFILE/bin${PATH:+:}$PATH" - # Export INFOPATH so that the updated info pages can be found - # and read by both /usr/bin/info and/or $GUIX_PROFILE/bin/info - # When INFOPATH is unset, add a trailing colon so that Emacs - # searches 'Info-default-directory-list'. - export INFOPATH="$_GUIX_PROFILE/share/info:$INFOPATH" -fi - -# GUIX_PROFILE: User's default profile -GUIX_PROFILE="$HOME/.guix-profile" -[ -L $GUIX_PROFILE ] || return -GUIX_LOCPATH="$GUIX_PROFILE/lib/locale" -export GUIX_PROFILE GUIX_LOCPATH - -[ -f "$GUIX_PROFILE/etc/profile" ] && . "$GUIX_PROFILE/etc/profile" - -# set XDG_DATA_DIRS to include Guix installations -export XDG_DATA_DIRS="$GUIX_PROFILE/share:${XDG_DATA_DIRS:-/usr/local/share/:/usr/share/}" -``` - -Please note that this will not take effect until the next shell or desktop -session (log out and log back in). - -### `guix pull` as root - -Before you do this, you need to read the section on [choosing your security -model][security-model] and adjust `guix` and `guix-daemon` flags according to -your choice, as invoking `guix pull` may pull substitutes from substitute -servers (which you may not want). - -As mentioned in a previous section, Guix expects -`${localstatedir}/guix/profiles/per-user/root/current-guix` to be populated with -`root`'s Guix profile, `guix pull`-ed and built by some former version of Guix. -However, this is not the case when we build from source. Therefore, we need to -perform a `guix pull` as `root`: - -```sh -sudo --login guix pull --branch=version- -# or -sudo --login guix pull --commit= -``` - -`guix pull` is quite a long process (especially if you're using -`--no-substitutes`). If you encounter build problems, please refer to the -[troubleshooting section](#troubleshooting). - -Note that running a bare `guix pull` with no commit or branch specified will -pull the latest commit on Guix's master branch, which is likely fine, but not -recommended. - -If you installed Guix from source, you may get an error like the following: -```sh -error: while creating symlink '/root/.config/guix/current' No such file or directory -``` -To resolve this, simply: -``` -sudo mkdir -p /root/.config/guix -``` -Then try the `guix pull` command again. - -After the `guix pull` finishes successfully, -`${localstatedir}/guix/profiles/per-user/root/current-guix` should be populated. - -#### Using the newly-pulled `guix` by restarting the daemon - -Depending on how you installed Guix, you should now make sure that your init -scripts and service configurations point to the newly-pulled `guix-daemon`. - -##### If you built Guix from source - -If you followed the instructions for [fixing argv\[0\]][fix-argv0], you can now -do the following: - -```sh -systemctl stop guix-daemon-original -systemctl disable guix-daemon-original - -systemctl enable guix-daemon -systemctl start guix-daemon -``` - -Remember to set `--no-substitutes` in `$libdir/systemd/system/guix-daemon.service` and other customizations if you used them for `guix-daemon-original.service`. - -##### If you installed Guix via the Debian/Ubuntu distribution packages - -You will need to create a `guix-daemon-latest` service which points to the new -`guix` rather than a pinned one. - -```sh -# Create guix-daemon-latest.service by modifying guix-daemon.service -sed -E -e "s|/usr/bin/guix-daemon|/var/guix/profiles/per-user/root/current-guix/bin/guix-daemon|" /etc/systemd/system/guix-daemon.service > /lib/systemd/system/guix-daemon-latest.service -chmod 664 /lib/systemd/system/guix-daemon-latest.service - -# Make systemd recognize the new service -systemctl daemon-reload - -# Make sure that the old guix-daemon.service is stopped and disabled -systemctl stop guix-daemon -systemctl disable guix-daemon - -# Make sure that the new guix-daemon-latest.service is started and enabled -systemctl enable guix-daemon-latest -systemctl start guix-daemon-latest -``` - -##### If you installed Guix via lantw44's Arch Linux AUR package - -At the time of writing (July 5th, 2021) the systemd unit for "updated Guix" is -`guix-daemon-latest.service`, therefore, you should do the following: - -```sh -systemctl stop guix-daemon -systemctl disable guix-daemon - -systemctl enable guix-daemon-latest -systemctl start guix-daemon-latest -``` - -##### Otherwise... - -Simply do: - -```sh -systemctl restart guix-daemon -``` - -### Checking everything - -If you followed all the steps above to make your Guix setup "prim and proper," -you can check that you did everything properly by running through this -checklist. - -1. `/etc/profile.d/guix.sh` should exist and be sourced at each shell login - -2. `guix describe` should not print `guix describe: error: failed to determine - origin`, but rather something like: - - ``` - Generation 38 Feb 22 2021 16:39:31 (current) - guix f350df4 - repository URL: https://codeberg.org/guix/guix.git - branch: version-1.2.0 - commit: f350df405fbcd5b9e27e6b6aa500da7f101f41e7 - ``` - -3. `guix-daemon` should be running from `${localstatedir}/guix/profiles/per-user/root/current-guix` - -# Troubleshooting - -## Derivation failed to build - -When you see a build failure like below: - -``` -building /gnu/store/...-foo-3.6.12.drv... -/ 'check' phasenote: keeping build directory `/tmp/guix-build-foo-3.6.12.drv-0' -builder for `/gnu/store/...-foo-3.6.12.drv' failed with exit code 1 -build of /gnu/store/...-foo-3.6.12.drv failed -View build log at '/var/log/guix/drvs/../...-foo-3.6.12.drv.bz2'. -cannot build derivation `/gnu/store/...-qux-7.69.1.drv': 1 dependencies couldn't be built -cannot build derivation `/gnu/store/...-bar-3.16.5.drv': 1 dependencies couldn't be built -cannot build derivation `/gnu/store/...-baz-2.0.5.drv': 1 dependencies couldn't be built -guix time-machine: error: build of `/gnu/store/...-baz-2.0.5.drv' failed -``` - -It means that `guix` failed to build a package named `foo`, which was a -dependency of `qux`, `bar`, and `baz`. Importantly, note that the last "failed" -line is not necessarily the root cause, the first "failed" line is. - -Most of the time, the build failure is due to a spurious test failure or the -package's build system/test suite breaking when running multi-threaded. To -rebuild _just_ this derivation in a single-threaded fashion (please don't forget -to add other `guix` flags like `--no-substitutes` as appropriate): - -```sh -$ guix build --cores=1 /gnu/store/...-foo-3.6.12.drv -``` - -If the single-threaded rebuild did not succeed, you may need to dig deeper. -You may view `foo`'s build logs in `less` like so (please replace paths with the -path you see in the build failure output): - -```sh -$ bzcat /var/log/guix/drvs/../...-foo-3.6.12.drv.bz2 | less -``` - -`foo`'s build directory is also preserved and available at -`/tmp/guix-build-foo-3.6.12.drv-0`. However, if you fail to build `foo` multiple -times, it may be `/tmp/...drv-1` or `/tmp/...drv-2`. Always consult the build -failure output for the most accurate, up-to-date information. - -### python(-minimal): [Errno 84] Invalid or incomplete multibyte or wide character - -This error occurs when your `$TMPDIR` (default: /tmp) exists on a filesystem -which rejects characters not present in the UTF-8 character code set. An example -is ZFS with the utf8only=on option set. - -More information: https://github.com/python/cpython/issues/81765 - -### openssl-1.1.1l and openssl-1.1.1n - -OpenSSL includes tests that will fail once some certificate has expired. -The workarounds from the GnuTLS section immediately below can be used. - -For openssl-1.1.1l use 2022-05-01 as the date. - -### GnuTLS: test-suite FAIL: status-request-revoked - -*The derivation is likely identified by: `/gnu/store/vhphki5sg9xkdhh2pbc8gi6vhpfzryf0-gnutls-3.6.12.drv`* - -This unfortunate error is most common for non-substitute builders who installed -Guix v1.2.0. The problem stems from the fact that one of GnuTLS's tests uses a -hardcoded certificate which expired on 2020-10-24. - -What's more unfortunate is that this GnuTLS derivation is somewhat special in -Guix's dependency graph and is not affected by the package transformation flags -like `--without-tests=`. - -The easiest solution for those encountering this problem is to install a newer -version of Guix. However, there are ways to work around this issue: - -#### Workaround 1: Using substitutes for this single derivation - -If you've authorized the official Guix build farm's key (more info -[here](./README.md#step-1-authorize-the-signing-keys)), then you can use -substitutes just for this single derivation by invoking the following: - -```sh -guix build --substitute-urls="https://ci.guix.gnu.org" /gnu/store/vhphki5sg9xkdhh2pbc8gi6vhpfzryf0-gnutls-3.6.12.drv -``` - -See [this section](./README.md#removing-authorized-keys) for instructions on how -to remove authorized keys if you don't want to keep the build farm's key -authorized. - -#### Workaround 2: Temporarily setting the system clock back - -This workaround was described [here](https://issues.guix.gnu.org/44559#5). - -Basically: - -1. Turn off NTP -2. Set system time to 2020-10-01 -3. guix build --no-substitutes /gnu/store/vhphki5sg9xkdhh2pbc8gi6vhpfzryf0-gnutls-3.6.12.drv -4. Set system time back to accurate current time -5. Turn NTP back on - -For example, - -```sh -sudo timedatectl set-ntp no -sudo date --set "01 oct 2020 15:00:00" -guix build /gnu/store/vhphki5sg9xkdhh2pbc8gi6vhpfzryf0-gnutls-3.6.12.drv -sudo timedatectl set-ntp yes -``` - -#### Workaround 3: Disable the tests in the Guix source code for this single derivation - -If all of the above workarounds fail, you can also disable the `tests` phase of -the derivation via the `arguments` option, as described in the official -[`package` -reference](https://guix.gnu.org/manual/en/html_node/package-Reference.html). - -For example, to disable the openssl-1.1 check phase: - -```diff -diff --git a/gnu/packages/tls.scm b/gnu/packages/tls.scm -index f1e844b..1077c4b 100644 ---- a/gnu/packages/tls.scm -+++ b/gnu/packages/tls.scm -@@ -494,4 +494,5 @@ (define-public openssl-1.1 - (arguments - `(#:parallel-tests? #f -+ #:tests? #f - #:test-target "test" -``` - -### coreutils: FAIL: tests/tail-2/inotify-dir-recreate - -The inotify-dir-create test fails on "remote" filesystems such as overlayfs -(Docker's default filesystem) due to the filesystem being mistakenly recognized -as non-remote. - -A relatively easy workaround to this is to make sure that a somewhat traditional -filesystem is mounted at `/tmp` (where `guix-daemon` performs its builds). For -Docker users, this might mean [using a volume][docker/volumes], [binding -mounting][docker/bind-mnt] from host, or (for those with enough RAM and swap) -[mounting a tmpfs][docker/tmpfs] using the `--tmpfs` flag. - -Please see the following links for more details: - -- An upstream coreutils bug has been filed: [debbugs#47940](https://debbugs.gnu.org/cgi/bugreport.cgi?bug=47940) -- A Guix bug detailing the underlying problem has been filed: [guix-issues#47935](https://issues.guix.gnu.org/47935), [guix-issues#49985](https://issues.guix.gnu.org/49985#5) -- A commit to skip this test is included since Guix 1.4.0: -[codeberg/guix@6ba1058](https://codeberg.org/guix/guix/commit/6ba1058df0c4ce5611c2367531ae5c3cdc729ab4) - - -[install-script]: #options-1-and-2-using-the-official-shell-installer-script-or-binary-tarball -[install-bin-tarball]: #options-1-and-2-using-the-official-shell-installer-script-or-binary-tarball -[install-fanquake-container]: #option-3-using-fanquakes-container-image -[install-distro-pkg]: #option-4-using-a-distribution-maintained-package -[install-source]: #option-5-building-from-source - -[fix-argv0]: #creating-and-starting-a-guix-daemon-original-service-with-a-fixed-argv0 -[security-model]: ./README.md#choosing-your-security-model - -[docker/volumes]: https://docs.docker.com/storage/volumes/ -[docker/bind-mnt]: https://docs.docker.com/storage/bind-mounts/ -[docker/tmpfs]: https://docs.docker.com/storage/tmpfs/ - -# Purging/Uninstalling Guix - -In the extraordinarily rare case where you messed up your Guix installation in -an irreversible way, you may want to completely purge Guix from your system and -start over. - -1. Uninstall Guix itself according to the way you installed it (e.g. `sudo apt - purge guix` for Ubuntu packaging, `sudo make uninstall` for a build from source). -2. Remove all build users and groups - - You may check for relevant users and groups using: - - ``` - getent passwd | grep guix - getent group | grep guix - ``` - - Then, you may remove users and groups using: - - ``` - sudo userdel - sudo groupdel - ``` - -3. Remove all possible Guix-related directories - - `/var/guix/` - - `/var/log/guix/` - - `/gnu/` - - `/etc/guix/` - - `/home/*/.config/guix/` - - `/home/*/.cache/guix/` - - `/home/*/.guix-profile/` - - `/root/.config/guix/` - - `/root/.cache/guix/` - - `/root/.guix-profile/` diff --git a/bench-ci/guix/README.md b/bench-ci/guix/README.md deleted file mode 100644 index 7f6b8232bba5..000000000000 --- a/bench-ci/guix/README.md +++ /dev/null @@ -1,430 +0,0 @@ -# Bootstrappable Bitcoin Core Builds - -This directory contains the files necessary to perform bootstrappable Bitcoin -Core builds. - -[Bootstrappability][b17e] furthers our binary security guarantees by allowing us -to _audit and reproduce_ our toolchain instead of blindly _trusting_ binary -downloads. - -We achieve bootstrappability by using Guix as a functional package manager. - -# Requirements - -Conservatively, you will need: - -- 16GB of free disk space on the partition that /gnu/store will reside in -- 8GB of free disk space **per platform triple** you're planning on building - (see the `HOSTS` [environment variable description][env-vars-list]) - -# Installation and Setup - -If you don't have Guix installed and set up, please follow the instructions in -[INSTALL.md](./INSTALL.md) - -# Usage - -If you haven't considered your security model yet, please read [the relevant -section](#choosing-your-security-model) before proceeding to perform a build. - -## Making the Xcode SDK available for macOS cross-compilation - -In order to perform a build for macOS (which is included in the default set of -platform triples to build), you'll need to extract the macOS SDK tarball using -tools found in the [`macdeploy` directory](../macdeploy/README.md#sdk-extraction). - -You can then either point to the SDK using the `SDK_PATH` environment variable: - -```sh -# Extract the SDK tarball to /path/to/parent/dir/of/extracted/SDK/Xcode---extracted-SDK-with-libcxx-headers -tar -C /path/to/parent/dir/of/extracted/SDK -xaf /path/to/Xcode---extracted-SDK-with-libcxx-headers.tar.gz - -# Indicate where to locate the SDK tarball -export SDK_PATH=/path/to/parent/dir/of/extracted/SDK -``` - -or extract it into `depends/SDKs`: - -```sh -mkdir -p depends/SDKs -tar -C depends/SDKs -xaf /path/to/SDK/tarball -``` - -## Building - -*The author highly recommends at least reading over the [common usage patterns -and examples](#common-guix-build-invocation-patterns-and-examples) section below -before starting a build. For a full list of customization options, see the -[recognized environment variables][env-vars-list] section.* - -To build Bitcoin Core reproducibly with all default options, invoke the -following from the top of a clean repository: - -```sh -./contrib/guix/guix-build -``` - -## Codesigning build outputs - -The `guix-codesign` command attaches codesignatures (produced by codesigners) to -existing non-codesigned outputs. Please see the [release process -documentation](/doc/release-process.md#codesigning) for more context. - -It respects many of the same environment variable flags as `guix-build`, with 2 -crucial differences: - -1. Since only Windows and macOS build outputs require codesigning, the `HOSTS` - environment variable will have a sane default value of `x86_64-w64-mingw32 - x86_64-apple-darwin arm64-apple-darwin` instead of all the platforms. -2. The `guix-codesign` command ***requires*** a `DETACHED_SIGS_REPO` flag. - * _**DETACHED_SIGS_REPO**_ - - Set the directory where detached codesignatures can be found for the current - Bitcoin Core version being built. - - _REQUIRED environment variable_ - -An invocation with all default options would look like: - -``` -env DETACHED_SIGS_REPO= ./contrib/guix/guix-codesign -``` - -## Cleaning intermediate work directories - -By default, `guix-build` leaves all intermediate files or "work directories" -(e.g. `depends/work`, `guix-build-*/distsrc-*`) intact at the end of a build so -that they are available to the user (to aid in debugging, etc.). However, these -directories usually take up a large amount of disk space. Therefore, a -`guix-clean` convenience script is provided which cleans the current `git` -worktree to save disk space: - -``` -./contrib/guix/guix-clean -``` - - -## Attesting to build outputs - -Much like how Gitian build outputs are attested to in a `gitian.sigs` -repository, Guix build outputs are attested to in the [`guix.sigs` -repository](https://github.com/bitcoin-core/guix.sigs). - -After you've cloned the `guix.sigs` repository, to attest to the current -worktree's commit/tag: - -``` -env GUIX_SIGS_REPO= SIGNER= ./contrib/guix/guix-attest -``` - -See `./contrib/guix/guix-attest --help` for more information on the various ways -`guix-attest` can be invoked. - -## Verifying build output attestations - -After at least one other signer has uploaded their signatures to the `guix.sigs` -repository: - -``` -git -C pull -env GUIX_SIGS_REPO= ./contrib/guix/guix-verify -``` - - -## Common `guix-build` invocation patterns and examples - -### Keeping caches and SDKs outside of the worktree - -If you perform a lot of builds and have a bunch of worktrees, you may find it -more efficient to keep the depends tree's download cache, build cache, and SDKs -outside of the worktrees to avoid duplicate downloads and unnecessary builds. To -help with this situation, the `guix-build` script honours the `SOURCES_PATH`, -`BASE_CACHE`, and `SDK_PATH` environment variables and will pass them on to the -depends tree so that you can do something like: - -```sh -env SOURCES_PATH="$HOME/depends-SOURCES_PATH" BASE_CACHE="$HOME/depends-BASE_CACHE" SDK_PATH="$HOME/macOS-SDKs" ./contrib/guix/guix-build -``` - -Note that the paths that these environment variables point to **must be -directories**, and **NOT symlinks to directories**. - -See the [recognized environment variables][env-vars-list] section for more -details. - -### Building a subset of platform triples - -Sometimes you only want to build a subset of the supported platform triples, in -which case you can override the default list by setting the space-separated -`HOSTS` environment variable: - -```sh -env HOSTS='x86_64-w64-mingw32 x86_64-apple-darwin' ./contrib/guix/guix-build -``` - -See the [recognized environment variables][env-vars-list] section for more -details. - -### Controlling the number of threads used by `guix` build commands - -Depending on your system's RAM capacity, you may want to decrease the number of -threads used to decrease RAM usage or vice versa. - -By default, the scripts under `./contrib/guix` will invoke all `guix` build -commands with `--cores="$JOBS"`. Note that `$JOBS` defaults to `$(nproc)` if not -specified. However, astute manual readers will also notice that `guix` build -commands also accept a `--max-jobs=` flag (which defaults to 1 if unspecified). - -Here is the difference between `--cores=` and `--max-jobs=`: - -> Note: When I say "derivation," think "package" - -`--cores=` - - - controls the number of CPU cores to build each derivation. This is the value - passed to `make`'s `--jobs=` flag. - -`--max-jobs=` - - - controls how many derivations can be built in parallel - - defaults to 1 - -Therefore, the default is for `guix` build commands to build one derivation at a -time, utilizing `$JOBS` threads. - -Specifying the `$JOBS` environment variable will only modify `--cores=`, but you -can also modify the value for `--max-jobs=` by specifying -`$ADDITIONAL_GUIX_COMMON_FLAGS`. For example, if you have a LOT of memory, you -may want to set: - -```sh -export ADDITIONAL_GUIX_COMMON_FLAGS='--max-jobs=8' -``` - -Which allows for a maximum of 8 derivations to be built at the same time, each -utilizing `$JOBS` threads. - -Or, if you'd like to avoid spurious build failures caused by issues with -parallelism within a single package, but would still like to build multiple -packages when the dependency graph allows for it, you may want to try: - -```sh -export JOBS=1 ADDITIONAL_GUIX_COMMON_FLAGS='--max-jobs=8' -``` - -See the [recognized environment variables][env-vars-list] section for more -details. - -## Recognized environment variables - -* _**HOSTS**_ - - Override the space-separated list of platform triples for which to perform a - bootstrappable build. - - _(defaults to "x86\_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu - riscv64-linux-gnu powerpc64-linux-gnu powerpc64le-linux-gnu - x86\_64-w64-mingw32 x86\_64-apple-darwin arm64-apple-darwin")_ - -* _**SOURCES_PATH**_ - - Set the depends tree download cache for sources. This is passed through to the - depends tree. Setting this to the same directory across multiple builds of the - depends tree can eliminate unnecessary redownloading of package sources. - - The path that this environment variable points to **must be a directory**, and - **NOT a symlink to a directory**. - -* _**BASE_CACHE**_ - - Set the depends tree cache for built packages. This is passed through to the - depends tree. Setting this to the same directory across multiple builds of the - depends tree can eliminate unnecessary building of packages. - - The path that this environment variable points to **must be a directory**, and - **NOT a symlink to a directory**. - -* _**SDK_PATH**_ - - Set the path where _extracted_ SDKs can be found. This is passed through to - the depends tree. Note that this should be set to the _parent_ directory of - the actual SDK (e.g. `SDK_PATH=$HOME/Downloads/macOS-SDKs` instead of - `$HOME/Downloads/macOS-SDKs/Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers`). - - The path that this environment variable points to **must be a directory**, and - **NOT a symlink to a directory**. - -* _**JOBS**_ - - Override the number of jobs to run simultaneously, you might want to do so on - a memory-limited machine. This may be passed to: - - - `guix` build commands as in `guix shell --cores="$JOBS"` - - `make` as in `make --jobs="$JOBS"` - - `cmake` as in `cmake --build build -j "$JOBS"` - - `xargs` as in `xargs -P"$JOBS"` - - See [here](#controlling-the-number-of-threads-used-by-guix-build-commands) for - more details. - - _(defaults to the value of `nproc` outside the container)_ - -* _**SOURCE_DATE_EPOCH**_ - - Override the reference UNIX timestamp used for bit-for-bit reproducibility, - the variable name conforms to [standard][r12e/source-date-epoch]. - - _(defaults to the output of `$(git log --format=%at -1)`)_ - -* _**V**_ - - If non-empty, will pass `V=1` to all `make` invocations, making `make` output - verbose. - - Note that any given value is ignored. The variable is only checked for - emptiness. More concretely, this means that `V=` (setting `V` to the empty - string) is interpreted the same way as not setting `V` at all, and that `V=0` - has the same effect as `V=1`. - -* _**SUBSTITUTE_URLS**_ - - A whitespace-delimited list of URLs from which to download pre-built packages. - A URL is only used if its signing key is authorized (refer to the [substitute - servers section](#option-1-building-with-substitutes) for more details). - -* _**ADDITIONAL_GUIX_COMMON_FLAGS**_ - - Additional flags to be passed to all `guix` commands. - -* _**ADDITIONAL_GUIX_TIMEMACHINE_FLAGS**_ - - Additional flags to be passed to `guix time-machine`. - -* _**ADDITIONAL_GUIX_ENVIRONMENT_FLAGS**_ - - Additional flags to be passed to the invocation of `guix shell` inside - `guix time-machine`. - -# Choosing your security model - -No matter how you installed Guix, you need to decide on your security model for -building packages with Guix. - -Guix allows us to achieve better binary security by using our CPU time to build -everything from scratch. However, it doesn't sacrifice user choice in pursuit of -this: users can decide whether or not to use **substitutes** (pre-built -packages). - -## Option 1: Building with substitutes - -### Step 1: Authorize the signing keys - -Depending on the installation procedure you followed, you may have already -authorized the Guix build farm key. In particular, the official shell installer -script asks you if you want the key installed, and the debian distribution -package authorized the key during installation. - -You can check the current list of authorized keys at `/etc/guix/acl`. - -At the time of writing, a `/etc/guix/acl` with just the Guix build farm key -authorized looks something like: - -```lisp -(acl - (entry - (public-key - (ecc - (curve Ed25519) - (q #8D156F295D24B0D9A86FA5741A840FF2D24F60F7B6C4134814AD55625971B394#) - ) - ) - (tag - (guix import) - ) - ) - ) -``` - -If you've determined that the official Guix build farm key hasn't been -authorized, and you would like to authorize it, run the following as root: - -``` -guix archive --authorize < /var/guix/profiles/per-user/root/current-guix/share/guix/ci.guix.gnu.org.pub -``` - -If -`/var/guix/profiles/per-user/root/current-guix/share/guix/ci.guix.gnu.org.pub` -doesn't exist, try: - -```sh -guix archive --authorize < /share/guix/ci.guix.gnu.org.pub -``` - -Where `` is likely: -- `/usr` if you installed from a distribution package -- `/usr/local` if you installed Guix from source and didn't supply any - prefix-modifying flags to Guix's `./configure` - -#### Removing authorized keys - -To remove previously authorized keys, simply edit `/etc/guix/acl` and remove the -`(entry (public-key ...))` entry. - -### Step 2: Specify the substitute servers - -Once its key is authorized, the official Guix build farm at -https://ci.guix.gnu.org is automatically used unless the `--no-substitutes` flag -is supplied. This default list of substitute servers is overridable both on a -`guix-daemon` level and when you invoke `guix` commands. See examples below for -the various ways of adding a substitute server after having [authorized -its signing key](#step-1-authorize-the-signing-keys). - -Change the **default list** of substitute servers by starting `guix-daemon` with -the `--substitute-urls` option (you will likely need to edit your init script): - -```sh -guix-daemon --substitute-urls='https://bordeaux.guix.gnu.org https://ci.guix.gnu.org' -``` - -Override the default list of substitute servers by passing the -`--substitute-urls` option for invocations of `guix` commands: - -```sh -guix --substitute-urls='https://bordeaux.guix.gnu.org https://ci.guix.gnu.org' -``` - -For scripts under `./contrib/guix`, set the `SUBSTITUTE_URLS` environment -variable: - -```sh -export SUBSTITUTE_URLS='https://bordeaux.guix.gnu.org https://ci.guix.gnu.org' -``` - -## Option 2: Disabling substitutes on an ad-hoc basis - -If you prefer not to use any substitutes, make sure to supply `--no-substitutes` -like in the following snippet. The first build will take a while, but the -resulting packages will be cached for future builds. - -For direct invocations of `guix`: -```sh -guix --no-substitutes -``` - -For the scripts under `./contrib/guix/`: -```sh -export ADDITIONAL_GUIX_COMMON_FLAGS='--no-substitutes' -``` - -## Option 3: Disabling substitutes by default - -`guix-daemon` accepts a `--no-substitutes` flag, which will make sure that, -unless otherwise overridden by a command line invocation, no substitutes will be -used. - -If you start `guix-daemon` using an init script, you can edit said script to -supply this flag. - -[b17e]: https://bootstrappable.org/ -[r12e/source-date-epoch]: https://reproducible-builds.org/docs/source-date-epoch/ -[env-vars-list]: #recognized-environment-variables diff --git a/bench-ci/guix/guix-attest b/bench-ci/guix/guix-attest deleted file mode 100755 index b0ef28dc3f92..000000000000 --- a/bench-ci/guix/guix-attest +++ /dev/null @@ -1,263 +0,0 @@ -#!/usr/bin/env bash -export LC_ALL=C -set -e -o pipefail - -# Source the common prelude, which: -# 1. Checks if we're at the top directory of the Bitcoin Core repository -# 2. Defines a few common functions and variables -# -# shellcheck source=libexec/prelude.bash -source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash" - - -################### -## Sanity Checks ## -################### - -################ -# Required non-builtin commands should be invokable -################ - -check_tools cat env basename mkdir diff sort - -if [ -z "$NO_SIGN" ]; then - # make it possible to override the gpg binary - GPG=${GPG:-gpg} - - # $GPG can contain extra arguments passed to the binary - # so let's check only the existence of arg[0] - # shellcheck disable=SC2206 - GPG_ARRAY=($GPG) - check_tools "${GPG_ARRAY[0]}" -fi - -################ -# Required env vars should be non-empty -################ - -cmd_usage() { -cat < \\ - SIGNER=GPG_KEY_NAME[=SIGNER_NAME] \\ - [ NO_SIGN=1 ] - ./contrib/guix/guix-attest - -Example w/o overriding signing name: - - env GUIX_SIGS_REPO=/home/achow101/guix.sigs \\ - SIGNER=achow101 \\ - ./contrib/guix/guix-attest - -Example overriding signing name: - - env GUIX_SIGS_REPO=/home/dongcarl/guix.sigs \\ - SIGNER=0x96AB007F1A7ED999=dongcarl \\ - ./contrib/guix/guix-attest - -Example w/o signing, just creating SHA256SUMS: - - env GUIX_SIGS_REPO=/home/achow101/guix.sigs \\ - SIGNER=achow101 \\ - NO_SIGN=1 \\ - ./contrib/guix/guix-attest - -EOF -} - -if [ -z "$GUIX_SIGS_REPO" ] || [ -z "$SIGNER" ]; then - cmd_usage - exit 1 -fi - -################ -# GUIX_SIGS_REPO should exist as a directory -################ - -if [ ! -d "$GUIX_SIGS_REPO" ]; then -cat << EOF -ERR: The specified GUIX_SIGS_REPO is not an existent directory: - - '$GUIX_SIGS_REPO' - -Hint: Please clone the guix.sigs repository and point to it with the - GUIX_SIGS_REPO environment variable. - -EOF -cmd_usage -exit 1 -fi - -################ -# The key specified in SIGNER should be usable -################ - -IFS='=' read -r gpg_key_name signer_name <<< "$SIGNER" -if [ -z "${signer_name}" ]; then - signer_name="$gpg_key_name" -fi - -if [ -z "$NO_SIGN" ] && ! ${GPG} --dry-run --list-secret-keys "${gpg_key_name}" >/dev/null 2>&1; then - echo "ERR: GPG can't seem to find any key named '${gpg_key_name}'" - exit 1 -fi - -################ -# We should be able to find at least one output -################ - -echo "Looking for build output SHA256SUMS fragments in ${OUTDIR_BASE}" - -shopt -s nullglob -sha256sum_fragments=( "$OUTDIR_BASE"/*/SHA256SUMS.part ) # This expands to an array of directories... -shopt -u nullglob - -noncodesigned_fragments=() -codesigned_fragments=() - -if (( ${#sha256sum_fragments[@]} )); then - echo "Found build output SHA256SUMS fragments:" - for outdir in "${sha256sum_fragments[@]}"; do - echo " '$outdir'" - case "$outdir" in - "$OUTDIR_BASE"/*-codesigned/SHA256SUMS.part) - codesigned_fragments+=("$outdir") - ;; - *) - noncodesigned_fragments+=("$outdir") - ;; - esac - done - echo -else - echo "ERR: Could not find any build output SHA256SUMS fragments in ${OUTDIR_BASE}" - exit 1 -fi - -############## -## Attest ## -############## - -# Usage: out_name $outdir -# -# HOST: The output directory being attested -# -out_name() { - basename "$(dirname "$1")" -} - -shasum_already_exists() { -cat < "$temp_noncodesigned" - if [ -e noncodesigned.SHA256SUMS ]; then - # The SHA256SUMS already exists, make sure it's exactly what we - # expect, error out if not - if diff -u noncodesigned.SHA256SUMS "$temp_noncodesigned"; then - echo "A noncodesigned.SHA256SUMS file already exists for '${VERSION}' and is up-to-date." - else - shasum_already_exists noncodesigned.SHA256SUMS - exit 1 - fi - else - mv "$temp_noncodesigned" noncodesigned.SHA256SUMS - fi - else - echo "ERR: No noncodesigned outputs found for '${VERSION}', exiting..." - exit 1 - fi - - temp_all="$(mktemp)" - trap 'rm -rf -- "$temp_all"' EXIT - - if (( ${#codesigned_fragments[@]} )); then - # Note: all.SHA256SUMS attests to all of $sha256sum_fragments, but is - # not needed if there are no $codesigned_fragments - cat "${sha256sum_fragments[@]}" \ - | sort -u \ - | sort -k2 \ - | basenameify_SHA256SUMS \ - > "$temp_all" - if [ -e all.SHA256SUMS ]; then - # The SHA256SUMS already exists, make sure it's exactly what we - # expect, error out if not - if diff -u all.SHA256SUMS "$temp_all"; then - echo "An all.SHA256SUMS file already exists for '${VERSION}' and is up-to-date." - else - shasum_already_exists all.SHA256SUMS - exit 1 - fi - else - mv "$temp_all" all.SHA256SUMS - fi - else - # It is fine to have the codesigned outputs be missing (perhaps the - # detached codesigs have not been published yet), just print a log - # message instead of erroring out - echo "INFO: No codesigned outputs found for '${VERSION}', skipping..." - fi - - if [ -z "$NO_SIGN" ]; then - echo "Signing SHA256SUMS to produce SHA256SUMS.asc" - for i in *.SHA256SUMS; do - if [ ! -e "$i".asc ]; then - ${GPG} --detach-sign \ - --digest-algo sha256 \ - --local-user "$gpg_key_name" \ - --armor \ - --output "$i".asc "$i" - else - echo "Signature already there" - fi - done - else - echo "Not signing SHA256SUMS as \$NO_SIGN is not empty" - fi - echo "" -) diff --git a/bench-ci/guix/guix-build b/bench-ci/guix/guix-build deleted file mode 100755 index 84d4f201b259..000000000000 --- a/bench-ci/guix/guix-build +++ /dev/null @@ -1,474 +0,0 @@ -#!/usr/bin/env bash -export LC_ALL=C -set -e -o pipefail - -# Source the common prelude, which: -# 1. Checks if we're at the top directory of the Bitcoin Core repository -# 2. Defines a few common functions and variables -# -# shellcheck source=libexec/prelude.bash -source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash" - - -################### -## SANITY CHECKS ## -################### - -################ -# Required non-builtin commands should be invocable -################ - -check_tools cat mkdir make getent curl git guix - -################ -# GUIX_BUILD_OPTIONS should be empty -################ -# -# GUIX_BUILD_OPTIONS is an environment variable recognized by guix commands that -# can perform builds. This seems like what we want instead of -# ADDITIONAL_GUIX_COMMON_FLAGS, but the value of GUIX_BUILD_OPTIONS is actually -# _appended_ to normal command-line options. Meaning that they will take -# precedence over the command-specific ADDITIONAL_GUIX__FLAGS. -# -# This seems like a poor user experience. Thus we check for GUIX_BUILD_OPTIONS's -# existence here and direct users of this script to use our (more flexible) -# custom environment variables. -if [ -n "$GUIX_BUILD_OPTIONS" ]; then -cat << EOF -Error: Environment variable GUIX_BUILD_OPTIONS is not empty: - '$GUIX_BUILD_OPTIONS' - -Unfortunately this script is incompatible with GUIX_BUILD_OPTIONS, please unset -GUIX_BUILD_OPTIONS and use ADDITIONAL_GUIX_COMMON_FLAGS to set build options -across guix commands or ADDITIONAL_GUIX__FLAGS to set build options for a -specific guix command. - -See contrib/guix/README.md for more details. -EOF -exit 1 -fi - -################ -# The git worktree should not be dirty -################ - -if ! git diff-index --quiet HEAD -- && [ -z "$FORCE_DIRTY_WORKTREE" ]; then -cat << EOF -ERR: The current git worktree is dirty, which may lead to broken builds. - - Aborting... - -Hint: To make your git worktree clean, You may want to: - 1. Commit your changes, - 2. Stash your changes, or - 3. Set the 'FORCE_DIRTY_WORKTREE' environment variable if you insist on - using a dirty worktree -EOF -exit 1 -fi - -mkdir -p "$VERSION_BASE" - -################ -# SOURCE_DATE_EPOCH should not unintentionally be set -################ - -check_source_date_epoch - -################ -# Build directories should not exist -################ - -# Default to building for all supported HOSTs (overridable by environment) -# powerpc64le-linux-gnu currently disabled due non-determinism issues across build arches. -export HOSTS="${HOSTS:-x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu powerpc64-linux-gnu - x86_64-w64-mingw32 - x86_64-apple-darwin arm64-apple-darwin}" - -# Usage: distsrc_for_host HOST -# -# HOST: The current platform triple we're building for -# -distsrc_for_host() { - echo "${DISTSRC_BASE}/distsrc-${VERSION}-${1}" -} - -# Accumulate a list of build directories that already exist... -hosts_distsrc_exists="" -for host in $HOSTS; do - if [ -e "$(distsrc_for_host "$host")" ]; then - hosts_distsrc_exists+=" ${host}" - fi -done - -if [ -n "$hosts_distsrc_exists" ]; then -# ...so that we can print them out nicely in an error message -cat << EOF -ERR: Build directories for this commit already exist for the following platform - triples you're attempting to build, probably because of previous builds. - Please remove, or otherwise deal with them prior to starting another build. - - Aborting... - -Hint: To blow everything away, you may want to use: - - $ ./contrib/guix/guix-clean - -Specifically, this will remove all files without an entry in the index, -excluding the SDK directory, the depends download cache, the depends built -packages cache, the garbage collector roots for Guix environments, and the -output directory. -EOF -for host in $hosts_distsrc_exists; do - echo " ${host} '$(distsrc_for_host "$host")'" -done -exit 1 -else - mkdir -p "$DISTSRC_BASE" -fi - -################ -# When building for darwin, the macOS SDK should exist -################ - -for host in $HOSTS; do - case "$host" in - *darwin*) - OSX_SDK="$(make -C "${PWD}/depends" --no-print-directory HOST="$host" print-OSX_SDK | sed 's@^[^=]\+=@@g')" - if [ -e "$OSX_SDK" ]; then - echo "Found macOS SDK at '${OSX_SDK}', using..." - break - else - echo "macOS SDK does not exist at '${OSX_SDK}', please place the extracted, untarred SDK there to perform darwin builds, or define SDK_PATH environment variable. Exiting..." - exit 1 - fi - ;; - esac -done - -################ -# VERSION_BASE should have enough space -################ - -avail_KiB="$(df -Pk "$VERSION_BASE" | sed 1d | tr -s ' ' | cut -d' ' -f4)" -total_required_KiB=0 -for host in $HOSTS; do - case "$host" in - *darwin*) required_KiB=440000 ;; - *mingw*) required_KiB=7600000 ;; - *) required_KiB=6400000 ;; - esac - total_required_KiB=$((total_required_KiB+required_KiB)) -done - -if (( total_required_KiB > avail_KiB )); then - total_required_GiB=$((total_required_KiB / 1048576)) - avail_GiB=$((avail_KiB / 1048576)) - echo "Performing a Bitcoin Core Guix build for the selected HOSTS requires ${total_required_GiB} GiB, however, only ${avail_GiB} GiB is available. Please free up some disk space before performing the build." - exit 1 -fi - -################ -# Check that we can connect to the guix-daemon -################ - -cat << EOF -Checking that we can connect to the guix-daemon... - -Hint: If this hangs, you may want to try turning your guix-daemon off and on - again. - -EOF -if ! guix gc --list-failures > /dev/null; then -cat << EOF - -ERR: Failed to connect to the guix-daemon, please ensure that one is running and - reachable. -EOF -exit 1 -fi - -# Developer note: we could use `guix repl` for this check and run: -# -# (import (guix store)) (close-connection (open-connection)) -# -# However, the internal API is likely to change more than the CLI invocation - -################ -# Services database must have basic entries -################ - -if ! getent services http https ftp > /dev/null 2>&1; then -cat << EOF -ERR: Your system's C library cannot find service database entries for at least - one of the following services: http, https, ftp. - -Hint: Most likely, /etc/services does not exist yet (common for docker images - and minimal distros), or you don't have permissions to access it. - - If /etc/services does not exist yet, you may want to install the - appropriate package for your distro which provides it. - - On Debian/Ubuntu: netbase - On Arch Linux: iana-etc - - For more information, see: getent(1), services(5) - -EOF - -fi - -######### -# SETUP # -######### - -# Determine the maximum number of jobs to run simultaneously (overridable by -# environment) -JOBS="${JOBS:-$(nproc)}" - -# Usage: host_to_commonname HOST -# -# HOST: The current platform triple we're building for -# -host_to_commonname() { - case "$1" in - *darwin*) echo osx ;; - *mingw*) echo win ;; - *linux*) echo linux ;; - *) exit 1 ;; - esac -} - -# Determine the reference time used for determinism (overridable by environment) -SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git -c log.showSignature=false log --format=%at -1)}" - -# Precious directories are those which should not be cleaned between successive -# guix builds -depends_precious_dir_names='SOURCES_PATH BASE_CACHE SDK_PATH' -precious_dir_names="${depends_precious_dir_names} OUTDIR_BASE PROFILES_BASE" - -# Usage: contains IFS-SEPARATED-LIST ITEM -contains() { - for i in ${1}; do - if [ "$i" = "${2}" ]; then - return 0 # Found! - fi - done - return 1 -} - -# If the user explicitly specified a precious directory, create it so we -# can map it into the container -for precious_dir_name in $precious_dir_names; do - precious_dir_path="${!precious_dir_name}" - if [ -n "$precious_dir_path" ]; then - if [ ! -e "$precious_dir_path" ]; then - mkdir -p "$precious_dir_path" - elif [ -L "$precious_dir_path" ]; then - echo "ERR: ${precious_dir_name} cannot be a symbolic link" - exit 1 - elif [ ! -d "$precious_dir_path" ]; then - echo "ERR: ${precious_dir_name} must be a directory" - exit 1 - fi - fi -done - -mkdir -p "$VAR_BASE" - -# Record the _effective_ values of precious directories such that guix-clean can -# avoid clobbering them if appropriate. -# -# shellcheck disable=SC2046,SC2086 -{ - # Get depends precious dir definitions from depends - make -C "${PWD}/depends" \ - --no-print-directory \ - -- $(printf "print-%s\n" $depends_precious_dir_names) - - # Get remaining precious dir definitions from the environment - for precious_dir_name in $precious_dir_names; do - precious_dir_path="${!precious_dir_name}" - if ! contains "$depends_precious_dir_names" "$precious_dir_name"; then - echo "${precious_dir_name}=${precious_dir_path}" - fi - done -} > "${VAR_BASE}/precious_dirs" - -# Make sure an output directory exists for our builds -OUTDIR_BASE="${OUTDIR_BASE:-${VERSION_BASE}/output}" -mkdir -p "$OUTDIR_BASE" - -# Download the depends sources now as we won't have internet access in the build -# container -for host in $HOSTS; do - make -C "${PWD}/depends" -j"$JOBS" download-"$(host_to_commonname "$host")" ${V:+V=1} ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} -done - -# Usage: outdir_for_host HOST SUFFIX -# -# HOST: The current platform triple we're building for -# -outdir_for_host() { - echo "${OUTDIR_BASE}/${1}${2:+-${2}}" -} - -# Usage: profiledir_for_host HOST SUFFIX -# -# HOST: The current platform triple we're building for -# -profiledir_for_host() { - echo "${PROFILES_BASE}/${1}${2:+-${2}}" -} - - -######### -# BUILD # -######### - -# Function to be called when building for host ${1} and the user interrupts the -# build -int_trap() { -cat << EOF -** INT received while building ${1}, you may want to clean up the relevant - work directories (e.g. distsrc-*) before rebuilding - -Hint: To blow everything away, you may want to use: - - $ ./contrib/guix/guix-clean - -Specifically, this will remove all files without an entry in the index, -excluding the SDK directory, the depends download cache, the depends built -packages cache, the garbage collector roots for Guix environments, and the -output directory. -EOF -} - -# Deterministically build Bitcoin Core -# shellcheck disable=SC2153 -for host in $HOSTS; do - - # Display proper warning when the user interrupts the build - trap 'int_trap ${host}' INT - - ( - # Required for 'contrib/guix/manifest.scm' to output the right manifest - # for the particular $HOST we're building for - export HOST="$host" - - # shellcheck disable=SC2030 -cat << EOF -INFO: Building ${VERSION:?not set} for platform triple ${HOST:?not set}: - ...using reference timestamp: ${SOURCE_DATE_EPOCH:?not set} - ...running at most ${JOBS:?not set} jobs - ...from worktree directory: '${PWD}' - ...bind-mounted in container to: '/bitcoin' - ...in build directory: '$(distsrc_for_host "$HOST")' - ...bind-mounted in container to: '$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")' - ...outputting in: '$(outdir_for_host "$HOST")' - ...bind-mounted in container to: '$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST")' - ADDITIONAL FLAGS (if set) - ADDITIONAL_GUIX_COMMON_FLAGS: ${ADDITIONAL_GUIX_COMMON_FLAGS} - ADDITIONAL_GUIX_ENVIRONMENT_FLAGS: ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} - ADDITIONAL_GUIX_TIMEMACHINE_FLAGS: ${ADDITIONAL_GUIX_TIMEMACHINE_FLAGS} -EOF - - # Run the build script 'contrib/guix/libexec/build.sh' in the build - # container specified by 'contrib/guix/manifest.scm'. - # - # Explanation of `guix shell` flags: - # - # --container run command within an isolated container - # - # Running in an isolated container minimizes build-time differences - # between machines and improves reproducibility - # - # --pure unset existing environment variables - # - # Same rationale as --container - # - # --no-cwd do not share current working directory with an - # isolated container - # - # When --container is specified, the default behavior is to share - # the current working directory with the isolated container at the - # same exact path (e.g. mapping '/home/satoshi/bitcoin/' to - # '/home/satoshi/bitcoin/'). This means that the $PWD inside the - # container becomes a source of irreproducibility. --no-cwd disables - # this behaviour. - # - # --share=SPEC for containers, share writable host file system - # according to SPEC - # - # --share="$PWD"=/bitcoin - # - # maps our current working directory to /bitcoin - # inside the isolated container, which we later cd - # into. - # - # While we don't want to map our current working directory to the - # same exact path (as this introduces irreproducibility), we do want - # it to be at a _fixed_ path _somewhere_ inside the isolated - # container so that we have something to build. '/bitcoin' was - # chosen arbitrarily. - # - # ${SOURCES_PATH:+--share="$SOURCES_PATH"} - # - # make the downloaded depends sources path available - # inside the isolated container - # - # The isolated container has no network access as it's in a - # different network namespace from the main machine, so we have to - # make the downloaded depends sources available to it. The sources - # should have been downloaded prior to this invocation. - # - # --keep-failed keep build tree of failed builds - # - # When builds of the Guix environment itself (not Bitcoin Core) - # fail, it is useful for the build tree to be kept for debugging - # purposes. - # - # ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} - # - # fetch substitute from SUBSTITUTE_URLS if they are - # authorized - # - # Depending on the user's security model, it may be desirable to use - # substitutes (pre-built packages) from servers that the user trusts. - # Please read the README.md in the same directory as this file for - # more information. - # - # shellcheck disable=SC2086,SC2031 - time-machine shell --manifest="${PWD}/bench-ci/guix/manifest.scm" \ - --container \ - --pure \ - --no-cwd \ - --share="$PWD"=/bitcoin \ - --share="$DISTSRC_BASE"=/distsrc-base \ - --share="$OUTDIR_BASE"=/outdir-base \ - --expose="$(git rev-parse --git-common-dir)" \ - ${SOURCES_PATH:+--share="$SOURCES_PATH"} \ - ${BASE_CACHE:+--share="$BASE_CACHE"} \ - ${SDK_PATH:+--share="$SDK_PATH"} \ - --cores="$JOBS" \ - --keep-failed \ - --fallback \ - --link-profile \ - --root="$(profiledir_for_host "${HOST}")" \ - ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ - ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ - -- env HOST="$host" \ - DISTNAME="$DISTNAME" \ - JOBS="$JOBS" \ - SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:?unable to determine value}" \ - ${V:+V=1} \ - ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} \ - ${BASE_CACHE:+BASE_CACHE="$BASE_CACHE"} \ - ${SDK_PATH:+SDK_PATH="$SDK_PATH"} \ - DISTSRC="$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")" \ - OUTDIR="$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST")" \ - DIST_ARCHIVE_BASE=/outdir-base/dist-archive \ - bash -c "cd /bitcoin && bash bench-ci/guix/libexec/build.sh" - ) - -done diff --git a/bench-ci/guix/guix-clean b/bench-ci/guix/guix-clean deleted file mode 100755 index 9af0a793cff7..000000000000 --- a/bench-ci/guix/guix-clean +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env bash -export LC_ALL=C -set -e -o pipefail - -# Source the common prelude, which: -# 1. Checks if we're at the top directory of the Bitcoin Core repository -# 2. Defines a few common functions and variables -# -# shellcheck source=libexec/prelude.bash -source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash" - - -################### -## Sanity Checks ## -################### - -################ -# Required non-builtin commands should be invokable -################ - -check_tools cat mkdir make git guix - - -############# -## Clean ## -############# - -# Usage: under_dir MAYBE_PARENT MAYBE_CHILD -# -# If MAYBE_CHILD is a subdirectory of MAYBE_PARENT, print the relative path -# from MAYBE_PARENT to MAYBE_CHILD. Otherwise, return 1 as the error code. -# -# NOTE: This does not perform any symlink-resolving or path canonicalization. -# -under_dir() { - local path_residue - path_residue="${2##"${1}"}" - if [ -z "$path_residue" ] || [ "$path_residue" = "$2" ]; then - return 1 - else - echo "$path_residue" - fi -} - -# Usage: dir_under_git_root MAYBE_CHILD -# -# If MAYBE_CHILD is under the current git repository and exists, print the -# relative path from the git repository's top-level directory to MAYBE_CHILD, -# otherwise, exit with an error code. -# -dir_under_git_root() { - local rv - rv="$(under_dir "$(git_root)" "$1")" - [ -n "$rv" ] && echo "$rv" -} - -shopt -s nullglob -found_precious_dirs_files=( "${version_base_prefix}"*/"${var_base_basename}/precious_dirs" ) # This expands to an array of directories... -shopt -u nullglob - -exclude_flags=() - -for precious_dirs_file in "${found_precious_dirs_files[@]}"; do - # Make sure the precious directories (e.g. SOURCES_PATH, BASE_CACHE, SDK_PATH) - # are excluded from git-clean - echo "Found precious_dirs file: '${precious_dirs_file}'" - - # Exclude the precious_dirs file itself - if dirs_file_exclude_fragment=$(dir_under_git_root "$(dirname "$precious_dirs_file")"); then - exclude_flags+=( --exclude="${dirs_file_exclude_fragment}/precious_dirs" ) - fi - - # Read each 'name=dir' pair from the precious_dirs file - while IFS='=' read -r name dir; do - # Add an exclusion flag if the precious directory is under the git root. - if under=$(dir_under_git_root "$dir"); then - echo "Avoiding ${name}: ${under}" - exclude_flags+=( --exclude="$under" ) - fi - done < "$precious_dirs_file" -done - -git clean -xdff "${exclude_flags[@]}" diff --git a/bench-ci/guix/guix-codesign b/bench-ci/guix/guix-codesign deleted file mode 100755 index ac7aae3a1802..000000000000 --- a/bench-ci/guix/guix-codesign +++ /dev/null @@ -1,384 +0,0 @@ -#!/usr/bin/env bash -export LC_ALL=C -set -e -o pipefail - -# Source the common prelude, which: -# 1. Checks if we're at the top directory of the Bitcoin Core repository -# 2. Defines a few common functions and variables -# -# shellcheck source=libexec/prelude.bash -source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash" - - -################### -## SANITY CHECKS ## -################### - -################ -# Required non-builtin commands should be invocable -################ - -check_tools cat mkdir git guix - -################ -# Required env vars should be non-empty -################ - -cmd_usage() { - cat < \\ - ./contrib/guix/guix-codesign - -EOF -} - -if [ -z "$DETACHED_SIGS_REPO" ]; then - cmd_usage - exit 1 -fi - -################ -# GUIX_BUILD_OPTIONS should be empty -################ -# -# GUIX_BUILD_OPTIONS is an environment variable recognized by guix commands that -# can perform builds. This seems like what we want instead of -# ADDITIONAL_GUIX_COMMON_FLAGS, but the value of GUIX_BUILD_OPTIONS is actually -# _appended_ to normal command-line options. Meaning that they will take -# precedence over the command-specific ADDITIONAL_GUIX__FLAGS. -# -# This seems like a poor user experience. Thus we check for GUIX_BUILD_OPTIONS's -# existence here and direct users of this script to use our (more flexible) -# custom environment variables. -if [ -n "$GUIX_BUILD_OPTIONS" ]; then -cat << EOF -Error: Environment variable GUIX_BUILD_OPTIONS is not empty: - '$GUIX_BUILD_OPTIONS' - -Unfortunately this script is incompatible with GUIX_BUILD_OPTIONS, please unset -GUIX_BUILD_OPTIONS and use ADDITIONAL_GUIX_COMMON_FLAGS to set build options -across guix commands or ADDITIONAL_GUIX__FLAGS to set build options for a -specific guix command. - -See contrib/guix/README.md for more details. -EOF -exit 1 -fi - -################ -# SOURCE_DATE_EPOCH should not unintentionally be set -################ - -check_source_date_epoch - -################ -# The codesignature git worktree should not be dirty -################ - -if ! git -C "$DETACHED_SIGS_REPO" diff-index --quiet HEAD -- && [ -z "$FORCE_DIRTY_WORKTREE" ]; then - cat << EOF -ERR: The DETACHED CODESIGNATURE git worktree is dirty, which may lead to broken builds. - - Aborting... - -Hint: To make your git worktree clean, You may want to: - 1. Commit your changes, - 2. Stash your changes, or - 3. Set the 'FORCE_DIRTY_WORKTREE' environment variable if you insist on - using a dirty worktree -EOF - exit 1 -fi - -################ -# Build directories should not exist -################ - -# Default to building for all supported HOSTs (overridable by environment) -export HOSTS="${HOSTS:-x86_64-w64-mingw32 x86_64-apple-darwin arm64-apple-darwin}" - -# Usage: distsrc_for_host HOST -# -# HOST: The current platform triple we're building for -# -distsrc_for_host() { - echo "${DISTSRC_BASE}/distsrc-${VERSION}-${1}-codesigned" -} - -# Accumulate a list of build directories that already exist... -hosts_distsrc_exists="" -for host in $HOSTS; do - if [ -e "$(distsrc_for_host "$host")" ]; then - hosts_distsrc_exists+=" ${host}" - fi -done - -if [ -n "$hosts_distsrc_exists" ]; then -# ...so that we can print them out nicely in an error message -cat << EOF -ERR: Build directories for this commit already exist for the following platform - triples you're attempting to build, probably because of previous builds. - Please remove, or otherwise deal with them prior to starting another build. - - Aborting... - -Hint: To blow everything away, you may want to use: - - $ ./contrib/guix/guix-clean - -Specifically, this will remove all files without an entry in the index, -excluding the SDK directory, the depends download cache, the depends built -packages cache, the garbage collector roots for Guix environments, and the -output directory. -EOF -for host in $hosts_distsrc_exists; do - echo " ${host} '$(distsrc_for_host "$host")'" -done -exit 1 -else - mkdir -p "$DISTSRC_BASE" -fi - - -################ -# Codesigning tarballs SHOULD exist -################ - -# Usage: outdir_for_host HOST SUFFIX -# -# HOST: The current platform triple we're building for -# -outdir_for_host() { - echo "${OUTDIR_BASE}/${1}${2:+-${2}}" -} - - -codesigning_tarball_for_host() { - case "$1" in - *mingw*) - echo "$(outdir_for_host "$1")/${DISTNAME}-win64-codesigning.tar.gz" - ;; - *darwin*) - echo "$(outdir_for_host "$1")/${DISTNAME}-${1}-codesigning.tar.gz" - ;; - *) - exit 1 - ;; - esac -} - -# Accumulate a list of build directories that already exist... -hosts_codesigning_tarball_missing="" -for host in $HOSTS; do - if [ ! -e "$(codesigning_tarball_for_host "$host")" ]; then - hosts_codesigning_tarball_missing+=" ${host}" - fi -done - -if [ -n "$hosts_codesigning_tarball_missing" ]; then - # ...so that we can print them out nicely in an error message - cat << EOF -ERR: Codesigning tarballs do not exist -... - -EOF -for host in $hosts_codesigning_tarball_missing; do - echo " ${host} '$(codesigning_tarball_for_host "$host")'" -done -exit 1 -fi - -################ -# Check that we can connect to the guix-daemon -################ - -cat << EOF -Checking that we can connect to the guix-daemon... - -Hint: If this hangs, you may want to try turning your guix-daemon off and on - again. - -EOF -if ! guix gc --list-failures > /dev/null; then - cat << EOF - -ERR: Failed to connect to the guix-daemon, please ensure that one is running and - reachable. -EOF - exit 1 -fi - -# Developer note: we could use `guix repl` for this check and run: -# -# (import (guix store)) (close-connection (open-connection)) -# -# However, the internal API is likely to change more than the CLI invocation - - -######### -# SETUP # -######### - -# Determine the maximum number of jobs to run simultaneously (overridable by -# environment) -JOBS="${JOBS:-$(nproc)}" - -# Determine the reference time used for determinism (overridable by environment) -SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git -c log.showSignature=false log --format=%at -1)}" - -# Make sure an output directory exists for our builds -OUTDIR_BASE="${OUTDIR_BASE:-${VERSION_BASE}/output}" -mkdir -p "$OUTDIR_BASE" - -# Usage: profiledir_for_host HOST SUFFIX -# -# HOST: The current platform triple we're building for -# -profiledir_for_host() { - echo "${PROFILES_BASE}/${1}${2:+-${2}}" -} - -######### -# BUILD # -######### - -# Function to be called when codesigning for host ${1} and the user interrupts -# the codesign -int_trap() { -cat << EOF -** INT received while codesigning ${1}, you may want to clean up the relevant - work directories (e.g. distsrc-*) before recodesigning - -Hint: To blow everything away, you may want to use: - - $ ./contrib/guix/guix-clean - -Specifically, this will remove all files without an entry in the index, -excluding the SDK directory, the depends download cache, the depends built -packages cache, the garbage collector roots for Guix environments, and the -output directory. -EOF -} - -# Deterministically build Bitcoin Core -# shellcheck disable=SC2153 -for host in $HOSTS; do - - # Display proper warning when the user interrupts the build - trap 'int_trap ${host}' INT - - ( - # Required for 'contrib/guix/manifest.scm' to output the right manifest - # for the particular $HOST we're building for - export HOST="$host" - - # shellcheck disable=SC2030 -cat << EOF -INFO: Codesigning ${VERSION:?not set} for platform triple ${HOST:?not set}: - ...using reference timestamp: ${SOURCE_DATE_EPOCH:?not set} - ...from worktree directory: '${PWD}' - ...bind-mounted in container to: '/bitcoin' - ...in build directory: '$(distsrc_for_host "$HOST")' - ...bind-mounted in container to: '$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")' - ...outputting in: '$(outdir_for_host "$HOST" codesigned)' - ...bind-mounted in container to: '$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST" codesigned)' - ...using detached signatures in: '${DETACHED_SIGS_REPO:?not set}' - ...bind-mounted in container to: '/detached-sigs' -EOF - - - # Run the build script 'contrib/guix/libexec/build.sh' in the build - # container specified by 'contrib/guix/manifest.scm'. - # - # Explanation of `guix shell` flags: - # - # --container run command within an isolated container - # - # Running in an isolated container minimizes build-time differences - # between machines and improves reproducibility - # - # --pure unset existing environment variables - # - # Same rationale as --container - # - # --no-cwd do not share current working directory with an - # isolated container - # - # When --container is specified, the default behavior is to share - # the current working directory with the isolated container at the - # same exact path (e.g. mapping '/home/satoshi/bitcoin/' to - # '/home/satoshi/bitcoin/'). This means that the $PWD inside the - # container becomes a source of irreproducibility. --no-cwd disables - # this behaviour. - # - # --share=SPEC for containers, share writable host file system - # according to SPEC - # - # --share="$PWD"=/bitcoin - # - # maps our current working directory to /bitcoin - # inside the isolated container, which we later cd - # into. - # - # While we don't want to map our current working directory to the - # same exact path (as this introduces irreproducibility), we do want - # it to be at a _fixed_ path _somewhere_ inside the isolated - # container so that we have something to build. '/bitcoin' was - # chosen arbitrarily. - # - # ${SOURCES_PATH:+--share="$SOURCES_PATH"} - # - # make the downloaded depends sources path available - # inside the isolated container - # - # The isolated container has no network access as it's in a - # different network namespace from the main machine, so we have to - # make the downloaded depends sources available to it. The sources - # should have been downloaded prior to this invocation. - # - # ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} - # - # fetch substitute from SUBSTITUTE_URLS if they are - # authorized - # - # Depending on the user's security model, it may be desirable to use - # substitutes (pre-built packages) from servers that the user trusts. - # Please read the README.md in the same directory as this file for - # more information. - # - # shellcheck disable=SC2086,SC2031 - time-machine shell --manifest="${PWD}/contrib/guix/manifest.scm" \ - --container \ - --pure \ - --no-cwd \ - --share="$PWD"=/bitcoin \ - --share="$DISTSRC_BASE"=/distsrc-base \ - --share="$OUTDIR_BASE"=/outdir-base \ - --share="$DETACHED_SIGS_REPO"=/detached-sigs \ - --expose="$(git rev-parse --git-common-dir)" \ - --expose="$(git -C "$DETACHED_SIGS_REPO" rev-parse --git-common-dir)" \ - ${SOURCES_PATH:+--share="$SOURCES_PATH"} \ - --cores="$JOBS" \ - --keep-failed \ - --fallback \ - --link-profile \ - --root="$(profiledir_for_host "${HOST}" codesigned)" \ - ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ - ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ - -- env HOST="$host" \ - DISTNAME="$DISTNAME" \ - JOBS="$JOBS" \ - SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:?unable to determine value}" \ - ${V:+V=1} \ - ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} \ - DISTSRC="$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")" \ - OUTDIR="$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST" codesigned)" \ - DIST_ARCHIVE_BASE=/outdir-base/dist-archive \ - DETACHED_SIGS_REPO=/detached-sigs \ - CODESIGNING_TARBALL="$(OUTDIR_BASE=/outdir-base && codesigning_tarball_for_host "$HOST")" \ - bash -c "cd /bitcoin && bash contrib/guix/libexec/codesign.sh" - ) - -done diff --git a/bench-ci/guix/guix-verify b/bench-ci/guix/guix-verify deleted file mode 100755 index 02ae022741ba..000000000000 --- a/bench-ci/guix/guix-verify +++ /dev/null @@ -1,174 +0,0 @@ -#!/usr/bin/env bash -export LC_ALL=C -set -e -o pipefail - -# Source the common prelude, which: -# 1. Checks if we're at the top directory of the Bitcoin Core repository -# 2. Defines a few common functions and variables -# -# shellcheck source=libexec/prelude.bash -source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash" - - -################### -## Sanity Checks ## -################### - -################ -# Required non-builtin commands should be invokable -################ - -check_tools cat diff gpg - -################ -# Required env vars should be non-empty -################ - -cmd_usage() { -cat < [ SIGNER= ] ./contrib/guix/guix-verify - -Example overriding signer's manifest to use as base - - env GUIX_SIGS_REPO=/home/dongcarl/guix.sigs SIGNER=achow101 ./contrib/guix/guix-verify - -EOF -} - -if [ -z "$GUIX_SIGS_REPO" ]; then - cmd_usage - exit 1 -fi - -################ -# GUIX_SIGS_REPO should exist as a directory -################ - -if [ ! -d "$GUIX_SIGS_REPO" ]; then -cat << EOF -ERR: The specified GUIX_SIGS_REPO is not an existent directory: - - '$GUIX_SIGS_REPO' - -Hint: Please clone the guix.sigs repository and point to it with the - GUIX_SIGS_REPO environment variable. - -EOF -cmd_usage -exit 1 -fi - -############## -## Verify ## -############## - -OUTSIGDIR_BASE="${GUIX_SIGS_REPO}/${VERSION}" -echo "Looking for signature directories in '${OUTSIGDIR_BASE}'" -echo "" - -# Usage: verify compare_manifest current_manifest -verify() { - local compare_manifest="$1" - local current_manifest="$2" - if ! gpg --quiet --batch --verify "$current_manifest".asc "$current_manifest" 1>&2; then - echo "ERR: Failed to verify GPG signature in '${current_manifest}'" - echo "" - echo "Hint: Either the signature is invalid or the public key is missing" - echo "" - failure=1 - elif ! diff --report-identical "$compare_manifest" "$current_manifest" 1>&2; then - echo "ERR: The SHA256SUMS attestation in these two directories differ:" - echo " '${compare_manifest}'" - echo " '${current_manifest}'" - echo "" - failure=1 - else - echo "Verified: '${current_manifest}'" - echo "" - fi -} - -shopt -s nullglob -all_noncodesigned=( "$OUTSIGDIR_BASE"/*/noncodesigned.SHA256SUMS ) -shopt -u nullglob - -echo "--------------------" -echo "" -if (( ${#all_noncodesigned[@]} )); then - compare_noncodesigned="${all_noncodesigned[0]}" - if [[ -n "$SIGNER" ]]; then - signer_noncodesigned="$OUTSIGDIR_BASE/$SIGNER/noncodesigned.SHA256SUMS" - if [[ -f "$signer_noncodesigned" ]]; then - echo "Using $SIGNER's manifest as the base to compare against" - compare_noncodesigned="$signer_noncodesigned" - else - echo "Unable to find $SIGNER's manifest, using the first one found" - fi - else - echo "No SIGNER provided, using the first manifest found" - fi - - for current_manifest in "${all_noncodesigned[@]}"; do - verify "$compare_noncodesigned" "$current_manifest" - done - - echo "DONE: Checking output signatures for noncodesigned.SHA256SUMS" - echo "" -else - echo "WARN: No signature directories with noncodesigned.SHA256SUMS found" - echo "" -fi - -shopt -s nullglob -all_all=( "$OUTSIGDIR_BASE"/*/all.SHA256SUMS ) -shopt -u nullglob - -echo "--------------------" -echo "" -if (( ${#all_all[@]} )); then - compare_all="${all_all[0]}" - if [[ -n "$SIGNER" ]]; then - signer_all="$OUTSIGDIR_BASE/$SIGNER/all.SHA256SUMS" - if [[ -f "$signer_all" ]]; then - echo "Using $SIGNER's manifest as the base to compare against" - compare_all="$signer_all" - else - echo "Unable to find $SIGNER's manifest, using the first one found" - fi - else - echo "No SIGNER provided, using the first manifest found" - fi - - for current_manifest in "${all_all[@]}"; do - verify "$compare_all" "$current_manifest" - done - - # Sanity check: there should be no entries that exist in - # noncodesigned.SHA256SUMS that doesn't exist in all.SHA256SUMS - if [[ "$(comm -23 <(sort "$compare_noncodesigned") <(sort "$compare_all") | wc -c)" -ne 0 ]]; then - echo "ERR: There are unique lines in noncodesigned.SHA256SUMS which" - echo " do not exist in all.SHA256SUMS, something went very wrong." - exit 1 - fi - - echo "DONE: Checking output signatures for all.SHA256SUMS" - echo "" -else - echo "WARN: No signature directories with all.SHA256SUMS found" - echo "" -fi - -echo "====================" -echo "" -if (( ${#all_noncodesigned[@]} + ${#all_all[@]} == 0 )); then - echo "ERR: Unable to perform any verifications as no signature directories" - echo " were found" - echo "" - exit 1 -fi - -if [ -n "$failure" ]; then - exit 1 -fi diff --git a/bench-ci/guix/libexec/build.sh b/bench-ci/guix/libexec/build.sh deleted file mode 100755 index 87ed1996cf38..000000000000 --- a/bench-ci/guix/libexec/build.sh +++ /dev/null @@ -1,416 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2019-2022 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -export LC_ALL=C -set -e -o pipefail -export TZ=UTC - -# Although Guix _does_ set umask when building its own packages (in our case, -# this is all packages in manifest.scm), it does not set it for `guix -# shell`. It does make sense for at least `guix shell --container` -# to set umask, so if that change gets merged upstream and we bump the -# time-machine to a commit which includes the aforementioned change, we can -# remove this line. -# -# This line should be placed before any commands which creates files. -umask 0022 - -if [ -n "$V" ]; then - # Print both unexpanded (-v) and expanded (-x) forms of commands as they are - # read from this file. - set -vx - # Set VERBOSE for CMake-based builds - export VERBOSE="$V" -fi - -# Check that required environment variables are set -cat << EOF -Required environment variables as seen inside the container: - DIST_ARCHIVE_BASE: ${DIST_ARCHIVE_BASE:?not set} - DISTNAME: ${DISTNAME:?not set} - HOST: ${HOST:?not set} - SOURCE_DATE_EPOCH: ${SOURCE_DATE_EPOCH:?not set} - JOBS: ${JOBS:?not set} - DISTSRC: ${DISTSRC:?not set} - OUTDIR: ${OUTDIR:?not set} -EOF - -ACTUAL_OUTDIR="${OUTDIR}" -OUTDIR="${DISTSRC}/output" - -##################### -# Environment Setup # -##################### - -# The depends folder also serves as a base-prefix for depends packages for -# $HOSTs after successfully building. -BASEPREFIX="${PWD}/depends" - -# Given a package name and an output name, return the path of that output in our -# current guix environment -store_path() { - grep --extended-regexp "/[^-]{32}-${1}-[^-]+${2:+-${2}}" "${GUIX_ENVIRONMENT}/manifest" \ - | head --lines=1 \ - | sed --expression='s|\x29*$||' \ - --expression='s|^[[:space:]]*"||' \ - --expression='s|"[[:space:]]*$||' -} - - -# Set environment variables to point the NATIVE toolchain to the right -# includes/libs -NATIVE_GCC="$(store_path gcc-toolchain)" - -unset LIBRARY_PATH -unset CPATH -unset C_INCLUDE_PATH -unset CPLUS_INCLUDE_PATH -unset OBJC_INCLUDE_PATH -unset OBJCPLUS_INCLUDE_PATH - -# Set native toolchain -build_CC="${NATIVE_GCC}/bin/gcc -isystem ${NATIVE_GCC}/include" -build_CXX="${NATIVE_GCC}/bin/g++ -isystem ${NATIVE_GCC}/include/c++ -isystem ${NATIVE_GCC}/include" -export C_INCLUDE_PATH="${NATIVE_GCC}/include" -export CPLUS_INCLUDE_PATH="${NATIVE_GCC}/include/c++:${NATIVE_GCC}/include" - -case "$HOST" in - *darwin*) export LIBRARY_PATH="${NATIVE_GCC}/lib" ;; # Required for native packages - *mingw*) export LIBRARY_PATH="${NATIVE_GCC}/lib" ;; - *) - NATIVE_GCC_STATIC="$(store_path gcc-toolchain static)" - export LIBRARY_PATH="${NATIVE_GCC}/lib:${NATIVE_GCC_STATIC}/lib" - ;; -esac - -# Set environment variables to point the CROSS toolchain to the right -# includes/libs for $HOST -case "$HOST" in - *mingw*) - # Determine output paths to use in CROSS_* environment variables - CROSS_GLIBC="$(store_path "mingw-w64-x86_64-winpthreads")" - CROSS_GCC="$(store_path "gcc-cross-${HOST}")" - CROSS_GCC_LIB_STORE="$(store_path "gcc-cross-${HOST}" lib)" - CROSS_GCC_LIBS=( "${CROSS_GCC_LIB_STORE}/lib/gcc/${HOST}"/* ) # This expands to an array of directories... - CROSS_GCC_LIB="${CROSS_GCC_LIBS[0]}" # ...we just want the first one (there should only be one) - - # The search path ordering is generally: - # 1. gcc-related search paths - # 2. libc-related search paths - # 2. kernel-header-related search paths (not applicable to mingw-w64 hosts) - export CROSS_C_INCLUDE_PATH="${CROSS_GCC_LIB}/include:${CROSS_GCC_LIB}/include-fixed:${CROSS_GLIBC}/include" - export CROSS_CPLUS_INCLUDE_PATH="${CROSS_GCC}/include/c++:${CROSS_GCC}/include/c++/${HOST}:${CROSS_GCC}/include/c++/backward:${CROSS_C_INCLUDE_PATH}" - export CROSS_LIBRARY_PATH="${CROSS_GCC_LIB_STORE}/lib:${CROSS_GCC_LIB}:${CROSS_GLIBC}/lib" - ;; - *darwin*) - # The CROSS toolchain for darwin uses the SDK and ignores environment variables. - # See depends/hosts/darwin.mk for more details. - ;; - *linux*) - CROSS_GLIBC="$(store_path "glibc-cross-${HOST}")" - CROSS_GLIBC_STATIC="$(store_path "glibc-cross-${HOST}" static)" - CROSS_KERNEL="$(store_path "linux-libre-headers-cross-${HOST}")" - CROSS_GCC="$(store_path "gcc-cross-${HOST}")" - CROSS_GCC_LIB_STORE="$(store_path "gcc-cross-${HOST}" lib)" - CROSS_GCC_LIBS=( "${CROSS_GCC_LIB_STORE}/lib/gcc/${HOST}"/* ) # This expands to an array of directories... - CROSS_GCC_LIB="${CROSS_GCC_LIBS[0]}" # ...we just want the first one (there should only be one) - - export CROSS_C_INCLUDE_PATH="${CROSS_GCC_LIB}/include:${CROSS_GCC_LIB}/include-fixed:${CROSS_GLIBC}/include:${CROSS_KERNEL}/include" - export CROSS_CPLUS_INCLUDE_PATH="${CROSS_GCC}/include/c++:${CROSS_GCC}/include/c++/${HOST}:${CROSS_GCC}/include/c++/backward:${CROSS_C_INCLUDE_PATH}" - export CROSS_LIBRARY_PATH="${CROSS_GCC_LIB_STORE}/lib:${CROSS_GCC_LIB}:${CROSS_GLIBC}/lib:${CROSS_GLIBC_STATIC}/lib" - ;; - *) - exit 1 ;; -esac - -# Sanity check CROSS_*_PATH directories -IFS=':' read -ra PATHS <<< "${CROSS_C_INCLUDE_PATH}:${CROSS_CPLUS_INCLUDE_PATH}:${CROSS_LIBRARY_PATH}" -for p in "${PATHS[@]}"; do - if [ -n "$p" ] && [ ! -d "$p" ]; then - echo "'$p' doesn't exist or isn't a directory... Aborting..." - exit 1 - fi -done - -# Disable Guix ld auto-rpath behavior -export GUIX_LD_WRAPPER_DISABLE_RPATH=yes - -# Make /usr/bin if it doesn't exist -[ -e /usr/bin ] || mkdir -p /usr/bin - -# Symlink env to a conventional path -[ -e /usr/bin/env ] || ln -s --no-dereference "$(command -v env)" /usr/bin/env - -# Determine the correct value for -Wl,--dynamic-linker for the current $HOST -case "$HOST" in - x86_64-linux-gnu) ;; - *linux*) - glibc_dynamic_linker=$( - case "$HOST" in - arm-linux-gnueabihf) echo /lib/ld-linux-armhf.so.3 ;; - aarch64-linux-gnu) echo /lib/ld-linux-aarch64.so.1 ;; - riscv64-linux-gnu) echo /lib/ld-linux-riscv64-lp64d.so.1 ;; - powerpc64-linux-gnu) echo /lib64/ld64.so.1;; - powerpc64le-linux-gnu) echo /lib64/ld64.so.2;; - *) exit 1 ;; - esac - ) - ;; -esac - -# Environment variables for determinism -export TAR_OPTIONS="--owner=0 --group=0 --numeric-owner --mtime='@${SOURCE_DATE_EPOCH}' --sort=name" -export TZ="UTC" - -#################### -# Depends Building # -#################### - -# Build the depends tree, overriding variables that assume multilib gcc -make -C depends --jobs="$JOBS" HOST="$HOST" \ - ${V:+V=1} \ - ${SOURCES_PATH+SOURCES_PATH="$SOURCES_PATH"} \ - ${BASE_CACHE+BASE_CACHE="$BASE_CACHE"} \ - ${SDK_PATH+SDK_PATH="$SDK_PATH"} \ - ${build_CC+build_CC="$build_CC"} \ - ${build_CXX+build_CXX="$build_CXX"} \ - x86_64_linux_CC=x86_64-linux-gnu-gcc \ - x86_64_linux_CXX=x86_64-linux-gnu-g++ \ - x86_64_linux_AR=x86_64-linux-gnu-gcc-ar \ - x86_64_linux_RANLIB=x86_64-linux-gnu-gcc-ranlib \ - x86_64_linux_NM=x86_64-linux-gnu-gcc-nm \ - x86_64_linux_STRIP=x86_64-linux-gnu-strip \ - NO_QT=1 \ - NO_QR=1 \ - NO_ZMQ=1 \ - NO_WALLET=1 \ - NO_BDB=1 \ - NO_USDT=1 - -case "$HOST" in - *darwin*) - # Unset now that Qt is built - unset LIBRARY_PATH - ;; -esac - -########################### -# Source Tarball Building # -########################### - -GIT_ARCHIVE="${DIST_ARCHIVE_BASE}/${DISTNAME}.tar.gz" - -# Create the source tarball if not already there -if [ ! -e "$GIT_ARCHIVE" ]; then - mkdir -p "$(dirname "$GIT_ARCHIVE")" - git archive --prefix="${DISTNAME}/" --output="$GIT_ARCHIVE" HEAD -fi - -mkdir -p "$OUTDIR" - -########################### -# Binary Tarball Building # -########################### - -# CONFIGFLAGS -CONFIGFLAGS="-DREDUCE_EXPORTS=ON -DBUILD_BENCH=OFF -DBUILD_GUI_TESTS=OFF -DBUILD_FUZZ_BINARY=OFF -DCMAKE_SKIP_RPATH=TRUE" - -# BENCHCOINFLAGS -BENCHCOINFLAGS="-DBUILD_CLI=OFF -DBUILD_TESTS=OFF -DCMAKE_CXX_FLAGS=-fno-omit-frame-pointer" - -# CFLAGS -HOST_CFLAGS="-O2 -g" -HOST_CFLAGS+=$(find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;) -case "$HOST" in - *mingw*) HOST_CFLAGS+=" -fno-ident" ;; - *darwin*) unset HOST_CFLAGS ;; -esac - -# CXXFLAGS -HOST_CXXFLAGS="$HOST_CFLAGS" - -case "$HOST" in - arm-linux-gnueabihf) HOST_CXXFLAGS="${HOST_CXXFLAGS} -Wno-psabi" ;; -esac - -# LDFLAGS -case "$HOST" in - x86_64-linux-gnu) HOST_LDFLAGS=" -static-pie -static-libgcc -Wl,-O2" ;; - *linux*) HOST_LDFLAGS="-Wl,--as-needed -Wl,--dynamic-linker=$glibc_dynamic_linker -static-libstdc++ -Wl,-O2" ;; - *mingw*) HOST_LDFLAGS="-Wl,--no-insert-timestamp" ;; -esac - -mkdir -p "$DISTSRC" -( - cd "$DISTSRC" - - # Extract the source tarball - tar --strip-components=1 -xf "${GIT_ARCHIVE}" - - # Configure this DISTSRC for $HOST - # shellcheck disable=SC2086 - env CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" LDFLAGS="${HOST_LDFLAGS}" \ - cmake -S . -B build \ - --toolchain "${BASEPREFIX}/${HOST}/toolchain.cmake" \ - -DWITH_CCACHE=OFF \ - -Werror=dev \ - ${CONFIGFLAGS} \ - ${BENCHCOINFLAGS} - - # Build Bitcoin Core - cmake --build build -j "$JOBS" ${V:+--verbose} - - # Perform basic security checks on a series of executables. - # cmake --build build -j 1 --target check-security ${V:+--verbose} - # Check that executables only contain allowed version symbols. - # cmake --build build -j 1 --target check-symbols ${V:+--verbose} - - mkdir -p "$OUTDIR" - - # Make the os-specific installers - case "$HOST" in - *mingw*) - cmake --build build -j "$JOBS" -t deploy ${V:+--verbose} - mv build/bitcoin-win64-setup.exe "${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe" - ;; - esac - - # Setup the directory where our Bitcoin Core build for HOST will be - # installed. This directory will also later serve as the input for our - # binary tarballs. - INSTALLPATH="${PWD}/installed/${DISTNAME}" - mkdir -p "${INSTALLPATH}" - # Install built Bitcoin Core to $INSTALLPATH - case "$HOST" in - *darwin*) - # This workaround can be dropped for CMake >= 3.27. - # See the upstream commit 689616785f76acd844fd448c51c5b2a0711aafa2. - find build -name 'cmake_install.cmake' -exec sed -i 's| -u -r | |g' {} + - - cmake --install build --strip --prefix "${INSTALLPATH}" ${V:+--verbose} - ;; - *) - cmake --install build --prefix "${INSTALLPATH}" ${V:+--verbose} - ;; - esac - - ( - cd installed - - case "$HOST" in - *darwin*) ;; - *) - # Split binaries from their debug symbols - { - find "${DISTNAME}/bin" "${DISTNAME}/libexec" -type f -executable -print0 - } | xargs -0 -P"$JOBS" -I{} "${DISTSRC}/build/split-debug.sh" {} {} {}.dbg - ;; - esac - - case "$HOST" in - *mingw*) - cp "${DISTSRC}/doc/README_windows.txt" "${DISTNAME}/readme.txt" - ;; - *linux*) - cp "${DISTSRC}/README.md" "${DISTNAME}/" - ;; - esac - - # copy over the example bitcoin.conf file. if contrib/devtools/gen-bitcoin-conf.sh - # has not been run before buildling, this file will be a stub - cp "${DISTSRC}/share/examples/bitcoin.conf" "${DISTNAME}/" - - cp -r "${DISTSRC}/share/rpcauth" "${DISTNAME}/share/" - - # Deterministically produce {non-,}debug binary tarballs ready - # for release - case "$HOST" in - *mingw*) - find "${DISTNAME}" -not -name "*.dbg" -print0 \ - | xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}" - find "${DISTNAME}" -not -name "*.dbg" \ - | sort \ - | zip -X@ "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}-unsigned.zip" \ - || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}-unsigned.zip" && exit 1 ) - find "${DISTNAME}" -name "*.dbg" -print0 \ - | xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}" - find "${DISTNAME}" -name "*.dbg" \ - | sort \ - | zip -X@ "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}-debug.zip" \ - || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}-debug.zip" && exit 1 ) - ;; - *linux*) - find "${DISTNAME}" -not -name "*.dbg" -print0 \ - | sort --zero-terminated \ - | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ - | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" \ - || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" && exit 1 ) - find "${DISTNAME}" -name "*.dbg" -print0 \ - | sort --zero-terminated \ - | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ - | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}-debug.tar.gz" \ - || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}-debug.tar.gz" && exit 1 ) - ;; - *darwin*) - find "${DISTNAME}" -print0 \ - | sort --zero-terminated \ - | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ - | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}-unsigned.tar.gz" \ - || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}-unsigned.tar.gz" && exit 1 ) - ;; - esac - ) # $DISTSRC/installed - - # Finally make tarballs for codesigning - case "$HOST" in - *mingw*) - cp -rf --target-directory=. contrib/windeploy - ( - cd ./windeploy - mkdir -p unsigned - cp --target-directory=unsigned/ "${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe" - cp -r --target-directory=unsigned/ "${INSTALLPATH}" - find unsigned/ -name "*.dbg" -print0 \ - | xargs -0r rm - find . -print0 \ - | sort --zero-terminated \ - | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ - | gzip -9n > "${OUTDIR}/${DISTNAME}-win64-codesigning.tar.gz" \ - || ( rm -f "${OUTDIR}/${DISTNAME}-win64-codesigning.tar.gz" && exit 1 ) - ) - ;; - *darwin*) - cmake --build build --target deploy ${V:+--verbose} - mv build/dist/bitcoin-macos-app.zip "${OUTDIR}/${DISTNAME}-${HOST}-unsigned.zip" - mkdir -p "unsigned-app-${HOST}" - cp --target-directory="unsigned-app-${HOST}" \ - contrib/macdeploy/detached-sig-create.sh - mv --target-directory="unsigned-app-${HOST}" build/dist - cp -r --target-directory="unsigned-app-${HOST}" "${INSTALLPATH}" - ( - cd "unsigned-app-${HOST}" - find . -print0 \ - | sort --zero-terminated \ - | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ - | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}-codesigning.tar.gz" \ - || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}-codesigning.tar.gz" && exit 1 ) - ) - ;; - esac -) # $DISTSRC - -rm -rf "$ACTUAL_OUTDIR" -mv --no-target-directory "$OUTDIR" "$ACTUAL_OUTDIR" \ - || ( rm -rf "$ACTUAL_OUTDIR" && exit 1 ) - -( - cd /outdir-base - { - echo "$GIT_ARCHIVE" - find "$ACTUAL_OUTDIR" -type f - } | xargs realpath --relative-base="$PWD" \ - | xargs sha256sum \ - | sort -k2 \ - | sponge "$ACTUAL_OUTDIR"/SHA256SUMS.part -) diff --git a/bench-ci/guix/libexec/codesign.sh b/bench-ci/guix/libexec/codesign.sh deleted file mode 100755 index fe86065350e9..000000000000 --- a/bench-ci/guix/libexec/codesign.sh +++ /dev/null @@ -1,153 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2021-2022 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -export LC_ALL=C -set -e -o pipefail - -# Environment variables for determinism -export TAR_OPTIONS="--owner=0 --group=0 --numeric-owner --mtime='@${SOURCE_DATE_EPOCH}' --sort=name" -export TZ=UTC - -# Although Guix _does_ set umask when building its own packages (in our case, -# this is all packages in manifest.scm), it does not set it for `guix -# shell`. It does make sense for at least `guix shell --container` -# to set umask, so if that change gets merged upstream and we bump the -# time-machine to a commit which includes the aforementioned change, we can -# remove this line. -# -# This line should be placed before any commands which creates files. -umask 0022 - -if [ -n "$V" ]; then - # Print both unexpanded (-v) and expanded (-x) forms of commands as they are - # read from this file. - set -vx - # Set VERBOSE for CMake-based builds - export VERBOSE="$V" -fi - -# Check that required environment variables are set -cat << EOF -Required environment variables as seen inside the container: - CODESIGNING_TARBALL: ${CODESIGNING_TARBALL:?not set} - DETACHED_SIGS_REPO: ${DETACHED_SIGS_REPO:?not set} - DIST_ARCHIVE_BASE: ${DIST_ARCHIVE_BASE:?not set} - DISTNAME: ${DISTNAME:?not set} - HOST: ${HOST:?not set} - SOURCE_DATE_EPOCH: ${SOURCE_DATE_EPOCH:?not set} - DISTSRC: ${DISTSRC:?not set} - OUTDIR: ${OUTDIR:?not set} -EOF - -ACTUAL_OUTDIR="${OUTDIR}" -OUTDIR="${DISTSRC}/output" - -git_head_version() { - local recent_tag - if recent_tag="$(git -C "$1" describe --exact-match HEAD 2> /dev/null)"; then - echo "${recent_tag#v}" - else - git -C "$1" rev-parse --short=12 HEAD - fi -} - -CODESIGNATURE_GIT_ARCHIVE="${DIST_ARCHIVE_BASE}/${DISTNAME}-codesignatures-$(git_head_version "$DETACHED_SIGS_REPO").tar.gz" - -# Create the codesignature tarball if not already there -if [ ! -e "$CODESIGNATURE_GIT_ARCHIVE" ]; then - mkdir -p "$(dirname "$CODESIGNATURE_GIT_ARCHIVE")" - git -C "$DETACHED_SIGS_REPO" archive --output="$CODESIGNATURE_GIT_ARCHIVE" HEAD -fi - -mkdir -p "$OUTDIR" - -mkdir -p "$DISTSRC" -( - cd "$DISTSRC" - - tar -xf "$CODESIGNING_TARBALL" - - mkdir -p codesignatures - tar -C codesignatures -xf "$CODESIGNATURE_GIT_ARCHIVE" - - case "$HOST" in - *mingw*) - # Apply detached codesignatures - WORKDIR=".tmp" - mkdir -p ${WORKDIR} - cp -r --target-directory="${WORKDIR}" "unsigned/${DISTNAME}" - find "${WORKDIR}/${DISTNAME}" -name "*.exe" -type f -exec rm {} \; - find unsigned/ -name "*.exe" -type f | while read -r bin - do - bin_base="$(realpath --relative-to=unsigned/ "${bin}")" - mkdir -p "${WORKDIR}/$(dirname "${bin_base}")" - osslsigncode attach-signature \ - -in "${bin}" \ - -out "${WORKDIR}/${bin_base/-unsigned}" \ - -CAfile "$GUIX_ENVIRONMENT/etc/ssl/certs/ca-certificates.crt" \ - -sigin codesignatures/win/"${bin_base}".pem - done - - # Move installer to outdir - cd "${WORKDIR}" - find . -name "*setup.exe" -print0 \ - | xargs -0r mv --target-directory="${OUTDIR}" - - # Make .zip from binaries - find "${DISTNAME}" -print0 \ - | xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}" - find "${DISTNAME}" \ - | sort \ - | zip -X@ "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}.zip" \ - || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}.zip" && exit 1 ) - ;; - *darwin*) - case "$HOST" in - arm64*) ARCH="arm64" ;; - x86_64*) ARCH="x86_64" ;; - esac - - # Apply detached codesignatures (in-place) - signapple apply dist/Bitcoin-Qt.app codesignatures/osx/"${HOST}"/dist/Bitcoin-Qt.app - find "${DISTNAME}" \( -wholename "*/bin/*" -o -wholename "*/libexec/*" \) -type f | while read -r bin - do - signapple apply "${bin}" "codesignatures/osx/${HOST}/${bin}.${ARCH}sign" - done - - # Make a .zip from dist/ - cd dist/ - find . -print0 \ - | xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}" - find . | sort \ - | zip -X@ "${OUTDIR}/${DISTNAME}-${HOST}.zip" - cd .. - - # Make a .tar.gz from bins - find "${DISTNAME}" -print0 \ - | sort --zero-terminated \ - | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ - | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" \ - || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" && exit 1 ) - ;; - *) - exit 1 - ;; - esac -) # $DISTSRC - -rm -rf "$ACTUAL_OUTDIR" -mv --no-target-directory "$OUTDIR" "$ACTUAL_OUTDIR" \ - || ( rm -rf "$ACTUAL_OUTDIR" && exit 1 ) - -( - cd /outdir-base - { - echo "$CODESIGNING_TARBALL" - echo "$CODESIGNATURE_GIT_ARCHIVE" - find "$ACTUAL_OUTDIR" -type f - } | xargs realpath --relative-base="$PWD" \ - | xargs sha256sum \ - | sort -k2 \ - | sponge "$ACTUAL_OUTDIR"/SHA256SUMS.part -) diff --git a/bench-ci/guix/libexec/prelude.bash b/bench-ci/guix/libexec/prelude.bash deleted file mode 100644 index 3cf568279f70..000000000000 --- a/bench-ci/guix/libexec/prelude.bash +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env bash -export LC_ALL=C -set -e -o pipefail - -# shellcheck source=contrib/shell/realpath.bash -source contrib/shell/realpath.bash - -# shellcheck source=contrib/shell/git-utils.bash -source contrib/shell/git-utils.bash - -# Source guix profile from the runner home directory -GUIX_PROFILE=/home/github-runner/.config/guix/current -. "$GUIX_PROFILE/etc/profile" || true -echo "Using the following guix command:" -command -v guix -echo "Guix command symlink points to:" -readlink -f "$(command -v guix)" -echo "Current Guix profile:" -echo "$GUIX_PROFILE" -echo "Profile generation info:" -guix describe - -################ -# Required non-builtin commands should be invocable -################ - -check_tools() { - for cmd in "$@"; do - if ! command -v "$cmd" > /dev/null 2>&1; then - echo "ERR: This script requires that '$cmd' is installed and available in your \$PATH" - exit 1 - fi - done -} - -################ -# SOURCE_DATE_EPOCH should not unintentionally be set -################ - -check_source_date_epoch() { - if [ -n "$SOURCE_DATE_EPOCH" ] && [ -z "$FORCE_SOURCE_DATE_EPOCH" ]; then - cat << EOF -ERR: Environment variable SOURCE_DATE_EPOCH is set which may break reproducibility. - - Aborting... - -Hint: You may want to: - 1. Unset this variable: \`unset SOURCE_DATE_EPOCH\` before rebuilding - 2. Set the 'FORCE_SOURCE_DATE_EPOCH' environment variable if you insist on - using your own epoch -EOF - exit 1 - fi -} - -check_tools cat env readlink dirname basename git - -################ -# We should be at the top directory of the repository -################ - -same_dir() { - local resolved1 resolved2 - resolved1="$(bash_realpath "${1}")" - resolved2="$(bash_realpath "${2}")" - [ "$resolved1" = "$resolved2" ] -} - -if ! same_dir "${PWD}" "$(git_root)"; then -cat << EOF -ERR: This script must be invoked from the top level of the git repository - -Hint: This may look something like: - env FOO=BAR ./contrib/guix/guix- - -EOF -exit 1 -fi - -################ -# Execute "$@" in a pinned, possibly older version of Guix, for reproducibility -# across time. -time-machine() { - # shellcheck disable=SC2086 - guix time-machine --url=https://github.com/fanquake/guix.git \ - --commit=5cb84f2013c5b1e48a7d0e617032266f1e6059e2 \ - --cores="$JOBS" \ - --keep-failed \ - --fallback \ - ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ - ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_TIMEMACHINE_FLAGS} \ - -- "$@" -} - - -################ -# Set common variables -################ - -VERSION="${FORCE_VERSION:-$(git_head_version)}" -DISTNAME="${DISTNAME:-bitcoin-${VERSION}}" - -version_base_prefix="${PWD}/guix-build-" -VERSION_BASE="${version_base_prefix}${VERSION}" # TOP - -DISTSRC_BASE="${DISTSRC_BASE:-${VERSION_BASE}}" - -OUTDIR_BASE="${OUTDIR_BASE:-${VERSION_BASE}/output}" - -var_base_basename="var" -VAR_BASE="${VAR_BASE:-${VERSION_BASE}/${var_base_basename}}" - -profiles_base_basename="profiles" -PROFILES_BASE="${PROFILES_BASE:-${VAR_BASE}/${profiles_base_basename}}" diff --git a/bench-ci/guix/manifest.scm b/bench-ci/guix/manifest.scm deleted file mode 100644 index f89eccc85360..000000000000 --- a/bench-ci/guix/manifest.scm +++ /dev/null @@ -1,610 +0,0 @@ -(use-modules (gnu packages) - ((gnu packages bash) #:select (bash-minimal)) - (gnu packages bison) - ((gnu packages certs) #:select (nss-certs)) - ((gnu packages cmake) #:select (cmake-minimal)) - (gnu packages commencement) - (gnu packages compression) - (gnu packages cross-base) - (gnu packages gawk) - (gnu packages gcc) - ((gnu packages installers) #:select (nsis-x86_64)) - ((gnu packages linux) #:select (linux-libre-headers-6.1)) - (gnu packages llvm) - (gnu packages mingw) - (gnu packages ninja) - (gnu packages pkg-config) - ((gnu packages python) #:select (python-minimal)) - ((gnu packages python-build) #:select (python-poetry-core)) - ((gnu packages python-crypto) #:select (python-asn1crypto)) - ((gnu packages python-science) #:select (python-scikit-build-core)) - ((gnu packages python-xyz) #:select (python-pydantic-2)) - ((gnu packages tls) #:select (openssl)) - ((gnu packages version-control) #:select (git-minimal)) - (guix build-system cmake) - (guix build-system gnu) - (guix build-system python) - (guix build-system pyproject) - (guix build-system trivial) - (guix download) - (guix gexp) - (guix git-download) - ((guix licenses) #:prefix license:) - (guix packages) - ((guix utils) #:select (cc-for-target substitute-keyword-arguments))) - -(define-syntax-rule (search-our-patches file-name ...) - "Return the list of absolute file names corresponding to each -FILE-NAME found in ./patches relative to the current file." - (parameterize - ((%patch-path (list (string-append (dirname (current-filename)) "/patches")))) - (list (search-patch file-name) ...))) - -(define building-on (string-append "--build=" (list-ref (string-split (%current-system) #\-) 0) "-guix-linux-gnu")) - -(define (make-cross-toolchain target - base-gcc-for-libc - base-kernel-headers - base-libc - base-gcc) - "Create a cross-compilation toolchain package for TARGET" - (let* ((xbinutils (cross-binutils target)) - ;; 1. Build a cross-compiling gcc without targeting any libc, derived - ;; from BASE-GCC-FOR-LIBC - (xgcc-sans-libc (cross-gcc target - #:xgcc base-gcc-for-libc - #:xbinutils xbinutils)) - ;; 2. Build cross-compiled kernel headers with XGCC-SANS-LIBC, derived - ;; from BASE-KERNEL-HEADERS - (xkernel (cross-kernel-headers target - #:linux-headers base-kernel-headers - #:xgcc xgcc-sans-libc - #:xbinutils xbinutils)) - ;; 3. Build a cross-compiled libc with XGCC-SANS-LIBC and XKERNEL, - ;; derived from BASE-LIBC - (xlibc (cross-libc target - #:libc base-libc - #:xgcc xgcc-sans-libc - #:xbinutils xbinutils - #:xheaders xkernel)) - ;; 4. Build a cross-compiling gcc targeting XLIBC, derived from - ;; BASE-GCC - (xgcc (cross-gcc target - #:xgcc base-gcc - #:xbinutils xbinutils - #:libc xlibc))) - ;; Define a meta-package that propagates the resulting XBINUTILS, XLIBC, and - ;; XGCC - (package - (name (string-append target "-toolchain")) - (version (package-version xgcc)) - (source #f) - (build-system trivial-build-system) - (arguments '(#:builder (begin (mkdir %output) #t))) - (propagated-inputs - (list xbinutils - xlibc - xgcc - `(,xlibc "static") - `(,xgcc "lib"))) - (synopsis (string-append "Complete GCC tool chain for " target)) - (description (string-append "This package provides a complete GCC tool -chain for " target " development.")) - (home-page (package-home-page xgcc)) - (license (package-license xgcc))))) - -(define base-gcc gcc-13) ;; 13.3.0 - -(define base-linux-kernel-headers linux-libre-headers-6.1) - -(define* (make-bitcoin-cross-toolchain target - #:key - (base-gcc-for-libc linux-base-gcc) - (base-kernel-headers base-linux-kernel-headers) - (base-libc glibc-2.31) - (base-gcc linux-base-gcc)) - "Convenience wrapper around MAKE-CROSS-TOOLCHAIN with default values -desirable for building Bitcoin Core release binaries." - (make-cross-toolchain target - base-gcc-for-libc - base-kernel-headers - base-libc - base-gcc)) - -(define (gcc-mingw-patches gcc) - (package-with-extra-patches gcc - (search-our-patches "gcc-remap-guix-store.patch"))) - -(define (binutils-mingw-patches binutils) - (package-with-extra-patches binutils - (search-our-patches "binutils-unaligned-default.patch"))) - -(define (winpthreads-patches mingw-w64-x86_64-winpthreads) - (package-with-extra-patches mingw-w64-x86_64-winpthreads - (search-our-patches "winpthreads-remap-guix-store.patch"))) - -(define (make-mingw-pthreads-cross-toolchain target) - "Create a cross-compilation toolchain package for TARGET" - (let* ((xbinutils (binutils-mingw-patches (cross-binutils target))) - (machine (substring target 0 (string-index target #\-))) - (pthreads-xlibc (winpthreads-patches (make-mingw-w64 machine - #:xgcc (cross-gcc target #:xgcc (gcc-mingw-patches base-gcc)) - #:with-winpthreads? #t))) - (pthreads-xgcc (cross-gcc target - #:xgcc (gcc-mingw-patches mingw-w64-base-gcc) - #:xbinutils xbinutils - #:libc pthreads-xlibc))) - ;; Define a meta-package that propagates the resulting XBINUTILS, XLIBC, and - ;; XGCC - (package - (name (string-append target "-posix-toolchain")) - (version (package-version pthreads-xgcc)) - (source #f) - (build-system trivial-build-system) - (arguments '(#:builder (begin (mkdir %output) #t))) - (propagated-inputs - (list xbinutils - pthreads-xlibc - pthreads-xgcc - `(,pthreads-xgcc "lib"))) - (synopsis (string-append "Complete GCC tool chain for " target)) - (description (string-append "This package provides a complete GCC tool -chain for " target " development.")) - (home-page (package-home-page pthreads-xgcc)) - (license (package-license pthreads-xgcc))))) - -;; While LIEF is packaged in Guix, we maintain our own package, -;; to simplify building, and more easily apply updates. -;; Moreover, the Guix's package uses cmake, which caused build -;; failure; see https://github.com/bitcoin/bitcoin/pull/27296. -(define-public python-lief - (package - (name "python-lief") - (version "0.16.6") - (source (origin - (method git-fetch) - (uri (git-reference - (url "https://github.com/lief-project/LIEF") - (commit version))) - (file-name (git-file-name name version)) - (sha256 - (base32 - "1pq9nagrnkl1x943bqnpiyxmkd9vk99znfxiwqp6vf012b50bz2a")) - (patches (search-our-patches "lief-scikit-0-9.patch")))) - (build-system pyproject-build-system) - (native-inputs (list cmake-minimal - ninja - python-scikit-build-core - python-pydantic-2)) - (arguments - (list - #:tests? #f ;needs network - #:phases #~(modify-phases %standard-phases - (add-before 'build 'set-pythonpath - (lambda _ - (setenv "PYTHONPATH" - (string-append (string-append (getcwd) "/api/python/backend") - ":" (or (getenv "PYTHONPATH") ""))))) - (add-after 'set-pythonpath 'change-directory - (lambda _ - (chdir "api/python")))))) - (home-page "https://github.com/lief-project/LIEF") - (synopsis "Library to instrument executable formats") - (description - "@code{python-lief} is a cross platform library which can parse, modify -and abstract ELF, PE and MachO formats.") - (license license:asl2.0))) - -(define osslsigncode - (package - (name "osslsigncode") - (version "2.5") - (source (origin - (method git-fetch) - (uri (git-reference - (url "https://github.com/mtrojnar/osslsigncode") - (commit version))) - (sha256 - (base32 - "1j47vwq4caxfv0xw68kw5yh00qcpbd56d7rq6c483ma3y7s96yyz")))) - (build-system cmake-build-system) - (inputs (list openssl)) - (home-page "https://github.com/mtrojnar/osslsigncode") - (synopsis "Authenticode signing and timestamping tool") - (description "osslsigncode is a small tool that implements part of the -functionality of the Microsoft tool signtool.exe - more exactly the Authenticode -signing and timestamping. But osslsigncode is based on OpenSSL and cURL, and -thus should be able to compile on most platforms where these exist.") - (license license:gpl3+))) ; license is with openssl exception - -(define-public python-elfesteem - (let ((commit "2eb1e5384ff7a220fd1afacd4a0170acff54fe56")) - (package - (name "python-elfesteem") - (version (git-version "0.1" "1" commit)) - (source - (origin - (method git-fetch) - (uri (git-reference - (url "https://github.com/LRGH/elfesteem") - (commit commit))) - (file-name (git-file-name name commit)) - (sha256 - (base32 - "07x6p8clh11z8s1n2kdxrqwqm2almgc5qpkcr9ckb6y5ivjdr5r6")))) - (build-system python-build-system) - ;; There are no tests, but attempting to run python setup.py test leads to - ;; PYTHONPATH problems, just disable the test - (arguments '(#:tests? #f)) - (home-page "https://github.com/LRGH/elfesteem") - (synopsis "ELF/PE/Mach-O parsing library") - (description "elfesteem parses ELF, PE and Mach-O files.") - (license license:lgpl2.1)))) - -(define-public python-oscrypto - (package - (name "python-oscrypto") - (version "1.3.0") - (source - (origin - (method git-fetch) - (uri (git-reference - (url "https://github.com/wbond/oscrypto") - (commit version))) - (file-name (git-file-name name version)) - (sha256 - (base32 - "1v5wkmzcyiqy39db8j2dvkdrv2nlsc48556h73x4dzjwd6kg4q0a")) - (patches (search-our-patches "oscrypto-hard-code-openssl.patch")))) - (build-system python-build-system) - (native-search-paths - (list (search-path-specification - (variable "SSL_CERT_FILE") - (file-type 'regular) - (separator #f) ;single entry - (files '("etc/ssl/certs/ca-certificates.crt"))))) - - (propagated-inputs - (list python-asn1crypto openssl)) - (arguments - `(#:phases - (modify-phases %standard-phases - (add-after 'unpack 'hard-code-path-to-libscrypt - (lambda* (#:key inputs #:allow-other-keys) - (let ((openssl (assoc-ref inputs "openssl"))) - (substitute* "oscrypto/__init__.py" - (("@GUIX_OSCRYPTO_USE_OPENSSL@") - (string-append openssl "/lib/libcrypto.so" "," openssl "/lib/libssl.so"))) - #t))) - (add-after 'unpack 'disable-broken-tests - (lambda _ - ;; This test is broken as there is no keyboard interrupt. - (substitute* "tests/test_trust_list.py" - (("^(.*)class TrustListTests" line indent) - (string-append indent - "@unittest.skip(\"Disabled by Guix\")\n" - line))) - (substitute* "tests/test_tls.py" - (("^(.*)class TLSTests" line indent) - (string-append indent - "@unittest.skip(\"Disabled by Guix\")\n" - line))) - #t)) - (replace 'check - (lambda _ - (invoke "python" "run.py" "tests") - #t))))) - (home-page "https://github.com/wbond/oscrypto") - (synopsis "Compiler-free Python crypto library backed by the OS") - (description "oscrypto is a compilation-free, always up-to-date encryption library for Python.") - (license license:expat))) - -(define-public python-oscryptotests - (package (inherit python-oscrypto) - (name "python-oscryptotests") - (propagated-inputs - (list python-oscrypto)) - (arguments - `(#:tests? #f - #:phases - (modify-phases %standard-phases - (add-after 'unpack 'hard-code-path-to-libscrypt - (lambda* (#:key inputs #:allow-other-keys) - (chdir "tests") - #t))))))) - -(define-public python-certvalidator - (let ((commit "a145bf25eb75a9f014b3e7678826132efbba6213")) - (package - (name "python-certvalidator") - (version (git-version "0.1" "1" commit)) - (source - (origin - (method git-fetch) - (uri (git-reference - (url "https://github.com/achow101/certvalidator") - (commit commit))) - (file-name (git-file-name name commit)) - (sha256 - (base32 - "1qw2k7xis53179lpqdqyylbcmp76lj7sagp883wmxg5i7chhc96k")))) - (build-system python-build-system) - (propagated-inputs - (list python-asn1crypto - python-oscrypto - python-oscryptotests)) ;; certvalidator tests import oscryptotests - (arguments - `(#:phases - (modify-phases %standard-phases - (add-after 'unpack 'disable-broken-tests - (lambda _ - (substitute* "tests/test_certificate_validator.py" - (("^(.*)class CertificateValidatorTests" line indent) - (string-append indent - "@unittest.skip(\"Disabled by Guix\")\n" - line))) - (substitute* "tests/test_crl_client.py" - (("^(.*)def test_fetch_crl" line indent) - (string-append indent - "@unittest.skip(\"Disabled by Guix\")\n" - line))) - (substitute* "tests/test_ocsp_client.py" - (("^(.*)def test_fetch_ocsp" line indent) - (string-append indent - "@unittest.skip(\"Disabled by Guix\")\n" - line))) - (substitute* "tests/test_registry.py" - (("^(.*)def test_build_paths" line indent) - (string-append indent - "@unittest.skip(\"Disabled by Guix\")\n" - line))) - (substitute* "tests/test_validate.py" - (("^(.*)def test_revocation_mode_hard" line indent) - (string-append indent - "@unittest.skip(\"Disabled by Guix\")\n" - line))) - (substitute* "tests/test_validate.py" - (("^(.*)def test_revocation_mode_soft" line indent) - (string-append indent - "@unittest.skip(\"Disabled by Guix\")\n" - line))) - #t)) - (replace 'check - (lambda _ - (invoke "python" "run.py" "tests") - #t))))) - (home-page "https://github.com/wbond/certvalidator") - (synopsis "Python library for validating X.509 certificates and paths") - (description "certvalidator is a Python library for validating X.509 -certificates or paths. Supports various options, including: validation at a -specific moment in time, whitelisting and revocation checks.") - (license license:expat)))) - -(define-public python-signapple - (let ((commit "85bfcecc33d2773bc09bc318cec0614af2c8e287")) - (package - (name "python-signapple") - (version (git-version "0.2.0" "1" commit)) - (source - (origin - (method git-fetch) - (uri (git-reference - (url "https://github.com/achow101/signapple") - (commit commit))) - (file-name (git-file-name name commit)) - (sha256 - (base32 - "17yqjll8nw83q6dhgqhkl7w502z5vy9sln8m6mlx0f1c10isg8yg")))) - (build-system pyproject-build-system) - (propagated-inputs - (list python-asn1crypto - python-oscrypto - python-certvalidator - python-elfesteem)) - (native-inputs (list python-poetry-core)) - ;; There are no tests, but attempting to run python setup.py test leads to - ;; problems, just disable the test - (arguments '(#:tests? #f)) - (home-page "https://github.com/achow101/signapple") - (synopsis "Mach-O binary signature tool") - (description "signapple is a Python tool for creating, verifying, and -inspecting signatures in Mach-O binaries.") - (license license:expat)))) - -(define-public mingw-w64-base-gcc - (package - (inherit base-gcc) - (arguments - (substitute-keyword-arguments (package-arguments base-gcc) - ((#:configure-flags flags) - `(append ,flags - ;; https://gcc.gnu.org/install/configure.html - (list "--enable-threads=posix", - "--enable-default-ssp=yes", - "--disable-gcov", - building-on))))))) - -(define-public linux-base-gcc - (package - (inherit base-gcc) - (arguments - (substitute-keyword-arguments (package-arguments base-gcc) - ((#:configure-flags flags) - `(append ,flags - ;; https://gcc.gnu.org/install/configure.html - (list "--enable-initfini-array=yes", - "--enable-default-ssp=yes", - "--enable-default-pie=yes", - "--enable-standard-branch-protection=yes", - "--enable-cet=yes", - "--disable-gcov", - "--disable-libsanitizer", - building-on))) - ((#:phases phases) - `(modify-phases ,phases - ;; Given a XGCC package, return a modified package that replace each instance of - ;; -rpath in the default system spec that's inserted by Guix with -rpath-link - (add-after 'pre-configure 'replace-rpath-with-rpath-link - (lambda _ - (substitute* (cons "gcc/config/rs6000/sysv4.h" - (find-files "gcc/config" - "^gnu-user.*\\.h$")) - (("-rpath=") "-rpath-link=")) - #t)))))))) - -(define-public glibc-2.31 - (let ((commit "7b27c450c34563a28e634cccb399cd415e71ebfe")) - (package - (inherit glibc) ;; 2.39 - (version "2.31") - (source (origin - (method git-fetch) - (uri (git-reference - (url "https://sourceware.org/git/glibc.git") - (commit commit))) - (file-name (git-file-name "glibc" commit)) - (sha256 - (base32 - "017qdpr5id7ddb4lpkzj2li1abvw916m3fc6n7nw28z4h5qbv2n0")) - (patches (search-our-patches "glibc-guix-prefix.patch" - "glibc-riscv-jumptarget.patch")))) - (arguments - (substitute-keyword-arguments (package-arguments glibc) - ((#:configure-flags flags) - `(append ,flags - ;; https://www.gnu.org/software/libc/manual/html_node/Configuring-and-compiling.html - (list "--enable-stack-protector=all", - "--enable-cet", - "--enable-bind-now", - "--disable-werror", - "--disable-timezone-tools", - "--disable-profile", - building-on))) - ((#:phases phases) - `(modify-phases ,phases - (add-before 'configure 'set-etc-rpc-installation-directory - (lambda* (#:key outputs #:allow-other-keys) - ;; Install the rpc data base file under `$out/etc/rpc'. - ;; Otherwise build will fail with "Permission denied." - ;; Can be removed when we are building 2.32 or later. - (let ((out (assoc-ref outputs "out"))) - (substitute* "sunrpc/Makefile" - (("^\\$\\(inst_sysconfdir\\)/rpc(.*)$" _ suffix) - (string-append out "/etc/rpc" suffix "\n")) - (("^install-others =.*$") - (string-append "install-others = " out "/etc/rpc\n"))))))))))))) - -(define-public glibc-2.42 - (let ((commit "71874f167aa5bb1538ff7e394beaacee28ebe65f")) - (package - (inherit glibc) ;; 2.39 - (version "2.42") - (source (origin - (method git-fetch) - (uri (git-reference - (url "https://sourceware.org/git/glibc.git") - (commit commit))) - (file-name (git-file-name "glibc" commit)) - (sha256 - (base32 - "1pfbk907fkbavg7grbvb5zlhd3y47f8jj3d2v1s5w7xjnn0ypigq")) - (patches (search-our-patches "glibc-2.42-guix-prefix.patch")))) - (arguments - (substitute-keyword-arguments (package-arguments glibc) - ((#:configure-flags flags) - `(append ,flags - ;; https://www.gnu.org/software/libc/manual/html_node/Configuring-and-compiling.html - (list "--enable-stack-protector=all", - "--enable-bind-now", - "--enable-fortify-source", - "--enable-cet=yes", - "--enable-nscd=no", - "--enable-static-nss=yes", - "--disable-timezone-tools", - "--disable-profile", - "--disable-werror", - building-on)))))))) - -;; The sponge tool from moreutils. -(define-public sponge - (package - (name "sponge") - (version "0.69") - (source (origin - (method url-fetch) - (uri (string-append - "https://git.joeyh.name/index.cgi/moreutils.git/snapshot/ - moreutils-" version ".tar.gz")) - (file-name (string-append "moreutils-" version ".tar.gz")) - (sha256 - (base32 - "1l859qnzccslvxlh5ghn863bkq2vgmqgnik6jr21b9kc6ljmsy8g")))) - (build-system gnu-build-system) - (arguments - (list #:phases - #~(modify-phases %standard-phases - (delete 'configure) - (replace 'install - (lambda* (#:key outputs #:allow-other-keys) - (let ((bin (string-append (assoc-ref outputs "out") "/bin"))) - (install-file "sponge" bin))))) - #:make-flags - #~(list "sponge" (string-append "CC=" #$(cc-for-target))))) - (home-page "https://joeyh.name/code/moreutils/") - (synopsis "Miscellaneous general-purpose command-line tools") - (description "Just sponge") - (license license:gpl2+))) - -(packages->manifest - (append - (list ;; The Basics - bash-minimal - which - coreutils-minimal - ;; File(system) inspection - grep - diffutils - findutils - ;; File transformation - patch - gawk - sed - sponge - ;; Compression and archiving - tar - gzip - xz - ;; Build tools - gcc-toolchain-13 - cmake-minimal - gnu-make - ninja - ;; Scripting - python-minimal ;; (3.10) - ;; Git - git-minimal - ;; Tests - python-lief) - (let ((target (getenv "HOST"))) - (cond ((string-suffix? "-mingw32" target) - (list zip - (make-mingw-pthreads-cross-toolchain "x86_64-w64-mingw32") - nsis-x86_64 - nss-certs - osslsigncode)) - ((string-contains target "x86_64-linux-") - (list (list gcc-toolchain-13 "static") - (make-bitcoin-cross-toolchain target - #:base-libc glibc-2.42))) - ((string-contains target "-linux-") - (list bison - pkg-config - (list gcc-toolchain-13 "static") - (make-bitcoin-cross-toolchain target))) - ((string-contains target "darwin") - (list clang-toolchain-19 - lld-19 - (make-lld-wrapper lld-19 #:lld-as-ld? #t) - python-signapple - zip)) - (else '()))))) diff --git a/bench-ci/guix/patches/binutils-unaligned-default.patch b/bench-ci/guix/patches/binutils-unaligned-default.patch deleted file mode 100644 index d1bc71aee142..000000000000 --- a/bench-ci/guix/patches/binutils-unaligned-default.patch +++ /dev/null @@ -1,22 +0,0 @@ -commit 6537181f59ed186a341db621812a6bc35e22eaf6 -Author: fanquake -Date: Wed Apr 10 12:15:52 2024 +0200 - - build: turn on -muse-unaligned-vector-move by default - - This allows us to avoid (more invasively) patching GCC, to avoid - unaligned instruction use. - -diff --git a/gas/config/tc-i386.c b/gas/config/tc-i386.c -index e0632681477..14a9653abdf 100644 ---- a/gas/config/tc-i386.c -+++ b/gas/config/tc-i386.c -@@ -801,7 +801,7 @@ static unsigned int no_cond_jump_promotion = 0; - static unsigned int sse2avx; - - /* Encode aligned vector move as unaligned vector move. */ --static unsigned int use_unaligned_vector_move; -+static unsigned int use_unaligned_vector_move = 1; - - /* Encode scalar AVX instructions with specific vector length. */ - static enum diff --git a/bench-ci/guix/patches/gcc-remap-guix-store.patch b/bench-ci/guix/patches/gcc-remap-guix-store.patch deleted file mode 100644 index a8b41d485b04..000000000000 --- a/bench-ci/guix/patches/gcc-remap-guix-store.patch +++ /dev/null @@ -1,20 +0,0 @@ -Without ffile-prefix-map, the debug symbols will contain paths for the -guix store which will include the hashes of each package. However, the -hash for the same package will differ when on different architectures. -In order to be reproducible regardless of the architecture used to build -the package, map all guix store prefixes to something fixed, e.g. /usr. - ---- a/libgcc/Makefile.in -+++ b/libgcc/Makefile.in -@@ -854,7 +854,7 @@ endif - # libgcc_eh.a, only LIB2ADDEH matters. If we do, only LIB2ADDEHSTATIC and - # LIB2ADDEHSHARED matter. (Usually all three are identical.) - --c_flags := -fexceptions -+c_flags := -fexceptions $(shell find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;) - - ifeq ($(enable_shared),yes) - --- -2.37.0 - diff --git a/bench-ci/guix/patches/glibc-2.42-guix-prefix.patch b/bench-ci/guix/patches/glibc-2.42-guix-prefix.patch deleted file mode 100644 index f2fc1b90f183..000000000000 --- a/bench-ci/guix/patches/glibc-2.42-guix-prefix.patch +++ /dev/null @@ -1,47 +0,0 @@ -Without -ffile-prefix-map, the debug symbols will contain paths for the -guix store which will include the hashes of each package. However, the -hash for the same package will differ when on different architectures. -In order to be reproducible regardless of the architecture used to build -the package, map all guix store prefixes to something fixed, e.g. /usr. - ---- a/Makeconfig -+++ b/Makeconfig -@@ -1074,6 +1074,10 @@ CPPFLAGS-.o = $(pic-default) - CFLAGS-.o = $(filter %frame-pointer,$(+cflags)) $(pie-default) - CFLAGS-.o += $(call elide-fortify-source,.o,$(routines_no_fortify)) - CFLAGS-.o += $(call elide-fortify-source,_chk.o,$(routines_no_fortify)) -+ -+# Map Guix store paths to /usr -+CFLAGS-.o += `find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;` -+ - libtype.o := lib%.a - object-suffixes += .o - ifeq (yes,$(build-shared)) -diff --git a/iconv/Makefile b/iconv/Makefile -index afb3fb7bdb..5acee345e0 100644 ---- a/iconv/Makefile -+++ b/iconv/Makefile -@@ -65,6 +65,9 @@ CFLAGS-gconv_cache.c += -DGCONV_DIR='"$(gconvdir)"' - CFLAGS-gconv_conf.c += -DGCONV_PATH='"$(gconvdir)"' - CFLAGS-iconvconfig.c += -DGCONV_PATH='"$(gconvdir)"' -DGCONV_DIR='"$(gconvdir)"' - -+# Map Guix store paths to /usr -+CFLAGS-.c += `find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;` -+ - # Set libof-* for each routine. - cpp-srcs-left := $(iconv_prog-modules) $(iconvconfig-modules) - lib := iconvprogs -diff --git a/posix/Makefile b/posix/Makefile -index 3d368b91f6..d79d8fb648 100644 ---- a/posix/Makefile -+++ b/posix/Makefile -@@ -590,6 +590,9 @@ CFLAGS-execlp.os = -fomit-frame-pointer - CFLAGS-nanosleep.c += -fexceptions -fasynchronous-unwind-tables - CFLAGS-fork.c = $(libio-mtsafe) $(config-cflags-wno-ignored-attributes) - -+# Map Guix store paths to /usr -+CFLAGS-.c += `find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;` -+ - tstgetopt-ARGS = -a -b -cfoobar --required foobar --optional=bazbug \ - --none random --col --color --colour - diff --git a/bench-ci/guix/patches/glibc-guix-prefix.patch b/bench-ci/guix/patches/glibc-guix-prefix.patch deleted file mode 100644 index 60e12ca52546..000000000000 --- a/bench-ci/guix/patches/glibc-guix-prefix.patch +++ /dev/null @@ -1,16 +0,0 @@ -Without ffile-prefix-map, the debug symbols will contain paths for the -guix store which will include the hashes of each package. However, the -hash for the same package will differ when on different architectures. -In order to be reproducible regardless of the architecture used to build -the package, map all guix store prefixes to something fixed, e.g. /usr. - ---- a/Makeconfig -+++ b/Makeconfig -@@ -1007,6 +1007,7 @@ object-suffixes := - CPPFLAGS-.o = $(pic-default) - # libc.a must be compiled with -fPIE/-fpie for static PIE. - CFLAGS-.o = $(filter %frame-pointer,$(+cflags)) $(pie-default) -+CFLAGS-.o += `find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;` - libtype.o := lib%.a - object-suffixes += .o - ifeq (yes,$(build-shared)) diff --git a/bench-ci/guix/patches/glibc-riscv-jumptarget.patch b/bench-ci/guix/patches/glibc-riscv-jumptarget.patch deleted file mode 100644 index 702959433d86..000000000000 --- a/bench-ci/guix/patches/glibc-riscv-jumptarget.patch +++ /dev/null @@ -1,57 +0,0 @@ -commit 68389203832ab39dd0dbaabbc4059e7fff51c29b -Author: Fangrui Song -Date: Thu Oct 28 11:39:49 2021 -0700 - - riscv: Fix incorrect jal with HIDDEN_JUMPTARGET - - A non-local STV_DEFAULT defined symbol is by default preemptible in a - shared object. j/jal cannot target a preemptible symbol. On other - architectures, such a jump instruction either causes PLT [BZ #18822], or - if short-ranged, sometimes rejected by the linker (but not by GNU ld's - riscv port [ld PR/28509]). - - Use HIDDEN_JUMPTARGET to target a non-preemptible symbol instead. - - With this patch, ld.so and libc.so can be linked with LLD if source - files are compiled/assembled with -mno-relax/-Wa,-mno-relax. - - Acked-by: Palmer Dabbelt - Reviewed-by: Adhemerval Zanella - -Can be dropped when we are using glibc 2.35 or later. - -diff --git a/sysdeps/riscv/setjmp.S b/sysdeps/riscv/setjmp.S -index 0b92016b31..bec7ff80f4 100644 ---- a/sysdeps/riscv/setjmp.S -+++ b/sysdeps/riscv/setjmp.S -@@ -21,7 +21,7 @@ - - ENTRY (_setjmp) - li a1, 0 -- j __sigsetjmp -+ j HIDDEN_JUMPTARGET (__sigsetjmp) - END (_setjmp) - ENTRY (setjmp) - li a1, 1 -diff --git a/sysdeps/unix/sysv/linux/riscv/setcontext.S b/sysdeps/unix/sysv/linux/riscv/setcontext.S -index 9510518750..e44a68aad4 100644 ---- a/sysdeps/unix/sysv/linux/riscv/setcontext.S -+++ b/sysdeps/unix/sysv/linux/riscv/setcontext.S -@@ -95,6 +95,7 @@ LEAF (__setcontext) - 99: j __syscall_error - - END (__setcontext) -+libc_hidden_def (__setcontext) - weak_alias (__setcontext, setcontext) - - LEAF (__start_context) -@@ -108,7 +109,7 @@ LEAF (__start_context) - /* Invoke subsequent context if present, else exit(0). */ - mv a0, s2 - beqz s2, 1f -- jal __setcontext --1: j exit -+ jal HIDDEN_JUMPTARGET (__setcontext) -+1: j HIDDEN_JUMPTARGET (exit) - - END (__start_context) diff --git a/bench-ci/guix/patches/lief-scikit-0-9.patch b/bench-ci/guix/patches/lief-scikit-0-9.patch deleted file mode 100644 index 71e617834f07..000000000000 --- a/bench-ci/guix/patches/lief-scikit-0-9.patch +++ /dev/null @@ -1,21 +0,0 @@ -Partially revert f23ced2f4ffc170d0a6f40ff4a1bee575e3447cf - -Restore compat with python-scikit-build-core 0.9.x -Can be dropped when using python-scikit-build-core >= 0.10.x - ---- a/api/python/backend/setup.py -+++ b/api/python/backend/setup.py -@@ -101,12 +101,12 @@ def _get_hooked_config(is_editable: bool) -> Optional[dict[str, Union[str, List[ - config_settings = { - "logging.level": "DEBUG", - "build-dir": config.build_dir, -- "build.targets": config.build.targets, - "install.strip": config.strip, - "backport.find-python": "0", - "wheel.py-api": config.build.py_api, - "cmake.source-dir": SRC_DIR.as_posix(), - "cmake.build-type": config.build.build_type, -+ "cmake.targets": config.build.targets, - "cmake.args": [ - *config.cmake_generator, - *config.get_cmake_args(is_editable), diff --git a/bench-ci/guix/patches/oscrypto-hard-code-openssl.patch b/bench-ci/guix/patches/oscrypto-hard-code-openssl.patch deleted file mode 100644 index 32027f2d09af..000000000000 --- a/bench-ci/guix/patches/oscrypto-hard-code-openssl.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/oscrypto/__init__.py b/oscrypto/__init__.py -index eb27313..371ab24 100644 ---- a/oscrypto/__init__.py -+++ b/oscrypto/__init__.py -@@ -302,3 +302,8 @@ def load_order(): - 'oscrypto._win.tls', - 'oscrypto.tls', - ] -+ -+ -+paths = '@GUIX_OSCRYPTO_USE_OPENSSL@'.split(',') -+assert len(paths) == 2, 'Value for OSCRYPTO_USE_OPENSSL env var must be two paths separated by a comma' -+use_openssl(*paths) diff --git a/bench-ci/guix/patches/winpthreads-remap-guix-store.patch b/bench-ci/guix/patches/winpthreads-remap-guix-store.patch deleted file mode 100644 index e1f1a6eba531..000000000000 --- a/bench-ci/guix/patches/winpthreads-remap-guix-store.patch +++ /dev/null @@ -1,17 +0,0 @@ -Without ffile-prefix-map, the debug symbols will contain paths for the -guix store which will include the hashes of each package. However, the -hash for the same package will differ when on different architectures. -In order to be reproducible regardless of the architecture used to build -the package, map all guix store prefixes to something fixed, e.g. /usr. - ---- a/mingw-w64-libraries/winpthreads/Makefile.in -+++ b/mingw-w64-libraries/winpthreads/Makefile.in -@@ -478,7 +478,7 @@ top_build_prefix = @top_build_prefix@ - top_builddir = @top_builddir@ - top_srcdir = @top_srcdir@ - SUBDIRS = . tests --AM_CFLAGS = -Wall -DWIN32_LEAN_AND_MEAN $(am__append_1) -+AM_CFLAGS = -Wall -DWIN32_LEAN_AND_MEAN $(am__append_1) $(shell find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;) - ACLOCAL_AMFLAGS = -I m4 - lib_LTLIBRARIES = libwinpthread.la - include_HEADERS = include/pthread.h include/sched.h include/semaphore.h include/pthread_unistd.h include/pthread_time.h include/pthread_compat.h include/pthread_signal.h diff --git a/bench-ci/guix/security-check.py b/bench-ci/guix/security-check.py deleted file mode 100755 index ac943e33aabd..000000000000 --- a/bench-ci/guix/security-check.py +++ /dev/null @@ -1,297 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2015-2022 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -''' -Perform basic security checks on a series of executables. -Exit status will be 0 if successful, and the program will be silent. -Otherwise the exit status will be 1 and it will log which executables failed which checks. - -Example usage: - - find ../path/to/guix/binaries -type f -executable | xargs python3 contrib/guix/security-check.py -''' -import re -import sys - -import lief - -def check_ELF_RELRO(binary) -> bool: - ''' - Check for read-only relocations. - GNU_RELRO program header must exist - Dynamic section must have BIND_NOW flag - ''' - have_gnu_relro = False - for segment in binary.segments: - # Note: not checking p_flags == PF_R: here as linkers set the permission differently - # This does not affect security: the permission flags of the GNU_RELRO program - # header are ignored, the PT_LOAD header determines the effective permissions. - # However, the dynamic linker need to write to this area so these are RW. - # Glibc itself takes care of mprotecting this area R after relocations are finished. - # See also https://marc.info/?l=binutils&m=1498883354122353 - if segment.type == lief.ELF.Segment.TYPE.GNU_RELRO: - have_gnu_relro = True - - have_bindnow = False - try: - flags = binary.get(lief.ELF.DynamicEntry.TAG.FLAGS) - if flags.has(lief.ELF.DynamicEntryFlags.FLAG.BIND_NOW): - have_bindnow = True - except Exception: - have_bindnow = False - - return have_gnu_relro and have_bindnow - -def check_ELF_CANARY(binary) -> bool: - ''' - Check for use of stack canary - ''' - return binary.has_symbol('__stack_chk_fail') - -def check_ELF_SEPARATE_CODE(binary): - ''' - Check that sections are appropriately separated in virtual memory, - based on their permissions. This checks for missing -Wl,-z,separate-code - and potentially other problems. - ''' - R = lief.ELF.Segment.FLAGS.R - W = lief.ELF.Segment.FLAGS.W - E = lief.ELF.Segment.FLAGS.X - EXPECTED_FLAGS = { - # Read + execute - '.init': R | E, - '.plt': R | E, - '.plt.got': R | E, - '.plt.sec': R | E, - '.text': R | E, - '.fini': R | E, - # Read-only data - '.interp': R, - '.note.gnu.property': R, - '.note.gnu.build-id': R, - '.note.ABI-tag': R, - '.gnu.hash': R, - '.dynsym': R, - '.dynstr': R, - '.gnu.version': R, - '.gnu.version_r': R, - '.rela.dyn': R, - '.rela.plt': R, - '.rodata': R, - '.eh_frame_hdr': R, - '.eh_frame': R, - '.qtmetadata': R, - '.gcc_except_table': R, - '.stapsdt.base': R, - # Writable data - '.init_array': R | W, - '.fini_array': R | W, - '.dynamic': R | W, - '.got': R | W, - '.data': R | W, - '.bss': R | W, - } - if binary.header.machine_type == lief.ELF.ARCH.PPC64: - # .plt is RW on ppc64 even with separate-code - EXPECTED_FLAGS['.plt'] = R | W - # For all LOAD program headers get mapping to the list of sections, - # and for each section, remember the flags of the associated program header. - flags_per_section = {} - for segment in binary.segments: - if segment.type == lief.ELF.Segment.TYPE.LOAD: - for section in segment.sections: - flags_per_section[section.name] = segment.flags - # Spot-check ELF LOAD program header flags per section - # If these sections exist, check them against the expected R/W/E flags - for (section, flags) in flags_per_section.items(): - if section in EXPECTED_FLAGS: - if int(EXPECTED_FLAGS[section]) != int(flags): - return False - return True - -def check_ELF_CONTROL_FLOW(binary) -> bool: - ''' - Check for control flow instrumentation - ''' - main = binary.get_function_address('main') - content = binary.get_content_from_virtual_address(main, 4, lief.Binary.VA_TYPES.AUTO) - - if content.tolist() == [243, 15, 30, 250]: # endbr64 - return True - return False - -def check_ELF_FORTIFY(binary) -> bool: - # no imported fortified funcs if we are fully static - # check could be changed to include all symbols - if binary.header.machine_type == lief.ELF.ARCH.X86_64: - return True - - # bitcoin wrapper does not currently contain any fortified functions - if '--monolithic' in binary.strings: - return True - - chk_funcs = set() - - for sym in binary.imported_symbols: - match = re.search(r'__[a-z]*_chk', sym.name) - if match: - chk_funcs.add(match.group(0)) - - # ignore stack-protector - chk_funcs.discard('__stack_chk') - - return len(chk_funcs) >= 1 - -def check_PE_DYNAMIC_BASE(binary) -> bool: - '''PIE: DllCharacteristics bit 0x40 signifies dynamicbase (ASLR)''' - return lief.PE.OptionalHeader.DLL_CHARACTERISTICS.DYNAMIC_BASE in binary.optional_header.dll_characteristics_lists - -# Must support high-entropy 64-bit address space layout randomization -# in addition to DYNAMIC_BASE to have secure ASLR. -def check_PE_HIGH_ENTROPY_VA(binary) -> bool: - '''PIE: DllCharacteristics bit 0x20 signifies high-entropy ASLR''' - return lief.PE.OptionalHeader.DLL_CHARACTERISTICS.HIGH_ENTROPY_VA in binary.optional_header.dll_characteristics_lists - -def check_PE_RELOC_SECTION(binary) -> bool: - '''Check for a reloc section. This is required for functional ASLR.''' - return binary.has_relocations - -def check_PE_CONTROL_FLOW(binary) -> bool: - ''' - Check for control flow instrumentation - ''' - main = binary.get_symbol('main').value - - section_addr = binary.section_from_rva(main).virtual_address - virtual_address = binary.optional_header.imagebase + section_addr + main - - content = binary.get_content_from_virtual_address(virtual_address, 4, lief.Binary.VA_TYPES.VA) - - if content.tolist() == [243, 15, 30, 250]: # endbr64 - return True - return False - -def check_PE_CANARY(binary) -> bool: - ''' - Check for use of stack canary - ''' - return binary.has_symbol('__stack_chk_fail') - -def check_MACHO_NOUNDEFS(binary) -> bool: - ''' - Check for no undefined references. - ''' - return binary.header.has(lief.MachO.Header.FLAGS.NOUNDEFS) - -def check_MACHO_FIXUP_CHAINS(binary) -> bool: - ''' - Check for use of chained fixups. - ''' - return binary.has_dyld_chained_fixups - -def check_MACHO_CANARY(binary) -> bool: - ''' - Check for use of stack canary - ''' - return binary.has_symbol('___stack_chk_fail') - -def check_PIE(binary) -> bool: - ''' - Check for position independent executable (PIE), - allowing for address space randomization. - ''' - return binary.is_pie - -def check_NX(binary) -> bool: - ''' - Check for no stack execution - ''' - - # binary.has_nx checks are only for the stack, but MachO binaries might - # have executable heaps. - if binary.format == lief.Binary.FORMATS.MACHO: - return binary.concrete.has_nx_stack and binary.concrete.has_nx_heap - else: - return binary.has_nx - -def check_MACHO_CONTROL_FLOW(binary) -> bool: - ''' - Check for control flow instrumentation - ''' - content = binary.get_content_from_virtual_address(binary.entrypoint, 4, lief.Binary.VA_TYPES.AUTO) - - if content.tolist() == [243, 15, 30, 250]: # endbr64 - return True - return False - -def check_MACHO_BRANCH_PROTECTION(binary) -> bool: - ''' - Check for branch protection instrumentation - ''' - content = binary.get_content_from_virtual_address(binary.entrypoint, 4, lief.Binary.VA_TYPES.AUTO) - - if content.tolist() == [95, 36, 3, 213]: # bti - return True - return False - -BASE_ELF = [ - ('FORTIFY', check_ELF_FORTIFY), - ('PIE', check_PIE), - ('NX', check_NX), - ('RELRO', check_ELF_RELRO), - ('CANARY', check_ELF_CANARY), - ('SEPARATE_CODE', check_ELF_SEPARATE_CODE), -] - -BASE_PE = [ - ('PIE', check_PIE), - ('DYNAMIC_BASE', check_PE_DYNAMIC_BASE), - ('HIGH_ENTROPY_VA', check_PE_HIGH_ENTROPY_VA), - ('NX', check_NX), - ('RELOC_SECTION', check_PE_RELOC_SECTION), - ('CONTROL_FLOW', check_PE_CONTROL_FLOW), - ('CANARY', check_PE_CANARY), -] - -BASE_MACHO = [ - ('NOUNDEFS', check_MACHO_NOUNDEFS), - ('CANARY', check_MACHO_CANARY), - ('FIXUP_CHAINS', check_MACHO_FIXUP_CHAINS), -] - -CHECKS = { - lief.Binary.FORMATS.ELF: { - lief.Header.ARCHITECTURES.X86_64: BASE_ELF + [('CONTROL_FLOW', check_ELF_CONTROL_FLOW)], - lief.Header.ARCHITECTURES.ARM: BASE_ELF, - lief.Header.ARCHITECTURES.ARM64: BASE_ELF, - lief.Header.ARCHITECTURES.PPC64: BASE_ELF, - lief.Header.ARCHITECTURES.RISCV: BASE_ELF, - }, - lief.Binary.FORMATS.PE: { - lief.Header.ARCHITECTURES.X86_64: BASE_PE, - }, - lief.Binary.FORMATS.MACHO: { - lief.Header.ARCHITECTURES.X86_64: BASE_MACHO + [('PIE', check_PIE), - ('NX', check_NX), - ('CONTROL_FLOW', check_MACHO_CONTROL_FLOW)], - lief.Header.ARCHITECTURES.ARM64: BASE_MACHO + [('BRANCH_PROTECTION', check_MACHO_BRANCH_PROTECTION)], - } -} - -if __name__ == '__main__': - retval: int = 0 - for filename in sys.argv[1:]: - binary = lief.parse(filename) - - etype = binary.format - arch = binary.abstract.header.architecture - - failed: list[str] = [] - for (name, func) in CHECKS[etype][arch]: - if not func(binary): - failed.append(name) - if failed: - print(f'{filename}: failed {" ".join(failed)}') - retval = 1 - sys.exit(retval) diff --git a/bench-ci/guix/symbol-check.py b/bench-ci/guix/symbol-check.py deleted file mode 100755 index 3d7a654c8589..000000000000 --- a/bench-ci/guix/symbol-check.py +++ /dev/null @@ -1,338 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2014 Wladimir J. van der Laan -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -''' -A script to check that release executables only contain certain symbols -and are only linked against allowed libraries. - -Example usage: - - find ../path/to/guix/binaries -type f -executable | xargs python3 contrib/guix/symbol-check.py -''' -import sys - -import lief - -# Debian 11 (Bullseye) EOL: 2026. https://wiki.debian.org/LTS -# -# - libgcc version 10.2.1 (https://packages.debian.org/bullseye/libgcc-s1) -# - libc version 2.31 (https://packages.debian.org/source/bullseye/glibc) -# -# Ubuntu 20.04 (Focal) EOL: 2030. https://wiki.ubuntu.com/ReleaseTeam -# -# - libgcc version 10.5.0 (https://packages.ubuntu.com/focal/libgcc1) -# - libc version 2.31 (https://packages.ubuntu.com/focal/libc6) -# -# CentOS Stream 9 EOL: 2027. https://www.centos.org/cl-vs-cs/#end-of-life -# -# - libgcc version 12.2.1 (https://mirror.stream.centos.org/9-stream/AppStream/x86_64/os/Packages/) -# - libc version 2.34 (https://mirror.stream.centos.org/9-stream/AppStream/x86_64/os/Packages/) -# -# See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html for more info. - -MAX_VERSIONS = { -'GCC': (7,0,0), -'GLIBC': { - lief.ELF.ARCH.X86_64: (0,0), - lief.ELF.ARCH.ARM: (2,31), - lief.ELF.ARCH.AARCH64:(2,31), - lief.ELF.ARCH.PPC64: (2,31), - lief.ELF.ARCH.RISCV: (2,31), -}, -'LIBATOMIC': (1,0), -'V': (0,5,0), # xkb (bitcoin-qt only) -} - -# Ignore symbols that are exported as part of every executable -IGNORE_EXPORTS = { -'environ', '_environ', '__environ', '_fini', '_init', 'stdin', -'stdout', 'stderr', '__libc_single_threaded', -} - -# Expected linker-loader names can be found here: -# https://sourceware.org/glibc/wiki/ABIList?action=recall&rev=16 -ELF_INTERPRETER_NAMES: dict[lief.ELF.ARCH, dict[lief.Header.ENDIANNESS, str]] = { - lief.ELF.ARCH.X86_64: { - lief.Header.ENDIANNESS.LITTLE: "", - }, - lief.ELF.ARCH.ARM: { - lief.Header.ENDIANNESS.LITTLE: "/lib/ld-linux-armhf.so.3", - }, - lief.ELF.ARCH.AARCH64: { - lief.Header.ENDIANNESS.LITTLE: "/lib/ld-linux-aarch64.so.1", - }, - lief.ELF.ARCH.PPC64: { - lief.Header.ENDIANNESS.BIG: "/lib64/ld64.so.1", - lief.Header.ENDIANNESS.LITTLE: "/lib64/ld64.so.2", - }, - lief.ELF.ARCH.RISCV: { - lief.Header.ENDIANNESS.LITTLE: "/lib/ld-linux-riscv64-lp64d.so.1", - }, -} - -ELF_ABIS: dict[lief.ELF.ARCH, dict[lief.Header.ENDIANNESS, list[int]]] = { - lief.ELF.ARCH.X86_64: { - lief.Header.ENDIANNESS.LITTLE: [3,2,0], - }, - lief.ELF.ARCH.ARM: { - lief.Header.ENDIANNESS.LITTLE: [3,2,0], - }, - lief.ELF.ARCH.AARCH64: { - lief.Header.ENDIANNESS.LITTLE: [3,7,0], - }, - lief.ELF.ARCH.PPC64: { - lief.Header.ENDIANNESS.LITTLE: [3,10,0], - lief.Header.ENDIANNESS.BIG: [3,2,0], - }, - lief.ELF.ARCH.RISCV: { - lief.Header.ENDIANNESS.LITTLE: [4,15,0], - }, -} - -# Allowed NEEDED libraries -ELF_ALLOWED_LIBRARIES = { -# bitcoind and bitcoin-qt -'libgcc_s.so.1', # GCC base support -'libc.so.6', # C library -'libpthread.so.0', # threading -'libm.so.6', # math library -'libatomic.so.1', -'ld-linux.so.2', # 32-bit dynamic linker -'ld-linux-aarch64.so.1', # 64-bit ARM dynamic linker -'ld-linux-armhf.so.3', # 32-bit ARM dynamic linker -'ld64.so.1', # POWER64 ABIv1 dynamic linker -'ld64.so.2', # POWER64 ABIv2 dynamic linker -'ld-linux-riscv64-lp64d.so.1', # 64-bit RISC-V dynamic linker -# bitcoin-qt only -'libxcb.so.1', # part of X11 -'libxkbcommon.so.0', # keyboard keymapping -'libxkbcommon-x11.so.0', # keyboard keymapping -'libfontconfig.so.1', # font support -'libfreetype.so.6', # font parsing -'libdl.so.2', # programming interface to dynamic linker -'libxcb-icccm.so.4', -'libxcb-image.so.0', -'libxcb-shm.so.0', -'libxcb-keysyms.so.1', -'libxcb-randr.so.0', -'libxcb-render-util.so.0', -'libxcb-render.so.0', -'libxcb-shape.so.0', -'libxcb-sync.so.1', -'libxcb-xfixes.so.0', -'libxcb-xkb.so.1', -} - -MACHO_ALLOWED_LIBRARIES = { -# bitcoind and bitcoin-qt -'libc++.1.dylib', # C++ Standard Library -'libSystem.B.dylib', # libc, libm, libpthread, libinfo -# bitcoin-qt only -'AppKit', # user interface -'ApplicationServices', # common application tasks. -'Carbon', # deprecated c back-compat API -'ColorSync', -'CoreFoundation', # low level func, data types -'CoreGraphics', # 2D rendering -'CoreServices', # operating system services -'CoreText', # interface for laying out text and handling fonts. -'CoreVideo', # video processing -'Foundation', # base layer functionality for apps/frameworks -'ImageIO', # read and write image file formats. -'IOKit', # user-space access to hardware devices and drivers. -'IOSurface', # cross process image/drawing buffers -'libobjc.A.dylib', # Objective-C runtime library -'Metal', # 3D graphics -'QuartzCore', # animation -'Security', # access control and authentication -'UniformTypeIdentifiers', # collection of types that map to MIME and file types -} - -PE_ALLOWED_LIBRARIES = { -'ADVAPI32.dll', # legacy security & registry -'bcrypt.dll', # newer security and identity API -'IPHLPAPI.DLL', # IP helper API -'KERNEL32.dll', # win32 base APIs -'msvcrt.dll', # C standard library for MSVC -'SHELL32.dll', # shell API -'WS2_32.dll', # sockets -# bitcoin-qt only -'api-ms-win-core-synch-l1-2-0.dll', # Synchronization Primitives API -'api-ms-win-core-winrt-l1-1-0.dll', # Windows Runtime API -'api-ms-win-core-winrt-string-l1-1-0.dll', # WinRT String API -'AUTHZ.dll', # Windows Authorization Framework -'comdlg32.dll', # Common Dialog Box Library -'d3d11.dll', # Direct3D 11 API -'d3d12.dll', # Direct3D 12 API -'d3d9.dll', # Direct3D 9 API -'dwmapi.dll', # desktop window manager -'DWrite.dll', # DirectX Typography Services -'dxgi.dll', # DirectX Graphics Infrastructure -'GDI32.dll', # graphics device interface -'IMM32.dll', # input method editor -'NETAPI32.dll', # network management -'ole32.dll', # component object model -'OLEAUT32.dll', # OLE Automation API -'SHLWAPI.dll', # light weight shell API -'USER32.dll', # user interface -'USERENV.dll', # user management -'UxTheme.dll', # visual style -'VERSION.dll', # version checking -'WINMM.dll', # WinMM audio API -'WTSAPI32.dll', # Remote Desktop -'SETUPAPI.dll', # Windows Setup API -'SHCORE.dll', # Stream Handler Core -} - -def check_version(max_versions, version, arch) -> bool: - (lib, _, ver) = version.rpartition('_') - ver = tuple([int(x) for x in ver.split('.')]) - if not lib in max_versions: - return False - if isinstance(max_versions[lib], tuple): - return ver <= max_versions[lib] - else: - return ver <= max_versions[lib][arch] - -def check_imported_symbols(binary) -> bool: - ok: bool = True - - for symbol in binary.imported_symbols: - if not symbol.imported: - continue - - version = symbol.symbol_version if symbol.has_version else None - - if version: - aux_version = version.symbol_version_auxiliary.name if version.has_auxiliary_version else None - if aux_version and not check_version(MAX_VERSIONS, aux_version, binary.header.machine_type): - print(f'{filename}: symbol {symbol.name} from unsupported version {version}') - ok = False - return ok - -def check_exported_symbols(binary) -> bool: - ok: bool = True - - for symbol in binary.dynamic_symbols: - if not symbol.exported: - continue - name = symbol.name - if binary.header.machine_type == lief.ELF.ARCH.RISCV or name in IGNORE_EXPORTS: - continue - print(f'{filename}: export of symbol {name} not allowed!') - ok = False - return ok - -def check_RUNPATH(binary) -> bool: - assert binary.get(lief.ELF.DynamicEntry.TAG.RUNPATH) is None - assert binary.get(lief.ELF.DynamicEntry.TAG.RPATH) is None - return True - -def check_ELF_libraries(binary) -> bool: - ok: bool = True - - if binary.header.machine_type == lief.ELF.ARCH.X86_64: - return len(binary.libraries) == 0 - - for library in binary.libraries: - if library not in ELF_ALLOWED_LIBRARIES: - print(f'{filename}: {library} is not in ALLOWED_LIBRARIES!') - ok = False - return ok - -def check_MACHO_libraries(binary) -> bool: - ok: bool = True - for dylib in binary.libraries: - split = dylib.name.split('/') - if split[-1] not in MACHO_ALLOWED_LIBRARIES: - print(f'{split[-1]} is not in ALLOWED_LIBRARIES!') - ok = False - return ok - -def check_MACHO_min_os(binary) -> bool: - if binary.build_version.minos == [14,0,0]: - return True - return False - -def check_MACHO_sdk(binary) -> bool: - if binary.build_version.sdk == [14, 0, 0]: - return True - return False - -def check_MACHO_lld(binary) -> bool: - if binary.build_version.tools[0].version == [19, 1, 4]: - return True - return False - -def check_PE_libraries(binary) -> bool: - ok: bool = True - for dylib in binary.libraries: - if dylib not in PE_ALLOWED_LIBRARIES: - print(f'{dylib} is not in ALLOWED_LIBRARIES!') - ok = False - return ok - -def check_PE_subsystem_version(binary) -> bool: - major: int = binary.optional_header.major_subsystem_version - minor: int = binary.optional_header.minor_subsystem_version - if major == 6 and minor == 2: - return True - return False - -def check_PE_application_manifest(binary) -> bool: - if not binary.has_resources: - # No resources at all. - return False - - rm = binary.resources_manager - return rm.has_manifest - -def check_ELF_interpreter(binary) -> bool: - expected_interpreter = ELF_INTERPRETER_NAMES[binary.header.machine_type][binary.abstract.header.endianness] - - return binary.concrete.interpreter == expected_interpreter - -def check_ELF_ABI(binary) -> bool: - expected_abi = ELF_ABIS[binary.header.machine_type][binary.abstract.header.endianness] - note = binary.concrete.get(lief.ELF.Note.TYPE.GNU_ABI_TAG) - assert note.abi == lief.ELF.NoteAbi.ABI.LINUX - return note.version == expected_abi - -CHECKS = { -lief.Binary.FORMATS.ELF: [ - ('IMPORTED_SYMBOLS', check_imported_symbols), - ('EXPORTED_SYMBOLS', check_exported_symbols), - ('LIBRARY_DEPENDENCIES', check_ELF_libraries), - ('INTERPRETER_NAME', check_ELF_interpreter), - ('ABI', check_ELF_ABI), - ('RUNPATH', check_RUNPATH), -], -lief.Binary.FORMATS.MACHO: [ - ('DYNAMIC_LIBRARIES', check_MACHO_libraries), - ('MIN_OS', check_MACHO_min_os), - ('SDK', check_MACHO_sdk), - ('LLD', check_MACHO_lld), -], -lief.Binary.FORMATS.PE: [ - ('DYNAMIC_LIBRARIES', check_PE_libraries), - ('SUBSYSTEM_VERSION', check_PE_subsystem_version), - ('APPLICATION_MANIFEST', check_PE_application_manifest), -] -} - -if __name__ == '__main__': - retval: int = 0 - for filename in sys.argv[1:]: - binary = lief.parse(filename) - - etype = binary.format - - failed: list[str] = [] - for (name, func) in CHECKS[etype]: - if not func(binary): - failed.append(name) - if failed: - print(f'{filename}: failed {" ".join(failed)}') - retval = 1 - sys.exit(retval) diff --git a/flake.nix b/flake.nix index f769290ebca5..e06fb057b38d 100644 --- a/flake.nix +++ b/flake.nix @@ -62,7 +62,7 @@ "-DWITH_ZMQ=OFF" ]; in - pkgs.stdenv.mkDerivation { + pkgs.ccacheStdenv.mkDerivation { inherit pname version @@ -149,10 +149,11 @@ default = pkgs.mkShell { buildInputs = [ # Benchmarking - pkgs.cargo-flamegraph + cargo-flamegraph pkgs.flamegraph pkgs.hyperfine pkgs.jq + pkgs.just pkgs.perf pkgs.perf-tools pkgs.util-linux diff --git a/justfile b/justfile index 86dbbab91115..5b32a5d7bf85 100644 --- a/justfile +++ b/justfile @@ -7,10 +7,9 @@ default: # Build base and head binaries for CI [group('ci')] -build-assumeutxo-binaries-guix base_commit head_commit: +build-binaries base_commit head_commit: #!/usr/bin/env bash set -euxo pipefail - unset SOURCE_DATE_EPOCH # needed to run on NixOS ./bench-ci/build_binaries.sh {{ base_commit }} {{ head_commit }} # Run uninstrumented benchmarks on mainnet diff --git a/shell.nix b/shell.nix deleted file mode 100644 index 35a9fdbf49c2..000000000000 --- a/shell.nix +++ /dev/null @@ -1,101 +0,0 @@ -# Copyright 0xB10C, willcl-ark -{ pkgs ? import - (fetchTarball "https://github.com/nixos/nixpkgs/archive/nixos-25.11.tar.gz") - { }, }: -let - inherit (pkgs.lib) optionals strings; - inherit (pkgs) stdenv; - - # Override the default cargo-flamegraph with a custom fork - cargo-flamegraph = pkgs.rustPlatform.buildRustPackage rec { - pname = - "flamegraph"; # Match the name in Cargo.toml, doesn't seem to work otherwise - version = "bitcoin-core"; - - src = pkgs.fetchFromGitHub { - owner = "willcl-ark"; - repo = "flamegraph"; - rev = "bitcoin-core"; - sha256 = "sha256-tQbr3MYfAiOxeT12V9au5KQK5X5JeGuV6p8GR/Sgen4="; - }; - - doCheck = false; - cargoHash = "sha256-QWPqTyTFSZNJNayNqLmsQSu0rX26XBKfdLROZ9tRjrg="; - - useFetchCargoVendor = true; - - nativeBuildInputs = - pkgs.lib.optionals stdenv.hostPlatform.isLinux [ pkgs.makeWrapper ]; - buildInputs = pkgs.lib.optionals stdenv.hostPlatform.isDarwin - [ pkgs.darwin.apple_sdk.frameworks.Security ]; - - postFixup = pkgs.lib.optionalString stdenv.hostPlatform.isLinux '' - wrapProgram $out/bin/cargo-flamegraph \ - --set-default PERF ${pkgs.linuxPackages.perf}/bin/perf - wrapProgram $out/bin/flamegraph \ - --set-default PERF ${pkgs.linuxPackages.perf}/bin/perf - ''; - }; - -in pkgs.mkShell { - nativeBuildInputs = with pkgs; [ - autoconf - automake - boost - ccache - clang_18 - cmake - libevent - libtool - pkg-config - sqlite - zeromq - ]; - buildInputs = with pkgs; [ - just - bash - git - shellcheck - python310 - uv - - # Benchmarking - cargo-flamegraph - flamegraph - hyperfine - jq - linuxKernel.packages.linux_6_6.perf - perf-tools - util-linux - - # Binary patching - patchelf - - # Guix - curl - getent - ]; - - shellHook = '' - echo "Bitcoin Core build nix-shell" - echo "" - echo "Setting up python venv" - - # fixes libstdc++ issues and libgl.so issues - export LD_LIBRARY_PATH=${stdenv.cc.cc.lib}/lib/:$LD_LIBRARY_PATH - - uv venv --python 3.10 - source .venv/bin/activate - uv pip install -r pyproject.toml - - patch-binary() { - if [ -z "$1" ]; then - echo "Usage: patch-binary " - return 1 - fi - patchelf --set-interpreter "$(cat $NIX_CC/nix-support/dynamic-linker)" "$1" - } - echo "Added patch-binary command" - echo " Usage: 'patch-binary '" - ''; -} From 39a9a6dcc2167a8ab6b2c09013a48a5d878e78d6 Mon Sep 17 00:00:00 2001 From: will Date: Mon, 8 Dec 2025 10:58:35 +0000 Subject: [PATCH 30/51] disable ccache for now --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index e06fb057b38d..3d0c2fa657b4 100644 --- a/flake.nix +++ b/flake.nix @@ -62,7 +62,7 @@ "-DWITH_ZMQ=OFF" ]; in - pkgs.ccacheStdenv.mkDerivation { + pkgs.stdenv.mkDerivation { inherit pname version From 31836d548b48005d8e3ec4c41b8063a339b62b05 Mon Sep 17 00:00:00 2001 From: will Date: Mon, 8 Dec 2025 11:19:19 +0000 Subject: [PATCH 31/51] increase build priority --- bench-ci/build_binaries.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench-ci/build_binaries.sh b/bench-ci/build_binaries.sh index 242456250d8d..4c42e5157a7e 100755 --- a/bench-ci/build_binaries.sh +++ b/bench-ci/build_binaries.sh @@ -25,7 +25,7 @@ for build in "base:${base_commit}" "head:${head_commit}"; do name="${build%%:*}" commit="${build#*:}" git checkout "$commit" - taskset -c 2-15 chrt -f 1 nix build -L + taskset -c 0-15 nix build -L cp "./result/bin/bitcoind" "./binaries/${name}/bitcoind" rm -rf "./result" done From c728ee9057a822e3fb6a3390afd0b2346d4634b2 Mon Sep 17 00:00:00 2001 From: will Date: Mon, 8 Dec 2025 12:04:31 +0000 Subject: [PATCH 32/51] use python runner --- .github/workflows/benchmark.yml | 141 ++++--- bench-ci/build_binaries.sh | 38 -- bench-ci/parse_and_plot.py | 247 ----------- bench-ci/prelude.sh | 78 ---- bench-ci/run-benchmark-instrumented.sh | 91 ---- bench-ci/run-benchmark.sh | 64 --- bench.py | 559 +++++++++++++++++++++++++ bench.toml | 30 ++ bench/README.md | 237 +++++++++++ bench/__init__.py | 3 + bench/analyze.py | 535 +++++++++++++++++++++++ bench/benchmark.py | 351 ++++++++++++++++ bench/build.py | 172 ++++++++ bench/capabilities.py | 162 +++++++ bench/config.py | 208 +++++++++ bench/report.py | 453 ++++++++++++++++++++ bench/utils.py | 259 ++++++++++++ flake.nix | 18 +- justfile | 93 +++- 19 files changed, 3130 insertions(+), 609 deletions(-) delete mode 100755 bench-ci/build_binaries.sh delete mode 100755 bench-ci/parse_and_plot.py delete mode 100644 bench-ci/prelude.sh delete mode 100755 bench-ci/run-benchmark-instrumented.sh delete mode 100755 bench-ci/run-benchmark.sh create mode 100755 bench.py create mode 100644 bench.toml create mode 100644 bench/README.md create mode 100644 bench/__init__.py create mode 100644 bench/analyze.py create mode 100644 bench/benchmark.py create mode 100644 bench/build.py create mode 100644 bench/capabilities.py create mode 100644 bench/config.py create mode 100644 bench/report.py create mode 100644 bench/utils.py diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 65e667e88fe1..46aa545e8cf5 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -3,6 +3,7 @@ on: pull_request: branches: - master + jobs: build-binaries: runs-on: [self-hosted, linux, x64] @@ -13,159 +14,175 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 1 + - name: Fetch base commit run: | - echo "CHECKOUT_COMMIT=$(git rev-parse HEAD)" >> "$GITHUB_ENV" + echo "HEAD_SHA=$(git rev-parse HEAD)" >> "$GITHUB_ENV" git fetch --depth=1 origin ${{ github.event.pull_request.base.sha }} + - name: Build both binaries - env: - CCACHE_DIR: /nix/var/cache/ccache run: | - mkdir -p ${{ runner.temp }}/binaries/base - mkdir -p ${{ runner.temp }}/binaries/head - nix develop --command bash -c ' - just build-binaries $BASE_SHA $CHECKOUT_COMMIT - cp binaries/base/bitcoind ${{ runner.temp }}/binaries/base/bitcoind - cp binaries/head/bitcoind ${{ runner.temp }}/binaries/head/bitcoind - ' + nix develop --command python3 bench.py build \ + --binaries-dir ${{ runner.temp }}/binaries \ + $BASE_SHA $HEAD_SHA + - name: Upload binaries uses: actions/upload-artifact@v4 with: name: bitcoind-binaries path: ${{ runner.temp }}/binaries/ + uninstrumented: needs: build-binaries strategy: matrix: include: - - network: mainnet - name: mainnet-default-uninstrumented + - name: mainnet-default-uninstrumented timeout: 600 - datadir_path: /data/pruned-840k dbcache: 450 - - network: mainnet - name: mainnet-large-uninstrumented + - name: mainnet-large-uninstrumented timeout: 600 - datadir_path: /data/pruned-840k dbcache: 32000 runs-on: [self-hosted, linux, x64] timeout-minutes: ${{ matrix.timeout }} env: - ORIGINAL_DATADIR: ${{ matrix.datadir_path }} + ORIGINAL_DATADIR: /data/pruned-840k BASE_SHA: ${{ github.event.pull_request.base.sha }} steps: - name: Checkout repo uses: actions/checkout@v4 with: fetch-depth: 1 + - name: Download binaries uses: actions/download-artifact@v4 with: name: bitcoind-binaries path: ${{ runner.temp }}/binaries + - name: Set binary permissions run: | chmod +x ${{ runner.temp }}/binaries/base/bitcoind chmod +x ${{ runner.temp }}/binaries/head/bitcoind + - name: Fetch base commit run: | - echo "CHECKOUT_COMMIT=$(git rev-parse HEAD)" >> "$GITHUB_ENV" + echo "HEAD_SHA=$(git rev-parse HEAD)" >> "$GITHUB_ENV" git fetch --depth=1 origin ${{ github.event.pull_request.base.sha }} - - name: Run AssumeUTXO ${{ matrix.network }} - env: - TMP_DATADIR: "${{ runner.temp }}/base_datadir" - BINARIES_DIR: "${{ runner.temp }}/binaries" + + - name: Run benchmark run: | - mkdir -p "$TMP_DATADIR" - nix develop --command just run-${{ matrix.network }}-ci $BASE_SHA $CHECKOUT_COMMIT $TMP_DATADIR $ORIGINAL_DATADIR ${{ runner.temp }}/results.json ${{ matrix.dbcache }} $BINARIES_DIR - - uses: actions/upload-artifact@v4 + nix develop --command python3 bench.py run \ + --profile ci \ + --binaries-dir ${{ runner.temp }}/binaries \ + --datadir $ORIGINAL_DATADIR \ + --tmp-datadir ${{ runner.temp }}/datadir \ + --output-dir ${{ runner.temp }}/output \ + --dbcache ${{ matrix.dbcache }} \ + $BASE_SHA $HEAD_SHA + + - name: Upload results + uses: actions/upload-artifact@v4 with: name: result-${{ matrix.name }} - path: "${{ runner.temp }}/results.json" - - name: Write GitHub and runner context files + path: ${{ runner.temp }}/output/results.json + + - name: Write context metadata env: GITHUB_CONTEXT: ${{ toJSON(github) }} RUNNER_CONTEXT: ${{ toJSON(runner) }} run: | - mkdir -p contexts - nix develop --command bash -c ' - echo "$GITHUB_CONTEXT" | jq "del(.token)" > contexts/github.json - echo "$RUNNER_CONTEXT" > contexts/runner.json - ' - - name: Upload context metadata as artifact + mkdir -p ${{ runner.temp }}/contexts + echo "$GITHUB_CONTEXT" | jq "del(.token)" > ${{ runner.temp }}/contexts/github.json + echo "$RUNNER_CONTEXT" > ${{ runner.temp }}/contexts/runner.json + + - name: Upload context metadata uses: actions/upload-artifact@v4 with: name: run-metadata-${{ matrix.name }} - path: ./contexts/ + path: ${{ runner.temp }}/contexts/ + instrumented: needs: build-binaries strategy: matrix: include: - - network: mainnet - name: mainnet-default-instrumented + - name: mainnet-default-instrumented timeout: 600 - datadir_path: /data/pruned-840k dbcache: 450 - - network: mainnet - name: mainnet-large-instrumented + - name: mainnet-large-instrumented timeout: 600 - datadir_path: /data/pruned-840k dbcache: 32000 runs-on: [self-hosted, linux, x64] timeout-minutes: ${{ matrix.timeout }} env: - ORIGINAL_DATADIR: ${{ matrix.datadir_path }} + ORIGINAL_DATADIR: /data/pruned-840k BASE_SHA: ${{ github.event.pull_request.base.sha }} steps: - name: Checkout repo uses: actions/checkout@v4 with: fetch-depth: 1 + - name: Download binaries uses: actions/download-artifact@v4 with: name: bitcoind-binaries path: ${{ runner.temp }}/binaries + - name: Set binary permissions run: | chmod +x ${{ runner.temp }}/binaries/base/bitcoind chmod +x ${{ runner.temp }}/binaries/head/bitcoind + - name: Fetch base commit run: | - echo "CHECKOUT_COMMIT=$(git rev-parse HEAD)" >> "$GITHUB_ENV" + echo "HEAD_SHA=$(git rev-parse HEAD)" >> "$GITHUB_ENV" git fetch --depth=1 origin ${{ github.event.pull_request.base.sha }} - - name: Run AssumeUTXO ${{ matrix.network }} - env: - TMP_DATADIR: "${{ runner.temp }}/base_datadir" - BINARIES_DIR: "${{ runner.temp }}/binaries" + + - name: Run instrumented benchmark run: | - mkdir -p "$TMP_DATADIR" - nix develop --command just run-${{ matrix.network }}-ci-instrumented $BASE_SHA $CHECKOUT_COMMIT $TMP_DATADIR $ORIGINAL_DATADIR ${{ runner.temp }}/results.json ${{ matrix.dbcache }} ${{ runner.temp }}/pngs $BINARIES_DIR - - uses: actions/upload-artifact@v4 + nix develop --command python3 bench.py run \ + --profile ci \ + --instrumented \ + --binaries-dir ${{ runner.temp }}/binaries \ + --datadir $ORIGINAL_DATADIR \ + --tmp-datadir ${{ runner.temp }}/datadir \ + --output-dir ${{ runner.temp }}/output \ + --dbcache ${{ matrix.dbcache }} \ + $BASE_SHA $HEAD_SHA + + - name: Upload results + uses: actions/upload-artifact@v4 with: name: result-${{ matrix.name }} - path: "${{ runner.temp }}/results.json" - - uses: actions/upload-artifact@v4 + path: ${{ runner.temp }}/output/results.json + + - name: Upload plots + uses: actions/upload-artifact@v4 with: name: pngs-${{ matrix.name }} - path: "${{ runner.temp }}/pngs/*.png" - - uses: actions/upload-artifact@v4 + path: ${{ runner.temp }}/output/plots/*.png + if-no-files-found: ignore + + - name: Upload flamegraphs + uses: actions/upload-artifact@v4 with: name: flamegraph-${{ matrix.name }} - path: "**/*-flamegraph.svg" - - name: Write GitHub and runner context files + path: ${{ runner.temp }}/output/*-flamegraph.svg + if-no-files-found: ignore + + - name: Write context metadata env: GITHUB_CONTEXT: ${{ toJSON(github) }} RUNNER_CONTEXT: ${{ toJSON(runner) }} run: | - mkdir -p contexts - nix develop --command bash -c ' - echo "$GITHUB_CONTEXT" | jq "del(.token)" > contexts/github.json - echo "$RUNNER_CONTEXT" > contexts/runner.json - ' - - name: Upload context metadata as artifact + mkdir -p ${{ runner.temp }}/contexts + echo "$GITHUB_CONTEXT" | jq "del(.token)" > ${{ runner.temp }}/contexts/github.json + echo "$RUNNER_CONTEXT" > ${{ runner.temp }}/contexts/runner.json + + - name: Upload context metadata uses: actions/upload-artifact@v4 with: name: run-metadata-${{ matrix.name }} - path: ./contexts/ + path: ${{ runner.temp }}/contexts/ diff --git a/bench-ci/build_binaries.sh b/bench-ci/build_binaries.sh deleted file mode 100755 index 4c42e5157a7e..000000000000 --- a/bench-ci/build_binaries.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env bash -set -euxo pipefail - -if [ $# -ne 2 ]; then - echo "Usage: $0 " - exit 1 -fi - -# Save current state of git -initial_ref=$(git symbolic-ref -q HEAD || git rev-parse HEAD) -if git symbolic-ref -q HEAD >/dev/null; then - initial_state="branch" - initial_branch=${initial_ref#refs/heads/} -else - initial_state="detached" -fi - -base_commit="$1" -head_commit="$2" - -mkdir -p binaries/base -mkdir -p binaries/head - -for build in "base:${base_commit}" "head:${head_commit}"; do - name="${build%%:*}" - commit="${build#*:}" - git checkout "$commit" - taskset -c 0-15 nix build -L - cp "./result/bin/bitcoind" "./binaries/${name}/bitcoind" - rm -rf "./result" -done - -# Restore initial git state -if [ "$initial_state" = "branch" ]; then - git checkout "$initial_branch" -else - git checkout "$initial_ref" -fi diff --git a/bench-ci/parse_and_plot.py b/bench-ci/parse_and_plot.py deleted file mode 100755 index 2a8a112cc4cf..000000000000 --- a/bench-ci/parse_and_plot.py +++ /dev/null @@ -1,247 +0,0 @@ -#!/usr/bin/env python3 -import sys -import os -import re -import datetime -import matplotlib.pyplot as plt -from collections import OrderedDict - - -def parse_updatetip_line(line): - match = re.match( - r'^([\d\-:TZ]+) UpdateTip: new best.+height=(\d+).+tx=(\d+).+cache=([\d.]+)MiB\((\d+)txo\)', - line - ) - if not match: - return None - iso_str, height_str, tx_str, cache_size_mb_str, cache_coins_count_str = match.groups() - parsed_datetime = datetime.datetime.strptime(iso_str, "%Y-%m-%dT%H:%M:%SZ") - return parsed_datetime, int(height_str), int(tx_str), float(cache_size_mb_str), int(cache_coins_count_str) - - -def parse_leveldb_compact_line(line): - match = re.match(r'^([\d\-:TZ]+) \[leveldb] Compacting.*files', line) - if not match: - return None - iso_str = match.groups()[0] - parsed_datetime = datetime.datetime.strptime(iso_str, "%Y-%m-%dT%H:%M:%SZ") - return parsed_datetime - - -def parse_leveldb_generated_table_line(line): - match = re.match(r'^([\d\-:TZ]+) \[leveldb] Generated table.*: (\d+) keys, (\d+) bytes', line) - if not match: - return None - iso_str, keys_count_str, bytes_count_str = match.groups() - parsed_datetime = datetime.datetime.strptime(iso_str, "%Y-%m-%dT%H:%M:%SZ") - return parsed_datetime, int(keys_count_str), int(bytes_count_str) - - -def parse_validation_txadd_line(line): - match = re.match(r'^([\d\-:TZ]+) \[validation] TransactionAddedToMempool: txid=.+wtxid=.+', line) - if not match: - return None - iso_str = match.groups()[0] - parsed_datetime = datetime.datetime.strptime(iso_str, "%Y-%m-%dT%H:%M:%SZ") - return parsed_datetime - - -def parse_coindb_write_batch_line(line): - match = re.match(r'^([\d\-:TZ]+) \[coindb] Writing (partial|final) batch of ([\d.]+) MiB', line) - if not match: - return None - iso_str, is_partial_str, size_mb_str = match.groups() - parsed_datetime = datetime.datetime.strptime(iso_str, "%Y-%m-%dT%H:%M:%SZ") - return parsed_datetime, is_partial_str, float(size_mb_str) - - -def parse_coindb_commit_line(line): - match = re.match(r'^([\d\-:TZ]+) \[coindb] Committed (\d+) changed transaction outputs', line) - if not match: - return None - iso_str, txout_count_str = match.groups() - parsed_datetime = datetime.datetime.strptime(iso_str, "%Y-%m-%dT%H:%M:%SZ") - return parsed_datetime, int(txout_count_str) - - -def parse_log_file(log_file): - with open(log_file, 'r', encoding='utf-8') as f: - update_tip_data = [] - leveldb_compact_data = [] - leveldb_gen_table_data = [] - validation_txadd_data = [] - coindb_write_batch_data = [] - coindb_commit_data = [] - - for line in f: - if result := parse_updatetip_line(line): - update_tip_data.append(result) - elif result := parse_leveldb_compact_line(line): - leveldb_compact_data.append(result) - elif result := parse_leveldb_generated_table_line(line): - leveldb_gen_table_data.append(result) - elif result := parse_validation_txadd_line(line): - validation_txadd_data.append(result) - elif result := parse_coindb_write_batch_line(line): - coindb_write_batch_data.append(result) - elif result := parse_coindb_commit_line(line): - coindb_commit_data.append(result) - - if not update_tip_data: - print("No UpdateTip entries found.") - sys.exit(0) - - assert all(update_tip_data[i][0] <= update_tip_data[i + 1][0] for i in - range(len(update_tip_data) - 1)), "UpdateTip entries are not sorted by time" - - return update_tip_data, leveldb_compact_data, leveldb_gen_table_data, validation_txadd_data, coindb_write_batch_data, coindb_commit_data - - -def generate_plot(x, y, x_label, y_label, title, output_file, is_height_based=False): - if not x or not y: - print(f"Skipping plot '{title}' as there is no data.") - return - - plt.figure(figsize=(30, 10)) - plt.plot(x, y) - plt.title(title, fontsize=20) - plt.xlabel(x_label, fontsize=16) - plt.ylabel(y_label, fontsize=16) - plt.grid(True) - - # Make sure the x-axis covers the full data range - min_x, max_x = min(x), max(x) - plt.xlim(min_x, max_x) - - # Add vertical lines for major protocol upgrades if this is a height-based plot - if is_height_based: - # Define all notable heights from the chainparams file - fork_heights = OrderedDict([ - ('BIP34', 227931), # Block v2, coinbase includes height - ('BIP66', 363725), # Strict DER signatures - ('BIP65', 388381), # OP_CHECKLOCKTIMEVERIFY - ('CSV', 419328), # BIP68, 112, 113 - OP_CHECKSEQUENCEVERIFY - ('Segwit', 481824), # BIP141, 143, 144, 145 - Segregated Witness - ('Taproot', 709632), # BIP341, 342 - Schnorr signatures & Taproot - ('Halving 1', 210000), # First halving - ('Halving 2', 420000), # Second halving - ('Halving 3', 630000), # Third halving - ('Halving 4', 840000), # Fourth halving - ]) - - # Colors for the different types of events - fork_colors = { - 'BIP34': 'blue', - 'BIP66': 'blue', - 'BIP65': 'blue', - 'CSV': 'blue', - 'Segwit': 'green', - 'Taproot': 'red', - 'Halving 1': 'purple', - 'Halving 2': 'purple', - 'Halving 3': 'purple', - 'Halving 4': 'purple', - } - - # Line styles for different types of events - fork_styles = { - 'BIP34': '--', - 'BIP66': '--', - 'BIP65': '--', - 'CSV': '--', - 'Segwit': '--', - 'Taproot': '--', - 'Halving 1': ':', - 'Halving 2': ':', - 'Halving 3': ':', - 'Halving 4': ':', - } - - max_y = max(y) - - # Position text labels at different heights to avoid overlap - text_positions = {} - position_increment = max_y * 0.05 - current_position = max_y * 0.9 - - # Add lines for forks that are in range - for fork_name, height in fork_heights.items(): - if min_x <= height <= max_x: - plt.axvline(x=height, color=fork_colors[fork_name], - linestyle=fork_styles[fork_name]) - - # Avoid label overlaps by staggering vertical positions - if height in text_positions: - # If this x position already has a label, adjust position - text_positions[height] -= position_increment - else: - text_positions[height] = current_position - current_position -= position_increment - if current_position < max_y * 0.1: - current_position = max_y * 0.9 # Reset if we're too low - - plt.text(height, text_positions[height], f'{fork_name} ({height})', - rotation=90, verticalalignment='top', - color=fork_colors[fork_name]) - - plt.xticks(rotation=90, fontsize=12) - plt.yticks(fontsize=12) - plt.tight_layout() - plt.savefig(output_file) - plt.close() - print(f"Saved plot to {output_file}") - - -if __name__ == "__main__": - if len(sys.argv) != 4: - print(f"Usage: {sys.argv[0]} ") - sys.exit(1) - - commit = sys.argv[1] - - log_file = sys.argv[2] - if not os.path.isfile(log_file): - print(f"File not found: {log_file}") - sys.exit(1) - - png_dir = sys.argv[3] - os.makedirs(png_dir, exist_ok=True) - - update_tip_data, leveldb_compact_data, leveldb_gen_table_data, validation_txadd_data, coindb_write_batch_data, coindb_commit_data = parse_log_file(log_file) - times, heights, tx_counts, cache_size, cache_count = zip(*update_tip_data) - float_minutes = [(t - times[0]).total_seconds() / 60 for t in times] - - generate_plot(float_minutes, heights, "Elapsed minutes", "Block Height", "Block Height vs Time", os.path.join(png_dir, f"{commit}-height_vs_time.png")) - generate_plot(heights, cache_size, "Block Height", "Cache Size (MiB)", "Cache Size vs Block Height", os.path.join(png_dir, f"{commit}-cache_vs_height.png"), is_height_based=True) - generate_plot(float_minutes, cache_size, "Elapsed minutes", "Cache Size (MiB)", "Cache Size vs Time", os.path.join(png_dir, f"{commit}-cache_vs_time.png")) - generate_plot(heights, tx_counts, "Block Height", "Transaction Count", "Transactions vs Block Height", os.path.join(png_dir, f"{commit}-tx_vs_height.png"), is_height_based=True) - generate_plot(heights, cache_count, "Block Height", "Coins Cache Size", "Coins Cache Size vs Height", os.path.join(png_dir, f"{commit}-coins_cache_vs_height.png"), is_height_based=True) - - # LevelDB Compaction and Generated Tables - if leveldb_compact_data: - leveldb_compact_times = [(t - times[0]).total_seconds() / 60 for t in leveldb_compact_data] - leveldb_compact_y = [1 for _ in leveldb_compact_times] # dummy y axis to mark compactions - generate_plot(leveldb_compact_times, leveldb_compact_y, "Elapsed minutes", "LevelDB Compaction", "LevelDB Compaction Events vs Time", os.path.join(png_dir, f"{commit}-leveldb_compact_vs_time.png")) - if leveldb_gen_table_data: - leveldb_gen_table_times, leveldb_gen_table_keys, leveldb_gen_table_bytes = zip(*leveldb_gen_table_data) - leveldb_gen_table_float_minutes = [(t - times[0]).total_seconds() / 60 for t in leveldb_gen_table_times] - generate_plot(leveldb_gen_table_float_minutes, leveldb_gen_table_keys, "Elapsed minutes", "Number of keys", "LevelDB Keys Generated vs Time", os.path.join(png_dir, f"{commit}-leveldb_gen_keys_vs_time.png")) - generate_plot(leveldb_gen_table_float_minutes, leveldb_gen_table_bytes, "Elapsed minutes", "Number of bytes", "LevelDB Bytes Generated vs Time", os.path.join(png_dir, f"{commit}-leveldb_gen_bytes_vs_time.png")) - - # validation mempool add transaction lines - if validation_txadd_data: - validation_txadd_times = [(t - times[0]).total_seconds() / 60 for t in validation_txadd_data] - validation_txadd_y = [1 for _ in validation_txadd_times] # dummy y axis to mark transaction additions - generate_plot(validation_txadd_times, validation_txadd_y, "Elapsed minutes", "Transaction Additions", "Transaction Additions to Mempool vs Time", os.path.join(png_dir, f"{commit}-validation_txadd_vs_time.png")) - - # coindb write batch lines - if coindb_write_batch_data: - coindb_write_batch_times, is_partial_strs, sizes_mb = zip(*coindb_write_batch_data) - coindb_write_batch_float_minutes = [(t - times[0]).total_seconds() / 60 for t in coindb_write_batch_times] - generate_plot(coindb_write_batch_float_minutes, sizes_mb, "Elapsed minutes", "Batch Size MiB", "Coin Database Partial/Final Write Batch Size vs Time", os.path.join(png_dir, f"{commit}-coindb_write_batch_size_vs_time.png")) - if coindb_commit_data: - coindb_commit_times, txout_counts = zip(*coindb_commit_data) - coindb_commit_float_minutes = [(t - times[0]).total_seconds() / 60 for t in coindb_commit_times] - generate_plot(coindb_commit_float_minutes, txout_counts, "Elapsed minutes", "Transaction Output Count", "Coin Database Transaction Output Committed vs Time", os.path.join(png_dir, f"{commit}-coindb_commit_txout_vs_time.png")) - - print("Plots saved!") diff --git a/bench-ci/prelude.sh b/bench-ci/prelude.sh deleted file mode 100644 index 98a5232b40fb..000000000000 --- a/bench-ci/prelude.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env bash -# Shared functions - -set -euxo pipefail - -clean_datadir() { - set -euxo pipefail - - local TMP_DATADIR="$1" - - mkdir -p "${TMP_DATADIR}" - - # If we're in CI, clean without confirmation - if [ -n "${CI:-}" ]; then - rm -Rf "${TMP_DATADIR:?}"/* - else - read -rp "Are you sure you want to delete everything in ${TMP_DATADIR}? [y/N] " response - if [[ "$response" =~ ^[Yy]$ ]]; then - rm -Rf "${TMP_DATADIR:?}"/* - else - echo "Aborting..." - exit 1 - fi - fi -} - -clean_logs() { - set -euxo pipefail - - local TMP_DATADIR="$1" - local logfile="${TMP_DATADIR}/debug.log" - - echo "Checking for ${logfile}" - if [ -e "${logfile}" ]; then - echo "Removing ${logfile}" - rm "${logfile}" - fi -} - -# Executes once before each *set* of timing runs. -setup_run() { - set -euxo pipefail - - local TMP_DATADIR="$1" - clean_datadir "${TMP_DATADIR}" -} - -# Executes before each timing run. -prepare_run() { - set -euxo pipefail - - local TMP_DATADIR="$1" - local ORIGINAL_DATADIR="$2" - - clean_datadir "${TMP_DATADIR}" - # Don't copy hidden files so use * - taskset -c 0-15 cp -r "$ORIGINAL_DATADIR"/* "$TMP_DATADIR" - # Clear page caches - /run/wrappers/bin/drop-caches - clean_logs "${TMP_DATADIR}" -} - -# Executes after the completion of all benchmarking runs for each individual -# command to be benchmarked. -cleanup_run() { - set -euxo pipefail - local TMP_DATADIR="$1" - clean_datadir "${TMP_DATADIR}" -} - -# Export all shared functions for use by hyperfine subshells -export_shared_functions() { - export -f clean_datadir - export -f clean_logs - export -f setup_run - export -f prepare_run - export -f cleanup_run -} diff --git a/bench-ci/run-benchmark-instrumented.sh b/bench-ci/run-benchmark-instrumented.sh deleted file mode 100755 index 1f4be0b0b2dd..000000000000 --- a/bench-ci/run-benchmark-instrumented.sh +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env bash - -set -euxo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "${SCRIPT_DIR}/prelude.sh" - -# Executed after each timing run - generates plots and handles flamegraph -conclude_run() { - set -euxo pipefail - - local commit="$1" - local TMP_DATADIR="$2" - local PNG_DIR="$3" - - # Search in subdirs e.g. $datadir/signet - debug_log=$(find "${TMP_DATADIR}" -name debug.log -print -quit) - if [ -n "${debug_log}" ]; then - echo "Generating plots from ${debug_log}" - if [ -x "bench-ci/parse_and_plot.py" ]; then - bench-ci/parse_and_plot.py "${commit}" "${debug_log}" "${PNG_DIR}" - else - ls -al "bench-ci/" - echo "parse_and_plot.py not found or not executable, skipping plot generation" - fi - else - ls -al "${TMP_DATADIR}/" - echo "debug.log not found, skipping plot generation" - fi - - # Move flamegraph if exists - if [ -e flamegraph.svg ]; then - mv flamegraph.svg "${commit}"-flamegraph.svg - fi -} - -run_benchmark() { - local base_commit="$1" - local head_commit="$2" - local TMP_DATADIR="$3" - local ORIGINAL_DATADIR="$4" - local results_file="$5" - local png_dir="$6" - local chain="$7" - local stop_at_height="$8" - local connect_address="$9" - local dbcache="${10}" - local BINARIES_DIR="${11}" - - # Export functions so they can be used by hyperfine - export_shared_functions - export -f conclude_run - - # Debug: Print all variables being used - echo "=== Debug Information ===" - echo "TMP_DATADIR: ${TMP_DATADIR}" - echo "ORIGINAL_DATADIR: ${ORIGINAL_DATADIR}" - echo "BINARIES_DIR: ${BINARIES_DIR}" - echo "base_commit: ${base_commit}" - echo "head_commit: ${head_commit}" - echo "results_file: ${results_file}" - echo "png_dir: ${png_dir}" - echo "chain: ${chain}" - echo "stop_at_height: ${stop_at_height}" - echo "connect_address: ${connect_address}" - echo "dbcache: ${dbcache}" - printf '\n' - - # Run hyperfine - hyperfine \ - --shell=bash \ - --setup "setup_run ${TMP_DATADIR}" \ - --prepare "prepare_run ${TMP_DATADIR} ${ORIGINAL_DATADIR}" \ - --conclude "conclude_run {commit} ${TMP_DATADIR} ${png_dir}" \ - --cleanup "cleanup_run ${TMP_DATADIR}" \ - --runs 1 \ - --export-json "${results_file}" \ - --show-output \ - --command-name "base (${base_commit})" \ - --command-name "head (${head_commit})" \ - "taskset -c 1 flamegraph --palette bitcoin --title 'bitcoind IBD@{commit}' -c 'record -F 101 --call-graph fp' -- taskset -c 2-15 chrt -o 0 ${BINARIES_DIR}/{commit}/bitcoind -datadir=${TMP_DATADIR} -connect=${connect_address} -daemon=0 -prune=10000 -chain=${chain} -stopatheight=${stop_at_height} -dbcache=${dbcache} -printtoconsole=0 -debug=coindb -debug=leveldb -debug=bench -debug=validation" \ - -L commit "base,head" -} - -# Main execution -if [ "$#" -ne 11 ]; then - echo "Usage: $0 base_commit head_commit TMP_DATADIR ORIGINAL_DATADIR results_dir png_dir chain stop_at_height connect_address dbcache BINARIES_DIR" - exit 1 -fi - -run_benchmark "$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8" "$9" "${10}" "${11}" diff --git a/bench-ci/run-benchmark.sh b/bench-ci/run-benchmark.sh deleted file mode 100755 index bce7857919ac..000000000000 --- a/bench-ci/run-benchmark.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env bash - -set -euxo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "${SCRIPT_DIR}/prelude.sh" - -# Executed after each timing run (no-op for uninstrumented) -conclude_run() { - set -euxo pipefail - return 0 -} - -run_benchmark() { - local base_commit="$1" - local head_commit="$2" - local TMP_DATADIR="$3" - local ORIGINAL_DATADIR="$4" - local results_file="$5" - local chain="$6" - local stop_at_height="$7" - local connect_address="$8" - local dbcache="${9}" - local BINARIES_DIR="${10}" - - # Export functions so they can be used by hyperfine - export_shared_functions - - # Debug: Print all variables being used - echo "=== Debug Information ===" - echo "TMP_DATADIR: ${TMP_DATADIR}" - echo "ORIGINAL_DATADIR: ${ORIGINAL_DATADIR}" - echo "BINARIES_DIR: ${BINARIES_DIR}" - echo "base_commit: ${base_commit}" - echo "head_commit: ${head_commit}" - echo "results_file: ${results_file}" - echo "chain: ${chain}" - echo "stop_at_height: ${stop_at_height}" - echo "connect_address: ${connect_address}" - echo "dbcache: ${dbcache}" - printf '\n' - - # Run hyperfine - hyperfine \ - --shell=bash \ - --setup "setup_run ${TMP_DATADIR}" \ - --prepare "prepare_run ${TMP_DATADIR} ${ORIGINAL_DATADIR}" \ - --cleanup "cleanup_run ${TMP_DATADIR}" \ - --runs 3 \ - --export-json "${results_file}" \ - --show-output \ - --command-name "base (${base_commit})" \ - --command-name "head (${head_commit})" \ - "taskset -c 2-15 chrt -o 0 ${BINARIES_DIR}/{commit}/bitcoind -datadir=${TMP_DATADIR} -connect=${connect_address} -daemon=0 -prune=10000 -chain=${chain} -stopatheight=${stop_at_height} -dbcache=${dbcache} -printtoconsole=0" \ - -L commit "base,head" -} - -# Main execution -if [ "$#" -ne 10 ]; then - echo "Usage: $0 base_commit head_commit TMP_DATADIR ORIGINAL_DATADIR results_dir chain stop_at_height connect_address dbcache BINARIES_DIR" - exit 1 -fi - -run_benchmark "$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8" "${9}" "${10}" diff --git a/bench.py b/bench.py new file mode 100755 index 000000000000..072f4ef95fc4 --- /dev/null +++ b/bench.py @@ -0,0 +1,559 @@ +#!/usr/bin/env python3 +"""Benchcoin - Bitcoin Core benchmarking toolkit. + +A unified CLI for building, benchmarking, analyzing, and reporting +on Bitcoin Core performance. + +Usage: + bench.py build BASE HEAD Build bitcoind at two commits + bench.py run BASE HEAD Run benchmark + bench.py analyze LOGFILE Generate plots from debug.log + bench.py report INPUT OUTPUT Generate HTML report + bench.py full BASE HEAD Complete pipeline: build → run → analyze +""" + +from __future__ import annotations + +import argparse +import logging +import sys +from pathlib import Path + +from bench.capabilities import detect_capabilities +from bench.config import build_config + +# Setup logging +logging.basicConfig( + level=logging.INFO, + format="%(levelname)s: %(message)s", +) +logger = logging.getLogger(__name__) + + +def cmd_build(args: argparse.Namespace) -> int: + """Build bitcoind at two commits.""" + from bench.build import BuildPhase + + capabilities = detect_capabilities() + config = build_config( + cli_args={ + "binaries_dir": args.binaries_dir, + "skip_existing": args.skip_existing, + "no_cpu_pinning": args.no_cpu_pinning, + "dry_run": args.dry_run, + "verbose": args.verbose, + }, + config_file=Path(args.config) if args.config else None, + profile=args.profile, + ) + + if args.verbose: + logging.getLogger().setLevel(logging.DEBUG) + + phase = BuildPhase(config, capabilities) + + try: + result = phase.run( + args.base_commit, + args.head_commit, + binaries_dir=Path(args.binaries_dir) if args.binaries_dir else None, + ) + logger.info(f"Built base binary: {result.base_binary}") + logger.info(f"Built head binary: {result.head_binary}") + return 0 + except Exception as e: + logger.error(f"Build failed: {e}") + return 1 + + +def cmd_run(args: argparse.Namespace) -> int: + """Run benchmark comparing two commits.""" + from bench.benchmark import BenchmarkPhase + + capabilities = detect_capabilities() + config = build_config( + cli_args={ + "datadir": args.datadir, + "tmp_datadir": args.tmp_datadir, + "binaries_dir": args.binaries_dir, + "output_dir": args.output_dir, + "stop_height": args.stop_height, + "dbcache": args.dbcache, + "runs": args.runs, + "connect": args.connect, + "chain": args.chain, + "instrumented": args.instrumented, + "no_cpu_pinning": args.no_cpu_pinning, + "no_cache_drop": args.no_cache_drop, + "dry_run": args.dry_run, + "verbose": args.verbose, + }, + config_file=Path(args.config) if args.config else None, + profile=args.profile, + ) + + if args.verbose: + logging.getLogger().setLevel(logging.DEBUG) + + # Validate config + errors = config.validate() + if errors: + for error in errors: + logger.error(error) + return 1 + + # Check binaries exist + binaries_dir = ( + Path(args.binaries_dir) if args.binaries_dir else Path(config.binaries_dir) + ) + base_binary = binaries_dir / "base" / "bitcoind" + head_binary = binaries_dir / "head" / "bitcoind" + + if not base_binary.exists(): + logger.error(f"Base binary not found: {base_binary}") + logger.error("Run 'bench.py build' first") + return 1 + + if not head_binary.exists(): + logger.error(f"Head binary not found: {head_binary}") + logger.error("Run 'bench.py build' first") + return 1 + + phase = BenchmarkPhase(config, capabilities) + + try: + result = phase.run( + base_commit=args.base_commit, + head_commit=args.head_commit, + base_binary=base_binary, + head_binary=head_binary, + datadir=Path(config.datadir), + output_dir=Path(config.output_dir), + ) + logger.info(f"Results saved to: {result.results_file}") + return 0 + except Exception as e: + logger.error(f"Benchmark failed: {e}") + if args.verbose: + import traceback + + traceback.print_exc() + return 1 + + +def cmd_analyze(args: argparse.Namespace) -> int: + """Generate plots from debug.log.""" + from bench.analyze import AnalyzePhase + + if args.verbose: + logging.getLogger().setLevel(logging.DEBUG) + + log_file = Path(args.log_file) + output_dir = Path(args.output_dir) + + if not log_file.exists(): + logger.error(f"Log file not found: {log_file}") + return 1 + + phase = AnalyzePhase() + + try: + result = phase.run( + commit=args.commit, + log_file=log_file, + output_dir=output_dir, + ) + logger.info(f"Generated {len(result.plots)} plots in {result.output_dir}") + return 0 + except Exception as e: + logger.error(f"Analysis failed: {e}") + if args.verbose: + import traceback + + traceback.print_exc() + return 1 + + +def cmd_report(args: argparse.Namespace) -> int: + """Generate HTML report from benchmark results.""" + from bench.report import ReportPhase + + if args.verbose: + logging.getLogger().setLevel(logging.DEBUG) + + input_dir = Path(args.input_dir) + output_dir = Path(args.output_dir) + + if not input_dir.exists(): + logger.error(f"Input directory not found: {input_dir}") + return 1 + + phase = ReportPhase() + + try: + result = phase.run( + input_dir=input_dir, + output_dir=output_dir, + title=args.title or "Benchmark Results", + ) + + # Print speedups + if result.speedups: + logger.info("Speedups:") + for network, speedup in result.speedups.items(): + sign = "+" if speedup > 0 else "" + logger.info(f" {network}: {sign}{speedup}%") + + return 0 + except Exception as e: + logger.error(f"Report generation failed: {e}") + if args.verbose: + import traceback + + traceback.print_exc() + return 1 + + +def cmd_full(args: argparse.Namespace) -> int: + """Run full pipeline: build → run → analyze.""" + from bench.analyze import AnalyzePhase + from bench.benchmark import BenchmarkPhase + from bench.build import BuildPhase + from bench.utils import find_debug_log + + capabilities = detect_capabilities() + config = build_config( + cli_args={ + "datadir": args.datadir, + "tmp_datadir": args.tmp_datadir, + "binaries_dir": args.binaries_dir, + "output_dir": args.output_dir, + "stop_height": args.stop_height, + "dbcache": args.dbcache, + "runs": args.runs, + "connect": args.connect, + "chain": args.chain, + "instrumented": args.instrumented, + "skip_existing": args.skip_existing, + "no_cpu_pinning": args.no_cpu_pinning, + "no_cache_drop": args.no_cache_drop, + "dry_run": args.dry_run, + "verbose": args.verbose, + }, + config_file=Path(args.config) if args.config else None, + profile=args.profile, + ) + + if args.verbose: + logging.getLogger().setLevel(logging.DEBUG) + + # Validate config + errors = config.validate() + if errors: + for error in errors: + logger.error(error) + return 1 + + output_dir = Path(config.output_dir) + binaries_dir = Path(config.binaries_dir) + + # Phase 1: Build + logger.info("=== Phase 1: Build ===") + build_phase = BuildPhase(config, capabilities) + + try: + build_result = build_phase.run( + args.base_commit, + args.head_commit, + binaries_dir=binaries_dir, + ) + except Exception as e: + logger.error(f"Build failed: {e}") + return 1 + + # Phase 2: Benchmark + logger.info("=== Phase 2: Benchmark ===") + benchmark_phase = BenchmarkPhase(config, capabilities) + + try: + benchmark_result = benchmark_phase.run( + base_commit=build_result.base_commit, + head_commit=build_result.head_commit, + base_binary=build_result.base_binary, + head_binary=build_result.head_binary, + datadir=Path(config.datadir), + output_dir=output_dir, + ) + except Exception as e: + logger.error(f"Benchmark failed: {e}") + return 1 + + # Phase 3: Analyze (for instrumented runs) + if config.instrumented: + logger.info("=== Phase 3: Analyze ===") + analyze_phase = AnalyzePhase(config) + + # Analyze base debug log + if benchmark_result.debug_log_base: + try: + analyze_phase.run( + commit=build_result.base_commit, + log_file=benchmark_result.debug_log_base, + output_dir=output_dir / "plots", + ) + except Exception as e: + logger.warning(f"Analysis for base failed: {e}") + + # Analyze head debug log + if benchmark_result.debug_log_head: + try: + analyze_phase.run( + commit=build_result.head_commit, + log_file=benchmark_result.debug_log_head, + output_dir=output_dir / "plots", + ) + except Exception as e: + logger.warning(f"Analysis for head failed: {e}") + + logger.info("=== Complete ===") + logger.info(f"Results: {benchmark_result.results_file}") + return 0 + + +def main() -> int: + """Main entry point.""" + parser = argparse.ArgumentParser( + description="Benchcoin - Bitcoin Core benchmarking toolkit", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__doc__, + ) + + # Global options + parser.add_argument( + "--config", + metavar="PATH", + help="Config file (default: bench.toml)", + ) + parser.add_argument( + "--profile", + choices=["quick", "full", "ci"], + default="full", + help="Configuration profile (default: full)", + ) + parser.add_argument( + "-v", + "--verbose", + action="store_true", + help="Verbose output", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Show what would be done without executing", + ) + + subparsers = parser.add_subparsers(dest="command", help="Commands") + + # Build command + build_parser = subparsers.add_parser("build", help="Build bitcoind at two commits") + build_parser.add_argument("base_commit", help="Base commit (for comparison)") + build_parser.add_argument("head_commit", help="Head commit (new code)") + build_parser.add_argument( + "--binaries-dir", + metavar="PATH", + help="Where to store binaries (default: ./binaries)", + ) + build_parser.add_argument( + "--skip-existing", + action="store_true", + help="Skip build if binary already exists", + ) + build_parser.add_argument( + "--no-cpu-pinning", + action="store_true", + help="Disable CPU affinity", + ) + build_parser.set_defaults(func=cmd_build) + + # Run command + run_parser = subparsers.add_parser("run", help="Run benchmark") + run_parser.add_argument("base_commit", help="Base commit hash") + run_parser.add_argument("head_commit", help="Head commit hash") + run_parser.add_argument( + "--datadir", + required=True, + metavar="PATH", + help="Source datadir with blockchain snapshot", + ) + run_parser.add_argument( + "--tmp-datadir", + metavar="PATH", + help="Temp datadir for benchmark runs", + ) + run_parser.add_argument( + "--binaries-dir", + metavar="PATH", + help="Location of pre-built binaries", + ) + run_parser.add_argument( + "--output-dir", + metavar="PATH", + help="Output directory for results", + ) + run_parser.add_argument( + "--stop-height", + type=int, + metavar="N", + help="Block height to stop at", + ) + run_parser.add_argument( + "--dbcache", + type=int, + metavar="N", + help="Database cache size in MB", + ) + run_parser.add_argument( + "--runs", + type=int, + metavar="N", + help="Number of benchmark iterations", + ) + run_parser.add_argument( + "--connect", + metavar="ADDR", + help="Connect address for sync", + ) + run_parser.add_argument( + "--chain", + choices=["main", "testnet", "signet", "regtest"], + help="Chain to use", + ) + run_parser.add_argument( + "--instrumented", + action="store_true", + help="Enable profiling (flamegraph + debug logging)", + ) + run_parser.add_argument( + "--no-cpu-pinning", + action="store_true", + help="Disable CPU affinity and scheduler priority", + ) + run_parser.add_argument( + "--no-cache-drop", + action="store_true", + help="Skip cache dropping between runs", + ) + run_parser.set_defaults(func=cmd_run) + + # Analyze command + analyze_parser = subparsers.add_parser( + "analyze", help="Generate plots from debug.log" + ) + analyze_parser.add_argument("commit", help="Commit hash (for naming)") + analyze_parser.add_argument("log_file", help="Path to debug.log") + analyze_parser.add_argument( + "--output-dir", + default="./plots", + metavar="PATH", + help="Output directory for plots", + ) + analyze_parser.set_defaults(func=cmd_analyze) + + # Report command + report_parser = subparsers.add_parser("report", help="Generate HTML report") + report_parser.add_argument("input_dir", help="Directory with results.json") + report_parser.add_argument("output_dir", help="Output directory for report") + report_parser.add_argument( + "--title", + help="Report title", + ) + report_parser.set_defaults(func=cmd_report) + + # Full command + full_parser = subparsers.add_parser( + "full", help="Full pipeline: build → run → analyze" + ) + full_parser.add_argument("base_commit", help="Base commit (for comparison)") + full_parser.add_argument("head_commit", help="Head commit (new code)") + full_parser.add_argument( + "--datadir", + required=True, + metavar="PATH", + help="Source datadir with blockchain snapshot", + ) + full_parser.add_argument( + "--tmp-datadir", + metavar="PATH", + help="Temp datadir for benchmark runs", + ) + full_parser.add_argument( + "--binaries-dir", + metavar="PATH", + help="Where to store binaries", + ) + full_parser.add_argument( + "--output-dir", + metavar="PATH", + help="Output directory for results", + ) + full_parser.add_argument( + "--stop-height", + type=int, + metavar="N", + help="Block height to stop at", + ) + full_parser.add_argument( + "--dbcache", + type=int, + metavar="N", + help="Database cache size in MB", + ) + full_parser.add_argument( + "--runs", + type=int, + metavar="N", + help="Number of benchmark iterations", + ) + full_parser.add_argument( + "--connect", + metavar="ADDR", + help="Connect address for sync", + ) + full_parser.add_argument( + "--chain", + choices=["main", "testnet", "signet", "regtest"], + help="Chain to use", + ) + full_parser.add_argument( + "--instrumented", + action="store_true", + help="Enable profiling (flamegraph + debug logging)", + ) + full_parser.add_argument( + "--skip-existing", + action="store_true", + help="Skip build if binary already exists", + ) + full_parser.add_argument( + "--no-cpu-pinning", + action="store_true", + help="Disable CPU affinity and scheduler priority", + ) + full_parser.add_argument( + "--no-cache-drop", + action="store_true", + help="Skip cache dropping between runs", + ) + full_parser.set_defaults(func=cmd_full) + + args = parser.parse_args() + + if not args.command: + parser.print_help() + return 1 + + return args.func(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bench.toml b/bench.toml new file mode 100644 index 000000000000..0513f4a0abf5 --- /dev/null +++ b/bench.toml @@ -0,0 +1,30 @@ +# Benchcoin configuration +# Values here override built-in defaults but are overridden by environment +# variables (BENCH_*) and CLI arguments. + +[defaults] +chain = "main" +dbcache = 450 +stop_height = 855000 +runs = 3 +# connect = "" # Empty or omit to use public P2P network + +[paths] +binaries_dir = "./binaries" +output_dir = "./bench-output" + +# Profiles override specific defaults +# Usage: bench.py --profile quick full HEAD~1 HEAD + +[profiles.quick] +stop_height = 1100 +runs = 3 + +[profiles.full] +stop_height = 855000 +runs = 3 + +[profiles.ci] +stop_height = 855000 +runs = 3 +connect = "148.251.128.115:33333" diff --git a/bench/README.md b/bench/README.md new file mode 100644 index 000000000000..9cf11ac9df4b --- /dev/null +++ b/bench/README.md @@ -0,0 +1,237 @@ +# Benchcoin + +A CLI for benchmarking Bitcoin Core IBD. + +## Quick Start + +```bash +# Quick smoke test on signet (requires nix) +nix develop --command python3 bench.py --profile quick full \ + --chain signet --datadir /path/to/signet/datadir HEAD~1 HEAD + +# Or use just (wraps nix develop) +just quick HEAD~1 HEAD /path/to/signet/datadir +``` + +## Requirements + +- **Nix** with flakes enabled (provides hyperfine, flamegraph, etc.) +- A blockchain datadir snapshot to benchmark against +- Two git commits to compare + +Optional (auto-detected, gracefully degrades without): +- `/run/wrappers/bin/drop-caches` (NixOS) - clears page cache between runs +- `taskset` / `chrt` - CPU affinity and scheduler priority + +## Commands + +``` +bench.py [GLOBAL_OPTIONS] COMMAND [OPTIONS] ARGS + +Global Options: + --profile {quick,full,ci} Configuration profile + --config PATH Custom config file + -v, --verbose Verbose output + --dry-run Show what would run + +Commands: + build Build bitcoind at two commits + run Run benchmark (requires pre-built binaries) + analyze Generate plots from debug.log + report Generate HTML report + full Complete pipeline: build → run → analyze +``` + +### build + +Build bitcoind binaries at two commits for comparison: + +```bash +python3 bench.py build HEAD~1 HEAD +python3 bench.py build --binaries-dir /tmp/bins abc123 def456 +python3 bench.py build --skip-existing HEAD~1 HEAD # reuse existing +``` + +### run + +Run hyperfine benchmark comparing two pre-built binaries: + +```bash +python3 bench.py run --datadir /data/snapshot HEAD~1 HEAD +python3 bench.py run --instrumented --datadir /data/snapshot HEAD~1 HEAD +``` + +Options: +- `--datadir PATH` - Source blockchain snapshot (required) +- `--tmp-datadir PATH` - Working directory (default: ./bench-output/tmp-datadir) +- `--stop-height N` - Block height to sync to +- `--dbcache N` - Database cache in MB +- `--runs N` - Number of iterations (default: 3, forced to 1 if instrumented) +- `--instrumented` - Enable flamegraph profiling and debug logging +- `--connect ADDR` - P2P node to sync from (empty = public network) +- `--chain {main,signet,testnet,regtest}` - Which chain +- `--no-cpu-pinning` - Disable taskset/chrt +- `--no-cache-drop` - Don't clear page cache between runs + +### analyze + +Generate plots from a debug.log file: + +```bash +python3 bench.py analyze abc123 /path/to/debug.log --output-dir ./plots +``` + +Generates PNG plots for: +- Block height vs time +- Cache size vs height/time +- Transaction count vs height +- LevelDB compaction events +- CoinDB write batches + +### report + +Generate HTML report from benchmark results: + +```bash +python3 bench.py report ./bench-output ./report +``` + +### full + +Run complete pipeline (build + run + analyze if instrumented): + +```bash +python3 bench.py --profile quick full --chain signet --datadir /tmp/signet HEAD~1 HEAD +python3 bench.py --profile full full --datadir /data/mainnet HEAD~1 HEAD +``` + +## Profiles + +Profiles set sensible defaults for common scenarios: + +| Profile | stop_height | runs | dbcache | connect | +|---------|-------------|------|---------|---------| +| quick | 1,500 | 1 | 450 | (public network) | +| full | 855,000 | 3 | 450 | (public network) | +| ci | 855,000 | 3 | 450 | 148.251.128.115:33333 | + +Override any profile setting with CLI flags: + +```bash +python3 bench.py --profile quick full --stop-height 5000 --datadir ... HEAD~1 HEAD +``` + +## Configuration + +Configuration is layered (lowest to highest priority): + +1. Built-in defaults +2. `bench.toml` (in repo root) +3. Environment variables (`BENCH_DATADIR`, `BENCH_DBCACHE`, etc.) +4. CLI arguments + +### bench.toml + +```toml +[defaults] +chain = "main" +dbcache = 450 +stop_height = 855000 +runs = 3 + +[paths] +binaries_dir = "./binaries" +output_dir = "./bench-output" + +[profiles.quick] +stop_height = 1500 +runs = 1 +dbcache = 450 + +[profiles.ci] +connect = "148.251.128.115:33333" +``` + +### Environment Variables + +```bash +export BENCH_DATADIR=/data/snapshot +export BENCH_DBCACHE=1000 +export BENCH_STOP_HEIGHT=100000 +``` + +## Justfile Recipes + +The justfile wraps common operations with `nix develop`: + +```bash +just quick HEAD~1 HEAD /path/to/datadir # Quick signet test +just full HEAD~1 HEAD /path/to/datadir # Full mainnet benchmark +just instrumented HEAD~1 HEAD /path/to/datadir # With flamegraphs +just build HEAD~1 HEAD # Build only +just run HEAD~1 HEAD /path/to/datadir # Run only (binaries must exist) +``` + +## Architecture + +``` +bench.py CLI entry point (argparse) +bench/ +├── config.py Layered configuration (TOML + env + CLI) +├── capabilities.py System capability detection +├── build.py Build phase (nix build) +├── benchmark.py Benchmark phase (hyperfine) +├── analyze.py Plot generation (matplotlib) +├── report.py HTML report generation +└── utils.py Git operations, datadir management +``` + +### Capability Detection + +The tool auto-detects system capabilities and gracefully degrades: + +```python +from bench.capabilities import detect_capabilities +caps = detect_capabilities() +# caps.has_hyperfine, caps.can_drop_caches, caps.can_pin_cpu, etc. +``` + +Missing optional features emit warnings but don't fail: + +``` +WARNING: drop-caches not available - cache won't be cleared between runs +WARNING: taskset not available - CPU affinity won't be set +``` + +Missing required features (hyperfine, flamegraph for instrumented) cause errors. + +### Hyperfine Integration + +The benchmark phase generates temporary shell scripts for hyperfine hooks: + +- `setup` - Clean tmp datadir (once before all runs) +- `prepare` - Copy snapshot, drop caches, clean logs (before each run) +- `cleanup` - Clean tmp datadir (after all runs per command) +- `conclude` - Collect flamegraph/logs (instrumented only, after each run) + +### Instrumented Mode + +When `--instrumented` is set: + +1. Wraps bitcoind in `flamegraph` for CPU profiling +2. Enables debug logging: `-debug=coindb -debug=leveldb -debug=bench -debug=validation` +3. Forces `runs=1` (profiling overhead makes multiple runs pointless) +4. Generates flamegraph SVGs and performance plots + +## CI Integration + +GitHub Actions workflows call bench.py directly (already in nix develop): + +```yaml +- run: | + nix develop --command python3 bench.py build \ + --binaries-dir ${{ runner.temp }}/binaries \ + $BASE_SHA $HEAD_SHA +``` + +CI-specific paths and the dedicated sync node are configured via `--profile ci`. diff --git a/bench/__init__.py b/bench/__init__.py new file mode 100644 index 000000000000..cb50424b155c --- /dev/null +++ b/bench/__init__.py @@ -0,0 +1,3 @@ +"""Benchcoin - Bitcoin Core benchmarking toolkit.""" + +__version__ = "0.1.0" diff --git a/bench/analyze.py b/bench/analyze.py new file mode 100644 index 000000000000..a31b807c422b --- /dev/null +++ b/bench/analyze.py @@ -0,0 +1,535 @@ +"""Analyze phase - parse debug.log and generate performance plots. + +Refactored from bench-ci/parse_and_plot.py for better structure and reusability. +""" + +from __future__ import annotations + +import datetime +import logging +import re +from collections import OrderedDict +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING + +# matplotlib is optional - gracefully handle if not installed +try: + import matplotlib.pyplot as plt + + HAS_MATPLOTLIB = True +except ImportError: + HAS_MATPLOTLIB = False + +if TYPE_CHECKING: + from .config import Config + +logger = logging.getLogger(__name__) + +# Bitcoin fork heights for plot annotations +FORK_HEIGHTS = OrderedDict( + [ + ("BIP34", 227931), # Block v2, coinbase includes height + ("BIP66", 363725), # Strict DER signatures + ("BIP65", 388381), # OP_CHECKLOCKTIMEVERIFY + ("CSV", 419328), # BIP68, 112, 113 - OP_CHECKSEQUENCEVERIFY + ("Segwit", 481824), # BIP141, 143, 144, 145 - Segregated Witness + ("Taproot", 709632), # BIP341, 342 - Schnorr signatures & Taproot + ("Halving 1", 210000), # First halving + ("Halving 2", 420000), # Second halving + ("Halving 3", 630000), # Third halving + ("Halving 4", 840000), # Fourth halving + ] +) + +FORK_COLORS = { + "BIP34": "blue", + "BIP66": "blue", + "BIP65": "blue", + "CSV": "blue", + "Segwit": "green", + "Taproot": "red", + "Halving 1": "purple", + "Halving 2": "purple", + "Halving 3": "purple", + "Halving 4": "purple", +} + +FORK_STYLES = { + "BIP34": "--", + "BIP66": "--", + "BIP65": "--", + "CSV": "--", + "Segwit": "--", + "Taproot": "--", + "Halving 1": ":", + "Halving 2": ":", + "Halving 3": ":", + "Halving 4": ":", +} + + +@dataclass +class UpdateTipEntry: + """Parsed UpdateTip log entry.""" + + timestamp: datetime.datetime + height: int + tx_count: int + cache_size_mb: float + cache_coins_count: int + + +@dataclass +class LevelDBCompactEntry: + """Parsed LevelDB compaction log entry.""" + + timestamp: datetime.datetime + + +@dataclass +class LevelDBGenTableEntry: + """Parsed LevelDB generated table log entry.""" + + timestamp: datetime.datetime + keys_count: int + bytes_count: int + + +@dataclass +class ValidationTxAddEntry: + """Parsed validation transaction added log entry.""" + + timestamp: datetime.datetime + + +@dataclass +class CoinDBWriteBatchEntry: + """Parsed coindb write batch log entry.""" + + timestamp: datetime.datetime + is_partial: bool + size_mb: float + + +@dataclass +class CoinDBCommitEntry: + """Parsed coindb commit log entry.""" + + timestamp: datetime.datetime + txout_count: int + + +@dataclass +class ParsedLog: + """All parsed data from a debug.log file.""" + + update_tip: list[UpdateTipEntry] + leveldb_compact: list[LevelDBCompactEntry] + leveldb_gen_table: list[LevelDBGenTableEntry] + validation_txadd: list[ValidationTxAddEntry] + coindb_write_batch: list[CoinDBWriteBatchEntry] + coindb_commit: list[CoinDBCommitEntry] + + +@dataclass +class AnalyzeResult: + """Result of the analyze phase.""" + + commit: str + output_dir: Path + plots: list[Path] + + +class LogParser: + """Parse bitcoind debug.log files.""" + + # Regex patterns + UPDATETIP_RE = re.compile( + r"^([\d\-:TZ]+) UpdateTip: new best.+height=(\d+).+tx=(\d+).+cache=([\d.]+)MiB\((\d+)txo\)" + ) + LEVELDB_COMPACT_RE = re.compile(r"^([\d\-:TZ]+) \[leveldb] Compacting.*files") + LEVELDB_GEN_TABLE_RE = re.compile( + r"^([\d\-:TZ]+) \[leveldb] Generated table.*: (\d+) keys, (\d+) bytes" + ) + VALIDATION_TXADD_RE = re.compile( + r"^([\d\-:TZ]+) \[validation] TransactionAddedToMempool: txid=.+wtxid=.+" + ) + COINDB_WRITE_BATCH_RE = re.compile( + r"^([\d\-:TZ]+) \[coindb] Writing (partial|final) batch of ([\d.]+) MiB" + ) + COINDB_COMMIT_RE = re.compile( + r"^([\d\-:TZ]+) \[coindb] Committed (\d+) changed transaction outputs" + ) + + @staticmethod + def parse_timestamp(iso_str: str) -> datetime.datetime: + """Parse ISO 8601 timestamp from log.""" + return datetime.datetime.strptime(iso_str, "%Y-%m-%dT%H:%M:%SZ") + + def parse_file(self, log_file: Path) -> ParsedLog: + """Parse a debug.log file and extract all relevant data.""" + update_tip: list[UpdateTipEntry] = [] + leveldb_compact: list[LevelDBCompactEntry] = [] + leveldb_gen_table: list[LevelDBGenTableEntry] = [] + validation_txadd: list[ValidationTxAddEntry] = [] + coindb_write_batch: list[CoinDBWriteBatchEntry] = [] + coindb_commit: list[CoinDBCommitEntry] = [] + + with open(log_file, "r", encoding="utf-8") as f: + for line in f: + if match := self.UPDATETIP_RE.match(line): + iso_str, height, tx, cache_mb, coins = match.groups() + update_tip.append( + UpdateTipEntry( + timestamp=self.parse_timestamp(iso_str), + height=int(height), + tx_count=int(tx), + cache_size_mb=float(cache_mb), + cache_coins_count=int(coins), + ) + ) + elif match := self.LEVELDB_COMPACT_RE.match(line): + leveldb_compact.append( + LevelDBCompactEntry( + timestamp=self.parse_timestamp(match.group(1)) + ) + ) + elif match := self.LEVELDB_GEN_TABLE_RE.match(line): + iso_str, keys, bytes_count = match.groups() + leveldb_gen_table.append( + LevelDBGenTableEntry( + timestamp=self.parse_timestamp(iso_str), + keys_count=int(keys), + bytes_count=int(bytes_count), + ) + ) + elif match := self.VALIDATION_TXADD_RE.match(line): + validation_txadd.append( + ValidationTxAddEntry( + timestamp=self.parse_timestamp(match.group(1)) + ) + ) + elif match := self.COINDB_WRITE_BATCH_RE.match(line): + iso_str, batch_type, size_mb = match.groups() + coindb_write_batch.append( + CoinDBWriteBatchEntry( + timestamp=self.parse_timestamp(iso_str), + is_partial=(batch_type == "partial"), + size_mb=float(size_mb), + ) + ) + elif match := self.COINDB_COMMIT_RE.match(line): + iso_str, txout_count = match.groups() + coindb_commit.append( + CoinDBCommitEntry( + timestamp=self.parse_timestamp(iso_str), + txout_count=int(txout_count), + ) + ) + + return ParsedLog( + update_tip=update_tip, + leveldb_compact=leveldb_compact, + leveldb_gen_table=leveldb_gen_table, + validation_txadd=validation_txadd, + coindb_write_batch=coindb_write_batch, + coindb_commit=coindb_commit, + ) + + +class PlotGenerator: + """Generate performance plots from parsed log data.""" + + def __init__(self, commit: str, output_dir: Path): + self.commit = commit + self.output_dir = output_dir + self.generated_plots: list[Path] = [] + + if not HAS_MATPLOTLIB: + raise RuntimeError( + "matplotlib is required for plot generation. " + "Install with: pip install matplotlib" + ) + + def generate_all(self, data: ParsedLog) -> list[Path]: + """Generate all plots from parsed data.""" + if not data.update_tip: + logger.warning("No UpdateTip entries found, skipping plot generation") + return [] + + # Verify entries are sorted by time + for i in range(len(data.update_tip) - 1): + if data.update_tip[i].timestamp > data.update_tip[i + 1].timestamp: + logger.warning("UpdateTip entries are not sorted by time") + break + + # Extract base time for elapsed calculations + base_time = data.update_tip[0].timestamp + + # Extract data series + times = [e.timestamp for e in data.update_tip] + heights = [e.height for e in data.update_tip] + tx_counts = [e.tx_count for e in data.update_tip] + cache_sizes = [e.cache_size_mb for e in data.update_tip] + cache_counts = [e.cache_coins_count for e in data.update_tip] + elapsed_minutes = [(t - base_time).total_seconds() / 60 for t in times] + + # Generate core plots + self._plot( + elapsed_minutes, + heights, + "Elapsed minutes", + "Block Height", + "Block Height vs Time", + f"{self.commit}-height_vs_time.png", + ) + + self._plot( + heights, + cache_sizes, + "Block Height", + "Cache Size (MiB)", + "Cache Size vs Block Height", + f"{self.commit}-cache_vs_height.png", + is_height_based=True, + ) + + self._plot( + elapsed_minutes, + cache_sizes, + "Elapsed minutes", + "Cache Size (MiB)", + "Cache Size vs Time", + f"{self.commit}-cache_vs_time.png", + ) + + self._plot( + heights, + tx_counts, + "Block Height", + "Transaction Count", + "Transactions vs Block Height", + f"{self.commit}-tx_vs_height.png", + is_height_based=True, + ) + + self._plot( + heights, + cache_counts, + "Block Height", + "Coins Cache Size", + "Coins Cache Size vs Height", + f"{self.commit}-coins_cache_vs_height.png", + is_height_based=True, + ) + + # LevelDB plots + if data.leveldb_compact: + compact_minutes = [ + (e.timestamp - base_time).total_seconds() / 60 + for e in data.leveldb_compact + ] + self._plot( + compact_minutes, + [1] * len(compact_minutes), + "Elapsed minutes", + "LevelDB Compaction", + "LevelDB Compaction Events vs Time", + f"{self.commit}-leveldb_compact_vs_time.png", + ) + + if data.leveldb_gen_table: + gen_minutes = [ + (e.timestamp - base_time).total_seconds() / 60 + for e in data.leveldb_gen_table + ] + gen_keys = [e.keys_count for e in data.leveldb_gen_table] + gen_bytes = [e.bytes_count for e in data.leveldb_gen_table] + + self._plot( + gen_minutes, + gen_keys, + "Elapsed minutes", + "Number of keys", + "LevelDB Keys Generated vs Time", + f"{self.commit}-leveldb_gen_keys_vs_time.png", + ) + + self._plot( + gen_minutes, + gen_bytes, + "Elapsed minutes", + "Number of bytes", + "LevelDB Bytes Generated vs Time", + f"{self.commit}-leveldb_gen_bytes_vs_time.png", + ) + + # Validation plots + if data.validation_txadd: + txadd_minutes = [ + (e.timestamp - base_time).total_seconds() / 60 + for e in data.validation_txadd + ] + self._plot( + txadd_minutes, + [1] * len(txadd_minutes), + "Elapsed minutes", + "Transaction Additions", + "Transaction Additions to Mempool vs Time", + f"{self.commit}-validation_txadd_vs_time.png", + ) + + # CoinDB plots + if data.coindb_write_batch: + batch_minutes = [ + (e.timestamp - base_time).total_seconds() / 60 + for e in data.coindb_write_batch + ] + batch_sizes = [e.size_mb for e in data.coindb_write_batch] + self._plot( + batch_minutes, + batch_sizes, + "Elapsed minutes", + "Batch Size MiB", + "Coin Database Partial/Final Write Batch Size vs Time", + f"{self.commit}-coindb_write_batch_size_vs_time.png", + ) + + if data.coindb_commit: + commit_minutes = [ + (e.timestamp - base_time).total_seconds() / 60 + for e in data.coindb_commit + ] + commit_txouts = [e.txout_count for e in data.coindb_commit] + self._plot( + commit_minutes, + commit_txouts, + "Elapsed minutes", + "Transaction Output Count", + "Coin Database Transaction Output Committed vs Time", + f"{self.commit}-coindb_commit_txout_vs_time.png", + ) + + return self.generated_plots + + def _plot( + self, + x: list, + y: list, + x_label: str, + y_label: str, + title: str, + filename: str, + is_height_based: bool = False, + ) -> None: + """Generate a single plot.""" + if not x or not y: + logger.debug(f"Skipping plot '{title}' - no data") + return + + plt.figure(figsize=(30, 10)) + plt.plot(x, y) + plt.title(title, fontsize=20) + plt.xlabel(x_label, fontsize=16) + plt.ylabel(y_label, fontsize=16) + plt.grid(True) + + min_x, max_x = min(x), max(x) + plt.xlim(min_x, max_x) + + # Add fork markers for height-based plots + if is_height_based: + self._add_fork_markers(min_x, max_x, max(y)) + + plt.xticks(rotation=90, fontsize=12) + plt.yticks(fontsize=12) + plt.tight_layout() + + output_path = self.output_dir / filename + plt.savefig(output_path) + plt.close() + + self.generated_plots.append(output_path) + logger.info(f"Saved plot: {output_path}") + + def _add_fork_markers(self, min_x: float, max_x: float, max_y: float) -> None: + """Add vertical lines for Bitcoin forks.""" + text_positions = {} + position_increment = max_y * 0.05 + current_position = max_y * 0.9 + + for fork_name, height in FORK_HEIGHTS.items(): + if min_x <= height <= max_x: + plt.axvline( + x=height, + color=FORK_COLORS[fork_name], + linestyle=FORK_STYLES[fork_name], + ) + + if height in text_positions: + text_positions[height] -= position_increment + else: + text_positions[height] = current_position + current_position -= position_increment + if current_position < max_y * 0.1: + current_position = max_y * 0.9 + + plt.text( + height, + text_positions[height], + f"{fork_name} ({height})", + rotation=90, + verticalalignment="top", + color=FORK_COLORS[fork_name], + ) + + +class AnalyzePhase: + """Analyze benchmark results and generate plots.""" + + def __init__(self, config: Config | None = None): + self.config = config + + def run( + self, + commit: str, + log_file: Path, + output_dir: Path, + ) -> AnalyzeResult: + """Analyze a debug.log and generate plots. + + Args: + commit: Commit hash (for naming) + log_file: Path to debug.log + output_dir: Where to save plots + + Returns: + AnalyzeResult with paths to generated plots + """ + if not HAS_MATPLOTLIB: + raise RuntimeError( + "matplotlib is required for plot generation. " + "Install with: pip install matplotlib" + ) + + if not log_file.exists(): + raise FileNotFoundError(f"Log file not found: {log_file}") + + output_dir.mkdir(parents=True, exist_ok=True) + + logger.info(f"Parsing log file: {log_file}") + parser = LogParser() + data = parser.parse_file(log_file) + + logger.info(f"Generating plots for {commit[:12]}") + generator = PlotGenerator(commit[:12], output_dir) + plots = generator.generate_all(data) + + logger.info(f"Generated {len(plots)} plots") + + return AnalyzeResult( + commit=commit, + output_dir=output_dir, + plots=plots, + ) diff --git a/bench/benchmark.py b/bench/benchmark.py new file mode 100644 index 000000000000..40022eeaee24 --- /dev/null +++ b/bench/benchmark.py @@ -0,0 +1,351 @@ +"""Benchmark phase - run hyperfine benchmarks comparing two bitcoind binaries.""" + +from __future__ import annotations + +import logging +import os +import shutil +import subprocess +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from .capabilities import Capabilities + from .config import Config + + +logger = logging.getLogger(__name__) + +# Debug flags for instrumented mode +INSTRUMENTED_DEBUG_FLAGS = ["coindb", "leveldb", "bench", "validation"] + + +@dataclass +class BenchmarkResult: + """Result of the benchmark phase.""" + + results_file: Path + base_commit: str + head_commit: str + instrumented: bool + flamegraph_base: Path | None = None + flamegraph_head: Path | None = None + debug_log_base: Path | None = None + debug_log_head: Path | None = None + + +class BenchmarkPhase: + """Run hyperfine benchmarks comparing two bitcoind binaries.""" + + def __init__( + self, + config: Config, + capabilities: Capabilities, + ): + self.config = config + self.capabilities = capabilities + self._temp_scripts: list[Path] = [] + + def run( + self, + base_commit: str, + head_commit: str, + base_binary: Path, + head_binary: Path, + datadir: Path, + output_dir: Path, + ) -> BenchmarkResult: + """Run benchmarks comparing base and head binaries. + + Args: + base_commit: Git hash of base commit + head_commit: Git hash of head commit + base_binary: Path to base bitcoind binary + head_binary: Path to head bitcoind binary + datadir: Source datadir with blockchain snapshot + output_dir: Where to store results + + Returns: + BenchmarkResult with paths to outputs + """ + # Check prerequisites + errors = self.capabilities.check_for_run(self.config.instrumented) + if errors: + raise RuntimeError("Benchmark prerequisites not met:\n" + "\n".join(errors)) + + # Log warnings about missing optional capabilities + for warning in self.capabilities.get_warnings(): + logger.warning(warning) + + # Setup directories + output_dir.mkdir(parents=True, exist_ok=True) + tmp_datadir = Path(self.config.tmp_datadir) + tmp_datadir.mkdir(parents=True, exist_ok=True) + + results_file = output_dir / "results.json" + + logger.info("Starting benchmark") + logger.info(f" Base: {base_commit[:12]}") + logger.info(f" Head: {head_commit[:12]}") + logger.info(f" Instrumented: {self.config.instrumented}") + logger.info(f" Runs: {self.config.runs}") + logger.info(f" Stop height: {self.config.stop_height}") + logger.info(f" dbcache: {self.config.dbcache}") + + try: + # Create hook scripts for hyperfine + setup_script = self._create_setup_script(tmp_datadir) + prepare_script = self._create_prepare_script(tmp_datadir, datadir) + cleanup_script = self._create_cleanup_script(tmp_datadir) + + # Build hyperfine command + cmd = self._build_hyperfine_cmd( + base_commit=base_commit, + head_commit=head_commit, + base_binary=base_binary, + head_binary=head_binary, + tmp_datadir=tmp_datadir, + results_file=results_file, + setup_script=setup_script, + prepare_script=prepare_script, + cleanup_script=cleanup_script, + output_dir=output_dir, + ) + + if self.config.dry_run: + logger.info(f"[DRY RUN] Would run: {' '.join(cmd)}") + return BenchmarkResult( + results_file=results_file, + base_commit=base_commit, + head_commit=head_commit, + instrumented=self.config.instrumented, + ) + + # Run hyperfine + logger.info("Running hyperfine...") + _result = subprocess.run(cmd, check=True) + + # Collect results + benchmark_result = BenchmarkResult( + results_file=results_file, + base_commit=base_commit, + head_commit=head_commit, + instrumented=self.config.instrumented, + ) + + # For instrumented runs, collect flamegraphs and debug logs + if self.config.instrumented: + base_fg = output_dir / f"{base_commit[:12]}-flamegraph.svg" + head_fg = output_dir / f"{head_commit[:12]}-flamegraph.svg" + base_log = output_dir / f"{base_commit[:12]}-debug.log" + head_log = output_dir / f"{head_commit[:12]}-debug.log" + + # Move flamegraphs from current directory if they exist + for src_name, dest in [ + ("base-flamegraph.svg", base_fg), + ("head-flamegraph.svg", head_fg), + ]: + src = Path(src_name) + if src.exists(): + shutil.move(str(src), str(dest)) + + if base_fg.exists(): + benchmark_result.flamegraph_base = base_fg + if head_fg.exists(): + benchmark_result.flamegraph_head = head_fg + if base_log.exists(): + benchmark_result.debug_log_base = base_log + if head_log.exists(): + benchmark_result.debug_log_head = head_log + + # Clean up tmp_datadir + if tmp_datadir.exists(): + logger.debug(f"Cleaning up tmp_datadir: {tmp_datadir}") + shutil.rmtree(tmp_datadir) + + return benchmark_result + + finally: + # Clean up temp scripts + for script in self._temp_scripts: + if script.exists(): + script.unlink() + self._temp_scripts.clear() + + def _create_temp_script(self, commands: list[str], name: str) -> Path: + """Create a temporary shell script.""" + content = "#!/usr/bin/env bash\nset -euxo pipefail\n" + content += "\n".join(commands) + "\n" + + fd, path = tempfile.mkstemp(suffix=".sh", prefix=f"bench_{name}_") + os.write(fd, content.encode()) + os.close(fd) + os.chmod(path, 0o755) + + script_path = Path(path) + self._temp_scripts.append(script_path) + return script_path + + def _create_setup_script(self, tmp_datadir: Path) -> Path: + """Create setup script (runs once before all timing runs).""" + commands = [ + f'mkdir -p "{tmp_datadir}"', + f'rm -rf "{tmp_datadir}"/*', + ] + return self._create_temp_script(commands, "setup") + + def _create_prepare_script(self, tmp_datadir: Path, original_datadir: Path) -> Path: + """Create prepare script (runs before each timing run).""" + commands = [ + f'rm -rf "{tmp_datadir}"/*', + ] + + # Copy datadir with optional CPU affinity + if self.capabilities.can_pin_cpu and not self.config.no_cpu_pinning: + commands.append( + f'taskset -c 0-15 cp -r "{original_datadir}"/* "{tmp_datadir}"' + ) + else: + commands.append(f'cp -r "{original_datadir}"/* "{tmp_datadir}"') + + # Drop caches if available + if self.capabilities.can_drop_caches and not self.config.no_cache_drop: + commands.append(self.capabilities.drop_caches_path) + + # Clean debug logs + commands.append( + f'find "{tmp_datadir}" -name debug.log -delete 2>/dev/null || true' + ) + + return self._create_temp_script(commands, "prepare") + + def _create_cleanup_script(self, tmp_datadir: Path) -> Path: + """Create cleanup script (runs after all timing runs for each command).""" + commands = [ + f'rm -rf "{tmp_datadir}"/*', + ] + return self._create_temp_script(commands, "cleanup") + + def _build_bitcoind_cmd( + self, + binary: Path, + tmp_datadir: Path, + ) -> str: + """Build the bitcoind command string for hyperfine.""" + parts = [] + + # Add flamegraph wrapper for instrumented mode + if self.config.instrumented: + # Flamegraph runs on core 1 + if self.capabilities.can_pin_cpu and not self.config.no_cpu_pinning: + parts.append("taskset -c 1") + parts.append("flamegraph") + parts.append("--palette bitcoin") + parts.append("--title 'bitcoind IBD'") + parts.append("-c 'record -F 101 --call-graph fp'") + parts.append("--") + + # Add CPU affinity for bitcoind (cores 2-15) + if self.capabilities.can_pin_cpu and not self.config.no_cpu_pinning: + parts.append("taskset -c 2-15") + + # Add scheduler priority + if self.capabilities.can_set_scheduler and not self.config.no_cpu_pinning: + parts.append("chrt -o 0") + + # Bitcoind command + parts.append(str(binary)) + parts.append(f"-datadir={tmp_datadir}") + parts.append(f"-dbcache={self.config.dbcache}") + parts.append(f"-stopatheight={self.config.stop_height}") + parts.append("-prune=10000") + parts.append(f"-chain={self.config.chain}") + parts.append("-daemon=0") + parts.append("-printtoconsole=0") + + if self.config.connect: + parts.append(f"-connect={self.config.connect}") + + # Debug flags for instrumented mode + if self.config.instrumented: + for flag in INSTRUMENTED_DEBUG_FLAGS: + parts.append(f"-debug={flag}") + + return " ".join(parts) + + def _build_hyperfine_cmd( + self, + base_commit: str, + head_commit: str, + base_binary: Path, + head_binary: Path, + tmp_datadir: Path, + results_file: Path, + setup_script: Path, + prepare_script: Path, + cleanup_script: Path, + output_dir: Path, + ) -> list[str]: + """Build the hyperfine command.""" + cmd = [ + "hyperfine", + "--shell=bash", + f"--setup={setup_script}", + f"--prepare={prepare_script}", + f"--cleanup={cleanup_script}", + f"--runs={self.config.runs}", + f"--export-json={results_file}", + "--show-output", + ] + + # For instrumented runs, we need separate conclude scripts per commit + # since hyperfine's parameter substitution doesn't work with --conclude + if self.config.instrumented: + base_conclude = self._create_conclude_script_for_commit( + base_commit[:12], tmp_datadir, output_dir + ) + head_conclude = self._create_conclude_script_for_commit( + head_commit[:12], tmp_datadir, output_dir + ) + # We'll handle conclude differently - see below + + # Command names + cmd.append(f"--command-name=base ({base_commit[:12]})") + cmd.append(f"--command-name=head ({head_commit[:12]})") + + # Build the actual commands to benchmark + base_cmd = self._build_bitcoind_cmd(base_binary, tmp_datadir) + head_cmd = self._build_bitcoind_cmd(head_binary, tmp_datadir) + + # For instrumented runs, append the conclude logic to each command + if self.config.instrumented: + base_cmd += f" && {base_conclude}" + head_cmd += f" && {head_conclude}" + + cmd.append(base_cmd) + cmd.append(head_cmd) + + return cmd + + def _create_conclude_script_for_commit( + self, + commit: str, + tmp_datadir: Path, + output_dir: Path, + ) -> str: + """Create inline conclude commands for a specific commit.""" + # Return shell commands to run after each benchmark + commands = [] + + # Move flamegraph if exists + commands.append(f'if [ -e flamegraph.svg ]; then mv flamegraph.svg "{output_dir}/{commit}-flamegraph.svg"; fi') + + # Copy debug log if exists + commands.append( + f'debug_log=$(find "{tmp_datadir}" -name debug.log -print -quit); ' + f'if [ -n "$debug_log" ]; then cp "$debug_log" "{output_dir}/{commit}-debug.log"; fi' + ) + + return " && ".join(commands) diff --git a/bench/build.py b/bench/build.py new file mode 100644 index 000000000000..feac1b0d5875 --- /dev/null +++ b/bench/build.py @@ -0,0 +1,172 @@ +"""Build phase - compile bitcoind at specified commits.""" + +from __future__ import annotations + +import logging +import shutil +import subprocess +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from .capabilities import Capabilities + from .config import Config + +from .utils import GitState, git_checkout, git_rev_parse + +logger = logging.getLogger(__name__) + + +@dataclass +class BuildResult: + """Result of the build phase.""" + + base_binary: Path + head_binary: Path + base_commit: str + head_commit: str + + +class BuildPhase: + """Build bitcoind binaries at two commits for comparison.""" + + def __init__( + self, + config: Config, + capabilities: Capabilities, + repo_path: Path | None = None, + ): + self.config = config + self.capabilities = capabilities + self.repo_path = repo_path or Path.cwd() + + def run( + self, + base_commit: str, + head_commit: str, + binaries_dir: Path | None = None, + ) -> BuildResult: + """Build bitcoind at both commits. + + Args: + base_commit: Git ref for base (comparison) commit + head_commit: Git ref for head (new) commit + binaries_dir: Where to store binaries (default: ./binaries) + + Returns: + BuildResult with paths to built binaries + """ + # Check prerequisites + errors = self.capabilities.check_for_build() + if errors: + raise RuntimeError("Build prerequisites not met:\n" + "\n".join(errors)) + + binaries_dir = binaries_dir or Path(self.config.binaries_dir) + + # Resolve commits to full hashes + base_hash = git_rev_parse(base_commit, self.repo_path) + head_hash = git_rev_parse(head_commit, self.repo_path) + + logger.info("Building binaries for comparison:") + logger.info(f" Base: {base_hash[:12]} ({base_commit})") + logger.info(f" Head: {head_hash[:12]} ({head_commit})") + + # Setup output directories + base_dir = binaries_dir / "base" + head_dir = binaries_dir / "head" + base_dir.mkdir(parents=True, exist_ok=True) + head_dir.mkdir(parents=True, exist_ok=True) + + base_binary = base_dir / "bitcoind" + head_binary = head_dir / "bitcoind" + + # Check if we can skip existing builds + if self.config.skip_existing: + if base_binary.exists() and head_binary.exists(): + logger.info( + "Both binaries exist and --skip-existing set, skipping build" + ) + return BuildResult( + base_binary=base_binary, + head_binary=head_binary, + base_commit=base_hash, + head_commit=head_hash, + ) + + # Save git state for restoration + git_state = GitState(self.repo_path) + git_state.save() + + try: + # Build both commits + builds = [ + ("base", base_hash, base_binary), + ("head", head_hash, head_binary), + ] + + for name, commit, output_path in builds: + if self.config.skip_existing and output_path.exists(): + logger.info(f"Skipping {name} build - binary exists") + continue + + self._build_commit(name, commit, output_path) + + finally: + # Always restore git state + git_state.restore() + + return BuildResult( + base_binary=base_binary, + head_binary=head_binary, + base_commit=base_hash, + head_commit=head_hash, + ) + + def _build_commit(self, name: str, commit: str, output_path: Path) -> None: + """Build bitcoind for a single commit.""" + logger.info(f"Building {name} ({commit[:12]})") + + if self.config.dry_run: + logger.info(f"[DRY RUN] Would build {commit[:12]} -> {output_path}") + return + + # Checkout the commit + git_checkout(commit, self.repo_path) + + # Build with nix + cmd = [] + + # Add CPU affinity if available + if self.capabilities.can_pin_cpu and not self.config.no_cpu_pinning: + cmd += ["taskset", "-c", "0-15"] + + cmd += ["nix", "build", "-L"] + + logger.debug(f"Running: {' '.join(cmd)}") + result = subprocess.run( + cmd, + cwd=self.repo_path, + ) + + if result.returncode != 0: + raise RuntimeError(f"Build failed for {name} ({commit[:12]})") + + # Copy binary to output location + nix_binary = self.repo_path / "result" / "bin" / "bitcoind" + if not nix_binary.exists(): + raise RuntimeError(f"Built binary not found at {nix_binary}") + + # Remove existing binary if present (may be read-only from nix) + if output_path.exists(): + output_path.chmod(0o755) + output_path.unlink() + + shutil.copy2(nix_binary, output_path) + output_path.chmod(0o755) # Ensure it's executable and writable + logger.info(f"Built {name} binary: {output_path}") + + # Clean up nix result symlink + result_link = self.repo_path / "result" + if result_link.is_symlink(): + result_link.unlink() diff --git a/bench/capabilities.py b/bench/capabilities.py new file mode 100644 index 000000000000..69ef2d1185c3 --- /dev/null +++ b/bench/capabilities.py @@ -0,0 +1,162 @@ +"""System capability detection for graceful degradation. + +Detects available tools and features, allowing the benchmark to run +on systems without all capabilities (with appropriate warnings). +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +from dataclasses import dataclass +from pathlib import Path + + +# Known paths for drop-caches on NixOS +DROP_CACHES_PATHS = [ + "/run/wrappers/bin/drop-caches", + "/usr/local/bin/drop-caches", +] + + +@dataclass +class Capabilities: + """Detected system capabilities.""" + + # Cache management + can_drop_caches: bool + drop_caches_path: str | None + + # CPU affinity and scheduling + can_pin_cpu: bool + can_set_scheduler: bool + + # Required tools + has_hyperfine: bool + has_flamegraph: bool + has_perf: bool + has_nix: bool + + # System info + cpu_count: int + is_nixos: bool + is_ci: bool + + def check_for_run(self, instrumented: bool = False) -> list[str]: + """Check if we have required capabilities for a benchmark run. + + Returns list of errors (empty if all good). + """ + errors = [] + + if not self.has_hyperfine: + errors.append("hyperfine not found in PATH (required for benchmarking)") + + if instrumented: + if not self.has_flamegraph: + errors.append( + "flamegraph not found in PATH (required for --instrumented)" + ) + if not self.has_perf: + errors.append("perf not found in PATH (required for --instrumented)") + + return errors + + def check_for_build(self) -> list[str]: + """Check if we have required capabilities for building. + + Returns list of errors (empty if all good). + """ + errors = [] + + if not self.has_nix: + errors.append("nix not found in PATH (required for building)") + + return errors + + def get_warnings(self) -> list[str]: + """Get warnings about missing optional capabilities.""" + warnings = [] + + if not self.can_drop_caches: + warnings.append( + "drop-caches not available - cache won't be cleared between runs" + ) + + if not self.can_pin_cpu: + warnings.append("taskset not available - CPU affinity won't be set") + + if not self.can_set_scheduler: + warnings.append("chrt not available - scheduler priority won't be set") + + return warnings + + +def _check_executable(name: str) -> bool: + """Check if an executable is available in PATH.""" + return shutil.which(name) is not None + + +def _find_drop_caches() -> str | None: + """Find drop-caches executable.""" + for path in DROP_CACHES_PATHS: + if Path(path).exists() and os.access(path, os.X_OK): + return path + return None + + +def _check_taskset() -> bool: + """Check if taskset is available and works.""" + if not _check_executable("taskset"): + return False + + # Try to run it to verify it works + try: + result = subprocess.run( + ["taskset", "-c", "0", "true"], + timeout=5, + ) + return result.returncode == 0 + except (subprocess.TimeoutExpired, OSError): + return False + + +def _check_chrt() -> bool: + """Check if chrt is available and works.""" + if not _check_executable("chrt"): + return False + + # Try to run it to verify it works + try: + result = subprocess.run( + ["chrt", "-o", "0", "true"], + timeout=5, + ) + return result.returncode == 0 + except (subprocess.TimeoutExpired, OSError): + return False + + +def _is_nixos() -> bool: + """Check if we're running on NixOS.""" + return Path("/etc/NIXOS").exists() + + +def detect_capabilities() -> Capabilities: + """Auto-detect system capabilities.""" + drop_caches_path = _find_drop_caches() + + return Capabilities( + can_drop_caches=drop_caches_path is not None, + drop_caches_path=drop_caches_path, + can_pin_cpu=_check_taskset(), + can_set_scheduler=_check_chrt(), + has_hyperfine=_check_executable("hyperfine"), + has_flamegraph=_check_executable("flamegraph"), + has_perf=_check_executable("perf"), + has_nix=_check_executable("nix"), + cpu_count=os.cpu_count() or 1, + is_nixos=_is_nixos(), + is_ci=os.environ.get("CI", "").lower() in ("true", "1", "yes"), + ) diff --git a/bench/config.py b/bench/config.py new file mode 100644 index 000000000000..2e0ea2c0967d --- /dev/null +++ b/bench/config.py @@ -0,0 +1,208 @@ +"""Configuration management for benchcoin. + +Layered configuration (lowest to highest priority): +1. Built-in defaults +2. bench.toml config file +3. Environment variables (BENCH_*) +4. CLI arguments +""" + +from __future__ import annotations + +import os +import tomllib +from dataclasses import dataclass +from pathlib import Path +from typing import Any + + +# Built-in defaults +DEFAULTS = { + "chain": "main", + "dbcache": 450, + "stop_height": 855000, + "runs": 3, + "connect": "", # Empty = use public P2P network + "binaries_dir": "./binaries", + "output_dir": "./bench-output", +} + +# Profile overrides +PROFILES = { + "quick": { + "stop_height": 1500, + "runs": 1, + }, + "full": { + "stop_height": 855000, + "runs": 3, + }, + "ci": { + "stop_height": 855000, + "runs": 3, + }, +} + +# Environment variable mapping +ENV_MAPPING = { + "BENCH_DATADIR": "datadir", + "BENCH_TMP_DATADIR": "tmp_datadir", + "BENCH_BINARIES_DIR": "binaries_dir", + "BENCH_OUTPUT_DIR": "output_dir", + "BENCH_STOP_HEIGHT": "stop_height", + "BENCH_DBCACHE": "dbcache", + "BENCH_CONNECT": "connect", + "BENCH_RUNS": "runs", + "BENCH_CHAIN": "chain", +} + + +@dataclass +class Config: + """Benchmark configuration.""" + + # Core benchmark settings + chain: str = "main" + dbcache: int = 450 + stop_height: int = 855000 + runs: int = 3 + connect: str = "" # Empty = use public P2P network + + # Paths + datadir: str | None = None + tmp_datadir: str | None = None + binaries_dir: str = "./binaries" + output_dir: str = "./bench-output" + + # Behavior flags + instrumented: bool = False + skip_existing: bool = False + no_cpu_pinning: bool = False + no_cache_drop: bool = False + verbose: bool = False + dry_run: bool = False + + # Profile used (for reference) + profile: str = "full" + + def __post_init__(self) -> None: + # If tmp_datadir not set, derive from output_dir + if self.tmp_datadir is None: + self.tmp_datadir = str(Path(self.output_dir) / "tmp-datadir") + + # Instrumented mode forces runs=1 + if self.instrumented and self.runs != 1: + self.runs = 1 + + def validate(self) -> list[str]: + """Validate configuration, return list of errors.""" + errors = [] + + if self.datadir is None: + errors.append("--datadir is required") + elif not Path(self.datadir).exists(): + errors.append(f"datadir does not exist: {self.datadir}") + + if self.stop_height < 1: + errors.append("stop_height must be positive") + + if self.dbcache < 1: + errors.append("dbcache must be positive") + + if self.runs < 1: + errors.append("runs must be positive") + + if self.chain not in ("main", "testnet", "signet", "regtest"): + errors.append(f"invalid chain: {self.chain}") + + return errors + + +def load_toml(path: Path) -> dict[str, Any]: + """Load configuration from TOML file.""" + if not path.exists(): + return {} + + with open(path, "rb") as f: + data = tomllib.load(f) + + # Flatten structure: merge [defaults] and [paths] into top level + result = {} + if "defaults" in data: + result.update(data["defaults"]) + if "paths" in data: + result.update(data["paths"]) + + return result + + +def load_env() -> dict[str, Any]: + """Load configuration from environment variables.""" + result = {} + + for env_var, config_key in ENV_MAPPING.items(): + value = os.environ.get(env_var) + if value is not None: + # Convert numeric values + if config_key in ("stop_height", "dbcache", "runs"): + try: + value = int(value) + except ValueError: + pass # Keep as string, will fail validation + result[config_key] = value + + return result + + +def apply_profile(config: dict[str, Any], profile_name: str) -> dict[str, Any]: + """Apply a named profile to configuration.""" + if profile_name not in PROFILES: + return config + + result = config.copy() + result.update(PROFILES[profile_name]) + result["profile"] = profile_name + return result + + +def build_config( + cli_args: dict[str, Any] | None = None, + config_file: Path | None = None, + profile: str = "full", +) -> Config: + """Build configuration from all sources. + + Priority (lowest to highest): + 1. Built-in defaults + 2. Config file (bench.toml) + 3. Profile overrides + 4. Environment variables + 5. CLI arguments + """ + # Start with defaults + config = DEFAULTS.copy() + + # Load config file + if config_file is None: + config_file = Path("bench.toml") + file_config = load_toml(config_file) + config.update(file_config) + + # Apply profile + config = apply_profile(config, profile) + + # Load environment variables + env_config = load_env() + config.update(env_config) + + # Apply CLI arguments (filter out None values) + if cli_args: + for key, value in cli_args.items(): + if value is not None: + config[key] = value + + # Build Config object (filter to only valid fields) + valid_fields = {f.name for f in Config.__dataclass_fields__.values()} + filtered = {k: v for k, v in config.items() if k in valid_fields} + + return Config(**filtered) diff --git a/bench/report.py b/bench/report.py new file mode 100644 index 000000000000..a41a035fbe9a --- /dev/null +++ b/bench/report.py @@ -0,0 +1,453 @@ +"""Report phase - generate HTML reports from benchmark results. + +Ported from the JavaScript logic in .github/workflows/publish-results.yml. +""" + +from __future__ import annotations + +import json +import logging +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +logger = logging.getLogger(__name__) + +# HTML template for individual run report +RUN_REPORT_TEMPLATE = """ + + + Benchmark Results + + + +
+

Benchmark Results

+
+

{title}

+ + +

Run Data

+
+ + + + + + + + + + + + + {run_data_rows} + +
NetworkCommandMean (s)Std DevUser (s)System (s)
+
+ + +

Speedup Summary

+
+ + + + + + + + + {speedup_rows} + +
NetworkSpeedup (%)
+
+ + + {graphs_section} +
+
+ +""" + +# HTML template for main index +INDEX_TEMPLATE = """ + + + Bitcoin Benchmark Results + + + +
+

Bitcoin Benchmark Results

+
+

Available Results

+
    + {run_list} +
+
+
+ +""" + + +@dataclass +class BenchmarkRun: + """Parsed benchmark run data.""" + + network: str + command: str + mean: float + stddev: float | None + user: float + system: float + parameters: dict[str, Any] = field(default_factory=dict) + + +@dataclass +class ReportResult: + """Result of report generation.""" + + output_dir: Path + index_file: Path + speedups: dict[str, float] + + +class ReportGenerator: + """Generate HTML reports from benchmark results.""" + + def __init__( + self, repo_url: str = "https://github.com/bitcoin-dev-tools/benchcoin" + ): + self.repo_url = repo_url + + def generate( + self, + input_dir: Path, + output_dir: Path, + title: str = "Benchmark Results", + ) -> ReportResult: + """Generate HTML report from benchmark artifacts. + + Args: + input_dir: Directory containing results.json and artifacts + output_dir: Where to write the HTML report + title: Title for the report + + Returns: + ReportResult with paths and speedup data + """ + output_dir.mkdir(parents=True, exist_ok=True) + + # Load results.json + results_file = input_dir / "results.json" + if not results_file.exists(): + raise FileNotFoundError(f"results.json not found in {input_dir}") + + with open(results_file) as f: + data = json.load(f) + + # Parse results + runs = self._parse_results(data) + + # Calculate speedups + speedups = self._calculate_speedups(runs) + + # Generate HTML + html = self._generate_html(runs, speedups, title, input_dir, output_dir) + + # Write report + index_file = output_dir / "index.html" + index_file.write_text(html) + logger.info(f"Generated report: {index_file}") + + # Copy artifacts (flamegraphs, plots) + self._copy_artifacts(input_dir, output_dir) + + return ReportResult( + output_dir=output_dir, + index_file=index_file, + speedups=speedups, + ) + + def generate_index( + self, + results_dir: Path, + output_file: Path, + ) -> None: + """Generate main index.html listing all available results. + + Args: + results_dir: Directory containing pr-* subdirectories + output_file: Where to write index.html + """ + runs = [] + + if results_dir.exists(): + for pr_dir in sorted(results_dir.iterdir()): + if pr_dir.is_dir() and pr_dir.name.startswith("pr-"): + pr_num = pr_dir.name.replace("pr-", "") + pr_runs = [] + for run_dir in sorted(pr_dir.iterdir()): + if run_dir.is_dir(): + pr_runs.append(run_dir.name) + if pr_runs: + runs.append((pr_num, pr_runs)) + + run_list_html = "" + for pr_num, pr_runs in runs: + run_links = "\n".join( + f'
  • Run {run}
    • ' + for run in pr_runs + ) + run_list_html += f""" +
  • PR #{pr_num} +
      + {run_links} +
    +
    • + """ + + html = INDEX_TEMPLATE.format(run_list=run_list_html) + output_file.write_text(html) + logger.info(f"Generated index: {output_file}") + + def _parse_results(self, data: dict) -> list[BenchmarkRun]: + """Parse results from hyperfine JSON output.""" + runs = [] + + # Handle both direct hyperfine output and combined results format + results = data.get("results", []) + + for result in results: + runs.append( + BenchmarkRun( + network=result.get("network", "default"), + command=result.get("command", ""), + mean=result.get("mean", 0), + stddev=result.get("stddev"), + user=result.get("user", 0), + system=result.get("system", 0), + parameters=result.get("parameters", {}), + ) + ) + + return runs + + def _calculate_speedups(self, runs: list[BenchmarkRun]) -> dict[str, float]: + """Calculate speedup percentages for each network.""" + speedups = {} + + # Group by network + by_network: dict[str, list[BenchmarkRun]] = {} + for run in runs: + if run.network not in by_network: + by_network[run.network] = [] + by_network[run.network].append(run) + + # Calculate speedup for each network + for network, network_runs in by_network.items(): + base_mean = None + head_mean = None + + for run in network_runs: + if "base" in run.command.lower(): + base_mean = run.mean + elif "head" in run.command.lower(): + head_mean = run.mean + + if base_mean and head_mean and base_mean > 0: + speedup = ((base_mean - head_mean) / base_mean) * 100 + speedups[network] = round(speedup, 1) + + return speedups + + def _generate_html( + self, + runs: list[BenchmarkRun], + speedups: dict[str, float], + title: str, + input_dir: Path, + output_dir: Path, + ) -> str: + """Generate the HTML report.""" + # Sort runs by network then by command (base first) + sorted_runs = sorted( + runs, + key=lambda r: (r.network, 0 if "base" in r.command.lower() else 1), + ) + + # Generate run data rows + run_data_rows = "" + for run in sorted_runs: + # Create commit link if there's a commit hash in the command + command_html = self._linkify_commit(run.command) + + stddev_str = f"{run.stddev:.3f}" if run.stddev else "N/A" + + run_data_rows += f""" + + {run.network} + {command_html} + {run.mean:.3f} + {stddev_str} + {run.user:.3f} + {run.system:.3f} + + """ + + # Generate speedup rows + speedup_rows = "" + for network, speedup in speedups.items(): + color_class = "" + if speedup > 0: + color_class = "text-green-600" + elif speedup < 0: + color_class = "text-red-600" + + speedup_rows += f""" + + {network} + {speedup}% + + """ + + # Generate graphs section + graphs_section = self._generate_graphs_section(runs, input_dir, output_dir) + + return RUN_REPORT_TEMPLATE.format( + title=title, + run_data_rows=run_data_rows, + speedup_rows=speedup_rows, + graphs_section=graphs_section, + ) + + def _linkify_commit(self, command: str) -> str: + """Convert commit hashes in command to links.""" + import re + + def replace_commit(match): + commit = match.group(1) + short_commit = commit[:8] if len(commit) > 8 else commit + return f'({short_commit})' + + return re.sub(r"\(([a-f0-9]{7,40})\)", replace_commit, command) + + def _generate_graphs_section( + self, + runs: list[BenchmarkRun], + input_dir: Path, + output_dir: Path, + ) -> str: + """Generate the flamegraphs and plots section.""" + graphs_html = "" + + for run in runs: + commit = run.parameters.get("commit", "") + if not commit: + # Try to extract from command + import re + + match = re.search(r"\(([a-f0-9]+)\)", run.command) + if match: + commit = match.group(1) + + if not commit: + continue + + short_commit = commit[:12] if len(commit) > 12 else commit + + # Check for flamegraph + flamegraph_name = f"{short_commit}-flamegraph.svg" + flamegraph_path = input_dir / flamegraph_name + + # Check for plots + plots_dir = input_dir / "plots" + plot_files = [] + if plots_dir.exists(): + plot_files = [ + p.name + for p in plots_dir.iterdir() + if p.name.startswith(f"{short_commit}-") and p.suffix == ".png" + ] + + if not flamegraph_path.exists() and not plot_files: + continue + + graphs_html += f""" +
    +

    {run.command}

    + """ + + if flamegraph_path.exists(): + graphs_html += f""" + + """ + + for plot in sorted(plot_files): + graphs_html += f""" + + {plot} + + """ + + graphs_html += "
    " + + if graphs_html: + return f""" +

    Flamegraphs and Plots

    + {graphs_html} + """ + + return "" + + def _copy_artifacts(self, input_dir: Path, output_dir: Path) -> None: + """Copy flamegraphs and plots to output directory.""" + import shutil + + # Skip if input and output are the same directory + if input_dir.resolve() == output_dir.resolve(): + logger.debug("Input and output are the same directory, skipping copy") + return + + # Copy flamegraphs + for svg in input_dir.glob("*-flamegraph.svg"): + dest = output_dir / svg.name + shutil.copy2(svg, dest) + logger.debug(f"Copied {svg.name}") + + # Copy plots directory + plots_dir = input_dir / "plots" + if plots_dir.exists(): + dest_plots = output_dir / "plots" + if dest_plots.exists(): + shutil.rmtree(dest_plots) + shutil.copytree(plots_dir, dest_plots) + logger.debug("Copied plots directory") + + +class ReportPhase: + """Generate reports from benchmark results.""" + + def __init__( + self, repo_url: str = "https://github.com/bitcoin-dev-tools/benchcoin" + ): + self.generator = ReportGenerator(repo_url) + + def run( + self, + input_dir: Path, + output_dir: Path, + title: str = "Benchmark Results", + ) -> ReportResult: + """Generate report from benchmark artifacts. + + Args: + input_dir: Directory containing results.json and artifacts + output_dir: Where to write the HTML report + title: Title for the report + + Returns: + ReportResult with paths and speedup data + """ + return self.generator.generate(input_dir, output_dir, title) diff --git a/bench/utils.py b/bench/utils.py new file mode 100644 index 000000000000..eda158833ebc --- /dev/null +++ b/bench/utils.py @@ -0,0 +1,259 @@ +"""Utility functions for git, datadir, and system operations.""" + +from __future__ import annotations + +import logging +import os +import shutil +import subprocess +import tempfile +from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from .capabilities import Capabilities + from .config import Config + +logger = logging.getLogger(__name__) + + +class GitError(Exception): + """Git operation failed.""" + + pass + + +class GitState: + """Saved git state for restoration after operations.""" + + def __init__(self, repo_path: Path | None = None): + self.repo_path = repo_path or Path.cwd() + self.original_branch: str | None = None + self.original_commit: str | None = None + self.was_detached: bool = False + + def save(self) -> None: + """Save current git state.""" + # Check if we're on a branch or detached HEAD + result = subprocess.run( + ["git", "symbolic-ref", "--short", "HEAD"], + capture_output=True, + text=True, + cwd=self.repo_path, + ) + + if result.returncode == 0: + self.original_branch = result.stdout.strip() + self.was_detached = False + else: + # Detached HEAD - save commit hash + result = subprocess.run( + ["git", "rev-parse", "HEAD"], + capture_output=True, + text=True, + check=True, + cwd=self.repo_path, + ) + self.original_commit = result.stdout.strip() + self.was_detached = True + + logger.debug( + f"Saved git state: branch={self.original_branch}, " + f"commit={self.original_commit}, detached={self.was_detached}" + ) + + def restore(self) -> None: + """Restore saved git state.""" + if self.original_branch: + logger.debug(f"Restoring branch: {self.original_branch}") + subprocess.run( + ["git", "checkout", self.original_branch], + check=True, + cwd=self.repo_path, + ) + elif self.original_commit: + logger.debug(f"Restoring detached HEAD: {self.original_commit}") + subprocess.run( + ["git", "checkout", self.original_commit], + check=True, + cwd=self.repo_path, + ) + + +def git_checkout(commit: str, repo_path: Path | None = None) -> None: + """Checkout a specific commit.""" + repo_path = repo_path or Path.cwd() + logger.info(f"Checking out {commit[:12]}") + + result = subprocess.run( + ["git", "checkout", commit], + cwd=repo_path, + capture_output=True, + text=True, + ) + + if result.returncode != 0: + raise GitError(f"Failed to checkout {commit}: {result.stderr}") + + +def git_rev_parse(ref: str, repo_path: Path | None = None) -> str: + """Resolve a git reference to a full commit hash.""" + repo_path = repo_path or Path.cwd() + + result = subprocess.run( + ["git", "rev-parse", ref], + cwd=repo_path, + capture_output=True, + text=True, + ) + + if result.returncode != 0: + raise GitError(f"Failed to resolve {ref}: {result.stderr}") + + return result.stdout.strip() + + +def clean_datadir(datadir: Path) -> None: + """Remove all contents from a data directory.""" + if not datadir.exists(): + return + + logger.debug(f"Cleaning datadir: {datadir}") + for item in datadir.iterdir(): + if item.is_dir(): + shutil.rmtree(item) + else: + item.unlink() + + +def copy_datadir(src: Path, dst: Path, capabilities: Capabilities) -> None: + """Copy blockchain data from source to destination. + + Uses taskset for CPU affinity if available. + """ + logger.info(f"Copying datadir: {src} -> {dst}") + + # Ensure destination exists + dst.mkdir(parents=True, exist_ok=True) + + # Build copy command + cmd = [] + if capabilities.can_pin_cpu: + cmd += ["taskset", "-c", "0-15"] + + cmd += ["cp", "-r"] + # Copy contents, not directory itself + cmd += [str(src) + "/.", str(dst)] + + subprocess.run(cmd, check=True) + + +def drop_caches(capabilities: Capabilities) -> bool: + """Drop filesystem caches if available. + + Returns True if caches were dropped, False if not available. + """ + if not capabilities.can_drop_caches or not capabilities.drop_caches_path: + logger.debug("Cache dropping not available, skipping") + return False + + logger.debug("Dropping filesystem caches") + subprocess.run([capabilities.drop_caches_path], check=True) + return True + + +def clean_debug_logs(datadir: Path) -> None: + """Remove debug.log files from datadir and subdirectories.""" + logger.debug(f"Cleaning debug logs in: {datadir}") + + for log_file in datadir.rglob("debug.log"): + log_file.unlink() + + +def find_debug_log(datadir: Path) -> Path | None: + """Find debug.log in datadir or subdirectories.""" + # Check common locations + candidates = [ + datadir / "debug.log", + datadir / "mainnet" / "debug.log", + datadir / "testnet3" / "debug.log", + datadir / "signet" / "debug.log", + datadir / "regtest" / "debug.log", + ] + + for candidate in candidates: + if candidate.exists(): + return candidate + + # Fallback: search recursively + for log_file in datadir.rglob("debug.log"): + return log_file + + return None + + +def create_temp_script(commands: list[str], name: str = "hook") -> Path: + """Create a temporary shell script for hyperfine hooks. + + Returns path to the script. + """ + script_content = "#!/usr/bin/env bash\nset -euxo pipefail\n" + script_content += "\n".join(commands) + "\n" + + # Create temp file that persists (caller is responsible for cleanup) + fd, path = tempfile.mkstemp(suffix=".sh", prefix=f"bench_{name}_") + os.write(fd, script_content.encode()) + os.close(fd) + os.chmod(path, 0o755) + + return Path(path) + + +def build_bitcoind_cmd( + binary: Path, + datadir: Path, + config: Config, + capabilities: Capabilities, + debug_flags: list[str] | None = None, +) -> list[str]: + """Build the bitcoind command with optional wrappers. + + Args: + binary: Path to bitcoind binary + datadir: Data directory + config: Benchmark configuration + capabilities: System capabilities + debug_flags: Optional debug flags for instrumented mode + """ + cmd = [] + + # Add CPU affinity if available and not disabled + if capabilities.can_pin_cpu and not config.no_cpu_pinning: + cmd += ["taskset", "-c", "2-15"] + + # Add scheduler priority if available + if capabilities.can_set_scheduler and not config.no_cpu_pinning: + cmd += ["chrt", "-o", "0"] + + # Add bitcoind with options + cmd += [ + str(binary), + f"-datadir={datadir}", + f"-dbcache={config.dbcache}", + f"-stopatheight={config.stop_height}", + "-prune=10000", + f"-chain={config.chain}", + "-daemon=0", + "-printtoconsole=0", + ] + + # Add connect address if specified + if config.connect: + cmd.append(f"-connect={config.connect}") + + # Add debug flags for instrumented mode + if debug_flags: + for flag in debug_flags: + cmd.append(f"-debug={flag}") + + return cmd diff --git a/flake.nix b/flake.nix index 3d0c2fa657b4..b42180629d1a 100644 --- a/flake.nix +++ b/flake.nix @@ -30,16 +30,16 @@ ]; CXXFlags = "${CFlags} -fno-omit-frame-pointer"; - nativeBuildInputs = with pkgs; [ - cmake - ninja - pkg-config - python3 + nativeBuildInputs = [ + pkgs.cmake + pkgs.ninja + pkgs.pkg-config + pkgs.python3 ]; - buildInputs = with pkgs; [ - boost188.dev - libevent.dev + buildInputs = [ + pkgs.boost188.dev + pkgs.libevent.dev ]; cmakeFlags = [ @@ -156,6 +156,8 @@ pkgs.just pkgs.perf pkgs.perf-tools + pkgs.python312 + pkgs.python312Packages.matplotlib pkgs.util-linux # Binary patching diff --git a/justfile b/justfile index 5b32a5d7bf85..51faef3160da 100644 --- a/justfile +++ b/justfile @@ -1,34 +1,85 @@ set shell := ["bash", "-uc"] -os := os() - default: just --list -# Build base and head binaries for CI +# ============================================================================ +# Local benchmarking commands +# ============================================================================ + +# Test instrumented run using signet (includes report generation) +[group('local')] +test-instrumented base head datadir: + nix develop --command python3 bench.py --profile quick full --chain signet --instrumented --skip-existing --datadir {{ datadir }} {{ base }} {{ head }} + nix develop --command python3 bench.py report bench-output/ bench-output/ + +# Test uninstrumented run using signet +[group('local')] +test-uninstrumented base head datadir: + nix develop --command python3 bench.py --profile quick full --chain signet --skip-existing --datadir {{ datadir }} {{ base }} {{ head }} + +# Full benchmark with instrumentation (flamegraphs + plots) +[group('local')] +instrumented base head datadir: + python3 bench.py --profile quick full --instrumented --datadir {{ datadir }} {{ base }} {{ head }} + +# Just build binaries (useful for incremental testing) +[group('local')] +build base head: + python3 bench.py build {{ base }} {{ head }} + +# Run benchmark with pre-built binaries +[group('local')] +run base head datadir: + python3 bench.py run --datadir {{ datadir }} {{ base }} {{ head }} + +# Generate plots from a debug.log file +[group('local')] +analyze commit logfile output_dir="./plots": + python3 bench.py analyze {{ commit }} {{ logfile }} --output-dir {{ output_dir }} + +# Generate HTML report from benchmark results +[group('local')] +report input_dir output_dir: + python3 bench.py report {{ input_dir }} {{ output_dir }} + +# ============================================================================ +# CI commands (called by GitHub Actions) +# ============================================================================ + +# Build binaries for CI [group('ci')] -build-binaries base_commit head_commit: - #!/usr/bin/env bash - set -euxo pipefail - ./bench-ci/build_binaries.sh {{ base_commit }} {{ head_commit }} +ci-build base_commit head_commit binaries_dir: + python3 bench.py build --binaries-dir {{ binaries_dir }} {{ base_commit }} {{ head_commit }} -# Run uninstrumented benchmarks on mainnet +# Run uninstrumented benchmarks for CI [group('ci')] -run-mainnet-ci base_commit head_commit TMP_DATADIR ORIGINAL_DATADIR results_file dbcache binaries_dir: - #!/usr/bin/env bash - set -euxo pipefail - unset SOURCE_DATE_EPOCH # needed to run on NixOS - ./bench-ci/run-benchmark.sh {{ base_commit }} {{ head_commit }} {{ TMP_DATADIR }} {{ ORIGINAL_DATADIR }} {{ results_file }} main 855000 "148.251.128.115:33333" {{ dbcache }} {{ binaries_dir }} +ci-run base_commit head_commit datadir tmp_datadir output_dir dbcache binaries_dir: + python3 bench.py --profile ci run \ + --binaries-dir {{ binaries_dir }} \ + --datadir {{ datadir }} \ + --tmp-datadir {{ tmp_datadir }} \ + --output-dir {{ output_dir }} \ + --dbcache {{ dbcache }} \ + {{ base_commit }} {{ head_commit }} -# Run instrumented benchmarks on mainnet +# Run instrumented benchmarks for CI [group('ci')] -run-mainnet-ci-instrumented base_commit head_commit TMP_DATADIR ORIGINAL_DATADIR results_file dbcache png_dir binaries_dir: - #!/usr/bin/env bash - set -euxo pipefail - unset SOURCE_DATE_EPOCH # needed to run on NixOS - ./bench-ci/run-benchmark-instrumented.sh {{ base_commit }} {{ head_commit }} {{ TMP_DATADIR }} {{ ORIGINAL_DATADIR }} {{ results_file }} {{ png_dir }} main 855000 "148.251.128.115:33333" {{ dbcache }} {{ binaries_dir }} +ci-run-instrumented base_commit head_commit datadir tmp_datadir output_dir dbcache binaries_dir: + python3 bench.py --profile ci run \ + --instrumented \ + --binaries-dir {{ binaries_dir }} \ + --datadir {{ datadir }} \ + --tmp-datadir {{ tmp_datadir }} \ + --output-dir {{ output_dir }} \ + --dbcache {{ dbcache }} \ + {{ base_commit }} {{ head_commit }} + +# ============================================================================ +# Git helpers +# ============================================================================ -# Cherry-pick commits from a bitcoin core PR onto this branch +# Cherry-pick commits from a Bitcoin Core PR onto this branch [group('git')] pick-pr pr_number: #!/usr/bin/env bash @@ -36,7 +87,7 @@ pick-pr pr_number: if ! git remote get-url upstream 2>/dev/null | grep -q "bitcoin/bitcoin"; then echo "Error: 'upstream' remote not found or doesn't point to bitcoin/bitcoin" - echo "Please add it with: `git remote add upstream https://github.com/bitcoin/bitcoin.git`" + echo "Please add it with: git remote add upstream https://github.com/bitcoin/bitcoin.git" exit 1 fi From 2677420f4e945ef628448d6711c61d77d939b61c Mon Sep 17 00:00:00 2001 From: will Date: Mon, 8 Dec 2025 14:53:05 +0000 Subject: [PATCH 33/51] fixup! use python runner --- .github/workflows/benchmark.yml | 6 ++---- bench.py | 30 +++++++++++++++++++++++++++++- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 46aa545e8cf5..d9861a0644f7 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -72,8 +72,7 @@ jobs: - name: Run benchmark run: | - nix develop --command python3 bench.py run \ - --profile ci \ + nix develop --command python3 bench.py --profile ci run \ --binaries-dir ${{ runner.temp }}/binaries \ --datadir $ORIGINAL_DATADIR \ --tmp-datadir ${{ runner.temp }}/datadir \ @@ -142,8 +141,7 @@ jobs: - name: Run instrumented benchmark run: | - nix develop --command python3 bench.py run \ - --profile ci \ + nix develop --command python3 bench.py --profile ci run \ --instrumented \ --binaries-dir ${{ runner.temp }}/binaries \ --datadir $ORIGINAL_DATADIR \ diff --git a/bench.py b/bench.py index 072f4ef95fc4..df2dd6108703 100755 --- a/bench.py +++ b/bench.py @@ -120,6 +120,7 @@ def cmd_run(args: argparse.Namespace) -> int: return 1 phase = BenchmarkPhase(config, capabilities) + output_dir = Path(config.output_dir) try: result = phase.run( @@ -128,9 +129,36 @@ def cmd_run(args: argparse.Namespace) -> int: base_binary=base_binary, head_binary=head_binary, datadir=Path(config.datadir), - output_dir=Path(config.output_dir), + output_dir=output_dir, ) logger.info(f"Results saved to: {result.results_file}") + + # For instrumented runs, also generate plots + if config.instrumented: + from bench.analyze import AnalyzePhase + + analyze_phase = AnalyzePhase(config) + + if result.debug_log_base: + try: + analyze_phase.run( + commit=args.base_commit, + log_file=result.debug_log_base, + output_dir=output_dir / "plots", + ) + except Exception as e: + logger.warning(f"Analysis for base failed: {e}") + + if result.debug_log_head: + try: + analyze_phase.run( + commit=args.head_commit, + log_file=result.debug_log_head, + output_dir=output_dir / "plots", + ) + except Exception as e: + logger.warning(f"Analysis for head failed: {e}") + return 0 except Exception as e: logger.error(f"Benchmark failed: {e}") From 6e2768bb5cc96811c805bab17ec450dff11c76c9 Mon Sep 17 00:00:00 2001 From: will Date: Mon, 8 Dec 2025 14:53:36 +0000 Subject: [PATCH 34/51] fixup! use python runner --- .github/workflows/publish-results.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/publish-results.yml b/.github/workflows/publish-results.yml index 11d0327192c5..a785374b4308 100644 --- a/.github/workflows/publish-results.yml +++ b/.github/workflows/publish-results.yml @@ -101,6 +101,11 @@ jobs: // Add network name to each result and collect means networkResults.results.forEach(result => { result.network = network; + // Extract commit from command string like "base (364a7bb8701e)" + const commitMatch = result.command.match(/\(([a-f0-9]+)\)/); + if (commitMatch) { + result.parameters = { commit: commitMatch[1] }; + } combinedResults.results.push(result); if (result.command.includes('base')) { baseMean = result.mean; From f815409e5539a2304492328c5f2391b7a5234ea7 Mon Sep 17 00:00:00 2001 From: will Date: Mon, 8 Dec 2025 15:20:26 +0000 Subject: [PATCH 35/51] fixup! use python runner --- bench/build.py | 9 +++------ bench/config.py | 1 + 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/bench/build.py b/bench/build.py index feac1b0d5875..b03093f8843a 100644 --- a/bench/build.py +++ b/bench/build.py @@ -134,13 +134,10 @@ def _build_commit(self, name: str, commit: str, output_path: Path) -> None: # Checkout the commit git_checkout(commit, self.repo_path) - # Build with nix + # Build with nix (use all available cores for faster builds) cmd = [] - - # Add CPU affinity if available - if self.capabilities.can_pin_cpu and not self.config.no_cpu_pinning: - cmd += ["taskset", "-c", "0-15"] - + if self.capabilities.can_pin_cpu: + cmd += ["taskset", "-c", f"2-{self.capabilities.cpu_count - 1}"] cmd += ["nix", "build", "-L"] logger.debug(f"Running: {' '.join(cmd)}") diff --git a/bench/config.py b/bench/config.py index 2e0ea2c0967d..a0e2926986fe 100644 --- a/bench/config.py +++ b/bench/config.py @@ -40,6 +40,7 @@ "ci": { "stop_height": 855000, "runs": 3, + "connect": "148.251.128.115:33333", }, } From e13bc9803189d547297cd161d0f4871d81609308 Mon Sep 17 00:00:00 2001 From: will Date: Mon, 8 Dec 2025 20:12:31 +0000 Subject: [PATCH 36/51] use nix shell for jq --- .github/workflows/benchmark.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index d9861a0644f7..31ed48b1bd10 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -92,7 +92,7 @@ jobs: RUNNER_CONTEXT: ${{ toJSON(runner) }} run: | mkdir -p ${{ runner.temp }}/contexts - echo "$GITHUB_CONTEXT" | jq "del(.token)" > ${{ runner.temp }}/contexts/github.json + echo "$GITHUB_CONTEXT" | nix develop --command jq "del(.token)" > ${{ runner.temp }}/contexts/github.json echo "$RUNNER_CONTEXT" > ${{ runner.temp }}/contexts/runner.json - name: Upload context metadata @@ -176,7 +176,7 @@ jobs: RUNNER_CONTEXT: ${{ toJSON(runner) }} run: | mkdir -p ${{ runner.temp }}/contexts - echo "$GITHUB_CONTEXT" | jq "del(.token)" > ${{ runner.temp }}/contexts/github.json + echo "$GITHUB_CONTEXT" | nix develop --command jq "del(.token)" > ${{ runner.temp }}/contexts/github.json echo "$RUNNER_CONTEXT" > ${{ runner.temp }}/contexts/runner.json - name: Upload context metadata From da4c9cf093a64698fcac706d06ff802963f47b6a Mon Sep 17 00:00:00 2001 From: will Date: Mon, 8 Dec 2025 20:45:13 +0000 Subject: [PATCH 37/51] remove isolated cpus --- bench.py | 18 ------------------ bench/README.md | 5 +---- bench/benchmark.py | 20 ++------------------ bench/build.py | 7 ++----- bench/capabilities.py | 44 ------------------------------------------- bench/config.py | 1 - bench/utils.py | 24 +++-------------------- 7 files changed, 8 insertions(+), 111 deletions(-) diff --git a/bench.py b/bench.py index df2dd6108703..d327cacd0993 100755 --- a/bench.py +++ b/bench.py @@ -39,7 +39,6 @@ def cmd_build(args: argparse.Namespace) -> int: cli_args={ "binaries_dir": args.binaries_dir, "skip_existing": args.skip_existing, - "no_cpu_pinning": args.no_cpu_pinning, "dry_run": args.dry_run, "verbose": args.verbose, }, @@ -83,7 +82,6 @@ def cmd_run(args: argparse.Namespace) -> int: "connect": args.connect, "chain": args.chain, "instrumented": args.instrumented, - "no_cpu_pinning": args.no_cpu_pinning, "no_cache_drop": args.no_cache_drop, "dry_run": args.dry_run, "verbose": args.verbose, @@ -263,7 +261,6 @@ def cmd_full(args: argparse.Namespace) -> int: "chain": args.chain, "instrumented": args.instrumented, "skip_existing": args.skip_existing, - "no_cpu_pinning": args.no_cpu_pinning, "no_cache_drop": args.no_cache_drop, "dry_run": args.dry_run, "verbose": args.verbose, @@ -396,11 +393,6 @@ def main() -> int: action="store_true", help="Skip build if binary already exists", ) - build_parser.add_argument( - "--no-cpu-pinning", - action="store_true", - help="Disable CPU affinity", - ) build_parser.set_defaults(func=cmd_build) # Run command @@ -461,11 +453,6 @@ def main() -> int: action="store_true", help="Enable profiling (flamegraph + debug logging)", ) - run_parser.add_argument( - "--no-cpu-pinning", - action="store_true", - help="Disable CPU affinity and scheduler priority", - ) run_parser.add_argument( "--no-cache-drop", action="store_true", @@ -562,11 +549,6 @@ def main() -> int: action="store_true", help="Skip build if binary already exists", ) - full_parser.add_argument( - "--no-cpu-pinning", - action="store_true", - help="Disable CPU affinity and scheduler priority", - ) full_parser.add_argument( "--no-cache-drop", action="store_true", diff --git a/bench/README.md b/bench/README.md index 9cf11ac9df4b..ca0d011303de 100644 --- a/bench/README.md +++ b/bench/README.md @@ -21,7 +21,6 @@ just quick HEAD~1 HEAD /path/to/signet/datadir Optional (auto-detected, gracefully degrades without): - `/run/wrappers/bin/drop-caches` (NixOS) - clears page cache between runs -- `taskset` / `chrt` - CPU affinity and scheduler priority ## Commands @@ -70,7 +69,6 @@ Options: - `--instrumented` - Enable flamegraph profiling and debug logging - `--connect ADDR` - P2P node to sync from (empty = public network) - `--chain {main,signet,testnet,regtest}` - Which chain -- `--no-cpu-pinning` - Disable taskset/chrt - `--no-cache-drop` - Don't clear page cache between runs ### analyze @@ -193,14 +191,13 @@ The tool auto-detects system capabilities and gracefully degrades: ```python from bench.capabilities import detect_capabilities caps = detect_capabilities() -# caps.has_hyperfine, caps.can_drop_caches, caps.can_pin_cpu, etc. +# caps.has_hyperfine, caps.can_drop_caches, etc. ``` Missing optional features emit warnings but don't fail: ``` WARNING: drop-caches not available - cache won't be cleared between runs -WARNING: taskset not available - CPU affinity won't be set ``` Missing required features (hyperfine, flamegraph for instrumented) cause errors. diff --git a/bench/benchmark.py b/bench/benchmark.py index 40022eeaee24..fae2cd7fee15 100644 --- a/bench/benchmark.py +++ b/bench/benchmark.py @@ -202,13 +202,8 @@ def _create_prepare_script(self, tmp_datadir: Path, original_datadir: Path) -> P f'rm -rf "{tmp_datadir}"/*', ] - # Copy datadir with optional CPU affinity - if self.capabilities.can_pin_cpu and not self.config.no_cpu_pinning: - commands.append( - f'taskset -c 0-15 cp -r "{original_datadir}"/* "{tmp_datadir}"' - ) - else: - commands.append(f'cp -r "{original_datadir}"/* "{tmp_datadir}"') + # Copy datadir + commands.append(f'cp -r "{original_datadir}"/* "{tmp_datadir}"') # Drop caches if available if self.capabilities.can_drop_caches and not self.config.no_cache_drop: @@ -238,23 +233,12 @@ def _build_bitcoind_cmd( # Add flamegraph wrapper for instrumented mode if self.config.instrumented: - # Flamegraph runs on core 1 - if self.capabilities.can_pin_cpu and not self.config.no_cpu_pinning: - parts.append("taskset -c 1") parts.append("flamegraph") parts.append("--palette bitcoin") parts.append("--title 'bitcoind IBD'") parts.append("-c 'record -F 101 --call-graph fp'") parts.append("--") - # Add CPU affinity for bitcoind (cores 2-15) - if self.capabilities.can_pin_cpu and not self.config.no_cpu_pinning: - parts.append("taskset -c 2-15") - - # Add scheduler priority - if self.capabilities.can_set_scheduler and not self.config.no_cpu_pinning: - parts.append("chrt -o 0") - # Bitcoind command parts.append(str(binary)) parts.append(f"-datadir={tmp_datadir}") diff --git a/bench/build.py b/bench/build.py index b03093f8843a..7b7c749b433f 100644 --- a/bench/build.py +++ b/bench/build.py @@ -134,11 +134,8 @@ def _build_commit(self, name: str, commit: str, output_path: Path) -> None: # Checkout the commit git_checkout(commit, self.repo_path) - # Build with nix (use all available cores for faster builds) - cmd = [] - if self.capabilities.can_pin_cpu: - cmd += ["taskset", "-c", f"2-{self.capabilities.cpu_count - 1}"] - cmd += ["nix", "build", "-L"] + # Build with nix + cmd = ["nix", "build", "-L"] logger.debug(f"Running: {' '.join(cmd)}") result = subprocess.run( diff --git a/bench/capabilities.py b/bench/capabilities.py index 69ef2d1185c3..b01ce2f3a711 100644 --- a/bench/capabilities.py +++ b/bench/capabilities.py @@ -28,10 +28,6 @@ class Capabilities: can_drop_caches: bool drop_caches_path: str | None - # CPU affinity and scheduling - can_pin_cpu: bool - can_set_scheduler: bool - # Required tools has_hyperfine: bool has_flamegraph: bool @@ -84,12 +80,6 @@ def get_warnings(self) -> list[str]: "drop-caches not available - cache won't be cleared between runs" ) - if not self.can_pin_cpu: - warnings.append("taskset not available - CPU affinity won't be set") - - if not self.can_set_scheduler: - warnings.append("chrt not available - scheduler priority won't be set") - return warnings @@ -106,38 +96,6 @@ def _find_drop_caches() -> str | None: return None -def _check_taskset() -> bool: - """Check if taskset is available and works.""" - if not _check_executable("taskset"): - return False - - # Try to run it to verify it works - try: - result = subprocess.run( - ["taskset", "-c", "0", "true"], - timeout=5, - ) - return result.returncode == 0 - except (subprocess.TimeoutExpired, OSError): - return False - - -def _check_chrt() -> bool: - """Check if chrt is available and works.""" - if not _check_executable("chrt"): - return False - - # Try to run it to verify it works - try: - result = subprocess.run( - ["chrt", "-o", "0", "true"], - timeout=5, - ) - return result.returncode == 0 - except (subprocess.TimeoutExpired, OSError): - return False - - def _is_nixos() -> bool: """Check if we're running on NixOS.""" return Path("/etc/NIXOS").exists() @@ -150,8 +108,6 @@ def detect_capabilities() -> Capabilities: return Capabilities( can_drop_caches=drop_caches_path is not None, drop_caches_path=drop_caches_path, - can_pin_cpu=_check_taskset(), - can_set_scheduler=_check_chrt(), has_hyperfine=_check_executable("hyperfine"), has_flamegraph=_check_executable("flamegraph"), has_perf=_check_executable("perf"), diff --git a/bench/config.py b/bench/config.py index a0e2926986fe..e17c0e31a921 100644 --- a/bench/config.py +++ b/bench/config.py @@ -78,7 +78,6 @@ class Config: # Behavior flags instrumented: bool = False skip_existing: bool = False - no_cpu_pinning: bool = False no_cache_drop: bool = False verbose: bool = False dry_run: bool = False diff --git a/bench/utils.py b/bench/utils.py index eda158833ebc..9cd28373d19d 100644 --- a/bench/utils.py +++ b/bench/utils.py @@ -127,21 +127,14 @@ def clean_datadir(datadir: Path) -> None: def copy_datadir(src: Path, dst: Path, capabilities: Capabilities) -> None: - """Copy blockchain data from source to destination. - - Uses taskset for CPU affinity if available. - """ + """Copy blockchain data from source to destination.""" logger.info(f"Copying datadir: {src} -> {dst}") # Ensure destination exists dst.mkdir(parents=True, exist_ok=True) # Build copy command - cmd = [] - if capabilities.can_pin_cpu: - cmd += ["taskset", "-c", "0-15"] - - cmd += ["cp", "-r"] + cmd = ["cp", "-r"] # Copy contents, not directory itself cmd += [str(src) + "/.", str(dst)] @@ -225,18 +218,7 @@ def build_bitcoind_cmd( capabilities: System capabilities debug_flags: Optional debug flags for instrumented mode """ - cmd = [] - - # Add CPU affinity if available and not disabled - if capabilities.can_pin_cpu and not config.no_cpu_pinning: - cmd += ["taskset", "-c", "2-15"] - - # Add scheduler priority if available - if capabilities.can_set_scheduler and not config.no_cpu_pinning: - cmd += ["chrt", "-o", "0"] - - # Add bitcoind with options - cmd += [ + cmd = [ str(binary), f"-datadir={datadir}", f"-dbcache={config.dbcache}", From 9c7fcd0794844421091d8260bc9437cc6b066199 Mon Sep 17 00:00:00 2001 From: will Date: Tue, 9 Dec 2025 09:43:28 +0000 Subject: [PATCH 38/51] fixup! use python runner --- bench.py | 13 ++-- bench/analyze.py | 7 --- bench/report.py | 8 +-- bench/utils.py | 150 +++-------------------------------------------- 4 files changed, 13 insertions(+), 165 deletions(-) diff --git a/bench.py b/bench.py index d327cacd0993..157072f95c28 100755 --- a/bench.py +++ b/bench.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 """Benchcoin - Bitcoin Core benchmarking toolkit. -A unified CLI for building, benchmarking, analyzing, and reporting -on Bitcoin Core performance. +A CLI for building, benchmarking, analyzing, and reporting on Bitcoin Core +performance. Usage: bench.py build BASE HEAD Build bitcoind at two commits @@ -22,7 +22,6 @@ from bench.capabilities import detect_capabilities from bench.config import build_config -# Setup logging logging.basicConfig( level=logging.INFO, format="%(levelname)s: %(message)s", @@ -93,14 +92,12 @@ def cmd_run(args: argparse.Namespace) -> int: if args.verbose: logging.getLogger().setLevel(logging.DEBUG) - # Validate config errors = config.validate() if errors: for error in errors: logger.error(error) return 1 - # Check binaries exist binaries_dir = ( Path(args.binaries_dir) if args.binaries_dir else Path(config.binaries_dir) ) @@ -135,7 +132,7 @@ def cmd_run(args: argparse.Namespace) -> int: if config.instrumented: from bench.analyze import AnalyzePhase - analyze_phase = AnalyzePhase(config) + analyze_phase = AnalyzePhase() if result.debug_log_base: try: @@ -245,7 +242,6 @@ def cmd_full(args: argparse.Namespace) -> int: from bench.analyze import AnalyzePhase from bench.benchmark import BenchmarkPhase from bench.build import BuildPhase - from bench.utils import find_debug_log capabilities = detect_capabilities() config = build_config( @@ -316,7 +312,7 @@ def cmd_full(args: argparse.Namespace) -> int: # Phase 3: Analyze (for instrumented runs) if config.instrumented: logger.info("=== Phase 3: Analyze ===") - analyze_phase = AnalyzePhase(config) + analyze_phase = AnalyzePhase() # Analyze base debug log if benchmark_result.debug_log_base: @@ -353,7 +349,6 @@ def main() -> int: epilog=__doc__, ) - # Global options parser.add_argument( "--config", metavar="PATH", diff --git a/bench/analyze.py b/bench/analyze.py index a31b807c422b..423b67465a83 100644 --- a/bench/analyze.py +++ b/bench/analyze.py @@ -11,7 +11,6 @@ from collections import OrderedDict from dataclasses import dataclass from pathlib import Path -from typing import TYPE_CHECKING # matplotlib is optional - gracefully handle if not installed try: @@ -21,9 +20,6 @@ except ImportError: HAS_MATPLOTLIB = False -if TYPE_CHECKING: - from .config import Config - logger = logging.getLogger(__name__) # Bitcoin fork heights for plot annotations @@ -488,9 +484,6 @@ def _add_fork_markers(self, min_x: float, max_x: float, max_y: float) -> None: class AnalyzePhase: """Analyze benchmark results and generate plots.""" - def __init__(self, config: Config | None = None): - self.config = config - def run( self, commit: str, diff --git a/bench/report.py b/bench/report.py index a41a035fbe9a..16445bde5246 100644 --- a/bench/report.py +++ b/bench/report.py @@ -7,6 +7,8 @@ import json import logging +import re +import shutil from dataclasses import dataclass, field from pathlib import Path from typing import Any @@ -324,8 +326,6 @@ def _generate_html( def _linkify_commit(self, command: str) -> str: """Convert commit hashes in command to links.""" - import re - def replace_commit(match): commit = match.group(1) short_commit = commit[:8] if len(commit) > 8 else commit @@ -346,8 +346,6 @@ def _generate_graphs_section( commit = run.parameters.get("commit", "") if not commit: # Try to extract from command - import re - match = re.search(r"\(([a-f0-9]+)\)", run.command) if match: commit = match.group(1) @@ -403,8 +401,6 @@ def _generate_graphs_section( def _copy_artifacts(self, input_dir: Path, output_dir: Path) -> None: """Copy flamegraphs and plots to output directory.""" - import shutil - # Skip if input and output are the same directory if input_dir.resolve() == output_dir.resolve(): logger.debug("Input and output are the same directory, skipping copy") diff --git a/bench/utils.py b/bench/utils.py index 9cd28373d19d..df454cf0644e 100644 --- a/bench/utils.py +++ b/bench/utils.py @@ -1,28 +1,14 @@ -"""Utility functions for git, datadir, and system operations.""" +"""Utility functions for git operations.""" from __future__ import annotations import logging -import os -import shutil import subprocess -import tempfile from pathlib import Path -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from .capabilities import Capabilities - from .config import Config logger = logging.getLogger(__name__) -class GitError(Exception): - """Git operation failed.""" - - pass - - class GitState: """Saved git state for restoration after operations.""" @@ -80,6 +66,12 @@ def restore(self) -> None: ) +class GitError(Exception): + """Git operation failed.""" + + pass + + def git_checkout(commit: str, repo_path: Path | None = None) -> None: """Checkout a specific commit.""" repo_path = repo_path or Path.cwd() @@ -111,131 +103,3 @@ def git_rev_parse(ref: str, repo_path: Path | None = None) -> str: raise GitError(f"Failed to resolve {ref}: {result.stderr}") return result.stdout.strip() - - -def clean_datadir(datadir: Path) -> None: - """Remove all contents from a data directory.""" - if not datadir.exists(): - return - - logger.debug(f"Cleaning datadir: {datadir}") - for item in datadir.iterdir(): - if item.is_dir(): - shutil.rmtree(item) - else: - item.unlink() - - -def copy_datadir(src: Path, dst: Path, capabilities: Capabilities) -> None: - """Copy blockchain data from source to destination.""" - logger.info(f"Copying datadir: {src} -> {dst}") - - # Ensure destination exists - dst.mkdir(parents=True, exist_ok=True) - - # Build copy command - cmd = ["cp", "-r"] - # Copy contents, not directory itself - cmd += [str(src) + "/.", str(dst)] - - subprocess.run(cmd, check=True) - - -def drop_caches(capabilities: Capabilities) -> bool: - """Drop filesystem caches if available. - - Returns True if caches were dropped, False if not available. - """ - if not capabilities.can_drop_caches or not capabilities.drop_caches_path: - logger.debug("Cache dropping not available, skipping") - return False - - logger.debug("Dropping filesystem caches") - subprocess.run([capabilities.drop_caches_path], check=True) - return True - - -def clean_debug_logs(datadir: Path) -> None: - """Remove debug.log files from datadir and subdirectories.""" - logger.debug(f"Cleaning debug logs in: {datadir}") - - for log_file in datadir.rglob("debug.log"): - log_file.unlink() - - -def find_debug_log(datadir: Path) -> Path | None: - """Find debug.log in datadir or subdirectories.""" - # Check common locations - candidates = [ - datadir / "debug.log", - datadir / "mainnet" / "debug.log", - datadir / "testnet3" / "debug.log", - datadir / "signet" / "debug.log", - datadir / "regtest" / "debug.log", - ] - - for candidate in candidates: - if candidate.exists(): - return candidate - - # Fallback: search recursively - for log_file in datadir.rglob("debug.log"): - return log_file - - return None - - -def create_temp_script(commands: list[str], name: str = "hook") -> Path: - """Create a temporary shell script for hyperfine hooks. - - Returns path to the script. - """ - script_content = "#!/usr/bin/env bash\nset -euxo pipefail\n" - script_content += "\n".join(commands) + "\n" - - # Create temp file that persists (caller is responsible for cleanup) - fd, path = tempfile.mkstemp(suffix=".sh", prefix=f"bench_{name}_") - os.write(fd, script_content.encode()) - os.close(fd) - os.chmod(path, 0o755) - - return Path(path) - - -def build_bitcoind_cmd( - binary: Path, - datadir: Path, - config: Config, - capabilities: Capabilities, - debug_flags: list[str] | None = None, -) -> list[str]: - """Build the bitcoind command with optional wrappers. - - Args: - binary: Path to bitcoind binary - datadir: Data directory - config: Benchmark configuration - capabilities: System capabilities - debug_flags: Optional debug flags for instrumented mode - """ - cmd = [ - str(binary), - f"-datadir={datadir}", - f"-dbcache={config.dbcache}", - f"-stopatheight={config.stop_height}", - "-prune=10000", - f"-chain={config.chain}", - "-daemon=0", - "-printtoconsole=0", - ] - - # Add connect address if specified - if config.connect: - cmd.append(f"-connect={config.connect}") - - # Add debug flags for instrumented mode - if debug_flags: - for flag in debug_flags: - cmd.append(f"-debug={flag}") - - return cmd From 419cd8640c1e6567752ed6ad44f0aaf5486aa531 Mon Sep 17 00:00:00 2001 From: will Date: Tue, 9 Dec 2025 09:48:37 +0000 Subject: [PATCH 39/51] fixup! use python runner --- bench/analyze.py | 9 +++++++++ bench/benchmark.py | 24 +++++++++++++++++++++++- bench/build.py | 11 +++++++++-- 3 files changed, 41 insertions(+), 3 deletions(-) diff --git a/bench/analyze.py b/bench/analyze.py index 423b67465a83..a5f52b4ce035 100644 --- a/bench/analyze.py +++ b/bench/analyze.py @@ -515,7 +515,16 @@ def run( parser = LogParser() data = parser.parse_file(log_file) + # Log parsed data summary + logger.info(f" UpdateTip entries: {len(data.update_tip)}") + logger.info(f" LevelDB compact entries: {len(data.leveldb_compact)}") + logger.info(f" LevelDB gen table entries: {len(data.leveldb_gen_table)}") + logger.info(f" Validation txadd entries: {len(data.validation_txadd)}") + logger.info(f" CoinDB write batch entries: {len(data.coindb_write_batch)}") + logger.info(f" CoinDB commit entries: {len(data.coindb_commit)}") + logger.info(f"Generating plots for {commit[:12]}") + logger.info(f" Output directory: {output_dir}") generator = PlotGenerator(commit[:12], output_dir) plots = generator.generate_all(data) diff --git a/bench/benchmark.py b/bench/benchmark.py index fae2cd7fee15..886245d11546 100644 --- a/bench/benchmark.py +++ b/bench/benchmark.py @@ -87,6 +87,9 @@ def run( results_file = output_dir / "results.json" logger.info("Starting benchmark") + logger.info(f" Output dir: {output_dir}") + logger.info(f" Temp datadir: {tmp_datadir}") + logger.info(f" Source datadir: {datadir}") logger.info(f" Base: {base_commit[:12]}") logger.info(f" Head: {head_commit[:12]}") logger.info(f" Instrumented: {self.config.instrumented}") @@ -114,6 +117,14 @@ def run( output_dir=output_dir, ) + # Log the commands being benchmarked + base_cmd = self._build_bitcoind_cmd(base_binary, tmp_datadir) + head_cmd = self._build_bitcoind_cmd(head_binary, tmp_datadir) + logger.info("Base command:") + logger.info(f" {base_cmd}") + logger.info("Head command:") + logger.info(f" {head_cmd}") + if self.config.dry_run: logger.info(f"[DRY RUN] Would run: {' '.join(cmd)}") return BenchmarkResult( @@ -123,8 +134,10 @@ def run( instrumented=self.config.instrumented, ) - # Run hyperfine + # Log the full hyperfine command logger.info("Running hyperfine...") + logger.info(f" Command: {' '.join(cmd[:7])} ...") # First few args + logger.debug(f" Full command: {' '.join(cmd)}") _result = subprocess.run(cmd, check=True) # Collect results @@ -137,6 +150,7 @@ def run( # For instrumented runs, collect flamegraphs and debug logs if self.config.instrumented: + logger.info("Collecting instrumented artifacts...") base_fg = output_dir / f"{base_commit[:12]}-flamegraph.svg" head_fg = output_dir / f"{head_commit[:12]}-flamegraph.svg" base_log = output_dir / f"{base_commit[:12]}-debug.log" @@ -149,16 +163,21 @@ def run( ]: src = Path(src_name) if src.exists(): + logger.info(f" Moving {src_name} -> {dest}") shutil.move(str(src), str(dest)) if base_fg.exists(): benchmark_result.flamegraph_base = base_fg + logger.info(f" Flamegraph (base): {base_fg}") if head_fg.exists(): benchmark_result.flamegraph_head = head_fg + logger.info(f" Flamegraph (head): {head_fg}") if base_log.exists(): benchmark_result.debug_log_base = base_log + logger.info(f" Debug log (base): {base_log}") if head_log.exists(): benchmark_result.debug_log_head = head_log + logger.info(f" Debug log (head): {head_log}") # Clean up tmp_datadir if tmp_datadir.exists(): @@ -186,6 +205,9 @@ def _create_temp_script(self, commands: list[str], name: str) -> Path: script_path = Path(path) self._temp_scripts.append(script_path) + logger.debug(f"Created {name} script: {script_path}") + for cmd in commands: + logger.debug(f" {cmd}") return script_path def _create_setup_script(self, tmp_datadir: Path) -> Path: diff --git a/bench/build.py b/bench/build.py index 7b7c749b433f..71e7e30b3288 100644 --- a/bench/build.py +++ b/bench/build.py @@ -71,6 +71,8 @@ def run( logger.info("Building binaries for comparison:") logger.info(f" Base: {base_hash[:12]} ({base_commit})") logger.info(f" Head: {head_hash[:12]} ({head_commit})") + logger.info(f" Repo: {self.repo_path}") + logger.info(f" Output: {binaries_dir}") # Setup output directories base_dir = binaries_dir / "base" @@ -132,12 +134,14 @@ def _build_commit(self, name: str, commit: str, output_path: Path) -> None: return # Checkout the commit + logger.info(f" Checking out {commit[:12]}...") git_checkout(commit, self.repo_path) # Build with nix cmd = ["nix", "build", "-L"] - logger.debug(f"Running: {' '.join(cmd)}") + logger.info(f" Running: {' '.join(cmd)}") + logger.info(f" Working directory: {self.repo_path}") result = subprocess.run( cmd, cwd=self.repo_path, @@ -151,6 +155,8 @@ def _build_commit(self, name: str, commit: str, output_path: Path) -> None: if not nix_binary.exists(): raise RuntimeError(f"Built binary not found at {nix_binary}") + logger.info(f" Copying {nix_binary} -> {output_path}") + # Remove existing binary if present (may be read-only from nix) if output_path.exists(): output_path.chmod(0o755) @@ -158,9 +164,10 @@ def _build_commit(self, name: str, commit: str, output_path: Path) -> None: shutil.copy2(nix_binary, output_path) output_path.chmod(0o755) # Ensure it's executable and writable - logger.info(f"Built {name} binary: {output_path}") + logger.info(f" Built {name} binary: {output_path}") # Clean up nix result symlink result_link = self.repo_path / "result" if result_link.is_symlink(): + logger.debug(f" Removing nix result symlink: {result_link}") result_link.unlink() From 14d1cf071658d37ca70bb5c6504e00951e53931a Mon Sep 17 00:00:00 2001 From: will Date: Tue, 9 Dec 2025 10:10:35 +0000 Subject: [PATCH 40/51] arbitrary bins --- .github/workflows/benchmark.yml | 12 +- bench.py | 380 ++++++++++++-------------------- bench/benchmark.py | 171 +++++++------- bench/build.py | 148 +++++++------ bench/capabilities.py | 1 - bench/compare.py | 180 +++++++++++++++ bench/report.py | 54 +++-- justfile | 49 ++-- 8 files changed, 558 insertions(+), 437 deletions(-) create mode 100644 bench/compare.py diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 31ed48b1bd10..f58da7c5bce5 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -23,8 +23,8 @@ jobs: - name: Build both binaries run: | nix develop --command python3 bench.py build \ - --binaries-dir ${{ runner.temp }}/binaries \ - $BASE_SHA $HEAD_SHA + -o ${{ runner.temp }}/binaries \ + $BASE_SHA:base $HEAD_SHA:head - name: Upload binaries uses: actions/upload-artifact@v4 @@ -73,12 +73,12 @@ jobs: - name: Run benchmark run: | nix develop --command python3 bench.py --profile ci run \ - --binaries-dir ${{ runner.temp }}/binaries \ --datadir $ORIGINAL_DATADIR \ --tmp-datadir ${{ runner.temp }}/datadir \ --output-dir ${{ runner.temp }}/output \ --dbcache ${{ matrix.dbcache }} \ - $BASE_SHA $HEAD_SHA + base:${{ runner.temp }}/binaries/base/bitcoind \ + head:${{ runner.temp }}/binaries/head/bitcoind - name: Upload results uses: actions/upload-artifact@v4 @@ -143,12 +143,12 @@ jobs: run: | nix develop --command python3 bench.py --profile ci run \ --instrumented \ - --binaries-dir ${{ runner.temp }}/binaries \ --datadir $ORIGINAL_DATADIR \ --tmp-datadir ${{ runner.temp }}/datadir \ --output-dir ${{ runner.temp }}/output \ --dbcache ${{ matrix.dbcache }} \ - $BASE_SHA $HEAD_SHA + base:${{ runner.temp }}/binaries/base/bitcoind \ + head:${{ runner.temp }}/binaries/head/bitcoind - name: Upload results uses: actions/upload-artifact@v4 diff --git a/bench.py b/bench.py index 157072f95c28..04acef1a0d51 100755 --- a/bench.py +++ b/bench.py @@ -5,11 +5,24 @@ performance. Usage: - bench.py build BASE HEAD Build bitcoind at two commits - bench.py run BASE HEAD Run benchmark - bench.py analyze LOGFILE Generate plots from debug.log + bench.py build COMMIT[:NAME]... Build bitcoind at one or more commits + bench.py run NAME:BINARY... Benchmark one or more binaries + bench.py analyze COMMIT LOGFILE Generate plots from debug.log + bench.py compare RESULTS... Compare benchmark results bench.py report INPUT OUTPUT Generate HTML report - bench.py full BASE HEAD Complete pipeline: build → run → analyze + +Examples: + # Build two commits + bench.py build HEAD~1:before HEAD:after + + # Benchmark built binaries + bench.py run before:./binaries/before/bitcoind after:./binaries/after/bitcoind --datadir /data + + # Compare results + bench.py compare ./bench-output/results.json + + # Generate HTML report + bench.py report ./bench-output ./report """ from __future__ import annotations @@ -30,13 +43,13 @@ def cmd_build(args: argparse.Namespace) -> int: - """Build bitcoind at two commits.""" + """Build bitcoind at one or more commits.""" from bench.build import BuildPhase capabilities = detect_capabilities() config = build_config( cli_args={ - "binaries_dir": args.binaries_dir, + "binaries_dir": args.output_dir, "skip_existing": args.skip_existing, "dry_run": args.dry_run, "verbose": args.verbose, @@ -52,12 +65,12 @@ def cmd_build(args: argparse.Namespace) -> int: try: result = phase.run( - args.base_commit, - args.head_commit, - binaries_dir=Path(args.binaries_dir) if args.binaries_dir else None, + args.commits, + output_dir=Path(args.output_dir) if args.output_dir else None, ) - logger.info(f"Built base binary: {result.base_binary}") - logger.info(f"Built head binary: {result.head_binary}") + logger.info(f"Built {len(result.binaries)} binary(ies):") + for binary in result.binaries: + logger.info(f" {binary.name}: {binary.path}") return 0 except Exception as e: logger.error(f"Build failed: {e}") @@ -65,15 +78,14 @@ def cmd_build(args: argparse.Namespace) -> int: def cmd_run(args: argparse.Namespace) -> int: - """Run benchmark comparing two commits.""" - from bench.benchmark import BenchmarkPhase + """Run benchmark on one or more binaries.""" + from bench.benchmark import BenchmarkPhase, parse_binary_spec capabilities = detect_capabilities() config = build_config( cli_args={ "datadir": args.datadir, "tmp_datadir": args.tmp_datadir, - "binaries_dir": args.binaries_dir, "output_dir": args.output_dir, "stop_height": args.stop_height, "dbcache": args.dbcache, @@ -98,31 +110,25 @@ def cmd_run(args: argparse.Namespace) -> int: logger.error(error) return 1 - binaries_dir = ( - Path(args.binaries_dir) if args.binaries_dir else Path(config.binaries_dir) - ) - base_binary = binaries_dir / "base" / "bitcoind" - head_binary = binaries_dir / "head" / "bitcoind" - - if not base_binary.exists(): - logger.error(f"Base binary not found: {base_binary}") - logger.error("Run 'bench.py build' first") + # Parse binary specs + try: + binaries = [parse_binary_spec(spec) for spec in args.binaries] + except ValueError as e: + logger.error(str(e)) return 1 - if not head_binary.exists(): - logger.error(f"Head binary not found: {head_binary}") - logger.error("Run 'bench.py build' first") - return 1 + # Validate binaries exist + for name, path in binaries: + if not path.exists(): + logger.error(f"Binary not found: {path} ({name})") + return 1 phase = BenchmarkPhase(config, capabilities) output_dir = Path(config.output_dir) try: result = phase.run( - base_commit=args.base_commit, - head_commit=args.head_commit, - base_binary=base_binary, - head_binary=head_binary, + binaries=binaries, datadir=Path(config.datadir), output_dir=output_dir, ) @@ -134,25 +140,16 @@ def cmd_run(args: argparse.Namespace) -> int: analyze_phase = AnalyzePhase() - if result.debug_log_base: - try: - analyze_phase.run( - commit=args.base_commit, - log_file=result.debug_log_base, - output_dir=output_dir / "plots", - ) - except Exception as e: - logger.warning(f"Analysis for base failed: {e}") - - if result.debug_log_head: - try: - analyze_phase.run( - commit=args.head_commit, - log_file=result.debug_log_head, - output_dir=output_dir / "plots", - ) - except Exception as e: - logger.warning(f"Analysis for head failed: {e}") + for binary_result in result.binaries: + if binary_result.debug_log: + try: + analyze_phase.run( + commit=binary_result.name, + log_file=binary_result.debug_log, + output_dir=output_dir / "plots", + ) + except Exception as e: + logger.warning(f"Analysis for {binary_result.name} failed: {e}") return 0 except Exception as e: @@ -164,6 +161,46 @@ def cmd_run(args: argparse.Namespace) -> int: return 1 +def cmd_compare(args: argparse.Namespace) -> int: + """Compare benchmark results from multiple files.""" + from bench.compare import ComparePhase + + if args.verbose: + logging.getLogger().setLevel(logging.DEBUG) + + results_files = [Path(f) for f in args.results_files] + + # Validate files exist + for f in results_files: + if not f.exists(): + logger.error(f"Results file not found: {f}") + return 1 + + phase = ComparePhase() + + try: + result = phase.run(results_files, baseline=args.baseline) + + # Output results + output_json = phase.to_json(result) + + if args.output: + output_path = Path(args.output) + output_path.write_text(output_json) + logger.info(f"Comparison saved to: {output_path}") + else: + print(output_json) + + return 0 + except Exception as e: + logger.error(f"Comparison failed: {e}") + if args.verbose: + import traceback + + traceback.print_exc() + return 1 + + def cmd_analyze(args: argparse.Namespace) -> int: """Generate plots from debug.log.""" from bench.analyze import AnalyzePhase @@ -237,110 +274,6 @@ def cmd_report(args: argparse.Namespace) -> int: return 1 -def cmd_full(args: argparse.Namespace) -> int: - """Run full pipeline: build → run → analyze.""" - from bench.analyze import AnalyzePhase - from bench.benchmark import BenchmarkPhase - from bench.build import BuildPhase - - capabilities = detect_capabilities() - config = build_config( - cli_args={ - "datadir": args.datadir, - "tmp_datadir": args.tmp_datadir, - "binaries_dir": args.binaries_dir, - "output_dir": args.output_dir, - "stop_height": args.stop_height, - "dbcache": args.dbcache, - "runs": args.runs, - "connect": args.connect, - "chain": args.chain, - "instrumented": args.instrumented, - "skip_existing": args.skip_existing, - "no_cache_drop": args.no_cache_drop, - "dry_run": args.dry_run, - "verbose": args.verbose, - }, - config_file=Path(args.config) if args.config else None, - profile=args.profile, - ) - - if args.verbose: - logging.getLogger().setLevel(logging.DEBUG) - - # Validate config - errors = config.validate() - if errors: - for error in errors: - logger.error(error) - return 1 - - output_dir = Path(config.output_dir) - binaries_dir = Path(config.binaries_dir) - - # Phase 1: Build - logger.info("=== Phase 1: Build ===") - build_phase = BuildPhase(config, capabilities) - - try: - build_result = build_phase.run( - args.base_commit, - args.head_commit, - binaries_dir=binaries_dir, - ) - except Exception as e: - logger.error(f"Build failed: {e}") - return 1 - - # Phase 2: Benchmark - logger.info("=== Phase 2: Benchmark ===") - benchmark_phase = BenchmarkPhase(config, capabilities) - - try: - benchmark_result = benchmark_phase.run( - base_commit=build_result.base_commit, - head_commit=build_result.head_commit, - base_binary=build_result.base_binary, - head_binary=build_result.head_binary, - datadir=Path(config.datadir), - output_dir=output_dir, - ) - except Exception as e: - logger.error(f"Benchmark failed: {e}") - return 1 - - # Phase 3: Analyze (for instrumented runs) - if config.instrumented: - logger.info("=== Phase 3: Analyze ===") - analyze_phase = AnalyzePhase() - - # Analyze base debug log - if benchmark_result.debug_log_base: - try: - analyze_phase.run( - commit=build_result.base_commit, - log_file=benchmark_result.debug_log_base, - output_dir=output_dir / "plots", - ) - except Exception as e: - logger.warning(f"Analysis for base failed: {e}") - - # Analyze head debug log - if benchmark_result.debug_log_head: - try: - analyze_phase.run( - commit=build_result.head_commit, - log_file=benchmark_result.debug_log_head, - output_dir=output_dir / "plots", - ) - except Exception as e: - logger.warning(f"Analysis for head failed: {e}") - - logger.info("=== Complete ===") - logger.info(f"Results: {benchmark_result.results_file}") - return 0 - - def main() -> int: """Main entry point.""" parser = argparse.ArgumentParser( @@ -375,11 +308,21 @@ def main() -> int: subparsers = parser.add_subparsers(dest="command", help="Commands") # Build command - build_parser = subparsers.add_parser("build", help="Build bitcoind at two commits") - build_parser.add_argument("base_commit", help="Base commit (for comparison)") - build_parser.add_argument("head_commit", help="Head commit (new code)") + build_parser = subparsers.add_parser( + "build", + help="Build bitcoind at one or more commits", + description="Build bitcoind binaries from git commits. " + "Each commit can optionally have a name suffix: COMMIT:NAME", + ) + build_parser.add_argument( + "commits", + nargs="+", + metavar="COMMIT[:NAME]", + help="Commit(s) to build. Format: COMMIT or COMMIT:NAME (e.g., HEAD:latest, abc123:v27)", + ) build_parser.add_argument( - "--binaries-dir", + "-o", + "--output-dir", metavar="PATH", help="Where to store binaries (default: ./binaries)", ) @@ -391,9 +334,18 @@ def main() -> int: build_parser.set_defaults(func=cmd_build) # Run command - run_parser = subparsers.add_parser("run", help="Run benchmark") - run_parser.add_argument("base_commit", help="Base commit hash") - run_parser.add_argument("head_commit", help="Head commit hash") + run_parser = subparsers.add_parser( + "run", + help="Run benchmark on one or more binaries", + description="Benchmark bitcoind binaries using hyperfine. " + "Each binary must have a name and path: NAME:PATH", + ) + run_parser.add_argument( + "binaries", + nargs="+", + metavar="NAME:PATH", + help="Binary(ies) to benchmark. Format: NAME:PATH (e.g., v27:./binaries/v27/bitcoind)", + ) run_parser.add_argument( "--datadir", required=True, @@ -406,14 +358,10 @@ def main() -> int: help="Temp datadir for benchmark runs", ) run_parser.add_argument( - "--binaries-dir", - metavar="PATH", - help="Location of pre-built binaries", - ) - run_parser.add_argument( + "-o", "--output-dir", metavar="PATH", - help="Output directory for results", + help="Output directory for results (default: ./bench-output)", ) run_parser.add_argument( "--stop-height", @@ -469,6 +417,32 @@ def main() -> int: ) analyze_parser.set_defaults(func=cmd_analyze) + # Compare command + compare_parser = subparsers.add_parser( + "compare", + help="Compare benchmark results from multiple files", + description="Load and compare results from one or more results.json files. " + "Calculates speedup percentages relative to a baseline.", + ) + compare_parser.add_argument( + "results_files", + nargs="+", + metavar="RESULTS_FILE", + help="results.json file(s) to compare", + ) + compare_parser.add_argument( + "--baseline", + metavar="NAME", + help="Name of the baseline entry (default: first entry)", + ) + compare_parser.add_argument( + "-o", + "--output", + metavar="FILE", + help="Output file for comparison JSON (default: stdout)", + ) + compare_parser.set_defaults(func=cmd_compare) + # Report command report_parser = subparsers.add_parser("report", help="Generate HTML report") report_parser.add_argument("input_dir", help="Directory with results.json") @@ -479,78 +453,6 @@ def main() -> int: ) report_parser.set_defaults(func=cmd_report) - # Full command - full_parser = subparsers.add_parser( - "full", help="Full pipeline: build → run → analyze" - ) - full_parser.add_argument("base_commit", help="Base commit (for comparison)") - full_parser.add_argument("head_commit", help="Head commit (new code)") - full_parser.add_argument( - "--datadir", - required=True, - metavar="PATH", - help="Source datadir with blockchain snapshot", - ) - full_parser.add_argument( - "--tmp-datadir", - metavar="PATH", - help="Temp datadir for benchmark runs", - ) - full_parser.add_argument( - "--binaries-dir", - metavar="PATH", - help="Where to store binaries", - ) - full_parser.add_argument( - "--output-dir", - metavar="PATH", - help="Output directory for results", - ) - full_parser.add_argument( - "--stop-height", - type=int, - metavar="N", - help="Block height to stop at", - ) - full_parser.add_argument( - "--dbcache", - type=int, - metavar="N", - help="Database cache size in MB", - ) - full_parser.add_argument( - "--runs", - type=int, - metavar="N", - help="Number of benchmark iterations", - ) - full_parser.add_argument( - "--connect", - metavar="ADDR", - help="Connect address for sync", - ) - full_parser.add_argument( - "--chain", - choices=["main", "testnet", "signet", "regtest"], - help="Chain to use", - ) - full_parser.add_argument( - "--instrumented", - action="store_true", - help="Enable profiling (flamegraph + debug logging)", - ) - full_parser.add_argument( - "--skip-existing", - action="store_true", - help="Skip build if binary already exists", - ) - full_parser.add_argument( - "--no-cache-drop", - action="store_true", - help="Skip cache dropping between runs", - ) - full_parser.set_defaults(func=cmd_full) - args = parser.parse_args() if not args.command: diff --git a/bench/benchmark.py b/bench/benchmark.py index 886245d11546..5dc25a0430e6 100644 --- a/bench/benchmark.py +++ b/bench/benchmark.py @@ -1,4 +1,4 @@ -"""Benchmark phase - run hyperfine benchmarks comparing two bitcoind binaries.""" +"""Benchmark phase - run hyperfine benchmarks on bitcoind binaries.""" from __future__ import annotations @@ -7,7 +7,7 @@ import shutil import subprocess import tempfile -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path from typing import TYPE_CHECKING @@ -22,22 +22,39 @@ INSTRUMENTED_DEBUG_FLAGS = ["coindb", "leveldb", "bench", "validation"] +@dataclass +class BinaryResult: + """Result for a single binary.""" + + name: str + flamegraph: Path | None = None + debug_log: Path | None = None + + @dataclass class BenchmarkResult: """Result of the benchmark phase.""" results_file: Path - base_commit: str - head_commit: str instrumented: bool - flamegraph_base: Path | None = None - flamegraph_head: Path | None = None - debug_log_base: Path | None = None - debug_log_head: Path | None = None + binaries: list[BinaryResult] = field(default_factory=list) + + +def parse_binary_spec(spec: str) -> tuple[str, Path]: + """Parse a binary spec like 'name:/path/to/binary'. + + Returns (name, path). + """ + if ":" not in spec: + raise ValueError(f"Invalid binary spec '{spec}': must be NAME:PATH") + name, path_str = spec.split(":", 1) + if not name: + raise ValueError(f"Invalid binary spec '{spec}': name cannot be empty") + return name, Path(path_str) class BenchmarkPhase: - """Run hyperfine benchmarks comparing two bitcoind binaries.""" + """Run hyperfine benchmarks on bitcoind binaries.""" def __init__( self, @@ -50,26 +67,28 @@ def __init__( def run( self, - base_commit: str, - head_commit: str, - base_binary: Path, - head_binary: Path, + binaries: list[tuple[str, Path]], datadir: Path, output_dir: Path, ) -> BenchmarkResult: - """Run benchmarks comparing base and head binaries. + """Run benchmarks on given binaries. Args: - base_commit: Git hash of base commit - head_commit: Git hash of head commit - base_binary: Path to base bitcoind binary - head_binary: Path to head bitcoind binary + binaries: List of (name, binary_path) tuples datadir: Source datadir with blockchain snapshot output_dir: Where to store results Returns: BenchmarkResult with paths to outputs """ + if not binaries: + raise ValueError("At least one binary is required") + + # Validate all binaries exist + for name, path in binaries: + if not path.exists(): + raise FileNotFoundError(f"Binary not found: {path} ({name})") + # Check prerequisites errors = self.capabilities.check_for_run(self.config.instrumented) if errors: @@ -90,8 +109,9 @@ def run( logger.info(f" Output dir: {output_dir}") logger.info(f" Temp datadir: {tmp_datadir}") logger.info(f" Source datadir: {datadir}") - logger.info(f" Base: {base_commit[:12]}") - logger.info(f" Head: {head_commit[:12]}") + logger.info(f" Binaries: {len(binaries)}") + for name, path in binaries: + logger.info(f" {name}: {path}") logger.info(f" Instrumented: {self.config.instrumented}") logger.info(f" Runs: {self.config.runs}") logger.info(f" Stop height: {self.config.stop_height}") @@ -105,10 +125,7 @@ def run( # Build hyperfine command cmd = self._build_hyperfine_cmd( - base_commit=base_commit, - head_commit=head_commit, - base_binary=base_binary, - head_binary=head_binary, + binaries=binaries, tmp_datadir=tmp_datadir, results_file=results_file, setup_script=setup_script, @@ -118,19 +135,15 @@ def run( ) # Log the commands being benchmarked - base_cmd = self._build_bitcoind_cmd(base_binary, tmp_datadir) - head_cmd = self._build_bitcoind_cmd(head_binary, tmp_datadir) - logger.info("Base command:") - logger.info(f" {base_cmd}") - logger.info("Head command:") - logger.info(f" {head_cmd}") + logger.info("Commands to benchmark:") + for name, path in binaries: + bitcoind_cmd = self._build_bitcoind_cmd(path, tmp_datadir) + logger.info(f" {name}: {bitcoind_cmd}") if self.config.dry_run: logger.info(f"[DRY RUN] Would run: {' '.join(cmd)}") return BenchmarkResult( results_file=results_file, - base_commit=base_commit, - head_commit=head_commit, instrumented=self.config.instrumented, ) @@ -138,46 +151,31 @@ def run( logger.info("Running hyperfine...") logger.info(f" Command: {' '.join(cmd[:7])} ...") # First few args logger.debug(f" Full command: {' '.join(cmd)}") - _result = subprocess.run(cmd, check=True) + subprocess.run(cmd, check=True) # Collect results benchmark_result = BenchmarkResult( results_file=results_file, - base_commit=base_commit, - head_commit=head_commit, instrumented=self.config.instrumented, ) # For instrumented runs, collect flamegraphs and debug logs if self.config.instrumented: logger.info("Collecting instrumented artifacts...") - base_fg = output_dir / f"{base_commit[:12]}-flamegraph.svg" - head_fg = output_dir / f"{head_commit[:12]}-flamegraph.svg" - base_log = output_dir / f"{base_commit[:12]}-debug.log" - head_log = output_dir / f"{head_commit[:12]}-debug.log" - - # Move flamegraphs from current directory if they exist - for src_name, dest in [ - ("base-flamegraph.svg", base_fg), - ("head-flamegraph.svg", head_fg), - ]: - src = Path(src_name) - if src.exists(): - logger.info(f" Moving {src_name} -> {dest}") - shutil.move(str(src), str(dest)) - - if base_fg.exists(): - benchmark_result.flamegraph_base = base_fg - logger.info(f" Flamegraph (base): {base_fg}") - if head_fg.exists(): - benchmark_result.flamegraph_head = head_fg - logger.info(f" Flamegraph (head): {head_fg}") - if base_log.exists(): - benchmark_result.debug_log_base = base_log - logger.info(f" Debug log (base): {base_log}") - if head_log.exists(): - benchmark_result.debug_log_head = head_log - logger.info(f" Debug log (head): {head_log}") + for name, _path in binaries: + binary_result = BinaryResult(name=name) + + flamegraph_file = output_dir / f"{name}-flamegraph.svg" + debug_log_file = output_dir / f"{name}-debug.log" + + if flamegraph_file.exists(): + binary_result.flamegraph = flamegraph_file + logger.info(f" Flamegraph ({name}): {flamegraph_file}") + if debug_log_file.exists(): + binary_result.debug_log = debug_log_file + logger.info(f" Debug log ({name}): {debug_log_file}") + + benchmark_result.binaries.append(binary_result) # Clean up tmp_datadir if tmp_datadir.exists(): @@ -283,10 +281,7 @@ def _build_bitcoind_cmd( def _build_hyperfine_cmd( self, - base_commit: str, - head_commit: str, - base_binary: Path, - head_binary: Path, + binaries: list[tuple[str, Path]], tmp_datadir: Path, results_file: Path, setup_script: Path, @@ -306,52 +301,42 @@ def _build_hyperfine_cmd( "--show-output", ] - # For instrumented runs, we need separate conclude scripts per commit - # since hyperfine's parameter substitution doesn't work with --conclude - if self.config.instrumented: - base_conclude = self._create_conclude_script_for_commit( - base_commit[:12], tmp_datadir, output_dir - ) - head_conclude = self._create_conclude_script_for_commit( - head_commit[:12], tmp_datadir, output_dir - ) - # We'll handle conclude differently - see below - - # Command names - cmd.append(f"--command-name=base ({base_commit[:12]})") - cmd.append(f"--command-name=head ({head_commit[:12]})") + # Add command names and build commands + for name, binary_path in binaries: + cmd.append(f"--command-name={name}") # Build the actual commands to benchmark - base_cmd = self._build_bitcoind_cmd(base_binary, tmp_datadir) - head_cmd = self._build_bitcoind_cmd(head_binary, tmp_datadir) + for name, binary_path in binaries: + bitcoind_cmd = self._build_bitcoind_cmd(binary_path, tmp_datadir) - # For instrumented runs, append the conclude logic to each command - if self.config.instrumented: - base_cmd += f" && {base_conclude}" - head_cmd += f" && {head_conclude}" + # For instrumented runs, append the conclude logic to each command + if self.config.instrumented: + conclude = self._create_conclude_commands(name, tmp_datadir, output_dir) + bitcoind_cmd += f" && {conclude}" - cmd.append(base_cmd) - cmd.append(head_cmd) + cmd.append(bitcoind_cmd) return cmd - def _create_conclude_script_for_commit( + def _create_conclude_commands( self, - commit: str, + name: str, tmp_datadir: Path, output_dir: Path, ) -> str: - """Create inline conclude commands for a specific commit.""" + """Create inline conclude commands for a specific binary.""" # Return shell commands to run after each benchmark commands = [] # Move flamegraph if exists - commands.append(f'if [ -e flamegraph.svg ]; then mv flamegraph.svg "{output_dir}/{commit}-flamegraph.svg"; fi') + commands.append( + f'if [ -e flamegraph.svg ]; then mv flamegraph.svg "{output_dir}/{name}-flamegraph.svg"; fi' + ) # Copy debug log if exists commands.append( f'debug_log=$(find "{tmp_datadir}" -name debug.log -print -quit); ' - f'if [ -n "$debug_log" ]; then cp "$debug_log" "{output_dir}/{commit}-debug.log"; fi' + f'if [ -n "$debug_log" ]; then cp "$debug_log" "{output_dir}/{name}-debug.log"; fi' ) return " && ".join(commands) diff --git a/bench/build.py b/bench/build.py index 71e7e30b3288..6187263a73de 100644 --- a/bench/build.py +++ b/bench/build.py @@ -18,18 +18,35 @@ logger = logging.getLogger(__name__) +@dataclass +class BuiltBinary: + """A single built binary.""" + + name: str + path: Path + commit: str + + @dataclass class BuildResult: """Result of the build phase.""" - base_binary: Path - head_binary: Path - base_commit: str - head_commit: str + binaries: list[BuiltBinary] + + +def parse_commit_spec(spec: str) -> tuple[str, str | None]: + """Parse a commit spec like 'abc123:name' or 'abc123'. + + Returns (commit, name) where name may be None. + """ + if ":" in spec: + commit, name = spec.split(":", 1) + return commit, name + return spec, None class BuildPhase: - """Build bitcoind binaries at two commits for comparison.""" + """Build bitcoind binaries at specified commits.""" def __init__( self, @@ -43,94 +60,101 @@ def __init__( def run( self, - base_commit: str, - head_commit: str, - binaries_dir: Path | None = None, + commit_specs: list[str], + output_dir: Path | None = None, ) -> BuildResult: - """Build bitcoind at both commits. + """Build bitcoind at given commits. Args: - base_commit: Git ref for base (comparison) commit - head_commit: Git ref for head (new) commit - binaries_dir: Where to store binaries (default: ./binaries) + commit_specs: List of commit specs like 'abc123:name' or 'abc123' + output_dir: Where to store binaries (default: ./binaries) Returns: - BuildResult with paths to built binaries + BuildResult with list of built binaries """ # Check prerequisites errors = self.capabilities.check_for_build() if errors: raise RuntimeError("Build prerequisites not met:\n" + "\n".join(errors)) - binaries_dir = binaries_dir or Path(self.config.binaries_dir) - - # Resolve commits to full hashes - base_hash = git_rev_parse(base_commit, self.repo_path) - head_hash = git_rev_parse(head_commit, self.repo_path) - - logger.info("Building binaries for comparison:") - logger.info(f" Base: {base_hash[:12]} ({base_commit})") - logger.info(f" Head: {head_hash[:12]} ({head_commit})") + output_dir = output_dir or Path(self.config.binaries_dir) + + # Parse commit specs and resolve to full hashes + commits: list[tuple[str, str, str]] = [] # (commit_hash, name, original_spec) + for spec in commit_specs: + commit, name = parse_commit_spec(spec) + commit_hash = git_rev_parse(commit, self.repo_path) + # Default name to short hash if not provided + if name is None: + name = commit_hash[:12] + commits.append((commit_hash, name, spec)) + + logger.info(f"Building {len(commits)} binary(ies):") + for commit_hash, name, spec in commits: + logger.info(f" {name}: {commit_hash[:12]} ({spec})") logger.info(f" Repo: {self.repo_path}") - logger.info(f" Output: {binaries_dir}") - - # Setup output directories - base_dir = binaries_dir / "base" - head_dir = binaries_dir / "head" - base_dir.mkdir(parents=True, exist_ok=True) - head_dir.mkdir(parents=True, exist_ok=True) - - base_binary = base_dir / "bitcoind" - head_binary = head_dir / "bitcoind" + logger.info(f" Output: {output_dir}") # Check if we can skip existing builds - if self.config.skip_existing: - if base_binary.exists() and head_binary.exists(): - logger.info( - "Both binaries exist and --skip-existing set, skipping build" - ) - return BuildResult( - base_binary=base_binary, - head_binary=head_binary, - base_commit=base_hash, - head_commit=head_hash, - ) + binaries_to_build: list[ + tuple[str, str, Path] + ] = [] # (commit_hash, name, output_path) + for commit_hash, name, _spec in commits: + binary_dir = output_dir / name + binary_dir.mkdir(parents=True, exist_ok=True) + binary_path = binary_dir / "bitcoind" + + if self.config.skip_existing and binary_path.exists(): + logger.info(f" Skipping {name} - binary exists") + else: + binaries_to_build.append((commit_hash, name, binary_path)) + + if not binaries_to_build: + logger.info("All binaries exist and --skip-existing set, skipping build") + return BuildResult( + binaries=[ + BuiltBinary( + name=name, + path=output_dir / name / "bitcoind", + commit=commit_hash, + ) + for commit_hash, name, _spec in commits + ] + ) # Save git state for restoration git_state = GitState(self.repo_path) git_state.save() - try: - # Build both commits - builds = [ - ("base", base_hash, base_binary), - ("head", head_hash, head_binary), - ] - - for name, commit, output_path in builds: - if self.config.skip_existing and output_path.exists(): - logger.info(f"Skipping {name} build - binary exists") - continue + built_binaries: list[BuiltBinary] = [] - self._build_commit(name, commit, output_path) + try: + for commit_hash, name, output_path in binaries_to_build: + self._build_commit(name, commit_hash, output_path) + built_binaries.append( + BuiltBinary(name=name, path=output_path, commit=commit_hash) + ) finally: # Always restore git state git_state.restore() - return BuildResult( - base_binary=base_binary, - head_binary=head_binary, - base_commit=base_hash, - head_commit=head_hash, - ) + # Include skipped binaries in result + all_binaries = [] + for commit_hash, name, _spec in commits: + binary_path = output_dir / name / "bitcoind" + all_binaries.append( + BuiltBinary(name=name, path=binary_path, commit=commit_hash) + ) + + return BuildResult(binaries=all_binaries) def _build_commit(self, name: str, commit: str, output_path: Path) -> None: """Build bitcoind for a single commit.""" logger.info(f"Building {name} ({commit[:12]})") if self.config.dry_run: - logger.info(f"[DRY RUN] Would build {commit[:12]} -> {output_path}") + logger.info(f" [DRY RUN] Would build {commit[:12]} -> {output_path}") return # Checkout the commit diff --git a/bench/capabilities.py b/bench/capabilities.py index b01ce2f3a711..31b6bd59f05f 100644 --- a/bench/capabilities.py +++ b/bench/capabilities.py @@ -8,7 +8,6 @@ import os import shutil -import subprocess from dataclasses import dataclass from pathlib import Path diff --git a/bench/compare.py b/bench/compare.py new file mode 100644 index 000000000000..fac328841634 --- /dev/null +++ b/bench/compare.py @@ -0,0 +1,180 @@ +"""Compare phase - compare benchmark results from multiple runs.""" + +from __future__ import annotations + +import json +import logging +from dataclasses import dataclass +from pathlib import Path + +logger = logging.getLogger(__name__) + + +@dataclass +class BenchmarkEntry: + """A single benchmark entry from results.json.""" + + command: str + mean: float + stddev: float | None + user: float + system: float + min: float + max: float + times: list[float] + + +@dataclass +class Comparison: + """Comparison of one entry against the baseline.""" + + name: str + mean: float + baseline_mean: float + speedup_percent: float + stddev: float | None + + +@dataclass +class CompareResult: + """Result of comparison.""" + + baseline: str + comparisons: list[Comparison] + + +class ComparePhase: + """Compare benchmark results from multiple results.json files.""" + + def run( + self, + results_files: list[Path], + baseline: str | None = None, + ) -> CompareResult: + """Compare benchmark results. + + Args: + results_files: List of results.json files to compare + baseline: Name of the baseline entry (default: first entry) + + Returns: + CompareResult with comparison data + """ + if not results_files: + raise ValueError("At least one results file is required") + + # Load all entries from all files + all_entries: list[BenchmarkEntry] = [] + for results_file in results_files: + if not results_file.exists(): + raise FileNotFoundError(f"Results file not found: {results_file}") + + logger.info(f"Loading results from: {results_file}") + with open(results_file) as f: + data = json.load(f) + + entries = self._parse_results(data) + logger.info(f" Found {len(entries)} entries") + all_entries.extend(entries) + + if not all_entries: + raise ValueError("No benchmark entries found in results files") + + # Determine baseline + if baseline is None: + baseline = all_entries[0].command + logger.info(f"Using baseline: {baseline}") + + # Find baseline entry + baseline_entry = None + for entry in all_entries: + if entry.command == baseline: + baseline_entry = entry + break + + if baseline_entry is None: + available = [e.command for e in all_entries] + raise ValueError( + f"Baseline '{baseline}' not found. Available: {', '.join(available)}" + ) + + # Calculate comparisons + comparisons: list[Comparison] = [] + for entry in all_entries: + if entry.command == baseline: + continue + + speedup = self._calculate_speedup(baseline_entry.mean, entry.mean) + comparisons.append( + Comparison( + name=entry.command, + mean=entry.mean, + baseline_mean=baseline_entry.mean, + speedup_percent=speedup, + stddev=entry.stddev, + ) + ) + + # Log results + logger.info("Comparison results:") + logger.info(f" Baseline ({baseline}): {baseline_entry.mean:.3f}s") + for comp in comparisons: + sign = "+" if comp.speedup_percent > 0 else "" + logger.info( + f" {comp.name}: {comp.mean:.3f}s ({sign}{comp.speedup_percent:.1f}%)" + ) + + return CompareResult( + baseline=baseline, + comparisons=comparisons, + ) + + def _parse_results(self, data: dict) -> list[BenchmarkEntry]: + """Parse results from hyperfine JSON output.""" + entries = [] + + results = data.get("results", []) + for result in results: + entries.append( + BenchmarkEntry( + command=result.get("command", "unknown"), + mean=result.get("mean", 0), + stddev=result.get("stddev"), + user=result.get("user", 0), + system=result.get("system", 0), + min=result.get("min", 0), + max=result.get("max", 0), + times=result.get("times", []), + ) + ) + + return entries + + def _calculate_speedup(self, baseline_mean: float, other_mean: float) -> float: + """Calculate speedup percentage. + + Positive = faster than baseline + Negative = slower than baseline + """ + if baseline_mean == 0: + return 0.0 + return round(((baseline_mean - other_mean) / baseline_mean) * 100, 1) + + def to_json(self, result: CompareResult) -> str: + """Convert comparison result to JSON.""" + return json.dumps( + { + "baseline": result.baseline, + "comparisons": [ + { + "name": c.name, + "mean": c.mean, + "baseline_mean": c.baseline_mean, + "speedup_percent": c.speedup_percent, + "stddev": c.stddev, + } + for c in result.comparisons + ], + }, + indent=2, + ) diff --git a/bench/report.py b/bench/report.py index 16445bde5246..25e3f9530be0 100644 --- a/bench/report.py +++ b/bench/report.py @@ -237,30 +237,29 @@ def _parse_results(self, data: dict) -> list[BenchmarkRun]: return runs def _calculate_speedups(self, runs: list[BenchmarkRun]) -> dict[str, float]: - """Calculate speedup percentages for each network.""" + """Calculate speedup percentages. + + Uses the first entry as baseline and compares all others against it. + Returns a dict mapping command name to speedup percentage. + """ speedups = {} - # Group by network - by_network: dict[str, list[BenchmarkRun]] = {} - for run in runs: - if run.network not in by_network: - by_network[run.network] = [] - by_network[run.network].append(run) + if len(runs) < 2: + return speedups - # Calculate speedup for each network - for network, network_runs in by_network.items(): - base_mean = None - head_mean = None + # Use first run as baseline + baseline = runs[0] + baseline_mean = baseline.mean - for run in network_runs: - if "base" in run.command.lower(): - base_mean = run.mean - elif "head" in run.command.lower(): - head_mean = run.mean + if baseline_mean <= 0: + return speedups - if base_mean and head_mean and base_mean > 0: - speedup = ((base_mean - head_mean) / base_mean) * 100 - speedups[network] = round(speedup, 1) + # Calculate speedup for each other run + for run in runs[1:]: + speedup = ((baseline_mean - run.mean) / baseline_mean) * 100 + # Use command name as key, extracting just the name part + name = run.command + speedups[name] = round(speedup, 1) return speedups @@ -300,17 +299,27 @@ def _generate_html( # Generate speedup rows speedup_rows = "" - for network, speedup in speedups.items(): + if sorted_runs: + # Add baseline row + baseline = sorted_runs[0] + speedup_rows += f""" + + {baseline.command} (baseline) + - + + """ + for name, speedup in speedups.items(): color_class = "" if speedup > 0: color_class = "text-green-600" elif speedup < 0: color_class = "text-red-600" + sign = "+" if speedup > 0 else "" speedup_rows += f""" - {network} - {speedup}% + {name} + {sign}{speedup}% """ @@ -326,6 +335,7 @@ def _generate_html( def _linkify_commit(self, command: str) -> str: """Convert commit hashes in command to links.""" + def replace_commit(match): commit = match.group(1) short_commit = commit[:8] if len(commit) > 8 else commit diff --git a/justfile b/justfile index 51faef3160da..d128c7e8b195 100644 --- a/justfile +++ b/justfile @@ -10,34 +10,55 @@ default: # Test instrumented run using signet (includes report generation) [group('local')] test-instrumented base head datadir: - nix develop --command python3 bench.py --profile quick full --chain signet --instrumented --skip-existing --datadir {{ datadir }} {{ base }} {{ head }} + nix develop --command python3 bench.py build --skip-existing {{ base }}:base {{ head }}:head + nix develop --command python3 bench.py --profile quick run \ + --chain signet \ + --instrumented \ + --datadir {{ datadir }} \ + base:./binaries/base/bitcoind \ + head:./binaries/head/bitcoind nix develop --command python3 bench.py report bench-output/ bench-output/ # Test uninstrumented run using signet [group('local')] test-uninstrumented base head datadir: - nix develop --command python3 bench.py --profile quick full --chain signet --skip-existing --datadir {{ datadir }} {{ base }} {{ head }} + nix develop --command python3 bench.py build --skip-existing {{ base }}:base {{ head }}:head + nix develop --command python3 bench.py --profile quick run \ + --chain signet \ + --datadir {{ datadir }} \ + base:./binaries/base/bitcoind \ + head:./binaries/head/bitcoind # Full benchmark with instrumentation (flamegraphs + plots) [group('local')] instrumented base head datadir: - python3 bench.py --profile quick full --instrumented --datadir {{ datadir }} {{ base }} {{ head }} + python3 bench.py build {{ base }}:base {{ head }}:head + python3 bench.py --profile quick run \ + --instrumented \ + --datadir {{ datadir }} \ + base:./binaries/base/bitcoind \ + head:./binaries/head/bitcoind # Just build binaries (useful for incremental testing) [group('local')] -build base head: - python3 bench.py build {{ base }} {{ head }} +build *commits: + python3 bench.py build {{ commits }} # Run benchmark with pre-built binaries [group('local')] -run base head datadir: - python3 bench.py run --datadir {{ datadir }} {{ base }} {{ head }} +run datadir *binaries: + python3 bench.py run --datadir {{ datadir }} {{ binaries }} # Generate plots from a debug.log file [group('local')] analyze commit logfile output_dir="./plots": python3 bench.py analyze {{ commit }} {{ logfile }} --output-dir {{ output_dir }} +# Compare benchmark results +[group('local')] +compare *results_files: + python3 bench.py compare {{ results_files }} + # Generate HTML report from benchmark results [group('local')] report input_dir output_dir: @@ -50,30 +71,30 @@ report input_dir output_dir: # Build binaries for CI [group('ci')] ci-build base_commit head_commit binaries_dir: - python3 bench.py build --binaries-dir {{ binaries_dir }} {{ base_commit }} {{ head_commit }} + python3 bench.py build -o {{ binaries_dir }} {{ base_commit }}:base {{ head_commit }}:head # Run uninstrumented benchmarks for CI [group('ci')] -ci-run base_commit head_commit datadir tmp_datadir output_dir dbcache binaries_dir: +ci-run datadir tmp_datadir output_dir dbcache binaries_dir: python3 bench.py --profile ci run \ - --binaries-dir {{ binaries_dir }} \ --datadir {{ datadir }} \ --tmp-datadir {{ tmp_datadir }} \ --output-dir {{ output_dir }} \ --dbcache {{ dbcache }} \ - {{ base_commit }} {{ head_commit }} + base:{{ binaries_dir }}/base/bitcoind \ + head:{{ binaries_dir }}/head/bitcoind # Run instrumented benchmarks for CI [group('ci')] -ci-run-instrumented base_commit head_commit datadir tmp_datadir output_dir dbcache binaries_dir: +ci-run-instrumented datadir tmp_datadir output_dir dbcache binaries_dir: python3 bench.py --profile ci run \ --instrumented \ - --binaries-dir {{ binaries_dir }} \ --datadir {{ datadir }} \ --tmp-datadir {{ tmp_datadir }} \ --output-dir {{ output_dir }} \ --dbcache {{ dbcache }} \ - {{ base_commit }} {{ head_commit }} + base:{{ binaries_dir }}/base/bitcoind \ + head:{{ binaries_dir }}/head/bitcoind # ============================================================================ # Git helpers From 766ee8dd3c11b742797a129026152eca67ab9e30 Mon Sep 17 00:00:00 2001 From: will Date: Tue, 9 Dec 2025 11:33:29 +0000 Subject: [PATCH 41/51] patch dynamic interpreter of binary if needed --- bench.toml | 2 +- bench/benchmark.py | 7 +++ bench/patchelf.py | 135 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 143 insertions(+), 1 deletion(-) create mode 100644 bench/patchelf.py diff --git a/bench.toml b/bench.toml index 0513f4a0abf5..9a61bd8048f0 100644 --- a/bench.toml +++ b/bench.toml @@ -17,7 +17,7 @@ output_dir = "./bench-output" # Usage: bench.py --profile quick full HEAD~1 HEAD [profiles.quick] -stop_height = 1100 +stop_height = 2000 runs = 3 [profiles.full] diff --git a/bench/benchmark.py b/bench/benchmark.py index 5dc25a0430e6..788e4e53e94d 100644 --- a/bench/benchmark.py +++ b/bench/benchmark.py @@ -11,6 +11,8 @@ from pathlib import Path from typing import TYPE_CHECKING +from .patchelf import ensure_binary_runnable + if TYPE_CHECKING: from .capabilities import Capabilities from .config import Config @@ -89,6 +91,11 @@ def run( if not path.exists(): raise FileNotFoundError(f"Binary not found: {path} ({name})") + # Ensure binaries can run on this system (patches guix binaries on NixOS) + for name, path in binaries: + if not ensure_binary_runnable(path): + raise RuntimeError(f"Binary {name} at {path} cannot be made runnable") + # Check prerequisites errors = self.capabilities.check_for_run(self.config.instrumented) if errors: diff --git a/bench/patchelf.py b/bench/patchelf.py new file mode 100644 index 000000000000..6da1e00867cf --- /dev/null +++ b/bench/patchelf.py @@ -0,0 +1,135 @@ +"""Patchelf utilities for fixing guix-built binaries on NixOS.""" + +from __future__ import annotations + +import logging +import os +import subprocess +from pathlib import Path + +logger = logging.getLogger(__name__) + + +def get_nix_interpreter() -> str | None: + """Get the path to the nix store's dynamic linker. + + Returns None if not on NixOS or can't find it. + """ + # Check if we're on NixOS + if not Path("/etc/NIXOS").exists(): + return None + + # Find the interpreter from the current glibc + # We can get this by checking what the current shell uses + try: + result = subprocess.run( + ["patchelf", "--print-interpreter", "/bin/sh"], + capture_output=True, + text=True, + ) + if result.returncode == 0: + interp = result.stdout.strip() + if interp and Path(interp).exists(): + return interp + except FileNotFoundError: + pass + + return None + + +def get_binary_interpreter(binary: Path) -> str | None: + """Get the interpreter (dynamic linker) of a binary.""" + try: + result = subprocess.run( + ["patchelf", "--print-interpreter", str(binary)], + capture_output=True, + text=True, + ) + if result.returncode == 0: + return result.stdout.strip() + except FileNotFoundError: + logger.debug("patchelf not found") + return None + + +def needs_patching(binary: Path) -> bool: + """Check if a binary needs to be patched for NixOS. + + Returns True if: + - We're on NixOS + - The binary has a non-nix interpreter (e.g., /lib64/ld-linux-x86-64.so.2) + """ + nix_interp = get_nix_interpreter() + if not nix_interp: + # Not on NixOS, no patching needed + return False + + binary_interp = get_binary_interpreter(binary) + if not binary_interp: + # Can't determine interpreter, assume no patching needed + return False + + # Check if the binary's interpreter is already in the nix store + if binary_interp.startswith("/nix/store/"): + return False + + # Binary uses a non-nix interpreter (e.g., /lib64/...) + return True + + +def patch_binary(binary: Path) -> bool: + """Patch a binary to use the nix store's dynamic linker. + + Returns True if patching was successful or not needed. + """ + if not needs_patching(binary): + logger.debug(f"Binary {binary} does not need patching") + return True + + nix_interp = get_nix_interpreter() + if not nix_interp: + logger.warning("Cannot patch binary: unable to find nix interpreter") + return False + + original_interp = get_binary_interpreter(binary) + logger.info(f"Patching binary: {binary}") + logger.info(f" Original interpreter: {original_interp}") + logger.info(f" New interpreter: {nix_interp}") + + # Make sure binary is writable + try: + os.chmod(binary, 0o755) + except OSError as e: + logger.warning(f"Could not make binary writable: {e}") + + try: + result = subprocess.run( + ["patchelf", "--set-interpreter", nix_interp, str(binary)], + capture_output=True, + text=True, + ) + if result.returncode != 0: + logger.error(f"patchelf failed: {result.stderr}") + return False + logger.info(" Patching successful") + return True + except FileNotFoundError: + logger.error("patchelf not found - install it or use nix develop") + return False + + +def ensure_binary_runnable(binary: Path) -> bool: + """Ensure a binary can run on this system. + + Patches the binary if necessary (on NixOS with non-nix binaries). + Returns True if the binary should be runnable. + """ + if not binary.exists(): + logger.error(f"Binary not found: {binary}") + return False + + # Check if patching is needed and do it + if needs_patching(binary): + return patch_binary(binary) + + return True From fedfd17c8789908e9a335e6b5f22d5614557fcff Mon Sep 17 00:00:00 2001 From: will Date: Tue, 9 Dec 2025 12:19:09 +0000 Subject: [PATCH 42/51] increase toml config prio --- bench/analyze.py | 3 ++- bench/config.py | 55 ++++++++++++++++++++++++++++++++++-------------- bench/report.py | 20 ++++++------------ 3 files changed, 47 insertions(+), 31 deletions(-) diff --git a/bench/analyze.py b/bench/analyze.py index a5f52b4ce035..baedd97d745c 100644 --- a/bench/analyze.py +++ b/bench/analyze.py @@ -432,7 +432,8 @@ def _plot( plt.grid(True) min_x, max_x = min(x), max(x) - plt.xlim(min_x, max_x) + if min_x < max_x: + plt.xlim(min_x, max_x) # Add fork markers for height-based plots if is_height_based: diff --git a/bench/config.py b/bench/config.py index e17c0e31a921..7991fee31bff 100644 --- a/bench/config.py +++ b/bench/config.py @@ -118,10 +118,14 @@ def validate(self) -> list[str]: return errors -def load_toml(path: Path) -> dict[str, Any]: - """Load configuration from TOML file.""" +def load_toml(path: Path) -> tuple[dict[str, Any], dict[str, dict[str, Any]]]: + """Load configuration from TOML file. + + Returns: + Tuple of (base_config, profiles_dict) + """ if not path.exists(): - return {} + return {}, {} with open(path, "rb") as f: data = tomllib.load(f) @@ -133,7 +137,10 @@ def load_toml(path: Path) -> dict[str, Any]: if "paths" in data: result.update(data["paths"]) - return result + # Extract profiles + profiles = data.get("profiles", {}) + + return result, profiles def load_env() -> dict[str, Any]: @@ -154,14 +161,29 @@ def load_env() -> dict[str, Any]: return result -def apply_profile(config: dict[str, Any], profile_name: str) -> dict[str, Any]: - """Apply a named profile to configuration.""" - if profile_name not in PROFILES: - return config +def apply_profile( + config: dict[str, Any], + profile_name: str, + toml_profiles: dict[str, dict[str, Any]] | None = None, +) -> dict[str, Any]: + """Apply a named profile to configuration. + Args: + config: Base configuration dict + profile_name: Name of profile to apply + toml_profiles: Profiles loaded from TOML file (override built-in) + """ result = config.copy() - result.update(PROFILES[profile_name]) result["profile"] = profile_name + + # Apply built-in profile first + if profile_name in PROFILES: + result.update(PROFILES[profile_name]) + + # Then apply TOML profile (overrides built-in) + if toml_profiles and profile_name in toml_profiles: + result.update(toml_profiles[profile_name]) + return result @@ -174,10 +196,11 @@ def build_config( Priority (lowest to highest): 1. Built-in defaults - 2. Config file (bench.toml) - 3. Profile overrides - 4. Environment variables - 5. CLI arguments + 2. Config file (bench.toml) base settings + 3. Built-in profile overrides + 4. Config file profile overrides + 5. Environment variables + 6. CLI arguments """ # Start with defaults config = DEFAULTS.copy() @@ -185,11 +208,11 @@ def build_config( # Load config file if config_file is None: config_file = Path("bench.toml") - file_config = load_toml(config_file) + file_config, toml_profiles = load_toml(config_file) config.update(file_config) - # Apply profile - config = apply_profile(config, profile) + # Apply profile (built-in first, then TOML overrides) + config = apply_profile(config, profile, toml_profiles) # Load environment variables env_config = load_env() diff --git a/bench/report.py b/bench/report.py index 25e3f9530be0..8574f75c4074 100644 --- a/bench/report.py +++ b/bench/report.py @@ -353,20 +353,12 @@ def _generate_graphs_section( graphs_html = "" for run in runs: - commit = run.parameters.get("commit", "") - if not commit: - # Try to extract from command - match = re.search(r"\(([a-f0-9]+)\)", run.command) - if match: - commit = match.group(1) - - if not commit: - continue - - short_commit = commit[:12] if len(commit) > 12 else commit + # Use the command/name directly (e.g., "base", "head") + # This is the name given to the binary in the benchmark + name = run.command # Check for flamegraph - flamegraph_name = f"{short_commit}-flamegraph.svg" + flamegraph_name = f"{name}-flamegraph.svg" flamegraph_path = input_dir / flamegraph_name # Check for plots @@ -376,7 +368,7 @@ def _generate_graphs_section( plot_files = [ p.name for p in plots_dir.iterdir() - if p.name.startswith(f"{short_commit}-") and p.suffix == ".png" + if p.name.startswith(f"{name}-") and p.suffix == ".png" ] if not flamegraph_path.exists() and not plot_files: @@ -384,7 +376,7 @@ def _generate_graphs_section( graphs_html += f"""
    -

    {run.command}

    +

    {name}

    """ if flamegraph_path.exists(): From fc55a9b7ece3299a963320df35ce488e72910c32 Mon Sep 17 00:00:00 2001 From: will Date: Tue, 9 Dec 2025 14:10:47 +0000 Subject: [PATCH 43/51] use report in publish job --- .github/workflows/publish-results.yml | 335 +++++--------------------- bench.py | 89 ++++++- bench/report.py | 241 ++++++++++++++++-- 3 files changed, 369 insertions(+), 296 deletions(-) diff --git a/.github/workflows/publish-results.yml b/.github/workflows/publish-results.yml index a785374b4308..72b1bdd39527 100644 --- a/.github/workflows/publish-results.yml +++ b/.github/workflows/publish-results.yml @@ -14,12 +14,20 @@ jobs: env: NETWORKS: "mainnet-default-instrumented,mainnet-large-instrumented,mainnet-default-uninstrumented,mainnet-large-uninstrumented" outputs: - speedups: ${{ steps.organize.outputs.speedups }} - pr-number: ${{ steps.organize.outputs.pr-number }} + speedups: ${{ steps.generate.outputs.speedups }} + pr-number: ${{ steps.metadata.outputs.pr-number }} + result-url: ${{ steps.generate.outputs.result-url }} steps: - uses: actions/checkout@v4 with: ref: gh-pages + + - name: Checkout benchcoin tools + uses: actions/checkout@v4 + with: + ref: master + path: benchcoin-tools + - name: Download artifacts env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -29,298 +37,86 @@ jobs: - name: Extract artifacts run: | for network in ${NETWORKS//,/ }; do + # Create network-specific directories with results if [ -d "result-${network}" ]; then mkdir -p "${network}-results" mv "result-${network}/results.json" "${network}-results/" fi + # Copy flamegraphs into network results directory if [ -d "flamegraph-${network}" ]; then - mkdir -p "${network}-flamegraph" - mv "flamegraph-${network}"/* "${network}-flamegraph/" + cp -r "flamegraph-${network}"/* "${network}-results/" 2>/dev/null || true + fi + + # Copy plots into network results directory + if [ -d "pngs-${network}" ]; then + mkdir -p "${network}-results/plots" + cp -r "pngs-${network}"/* "${network}-results/plots/" 2>/dev/null || true fi + # Keep metadata separate for extraction if [ -d "run-metadata-${network}" ]; then mkdir -p "${network}-metadata" mv "run-metadata-${network}"/* "${network}-metadata/" fi + done - if [ -d "pngs-${network}" ]; then - mkdir -p "${network}-plots" - mv "pngs-${network}"/* "${network}-plots/" + - name: Extract metadata + id: metadata + run: | + # Find PR number and run ID from any available metadata + for network in ${NETWORKS//,/ }; do + if [ -f "${network}-metadata/github.json" ]; then + PR_NUMBER=$(jq -r '.event.pull_request.number // "main"' "${network}-metadata/github.json") + RUN_ID=$(jq -r '.run_id' "${network}-metadata/github.json") + echo "pr-number=${PR_NUMBER}" >> $GITHUB_OUTPUT + echo "run-id=${RUN_ID}" >> $GITHUB_OUTPUT + echo "Found metadata: PR=${PR_NUMBER}, Run=${RUN_ID}" + break fi done - - name: Organize results - id: organize - uses: actions/github-script@v7 - with: - script: | - const fs = require('fs'); - const path = require('path'); - const networks = process.env.NETWORKS.split(','); - let prNumber = 'main'; - let runId; - - // First, extract metadata and get PR number - for (const network of networks) { - if (fs.existsSync(`${network}-metadata/github.json`)) { - const metadata = JSON.parse(fs.readFileSync(`${network}-metadata/github.json`, 'utf8')); - prNumber = metadata.event.pull_request?.number || prNumber; - runId = metadata.run_id; - } - } - - if (!runId) { - console.error('No valid metadata found for any network'); - process.exit(1); - } - - // Create directory structure - const resultDir = `results/pr-${prNumber}/${runId}`; - fs.mkdirSync(resultDir, { recursive: true }); - - // Now copy metadata files - for (const network of networks) { - if (fs.existsSync(`${network}-metadata/github.json`)) { - const metadataDir = `${resultDir}/${network}-metadata`; - fs.mkdirSync(metadataDir, { recursive: true }); - fs.copyFileSync(`${network}-metadata/github.json`, `${metadataDir}/github.json`); - } - } - - // Process each network's results - const combinedResults = { - results: [], - speedups: {} - }; - - for (const network of networks) { - if (fs.existsSync(`${network}-results`)) { - const networkResults = JSON.parse(fs.readFileSync(`${network}-results/results.json`, 'utf8')); - let baseMean, headMean; - - // Add network name to each result and collect means - networkResults.results.forEach(result => { - result.network = network; - // Extract commit from command string like "base (364a7bb8701e)" - const commitMatch = result.command.match(/\(([a-f0-9]+)\)/); - if (commitMatch) { - result.parameters = { commit: commitMatch[1] }; - } - combinedResults.results.push(result); - if (result.command.includes('base')) { - baseMean = result.mean; - } else if (result.command.includes('head')) { - headMean = result.mean; - } - }); - - // Calculate speedup if we have both measurements - if (baseMean && headMean) { - const speedup = baseMean > 0 ? ((baseMean - headMean) / baseMean * 100).toFixed(1) : 'N/A'; - combinedResults.speedups[network] = speedup; - } - - // Move flamegraphs - if (fs.existsSync(`${network}-flamegraph`)) { - fs.readdirSync(`${network}-flamegraph`).forEach(file => { - const sourceFile = `${network}-flamegraph/${file}`; - const targetFile = `${resultDir}/${network}-${file}`; - fs.copyFileSync(sourceFile, targetFile); - }); - } - // Move plots - if (fs.existsSync(`${network}-plots`)) { - const targetPlotsDir = `${resultDir}/${network}-plots`; - fs.mkdirSync(targetPlotsDir, { recursive: true }); - fs.readdirSync(`${network}-plots`).forEach(plot => { - const sourcePlot = `${network}-plots/${plot}`; - const targetPlot = `${targetPlotsDir}/${plot}`; - fs.copyFileSync(sourcePlot, targetPlot); - }); - } - } - } - - // Write combined results - fs.writeFileSync(`${resultDir}/results.json`, JSON.stringify(combinedResults, null, 2)); - - // Create index.html for this run - // Sort results by network then by command type (base first) - const sortedResults = combinedResults.results.sort((a, b) => { - if (a.network !== b.network) return a.network.localeCompare(b.network); - const aIsBase = a.command.includes('base'); - const bIsBase = b.command.includes('base'); - return bIsBase - aIsBase; // base first - }); - - const indexHtml = ` - - - Benchmark Results - - - -
    -

    Benchmark Results

    -
    -

    PR #${prNumber} - Run ${runId}

    - - -

    Run Data

    -
    - - - - - - - - - - - - - ${sortedResults.map(result => ` - - - - - - - - - `).join('')} - -
    NetworkCommandMean (s)Std DevUser (s)System (s)
    ${result.network} - ${result.command.replace( - /\((\w+)\)/, - (_, commit) => `(${commit.slice(0, 8)})` - )} - ${result.mean.toFixed(3)}${result.stddev?.toFixed(3) || 'N/A'}${result.user.toFixed(3)}${result.system.toFixed(3)}
    -
    - - -

    Speedup Summary

    -
    - - - - - - - - - ${Object.entries(combinedResults.speedups).map(([network, speedup]) => ` - - - - - `).join('')} - -
    NetworkSpeedup (%)
    ${network}${speedup}%
    -
    - - - ${networks.filter(network => network.includes('instrumented')).map(network => { - const networkResults = combinedResults.results.filter(r => r.network === network); - const graphsHtml = networkResults.map(result => { - const flameGraphFile = `${network}-${result.parameters.commit}-flamegraph.svg`; - const flameGraphPath = `${resultDir}/${network}-${result.parameters.commit}-flamegraph.svg`; - - const plotDir = `${resultDir}/${network}-plots`; - const plots = fs.existsSync(plotDir) - ? fs.readdirSync(plotDir) - .filter(plot => plot.startsWith(`${result.parameters.commit}-`)) - .map(plot => ` - - ${plot} - - `) - .join('') - : ''; - - if (!fs.existsSync(flameGraphPath) && !plots) return ''; - - return ` -
    -

    ${result.command.replace(/\((\w+)\)/, (_, commit) => `(${commit.slice(0, 8)})`)}

    - ${fs.existsSync(flameGraphPath) ? ` - - ` : ''} - ${plots} -
    - `; - }).join(''); - - if (!graphsHtml.trim()) return ''; - - return ` -
    -

    ${network} Graphs

    - ${graphsHtml} -
    - `; - }).join('')} -
    -
    - - `; + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' - fs.writeFileSync(`${resultDir}/index.html`, indexHtml); + - name: Generate report + id: generate + env: + PR_NUMBER: ${{ steps.metadata.outputs.pr-number }} + RUN_ID: ${{ steps.metadata.outputs.run-id }} + run: | + cd benchcoin-tools - // Update main index.html - const prs = fs.readdirSync('results') - .filter(dir => dir.startsWith('pr-')) - .map(dir => ({ - pr: dir.replace('pr-', ''), - runs: fs.readdirSync(`results/${dir}`) - })); + # Build network arguments + NETWORK_ARGS="" + for network in ${NETWORKS//,/ }; do + if [ -d "../${network}-results" ]; then + NETWORK_ARGS="${NETWORK_ARGS} --network ${network}:../${network}-results" + fi + done - const mainIndexHtml = ` - - - Bitcoin Benchmark Results - - - -
    -

    Bitcoin Benchmark Results

    -
    -

    Available Results

    -
      - ${prs.map(({pr, runs}) => ` -
    • PR #${pr} -
        - ${runs.map(run => ` -
      • Run ${run}
        • - `).join('')} -
      -
      • - `).join('')} -
    -
    -
    - - `; + # Generate report + python3 bench.py report \ + ${NETWORK_ARGS} \ + --pr-number "${PR_NUMBER}" \ + --run-id "${RUN_ID}" \ + --update-index \ + "../results/pr-${PR_NUMBER}/${RUN_ID}" - fs.writeFileSync('index.html', mainIndexHtml); + # Read speedups from generated results.json + SPEEDUPS=$(jq -r '.speedups | to_entries | map(select(.key | contains("uninstrumented"))) | map("\(.key): \(.value)%") | join(", ")' "../results/pr-${PR_NUMBER}/${RUN_ID}/results.json") + echo "speedups=${SPEEDUPS}" >> $GITHUB_OUTPUT - // Set outputs for use in PR comment - const resultUrl = `https://${context.repo.owner}.github.io/${context.repo.name}/results/pr-${prNumber}/${runId}/index.html`; - const speedupString = Object.entries(combinedResults.speedups) - .filter(([network]) => network.includes('uninstrumented')) - .map(([network, speedup]) => `${network}: ${speedup}%`) - .join(', '); + RESULT_URL="https://${{ github.repository_owner }}.github.io/${{ github.event.repository.name }}/results/pr-${PR_NUMBER}/${RUN_ID}/index.html" + echo "result-url=${RESULT_URL}" >> $GITHUB_OUTPUT - core.setOutput('result-url', resultUrl); - core.setOutput('speedups', speedupString); - core.setOutput('pr-number', prNumber); - return { url: resultUrl, speedups: speedupString }; - name: Upload Pages artifact uses: actions/upload-pages-artifact@v3 with: path: results + - name: Commit and push to gh-pages run: | git config --global user.name "github-actions[bot]" @@ -328,6 +124,7 @@ jobs: git add results/ index.html git commit -m "Update benchmark results from run ${{ github.event.workflow_run.id }}" git push origin gh-pages + comment-pr: needs: build runs-on: ubuntu-latest @@ -342,5 +139,5 @@ jobs: run: | gh pr comment ${{ needs.build.outputs.pr-number }} \ --repo ${{ github.repository }} \ - --body "📊 Benchmark results for this run (${{ github.event.workflow_run.id }}) will be available at: https://${{ github.repository_owner }}.github.io/${{ github.event.repository.name }}/results/pr-${{ needs.build.outputs.pr-number }}/${{ github.event.workflow_run.id }}/index.html after the github pages \"build and deployment\" action has completed. + --body "📊 Benchmark results for this run (${{ github.event.workflow_run.id }}) will be available at: ${{ needs.build.outputs.result-url }} after the github pages \"build and deployment\" action has completed. 🚀 Speedups: ${{ needs.build.outputs.speedups }}" diff --git a/bench.py b/bench.py index 04acef1a0d51..aba690a25ca2 100755 --- a/bench.py +++ b/bench.py @@ -241,21 +241,52 @@ def cmd_report(args: argparse.Namespace) -> int: if args.verbose: logging.getLogger().setLevel(logging.DEBUG) - input_dir = Path(args.input_dir) output_dir = Path(args.output_dir) - - if not input_dir.exists(): - logger.error(f"Input directory not found: {input_dir}") - return 1 - phase = ReportPhase() try: - result = phase.run( - input_dir=input_dir, - output_dir=output_dir, - title=args.title or "Benchmark Results", - ) + # CI multi-network mode + if args.networks: + network_dirs = {} + for spec in args.networks: + if ":" not in spec: + logger.error(f"Invalid network spec '{spec}': must be NETWORK:PATH") + return 1 + network, path = spec.split(":", 1) + network_dirs[network] = Path(path) + + # Validate directories exist + for network, path in network_dirs.items(): + if not path.exists(): + logger.error(f"Network directory not found: {path} ({network})") + return 1 + + result = phase.run_multi_network( + network_dirs=network_dirs, + output_dir=output_dir, + title=args.title or "Benchmark Results", + pr_number=args.pr_number, + run_id=args.run_id, + ) + + # Update main index if we have a results directory + if args.update_index: + results_base = output_dir.parent.parent # Go up from pr-N/run-id + if results_base.exists(): + phase.update_index(results_base, results_base.parent / "index.html") + else: + # Standard single-directory mode + input_dir = Path(args.input_dir) + + if not input_dir.exists(): + logger.error(f"Input directory not found: {input_dir}") + return 1 + + result = phase.run( + input_dir=input_dir, + output_dir=output_dir, + title=args.title or "Benchmark Results", + ) # Print speedups if result.speedups: @@ -444,13 +475,45 @@ def main() -> int: compare_parser.set_defaults(func=cmd_compare) # Report command - report_parser = subparsers.add_parser("report", help="Generate HTML report") - report_parser.add_argument("input_dir", help="Directory with results.json") + report_parser = subparsers.add_parser( + "report", + help="Generate HTML report", + description="Generate HTML report from benchmark results. " + "Use --network for multi-network CI reports.", + ) + report_parser.add_argument( + "input_dir", + nargs="?", + help="Directory with results.json (for single-network mode)", + ) report_parser.add_argument("output_dir", help="Output directory for report") report_parser.add_argument( "--title", help="Report title", ) + # CI multi-network options + report_parser.add_argument( + "--network", + dest="networks", + action="append", + metavar="NAME:PATH", + help="Network results directory (repeatable, e.g., --network mainnet:./mainnet-results)", + ) + report_parser.add_argument( + "--pr-number", + metavar="N", + help="PR number (for CI reports)", + ) + report_parser.add_argument( + "--run-id", + metavar="ID", + help="Run ID (for CI reports)", + ) + report_parser.add_argument( + "--update-index", + action="store_true", + help="Update main index.html (for CI reports)", + ) report_parser.set_defaults(func=cmd_report) args = parser.parse_args() diff --git a/bench/report.py b/bench/report.py index 8574f75c4074..7f95d0c1ea47 100644 --- a/bench/report.py +++ b/bench/report.py @@ -122,6 +122,103 @@ def __init__( ): self.repo_url = repo_url + def generate_multi_network( + self, + network_dirs: dict[str, Path], + output_dir: Path, + title: str = "Benchmark Results", + pr_number: str | None = None, + run_id: str | None = None, + ) -> ReportResult: + """Generate HTML report from multiple network benchmark results. + + Args: + network_dirs: Dict mapping network name to directory containing results.json + output_dir: Where to write the HTML report + title: Title for the report + pr_number: PR number (for CI reports) + run_id: Run ID (for CI reports) + + Returns: + ReportResult with paths and speedup data + """ + output_dir.mkdir(parents=True, exist_ok=True) + + # Combine results from all networks + all_runs: list[BenchmarkRun] = [] + for network, input_dir in network_dirs.items(): + results_file = input_dir / "results.json" + if not results_file.exists(): + logger.warning( + f"results.json not found in {input_dir} for network {network}" + ) + continue + + with open(results_file) as f: + data = json.load(f) + + # Parse and add network to each run + for result in data.get("results", []): + all_runs.append( + BenchmarkRun( + network=network, + command=result.get("command", ""), + mean=result.get("mean", 0), + stddev=result.get("stddev"), + user=result.get("user", 0), + system=result.get("system", 0), + parameters=result.get("parameters", {}), + ) + ) + + # Copy artifacts from this network + self._copy_network_artifacts(network, input_dir, output_dir) + + if not all_runs: + raise ValueError("No benchmark results found in any network directory") + + # Calculate speedups per network + speedups = self._calculate_speedups_per_network(all_runs) + + # Build title with PR/run info if provided + full_title = title + if pr_number and run_id: + full_title = f"PR #{pr_number} - Run {run_id}" + + # Generate HTML + html = self._generate_html( + all_runs, speedups, full_title, output_dir, output_dir + ) + + # Write report + index_file = output_dir / "index.html" + index_file.write_text(html) + logger.info(f"Generated report: {index_file}") + + # Write combined results.json + combined_results = { + "results": [ + { + "network": run.network, + "command": run.command, + "mean": run.mean, + "stddev": run.stddev, + "user": run.user, + "system": run.system, + } + for run in all_runs + ], + "speedups": speedups, + } + results_file = output_dir / "results.json" + results_file.write_text(json.dumps(combined_results, indent=2)) + + return ReportResult( + output_dir=output_dir, + index_file=index_file, + speedups=speedups, + ) + def generate( self, input_dir: Path, @@ -263,6 +360,59 @@ def _calculate_speedups(self, runs: list[BenchmarkRun]) -> dict[str, float]: return speedups + def _calculate_speedups_per_network( + self, runs: list[BenchmarkRun] + ) -> dict[str, float]: + """Calculate speedup percentages per network. + + For each network, uses 'base' as baseline and calculates speedup for 'head'. + Returns a dict mapping network name to speedup percentage. + """ + speedups = {} + + # Group runs by network + networks: dict[str, list[BenchmarkRun]] = {} + for run in runs: + if run.network not in networks: + networks[run.network] = [] + networks[run.network].append(run) + + # Calculate speedup for each network + for network, network_runs in networks.items(): + base_mean = None + head_mean = None + + for run in network_runs: + if run.command == "base": + base_mean = run.mean + elif run.command == "head": + head_mean = run.mean + + if base_mean and head_mean and base_mean > 0: + speedup = ((base_mean - head_mean) / base_mean) * 100 + speedups[network] = round(speedup, 1) + + return speedups + + def _copy_network_artifacts( + self, network: str, input_dir: Path, output_dir: Path + ) -> None: + """Copy artifacts from a network directory with network prefix.""" + # Copy flamegraphs with network prefix + for svg in input_dir.glob("*-flamegraph.svg"): + dest = output_dir / f"{network}-{svg.name}" + shutil.copy2(svg, dest) + logger.debug(f"Copied {svg.name} as {dest.name}") + + # Copy plots directory with network prefix + plots_dir = input_dir / "plots" + if plots_dir.exists(): + dest_plots = output_dir / f"{network}-plots" + if dest_plots.exists(): + shutil.rmtree(dest_plots) + shutil.copytree(plots_dir, dest_plots) + logger.debug(f"Copied plots to {dest_plots}") + def _generate_html( self, runs: list[BenchmarkRun], @@ -354,40 +504,70 @@ def _generate_graphs_section( for run in runs: # Use the command/name directly (e.g., "base", "head") - # This is the name given to the binary in the benchmark name = run.command + network = run.network + + # Check for flamegraph - try both with and without network prefix + # Network-prefixed: {network}-{name}-flamegraph.svg (for multi-network reports) + # Non-prefixed: {name}-flamegraph.svg (for single-network reports) + flamegraph_name = None + flamegraph_path = None - # Check for flamegraph - flamegraph_name = f"{name}-flamegraph.svg" - flamegraph_path = input_dir / flamegraph_name + network_prefixed = f"{network}-{name}-flamegraph.svg" + non_prefixed = f"{name}-flamegraph.svg" - # Check for plots - plots_dir = input_dir / "plots" + if (output_dir / network_prefixed).exists(): + flamegraph_name = network_prefixed + flamegraph_path = output_dir / network_prefixed + elif (input_dir / non_prefixed).exists(): + flamegraph_name = non_prefixed + flamegraph_path = input_dir / non_prefixed + + # Check for plots - try both network-prefixed and non-prefixed directories plot_files = [] - if plots_dir.exists(): + plots_dir = None + + network_plots_dir = output_dir / f"{network}-plots" + regular_plots_dir = input_dir / "plots" + + if network_plots_dir.exists(): + plots_dir = network_plots_dir + plot_files = [ + p.name + for p in plots_dir.iterdir() + if p.name.startswith(f"{name}-") and p.suffix == ".png" + ] + elif regular_plots_dir.exists(): + plots_dir = regular_plots_dir plot_files = [ p.name for p in plots_dir.iterdir() if p.name.startswith(f"{name}-") and p.suffix == ".png" ] - if not flamegraph_path.exists() and not plot_files: + if not flamegraph_path and not plot_files: continue + # Build display label + display_label = f"{network} - {name}" if network != "default" else name + graphs_html += f"""
    -

    {name}

    +

    {display_label}

    """ - if flamegraph_path.exists(): + if flamegraph_path: graphs_html += f""" """ - for plot in sorted(plot_files): - graphs_html += f""" - - {plot} + if plot_files and plots_dir: + # Determine the relative path for plots + plots_rel_path = plots_dir.name + for plot in sorted(plot_files): + graphs_html += f""" + + {plot} """ @@ -449,3 +629,36 @@ def run( ReportResult with paths and speedup data """ return self.generator.generate(input_dir, output_dir, title) + + def run_multi_network( + self, + network_dirs: dict[str, Path], + output_dir: Path, + title: str = "Benchmark Results", + pr_number: str | None = None, + run_id: str | None = None, + ) -> ReportResult: + """Generate report from multiple network benchmark results. + + Args: + network_dirs: Dict mapping network name to directory containing results.json + output_dir: Where to write the HTML report + title: Title for the report + pr_number: PR number (for CI reports) + run_id: Run ID (for CI reports) + + Returns: + ReportResult with paths and speedup data + """ + return self.generator.generate_multi_network( + network_dirs, output_dir, title, pr_number, run_id + ) + + def update_index(self, results_dir: Path, output_file: Path) -> None: + """Update the main index.html listing all results. + + Args: + results_dir: Directory containing pr-* subdirectories + output_file: Where to write index.html + """ + self.generator.generate_index(results_dir, output_file) From b590629dbcbac7bec4360a719eedf6a29542145a Mon Sep 17 00:00:00 2001 From: will Date: Tue, 9 Dec 2025 14:11:38 +0000 Subject: [PATCH 44/51] deleteme --- bench.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench.toml b/bench.toml index 9a61bd8048f0..7bd38faa317b 100644 --- a/bench.toml +++ b/bench.toml @@ -25,6 +25,6 @@ stop_height = 855000 runs = 3 [profiles.ci] -stop_height = 855000 +stop_height = 845000 runs = 3 connect = "148.251.128.115:33333" From a86b795fb9768aa42bbded4658894fd29c2318d4 Mon Sep 17 00:00:00 2001 From: Andrew Toth Date: Wed, 6 Nov 2024 19:51:34 -0500 Subject: [PATCH 45/51] coins: allow emplacing non-dirty coins internally --- src/coins.cpp | 6 ++++-- src/coins.h | 7 ++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/coins.cpp b/src/coins.cpp index 554a3ebe962b..82619877370d 100644 --- a/src/coins.cpp +++ b/src/coins.cpp @@ -110,12 +110,14 @@ void CCoinsViewCache::AddCoin(const COutPoint &outpoint, Coin&& coin, bool possi (bool)it->second.coin.IsCoinBase()); } -void CCoinsViewCache::EmplaceCoinInternalDANGER(COutPoint&& outpoint, Coin&& coin) { +void CCoinsViewCache::EmplaceCoinInternalDANGER(COutPoint&& outpoint, Coin&& coin, bool set_dirty) { const auto mem_usage{coin.DynamicMemoryUsage()}; auto [it, inserted] = cacheCoins.try_emplace(std::move(outpoint), std::move(coin)); if (inserted) { - CCoinsCacheEntry::SetDirty(*it, m_sentinel); cachedCoinsUsage += mem_usage; + if (set_dirty) { + CCoinsCacheEntry::SetDirty(*it, m_sentinel); + } } } diff --git a/src/coins.h b/src/coins.h index 2fcc764a3fdf..6ceeac3ce2dc 100644 --- a/src/coins.h +++ b/src/coins.h @@ -421,12 +421,13 @@ class CCoinsViewCache : public CCoinsViewBacked /** * Emplace a coin into cacheCoins without performing any checks, marking - * the emplaced coin as dirty. + * the emplaced coin as dirty unless `set_dirty` is `false`. * - * NOT FOR GENERAL USE. Used only when loading coins from a UTXO snapshot. + * NOT FOR GENERAL USE. Used when loading coins from a UTXO snapshot, and + * in the InputFetcher. * @sa ChainstateManager::PopulateAndValidateSnapshot() */ - void EmplaceCoinInternalDANGER(COutPoint&& outpoint, Coin&& coin); + void EmplaceCoinInternalDANGER(COutPoint&& outpoint, Coin&& coin, bool set_dirty = true); /** * Spend a coin. Pass moveto in order to get the deleted data. From 5e9823a40bc55ccbb4956842898ddf641ced5c83 Mon Sep 17 00:00:00 2001 From: Andrew Toth Date: Wed, 6 Nov 2024 19:51:57 -0500 Subject: [PATCH 46/51] coins: add inputfetcher --- src/inputfetcher.h | 246 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 246 insertions(+) create mode 100644 src/inputfetcher.h diff --git a/src/inputfetcher.h b/src/inputfetcher.h new file mode 100644 index 000000000000..5b89fd0ebe87 --- /dev/null +++ b/src/inputfetcher.h @@ -0,0 +1,246 @@ +// Copyright (c) 2024-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_INPUTFETCHER_H +#define BITCOIN_INPUTFETCHER_H + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +/** + * Input fetcher for fetching inputs from the CoinsDB and inserting + * into the CoinsTip. + * + * The main thread loops through the block and writes all input prevouts to a + * global vector. It then wakes all workers and starts working as well. Each + * thread assigns itself a range of outpoints from the shared vector, and + * fetches the coins from disk. The outpoint and coin pairs are written to a + * thread local vector of pairs. Once all outpoints are fetched, the main thread + * loops through all thread local vectors and writes the pairs to the cache. + */ +class InputFetcher +{ +private: + //! Mutex to protect the inner state + Mutex m_mutex{}; + //! Worker threads block on this when out of work + std::condition_variable m_worker_cv{}; + //! Main thread blocks on this when out of work + std::condition_variable m_main_cv{}; + + /** + * The outpoints to be fetched from disk. + * This is written to on the main thread, then read from all worker + * threads only after the main thread is done writing. Hence, it doesn't + * need to be guarded by a lock. + */ + std::vector m_outpoints{}; + /** + * The index of the last outpoint that is being fetched. Workers assign + * themselves a range of outpoints to fetch from m_outpoints. They will use + * this index as the end of their range, and then set this index to the + * beginning of the range they take for the next worker. Once it gets to + * zero, all outpoints have been assigned and the next worker will wait. + */ + size_t m_last_outpoint_index GUARDED_BY(m_mutex){0}; + + //! The set of txids of the transactions in the current block being fetched. + std::unordered_set m_txids{}; + //! The vector of thread local vectors of pairs to be written to the cache. + std::vector>> m_pairs{}; + + /** + * Number of outpoint fetches that haven't completed yet. + * This includes outpoints that have already been assigned, but are still in + * the worker's own batches. + */ + int32_t m_in_flight_outpoints_count GUARDED_BY(m_mutex){0}; + //! The number of worker threads that are waiting on m_worker_cv + int32_t m_idle_worker_count GUARDED_BY(m_mutex){0}; + //! The maximum number of outpoints to be assigned in one batch + const int32_t m_batch_size; + //! DB coins view to fetch from. + const CCoinsView* m_db{nullptr}; + //! The cache to check if we already have this input. + const CCoinsViewCache* m_cache{nullptr}; + + std::vector m_worker_threads; + bool m_request_stop GUARDED_BY(m_mutex){false}; + + //! Internal function that does the fetching from disk. + void Loop(int32_t index, bool is_main_thread = false) noexcept EXCLUSIVE_LOCKS_REQUIRED(!m_mutex) + { + auto local_batch_size{0}; + auto end_index{0}; + auto& cond{is_main_thread ? m_main_cv : m_worker_cv}; + do { + { + WAIT_LOCK(m_mutex, lock); + // first do the clean-up of the previous loop run (allowing us to do + // it in the same critsect) local_batch_size will only be + // truthy after first run. + if (local_batch_size) { + m_in_flight_outpoints_count -= local_batch_size; + if (!is_main_thread && m_in_flight_outpoints_count == 0) { + m_main_cv.notify_one(); + } + } + + // logically, the do loop starts here + while (m_last_outpoint_index == 0) { + if ((is_main_thread && m_in_flight_outpoints_count == 0) || m_request_stop) { + return; + } + ++m_idle_worker_count; + cond.wait(lock); + --m_idle_worker_count; + } + + // Assign a batch of outpoints to this thread + local_batch_size = std::max(1, std::min(m_batch_size, + static_cast(m_last_outpoint_index / + (m_worker_threads.size() + 1 + m_idle_worker_count)))); + end_index = m_last_outpoint_index; + m_last_outpoint_index -= local_batch_size; + } + + auto& local_pairs{m_pairs[index]}; + local_pairs.reserve(local_pairs.size() + local_batch_size); + try { + for (auto i{end_index - local_batch_size}; i < end_index; ++i) { + const auto& outpoint{m_outpoints[i]}; + // If an input spends an outpoint from earlier in the + // block, it won't be in the cache yet but it also won't be + // in the db either. + if (m_txids.contains(outpoint.hash)) { + continue; + } + if (m_cache->HaveCoinInCache(outpoint)) { + continue; + } + if (auto coin{m_db->GetCoin(outpoint)}; coin) { + local_pairs.emplace_back(outpoint, std::move(*coin)); + } else { + // Missing an input. This block will fail validation. + // Skip remaining outpoints and continue so main thread + // can proceed. + LOCK(m_mutex); + m_in_flight_outpoints_count -= m_last_outpoint_index; + m_last_outpoint_index = 0; + break; + } + } + } catch (const std::runtime_error&) { + // Database error. This will be handled later in validation. + // Skip remaining outpoints and continue so main thread + // can proceed. + LOCK(m_mutex); + m_in_flight_outpoints_count -= m_last_outpoint_index; + m_last_outpoint_index = 0; + } + } while (true); + } + +public: + + //! Create a new input fetcher + explicit InputFetcher(int32_t batch_size, int32_t worker_thread_count) noexcept + : m_batch_size(batch_size) + { + if (worker_thread_count < 1) { + // Don't do anything if there are no worker threads. + return; + } + m_pairs.reserve(worker_thread_count + 1); + for (auto n{0}; n < worker_thread_count + 1; ++n) { + m_pairs.emplace_back(); + } + m_worker_threads.reserve(worker_thread_count); + for (auto n{0}; n < worker_thread_count; ++n) { + m_worker_threads.emplace_back([this, n]() { + util::ThreadRename(strprintf("inputfetch.%i", n)); + Loop(n); + }); + } + } + + // Since this class manages its own resources, which is a thread + // pool `m_worker_threads`, copy and move operations are not appropriate. + InputFetcher(const InputFetcher&) = delete; + InputFetcher& operator=(const InputFetcher&) = delete; + InputFetcher(InputFetcher&&) = delete; + InputFetcher& operator=(InputFetcher&&) = delete; + + //! Fetch all block inputs from db, and insert into cache. + void FetchInputs(CCoinsViewCache& cache, + const CCoinsView& db, + const CBlock& block) noexcept + EXCLUSIVE_LOCKS_REQUIRED(!m_mutex) + { + if (m_worker_threads.empty() || block.vtx.size() <= 1) { + return; + } + + // Set the db and cache to use for this block. + m_db = &db; + m_cache = &cache; + + // Loop through the inputs of the block and add them to the queue + m_txids.reserve(block.vtx.size() - 1); + for (const auto& tx : block.vtx) { + if (tx->IsCoinBase()) { + continue; + } + m_outpoints.reserve(m_outpoints.size() + tx->vin.size()); + for (const auto& in : tx->vin) { + m_outpoints.emplace_back(in.prevout); + } + m_txids.emplace(tx->GetHash()); + } + { + LOCK(m_mutex); + m_in_flight_outpoints_count = m_outpoints.size(); + m_last_outpoint_index = m_outpoints.size(); + } + m_worker_cv.notify_all(); + + // Have the main thread work too while we wait for other threads + Loop(m_worker_threads.size(), /*is_main_thread=*/true); + + // At this point all threads are done writing to m_pairs, so we can + // safely read from it and insert the fetched coins into the cache. + for (auto& local_pairs : m_pairs) { + for (auto&& [outpoint, coin] : local_pairs) { + cache.EmplaceCoinInternalDANGER(std::move(outpoint), + std::move(coin), + /*set_dirty=*/false); + } + local_pairs.clear(); + } + m_txids.clear(); + m_outpoints.clear(); + } + + ~InputFetcher() + { + WITH_LOCK(m_mutex, m_request_stop = true); + m_worker_cv.notify_all(); + for (std::thread& t : m_worker_threads) { + t.join(); + } + } +}; + +#endif // BITCOIN_INPUTFETCHER_H From 950eaf962feeec7495021ead77ec2ddc0fb01dba Mon Sep 17 00:00:00 2001 From: Andrew Toth Date: Wed, 6 Nov 2024 19:52:08 -0500 Subject: [PATCH 47/51] tests: add inputfetcher tests --- src/test/CMakeLists.txt | 1 + src/test/inputfetcher_tests.cpp | 191 ++++++++++++++++++++++++++++++++ 2 files changed, 192 insertions(+) create mode 100644 src/test/inputfetcher_tests.cpp diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index 9528004e988e..9927e8f0b681 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -50,6 +50,7 @@ add_executable(test_bitcoin headers_sync_chainwork_tests.cpp httpserver_tests.cpp i2p_tests.cpp + inputfetcher_tests.cpp interfaces_tests.cpp key_io_tests.cpp key_tests.cpp diff --git a/src/test/inputfetcher_tests.cpp b/src/test/inputfetcher_tests.cpp new file mode 100644 index 000000000000..957a7bf8fb0e --- /dev/null +++ b/src/test/inputfetcher_tests.cpp @@ -0,0 +1,191 @@ +// Copyright (c) 2024-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + +BOOST_AUTO_TEST_SUITE(inputfetcher_tests) + +struct InputFetcherTest : BasicTestingSetup { +private: + std::unique_ptr m_fetcher{nullptr}; + std::unique_ptr m_block{nullptr}; + + CBlock CreateBlock(int32_t num_txs) + { + CBlock block; + CMutableTransaction coinbase; + coinbase.vin.emplace_back(); + block.vtx.push_back(MakeTransactionRef(coinbase)); + + Txid prevhash{Txid::FromUint256(uint256(1))}; + + for (auto i{1}; i < num_txs; ++i) { + CMutableTransaction tx; + const auto txid{m_rng.randbool() ? Txid::FromUint256(uint256(i)) : prevhash}; + tx.vin.emplace_back(COutPoint(txid, 0)); + prevhash = tx.GetHash(); + block.vtx.push_back(MakeTransactionRef(tx)); + } + + return block; + } + +public: + explicit InputFetcherTest(const ChainType chainType = ChainType::MAIN, + TestOpts opts = {}) + : BasicTestingSetup{chainType, opts} + { + SeedRandomForTest(SeedRand::ZEROS); + + const auto cores{GetNumCores()}; + const auto num_txs{m_rng.randrange(cores * 10)}; + m_block = std::make_unique(CreateBlock(num_txs)); + const auto batch_size{m_rng.randrange(m_block->vtx.size() * 2)}; + const auto worker_threads{m_rng.randrange(cores * 2) + 1}; + m_fetcher = std::make_unique(batch_size, worker_threads); + } + + InputFetcher& getFetcher() { return *m_fetcher; } + const CBlock& getBlock() { return *m_block; } +}; + +BOOST_FIXTURE_TEST_CASE(fetch_inputs, InputFetcherTest) +{ + const auto& block{getBlock()}; + for (auto i{0}; i < 3; ++i) { + CCoinsView dummy; + CCoinsViewCache db(&dummy); + + for (const auto& tx : block.vtx) { + for (const auto& in : tx->vin) { + auto outpoint{in.prevout}; + Coin coin{}; + coin.out.nValue = 1; + db.EmplaceCoinInternalDANGER(std::move(outpoint), std::move(coin)); + } + } + + CCoinsViewCache cache(&db); + getFetcher().FetchInputs(cache, db, block); + + std::unordered_set txids{}; + txids.reserve(block.vtx.size() - 1); + + for (const auto& tx : block.vtx) { + if (tx->IsCoinBase()) { + BOOST_CHECK(!cache.HaveCoinInCache(tx->vin[0].prevout)); + } else { + for (const auto& in : tx->vin) { + const auto& outpoint{in.prevout}; + const auto have{cache.HaveCoinInCache(outpoint)}; + const auto should_have{!txids.contains(outpoint.hash)}; + BOOST_CHECK(should_have ? have : !have); + } + txids.emplace(tx->GetHash()); + } + } + } +} + +// Test for the case where a block spends coins that are spent in the cache, but +// the spentness has not been flushed to the db. So the input fetcher will fetch +// the coin from the db since HaveCoinInCache will return false for an existing +// but spent coin. However, the fetched coin will fail to be inserted into the +// cache because the emplace call in EmplaceCoinInternalDANGER will not insert +// the unspent coin due to the collision with the already spent coin in the map. +BOOST_FIXTURE_TEST_CASE(fetch_no_double_spend, InputFetcherTest) +{ + const auto& block{getBlock()}; + for (auto i{0}; i < 3; ++i) { + CCoinsView dummy; + CCoinsViewCache db(&dummy); + + for (const auto& tx : block.vtx) { + for (const auto& in : tx->vin) { + auto outpoint{in.prevout}; + Coin coin{}; + coin.out.nValue = 1; + db.EmplaceCoinInternalDANGER(std::move(outpoint), std::move(coin)); + } + } + + CCoinsViewCache cache(&db); + + // Add all inputs as spent already in cache + for (const auto& tx : block.vtx) { + for (const auto& in : tx->vin) { + auto outpoint{in.prevout}; + Coin coin{}; + cache.EmplaceCoinInternalDANGER(std::move(outpoint), std::move(coin)); + } + } + + getFetcher().FetchInputs(cache, db, block); + + // Coins are still spent, even though they exist unspent in the parent db + for (const auto& tx : block.vtx) { + for (const auto& in : tx->vin) { + BOOST_CHECK(!cache.HaveCoinInCache(in.prevout)); + } + } + } +} + +BOOST_FIXTURE_TEST_CASE(fetch_no_inputs, InputFetcherTest) +{ + const auto& block{getBlock()}; + for (auto i{0}; i < 3; ++i) { + CCoinsView db; + CCoinsViewCache cache(&db); + getFetcher().FetchInputs(cache, db, block); + + for (const auto& tx : block.vtx) { + for (const auto& in : tx->vin) { + BOOST_CHECK(!cache.HaveCoinInCache(in.prevout)); + } + } + } +} + +class ThrowCoinsView : public CCoinsView +{ + std::optional GetCoin(const COutPoint& outpoint) const override + { + throw std::runtime_error("database error"); + } +}; + +BOOST_FIXTURE_TEST_CASE(fetch_input_exceptions, InputFetcherTest) +{ + const auto& block{getBlock()}; + for (auto i{0}; i < 3; ++i) { + ThrowCoinsView db; + CCoinsViewCache cache(&db); + getFetcher().FetchInputs(cache, db, block); + + for (const auto& tx : block.vtx) { + for (const auto& in : tx->vin) { + BOOST_CHECK(!cache.HaveCoinInCache(in.prevout)); + } + } + } +} + +BOOST_AUTO_TEST_SUITE_END() From 578515344e0f78d0f9bea4330eed3a4ff7c48fe5 Mon Sep 17 00:00:00 2001 From: Andrew Toth Date: Wed, 6 Nov 2024 19:52:18 -0500 Subject: [PATCH 48/51] bench: add inputfetcher bench --- src/bench/CMakeLists.txt | 1 + src/bench/inputfetcher.cpp | 57 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 src/bench/inputfetcher.cpp diff --git a/src/bench/CMakeLists.txt b/src/bench/CMakeLists.txt index e0e03b1df7cc..9d03f075a750 100644 --- a/src/bench/CMakeLists.txt +++ b/src/bench/CMakeLists.txt @@ -29,6 +29,7 @@ add_executable(bench_bitcoin gcs_filter.cpp hashpadding.cpp index_blockfilter.cpp + inputfetcher.cpp load_external.cpp lockedpool.cpp logging.cpp diff --git a/src/bench/inputfetcher.cpp b/src/bench/inputfetcher.cpp new file mode 100644 index 000000000000..66be4a6ff593 --- /dev/null +++ b/src/bench/inputfetcher.cpp @@ -0,0 +1,57 @@ +// Copyright (c) 2024-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static constexpr auto QUEUE_BATCH_SIZE{128}; +static constexpr auto DELAY{2ms}; + +//! Simulates a DB by adding a delay when calling GetCoin +class DelayedCoinsView : public CCoinsView +{ +private: + std::chrono::milliseconds m_delay; + +public: + DelayedCoinsView(std::chrono::milliseconds delay) : m_delay(delay) {} + + std::optional GetCoin(const COutPoint& outpoint) const override + { + UninterruptibleSleep(m_delay); + return Coin{}; + } + + bool BatchWrite(CoinsViewCacheCursor& cursor, const uint256 &hashBlock) override { return true; } +}; + +static void InputFetcherBenchmark(benchmark::Bench& bench) +{ + DataStream stream{benchmark::data::block413567}; + CBlock block; + stream >> TX_WITH_WITNESS(block); + + DelayedCoinsView db(DELAY); + CCoinsViewCache cache(&db); + + // The main thread should be counted to prevent thread oversubscription, and + // to decrease the variance of benchmark results. + const auto worker_threads_num{GetNumCores() - 1}; + InputFetcher fetcher{QUEUE_BATCH_SIZE, worker_threads_num}; + + bench.run([&] { + const auto ok{cache.Flush()}; + assert(ok); + fetcher.FetchInputs(cache, db, block); + }); +} + +BENCHMARK(InputFetcherBenchmark, benchmark::PriorityLevel::HIGH); From d10517588bcd51e7eb82466ac528cd95cfbce0a9 Mon Sep 17 00:00:00 2001 From: Andrew Toth Date: Sat, 16 Nov 2024 14:33:30 -0500 Subject: [PATCH 49/51] fuzz: add inputfetcher fuzz harness --- src/test/fuzz/CMakeLists.txt | 1 + src/test/fuzz/inputfetcher.cpp | 153 +++++++++++++++++++++++++++++++++ 2 files changed, 154 insertions(+) create mode 100644 src/test/fuzz/inputfetcher.cpp diff --git a/src/test/fuzz/CMakeLists.txt b/src/test/fuzz/CMakeLists.txt index 607723b978ae..5abc124f6310 100644 --- a/src/test/fuzz/CMakeLists.txt +++ b/src/test/fuzz/CMakeLists.txt @@ -54,6 +54,7 @@ add_executable(fuzz hex.cpp http_request.cpp i2p.cpp + inputfetcher.cpp integer.cpp key.cpp key_io.cpp diff --git a/src/test/fuzz/inputfetcher.cpp b/src/test/fuzz/inputfetcher.cpp new file mode 100644 index 000000000000..ca3c2f7509d1 --- /dev/null +++ b/src/test/fuzz/inputfetcher.cpp @@ -0,0 +1,153 @@ +// Copyright (c) 2024-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +using DbMap = std::map, bool>>; + +class DbCoinsView : public CCoinsView +{ +private: + DbMap& m_map; + +public: + DbCoinsView(DbMap& map) : m_map(map) {} + + std::optional GetCoin(const COutPoint& outpoint) const override + { + const auto it{m_map.find(outpoint)}; + assert(it != m_map.end()); + const auto [coin, err] = it->second; + if (err) { + throw std::runtime_error("database error"); + } + return coin; + } +}; + +class NoAccessCoinsView : public CCoinsView +{ +public: + std::optional GetCoin(const COutPoint& outpoint) const override + { + abort(); + } +}; + +FUZZ_TARGET(inputfetcher) +{ + SeedRandomStateForTest(SeedRand::ZEROS); + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + + const auto batch_size{ + fuzzed_data_provider.ConsumeIntegralInRange(0, 1024)}; + const auto worker_threads{ + fuzzed_data_provider.ConsumeIntegralInRange(2, 4)}; + InputFetcher fetcher{batch_size, worker_threads}; + + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { + CBlock block; + Txid prevhash{Txid::FromUint256(ConsumeUInt256(fuzzed_data_provider))}; + + DbMap db_map{}; + std::map cache_map{}; + + DbCoinsView db(db_map); + + NoAccessCoinsView back; + CCoinsViewCache cache(&back); + + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), static_cast(batch_size * worker_threads * 2)) { + CMutableTransaction tx; + + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10) { + const auto txid{fuzzed_data_provider.ConsumeBool() + ? Txid::FromUint256(ConsumeUInt256(fuzzed_data_provider)) + : prevhash}; + const auto index{fuzzed_data_provider.ConsumeIntegral()}; + const COutPoint outpoint(txid, index); + + tx.vin.emplace_back(outpoint); + + std::optional maybe_coin; + if (fuzzed_data_provider.ConsumeBool()) { + Coin coin{}; + coin.fCoinBase = fuzzed_data_provider.ConsumeBool(); + coin.nHeight = + fuzzed_data_provider.ConsumeIntegralInRange( + 0, std::numeric_limits::max()); + coin.out.nValue = ConsumeMoney(fuzzed_data_provider); + maybe_coin = coin; + } else { + maybe_coin = std::nullopt; + } + db_map.try_emplace(outpoint, std::make_pair( + maybe_coin, + fuzzed_data_provider.ConsumeBool())); + + // Add the coin to the cache + if (fuzzed_data_provider.ConsumeBool()) { + Coin coin{}; + coin.fCoinBase = fuzzed_data_provider.ConsumeBool(); + coin.nHeight = + fuzzed_data_provider.ConsumeIntegralInRange( + 0, std::numeric_limits::max()); + coin.out.nValue = + fuzzed_data_provider.ConsumeIntegralInRange( + -1, MAX_MONEY); + cache_map.try_emplace(outpoint, coin); + cache.EmplaceCoinInternalDANGER( + COutPoint(outpoint), + std::move(coin), + /*set_dirty=*/fuzzed_data_provider.ConsumeBool()); + } + } + + prevhash = tx.GetHash(); + block.vtx.push_back(MakeTransactionRef(tx)); + } + + fetcher.FetchInputs(cache, db, block); + + for (const auto& [outpoint, pair] : db_map) { + // Check pre-existing coins in the cache have not been updated + const auto it{cache_map.find(outpoint)}; + if (it != cache_map.end()) { + const auto& cache_coin{it->second}; + const auto& coin{cache.AccessCoin(outpoint)}; + assert(coin.IsSpent() == cache_coin.IsSpent()); + assert(coin.fCoinBase == cache_coin.fCoinBase); + assert(coin.nHeight == cache_coin.nHeight); + assert(coin.out == cache_coin.out); + continue; + } + + if (!cache.HaveCoinInCache(outpoint)) { + continue; + } + + const auto& [maybe_coin, err] = pair; + assert(maybe_coin && !err); + + // Check any newly added coins in the cache are the same as the db + const auto& coin{cache.AccessCoin(outpoint)}; + assert(!coin.IsSpent()); + assert(coin.fCoinBase == (*maybe_coin).fCoinBase); + assert(coin.nHeight == (*maybe_coin).nHeight); + assert(coin.out == (*maybe_coin).out); + } + } +} From 4e339c3eb3ec524cb77e4986bb55347d9623da7b Mon Sep 17 00:00:00 2001 From: Andrew Toth Date: Wed, 17 Sep 2025 14:24:07 -0400 Subject: [PATCH 50/51] tests: set par=1 in feature_proxy to limit thread spawning --- test/functional/feature_proxy.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/test/functional/feature_proxy.py b/test/functional/feature_proxy.py index ba8a0212a6a0..cf39f66c520f 100755 --- a/test/functional/feature_proxy.py +++ b/test/functional/feature_proxy.py @@ -136,6 +136,9 @@ def setup_nodes(self): if self.have_unix_sockets: args[5] = ['-listen', f'-proxy=unix:{socket_path}'] args[6] = ['-listen', f'-onion=unix:{socket_path}'] + # Keep validation threads low to avoid CI thread/pid limits. + # Ensure even empty arg lists get '-par=1'. + args = [a + ['-par=1'] if a else ['-par=1'] for a in args] self.add_nodes(self.num_nodes, extra_args=args) self.start_nodes() @@ -379,42 +382,42 @@ def networks_dict(d): self.stop_node(1) self.log.info("Test passing invalid -proxy hostname raises expected init error") - self.nodes[1].extra_args = ["-proxy=abc..abc:23456"] + self.nodes[1].extra_args = ["-proxy=abc..abc:23456", "-par=1"] msg = "Error: Invalid -proxy address or hostname: 'abc..abc:23456'" self.nodes[1].assert_start_raises_init_error(expected_msg=msg) self.log.info("Test passing invalid -proxy port raises expected init error") - self.nodes[1].extra_args = ["-proxy=192.0.0.1:def"] + self.nodes[1].extra_args = ["-proxy=192.0.0.1:def", "-par=1"] msg = "Error: Invalid port specified in -proxy: '192.0.0.1:def'" self.nodes[1].assert_start_raises_init_error(expected_msg=msg) self.log.info("Test passing invalid -onion hostname raises expected init error") - self.nodes[1].extra_args = ["-onion=xyz..xyz:23456"] + self.nodes[1].extra_args = ["-onion=xyz..xyz:23456", "-par=1"] msg = "Error: Invalid -onion address or hostname: 'xyz..xyz:23456'" self.nodes[1].assert_start_raises_init_error(expected_msg=msg) self.log.info("Test passing invalid -onion port raises expected init error") - self.nodes[1].extra_args = ["-onion=192.0.0.1:def"] + self.nodes[1].extra_args = ["-onion=192.0.0.1:def", "-par=1"] msg = "Error: Invalid port specified in -onion: '192.0.0.1:def'" self.nodes[1].assert_start_raises_init_error(expected_msg=msg) self.log.info("Test passing invalid -i2psam hostname raises expected init error") - self.nodes[1].extra_args = ["-i2psam=def..def:23456"] + self.nodes[1].extra_args = ["-i2psam=def..def:23456", "-par=1"] msg = "Error: Invalid -i2psam address or hostname: 'def..def:23456'" self.nodes[1].assert_start_raises_init_error(expected_msg=msg) self.log.info("Test passing invalid -i2psam port raises expected init error") - self.nodes[1].extra_args = ["-i2psam=192.0.0.1:def"] + self.nodes[1].extra_args = ["-i2psam=192.0.0.1:def", "-par=1"] msg = "Error: Invalid port specified in -i2psam: '192.0.0.1:def'" self.nodes[1].assert_start_raises_init_error(expected_msg=msg) self.log.info("Test passing invalid -onlynet=i2p without -i2psam raises expected init error") - self.nodes[1].extra_args = ["-onlynet=i2p"] + self.nodes[1].extra_args = ["-onlynet=i2p", "-par=1"] msg = "Error: Outbound connections restricted to i2p (-onlynet=i2p) but -i2psam is not provided" self.nodes[1].assert_start_raises_init_error(expected_msg=msg) self.log.info("Test passing invalid -onlynet=cjdns without -cjdnsreachable raises expected init error") - self.nodes[1].extra_args = ["-onlynet=cjdns"] + self.nodes[1].extra_args = ["-onlynet=cjdns", "-par=1"] msg = "Error: Outbound connections restricted to CJDNS (-onlynet=cjdns) but -cjdnsreachable is not provided" self.nodes[1].assert_start_raises_init_error(expected_msg=msg) @@ -424,11 +427,11 @@ def networks_dict(d): "the proxy for reaching the Tor network is explicitly forbidden: -onion=0" ) for arg in ["-onion=0", "-noonion"]: - self.nodes[1].extra_args = ["-onlynet=onion", arg] + self.nodes[1].extra_args = ["-onlynet=onion", "-par=1", arg] self.nodes[1].assert_start_raises_init_error(expected_msg=msg) self.log.info("Test passing -onlynet=onion without -proxy, -onion or -listenonion raises expected init error") - self.nodes[1].extra_args = ["-onlynet=onion", "-listenonion=0"] + self.nodes[1].extra_args = ["-onlynet=onion", "-listenonion=0", "-par=1"] msg = ( "Error: Outbound connections restricted to Tor (-onlynet=onion) but the proxy for " "reaching the Tor network is not provided: none of -proxy, -onion or -listenonion is given" @@ -436,11 +439,11 @@ def networks_dict(d): self.nodes[1].assert_start_raises_init_error(expected_msg=msg) self.log.info("Test passing -onlynet=onion without -proxy or -onion but with -listenonion=1 is ok") - self.start_node(1, extra_args=["-onlynet=onion", "-listenonion=1"]) + self.start_node(1, extra_args=["-onlynet=onion", "-listenonion=1", "-par=1"]) self.stop_node(1) self.log.info("Test passing unknown network to -onlynet raises expected init error") - self.nodes[1].extra_args = ["-onlynet=abc"] + self.nodes[1].extra_args = ["-onlynet=abc", "-par=1"] msg = "Error: Unknown network specified in -onlynet: 'abc'" self.nodes[1].assert_start_raises_init_error(expected_msg=msg) @@ -486,7 +489,7 @@ def networks_dict(d): self.stop_node(1) self.log.info("Test passing too-long unix path to -proxy raises init error") - self.nodes[1].extra_args = [f"-proxy=unix:{'x' * 1000}"] + self.nodes[1].extra_args = [f"-proxy=unix:{'x' * 1000}", "-par=1"] if self.have_unix_sockets: msg = f"Error: Invalid -proxy address or hostname: 'unix:{'x' * 1000}'" else: From 1528235621c9f805d5bd86e90bb566fcf13846e9 Mon Sep 17 00:00:00 2001 From: Andrew Toth Date: Wed, 6 Nov 2024 19:52:34 -0500 Subject: [PATCH 51/51] validation: fetch block inputs in parallel --- src/validation.cpp | 3 +++ src/validation.h | 3 +++ 2 files changed, 6 insertions(+) diff --git a/src/validation.cpp b/src/validation.cpp index 73b3e91dd20b..1e0de94edca6 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -3086,6 +3086,8 @@ bool Chainstate::ConnectTip( LogDebug(BCLog::BENCH, " - Load block from disk: %.2fms\n", Ticks(time_2 - time_1)); { + m_chainman.GetInputFetcher().FetchInputs(CoinsTip(), CoinsDB(), *block_to_connect); + CCoinsViewCache view(&CoinsTip()); bool rv = ConnectBlock(*block_to_connect, state, pindexNew, view); if (m_chainman.m_options.signals) { @@ -6242,6 +6244,7 @@ static ChainstateManager::Options&& Flatten(ChainstateManager::Options&& opts) ChainstateManager::ChainstateManager(const util::SignalInterrupt& interrupt, Options options, node::BlockManager::Options blockman_options) : m_script_check_queue{/*batch_size=*/128, std::clamp(options.worker_threads_num, 0, MAX_SCRIPTCHECK_THREADS)}, + m_input_fetcher{/*batch_size=*/128, std::clamp(options.worker_threads_num, 0, MAX_SCRIPTCHECK_THREADS)}, m_interrupt{interrupt}, m_options{Flatten(std::move(options))}, m_blockman{interrupt, std::move(blockman_options)}, diff --git a/src/validation.h b/src/validation.h index cd448f3ca9eb..4f25b804c26f 100644 --- a/src/validation.h +++ b/src/validation.h @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -979,6 +980,7 @@ class ChainstateManager //! A queue for script verifications that have to be performed by worker threads. CCheckQueue m_script_check_queue; + InputFetcher m_input_fetcher; //! Timers and counters used for benchmarking validation in both background //! and active chainstates. @@ -1343,6 +1345,7 @@ class ChainstateManager void RecalculateBestHeader() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); CCheckQueue& GetCheckQueue() { return m_script_check_queue; } + InputFetcher& GetInputFetcher() { return m_input_fetcher; } ~ChainstateManager(); };