diff --git a/.github/workflows/gossipsub-interop-pr.yml b/.github/workflows/gossipsub-interop-pr.yml index 6f0a49490..48643e9c5 100644 --- a/.github/workflows/gossipsub-interop-pr.yml +++ b/.github/workflows/gossipsub-interop-pr.yml @@ -27,6 +27,11 @@ jobs: - name: Set up Rust uses: dtolnay/rust-toolchain@stable + - name: Set up Nim + run: | + curl https://nim-lang.org/choosenim/init.sh -sSf | sh -s -- -y + echo "$HOME/.nimble/bin" >> $GITHUB_PATH + - name: Install uv run: | curl -LsSf https://astral.sh/uv/install.sh | sh @@ -60,3 +65,12 @@ jobs: - name: Run gossipsub interop tests working-directory: gossipsub-interop run: make test + continue-on-error: true + + - name: Upload Shadow output + if: always() + uses: actions/upload-artifact@v4 + with: + name: shadow-output + path: gossipsub-interop/shadow-outputs/ + retention-days: 5 diff --git a/gossipsub-interop/.gitignore b/gossipsub-interop/.gitignore index 26b0c9768..7e60de738 100644 --- a/gossipsub-interop/.gitignore +++ b/gossipsub-interop/.gitignore @@ -21,3 +21,4 @@ backup timelines experiment-results/** __pycache__ +nim-libp2p-src/ diff --git a/gossipsub-interop/Makefile b/gossipsub-interop/Makefile index 8e316b16a..90a60bf7b 100644 --- a/gossipsub-interop/Makefile +++ b/gossipsub-interop/Makefile @@ -1,7 +1,28 @@ # Default target all: binaries -binaries: +NIM_LIBP2P_COMMIT = 22ab0402cd86f88fba0103f7946311d93e2566cd +nim-libp2p/gossipsub-bin: + @echo "Building nim-libp2p from commit: $(NIM_LIBP2P_COMMIT)" + # Clone nim-libp2p (if not already present) and build gossipsub binary + @if [ ! -d "nim-libp2p-src" ]; then \ + git clone https://github.com/vacp2p/nim-libp2p.git nim-libp2p-src && \ + cd nim-libp2p-src && git checkout $(NIM_LIBP2P_COMMIT); \ + fi + @mkdir -p nim-libp2p + cd nim-libp2p-src && nimble install_pinned + cd nim-libp2p-src && nim c -d:release \ + --skipProjCfg --skipParentCfg \ + --NimblePath:./nimbledeps/pkgs2 \ + -p:. --mm:refc \ + -d:chronicles_colors=None \ + -d:chronicles_log_level=TRACE \ + -d:chronicles_default_output_device=stderr \ + --threads:on \ + -o:$(CURDIR)/nim-libp2p/gossipsub-bin \ + ./interop/gossipsub/peer.nim + +binaries: nim-libp2p/gossipsub-bin cd go-libp2p && go build -linkshared -o gossipsub-bin cd rust-libp2p && cargo build cd jvm-libp2p && ./gradlew installDist @@ -9,6 +30,8 @@ binaries: # Clean all generated shadow simulation files clean: rm -rf shadow-outputs || true + rm -rf nim-libp2p-src || true + rm -f nim-libp2p/gossipsub-bin || true rm latest rm plots/* || true @@ -16,56 +39,84 @@ test: test-partial-messages test-subnet-blob test-partial-messages: # Testing partial messages - @echo "Testing partial messages" - @uv run run.py --node_count 32 --composition "rust-and-go" --scenario "partial-messages" && uv run checks/partial_messages.py latest --count 1 + @echo "Testing partial messages (rust-and-go)" + @uv run run.py --node_count 32 --composition rust go --scenario "partial-messages" && uv run checks/partial_messages.py latest --count 1 - @echo "Testing partial messages chain" - @uv run run.py --node_count 8 --composition "rust-and-go" --scenario "partial-messages-chain" && uv run checks/partial_messages.py latest --count 16 + @echo "Testing partial messages (nim-and-go)" + @uv run run.py --node_count 32 --composition nim go --scenario "partial-messages" && uv run checks/partial_messages.py latest --count 1 - @echo "Testing fanout" - uv run run.py --node_count 8 --composition "rust-and-go" --scenario "partial-messages-fanout" && uv run checks/partial_messages.py latest/ - uv run run.py --node_count 8 --seed 1 --composition "rust-and-go" --scenario "partial-messages-fanout" && uv run checks/partial_messages.py latest/ - uv run run.py --node_count 8 --seed 2 --composition "rust-and-go" --scenario "partial-messages-fanout" && uv run checks/partial_messages.py latest/ - uv run run.py --node_count 8 --seed 3 --composition "rust-and-go" --scenario "partial-messages-fanout" && uv run checks/partial_messages.py latest/ + @echo "Testing partial messages (nim-and-rust)" + @uv run run.py --node_count 32 --composition nim rust --scenario "partial-messages" && uv run checks/partial_messages.py latest --count 1 + + @echo "Testing partial messages chain (rust-and-go)" + @uv run run.py --node_count 8 --composition rust go --scenario "partial-messages-chain" && uv run checks/partial_messages.py latest --count 16 + + @echo "Testing partial messages chain (nim-and-go)" + @uv run run.py --node_count 8 --composition nim go --scenario "partial-messages-chain" && uv run checks/partial_messages.py latest --count 16 + + @echo "Testing partial messages chain (nim-and-rust)" + @uv run run.py --node_count 8 --composition nim rust --scenario "partial-messages-chain" && uv run checks/partial_messages.py latest --count 16 + + @echo "Testing fanout (rust-and-go)" + uv run run.py --node_count 8 --composition rust go --scenario "partial-messages-fanout" && uv run checks/partial_messages.py latest/ + uv run run.py --node_count 8 --seed 1 --composition rust go --scenario "partial-messages-fanout" && uv run checks/partial_messages.py latest/ + uv run run.py --node_count 8 --seed 2 --composition rust go --scenario "partial-messages-fanout" && uv run checks/partial_messages.py latest/ + uv run run.py --node_count 8 --seed 3 --composition rust go --scenario "partial-messages-fanout" && uv run checks/partial_messages.py latest/ + + @echo "Testing fanout (nim-and-go)" + uv run run.py --node_count 8 --composition nim go --scenario "partial-messages-fanout" && uv run checks/partial_messages.py latest/ + uv run run.py --node_count 8 --seed 1 --composition nim go --scenario "partial-messages-fanout" && uv run checks/partial_messages.py latest/ + uv run run.py --node_count 8 --seed 2 --composition nim go --scenario "partial-messages-fanout" && uv run checks/partial_messages.py latest/ + uv run run.py --node_count 8 --seed 3 --composition nim go --scenario "partial-messages-fanout" && uv run checks/partial_messages.py latest/ + + @echo "Testing fanout (nim-and-rust)" + uv run run.py --node_count 8 --composition nim rust --scenario "partial-messages-fanout" && uv run checks/partial_messages.py latest/ + uv run run.py --node_count 8 --seed 1 --composition nim rust --scenario "partial-messages-fanout" && uv run checks/partial_messages.py latest/ + uv run run.py --node_count 8 --seed 2 --composition nim rust --scenario "partial-messages-fanout" && uv run checks/partial_messages.py latest/ + uv run run.py --node_count 8 --seed 3 --composition nim rust --scenario "partial-messages-fanout" && uv run checks/partial_messages.py latest/ test-subnet-blob: # Testing subnet blob scenario @echo "Testing subnet blob messages" @echo "Testing single implementations" - uv run run.py --node_count 32 --composition "all-go" && uv run checks/subnet_blob_msg.py latest/ - uv run run.py --node_count 32 --composition "all-rust" && uv run checks/subnet_blob_msg.py latest/ - uv run run.py --node_count 32 --composition "all-jvm" && uv run checks/subnet_blob_msg.py latest/ + uv run run.py --node_count 32 --composition go && uv run checks/subnet_blob_msg.py latest/ + uv run run.py --node_count 32 --composition rust && uv run checks/subnet_blob_msg.py latest/ + uv run run.py --node_count 32 --composition jvm && uv run checks/subnet_blob_msg.py latest/ + uv run run.py --node_count 32 --composition nim && uv run checks/subnet_blob_msg.py latest/ @echo "Testing impl pairs" - uv run run.py --node_count 32 --composition "rust-and-go" && uv run checks/subnet_blob_msg.py latest/ - uv run run.py --node_count 32 --composition "jvm-and-go" && uv run checks/subnet_blob_msg.py latest/ - uv run run.py --node_count 32 --composition "jvm-and-rust" && uv run checks/subnet_blob_msg.py latest/ + uv run run.py --node_count 32 --composition rust go && uv run checks/subnet_blob_msg.py latest/ + uv run run.py --node_count 32 --composition jvm go && uv run checks/subnet_blob_msg.py latest/ + uv run run.py --node_count 32 --composition jvm rust && uv run checks/subnet_blob_msg.py latest/ + uv run run.py --node_count 32 --composition nim go && uv run checks/subnet_blob_msg.py latest/ + uv run run.py --node_count 32 --composition nim rust && uv run checks/subnet_blob_msg.py latest/ + uv run run.py --node_count 32 --composition nim jvm && uv run checks/subnet_blob_msg.py latest/ @echo "Testing all" - uv run run.py --node_count 32 --composition "all-three" && uv run checks/subnet_blob_msg.py latest/ + uv run run.py --node_count 32 --composition go rust jvm nim && uv run checks/subnet_blob_msg.py latest/ test-go: # Testing partial messages @echo "Testing partial messages" - @uv run run.py --node_count 8 --composition "all-go" --scenario "partial-messages" && uv run checks/partial_messages.py latest --count 1 + @uv run run.py --node_count 8 --composition go --scenario "partial-messages" && uv run checks/partial_messages.py latest --count 1 @echo "Testing partial messages chain" - @uv run run.py --node_count 8 --composition "all-go" --scenario "partial-messages-chain" && uv run checks/partial_messages.py latest --count 16 + @uv run run.py --node_count 8 --composition go --scenario "partial-messages-chain" && uv run checks/partial_messages.py latest --count 16 @echo "Testing fanout" - @uv run run.py --node_count 2 --composition "all-go" --scenario "partial-messages-fanout" && uv run checks/partial_messages.py latest/ + @uv run run.py --node_count 2 --composition go --scenario "partial-messages-fanout" && uv run checks/partial_messages.py latest/ test-rust-only: # Testing partial messages @echo "Testing partial messages" - @uv run run.py --node_count 8 --composition "all-rust" --scenario "partial-messages" && uv run checks/partial_messages.py latest --count 1 + @uv run run.py --node_count 8 --composition rust --scenario "partial-messages" && uv run checks/partial_messages.py latest --count 1 @echo "Testing partial messages chain" - @uv run run.py --node_count 8 --composition "all-rust" --scenario "partial-messages-chain" && uv run checks/partial_messages.py latest --count 16 + @uv run run.py --node_count 8 --composition rust --scenario "partial-messages-chain" && uv run checks/partial_messages.py latest --count 16 @echo "Testing fanout" - @uv run run.py --node_count 2 --composition "all-rust" --scenario "partial-messages-fanout" && uv run checks/partial_messages.py latest/ + @uv run run.py --node_count 2 --composition rust --scenario "partial-messages-fanout" && uv run checks/partial_messages.py latest/ diff --git a/gossipsub-interop/README.md b/gossipsub-interop/README.md index 310074788..295340f5c 100644 --- a/gossipsub-interop/README.md +++ b/gossipsub-interop/README.md @@ -50,7 +50,7 @@ nodes with default GossipSub parameters and sending large messages to a network of 700 nodes: ```bash -uv run run.py --node_count 700 --composition "rust-and-go" --scenario "subnet-blob-msg" +uv run run.py --node_count 700 --composition rust go --scenario "subnet-blob-msg" ``` The definitions of the experiment, composition, and scenarios are defined in `experiment.py`. @@ -59,7 +59,7 @@ After running an experiment all the results and configuration needed to reproduce the test are saved in an output folder which, by default, is named by the specific scenario, node count, and composition. For the above example, the output folder is -`subnet-blob-msg-700-rust-and-go.data`. This output folder contains the following files: +`subnet-blob-msg-700-rust-go.data`. This output folder contains the following files: - shadow.yaml: The Shadow config defining the binaries and network. - graph.gml: The graph of the network links for Shadow. @@ -74,14 +74,14 @@ To build the implementation reference `./test-specs/implementation.md`. After implementing it, make sure to add build commands in the Makefile's `binaries` recipe. -Finally, add it to the `composition` function in `experiment.py`. +Finally, add an entry to the `IMPLEMENTATIONS` dict in `experiment.py` mapping a short name (e.g. `nim`) to the binary path. It then becomes available to `--composition` automatically. ## Examples Minimal test of partial messages ```bash -uv run run.py --node_count 2 --composition "all-go" --scenario "partial-messages" && uv run checks/partial_messages.py latest/ +uv run run.py --node_count 2 --composition go --scenario "partial-messages" && uv run checks/partial_messages.py latest/ ``` That command runs the shadow simulation and then verifies the stdout logs have the expected message. diff --git a/gossipsub-interop/experiment.py b/gossipsub-interop/experiment.py index 338ba1221..7ecca96fb 100644 --- a/gossipsub-interop/experiment.py +++ b/gossipsub-interop/experiment.py @@ -340,61 +340,33 @@ def scenario( return ExperimentParams(script=instructions) -def composition(preset_name: str) -> List[Binary]: - match preset_name: - case "all-go": - return [Binary("go-libp2p/gossipsub-bin", percent_of_nodes=100)] - case "all-rust": - # Always use debug. We don't measure compute performance here. - return [ - Binary( - "rust-libp2p/target/debug/rust-libp2p-gossip", percent_of_nodes=100 - ) - ] - case "rust-and-go": - return [ - Binary( - "rust-libp2p/target/debug/rust-libp2p-gossip", percent_of_nodes=50 - ), - Binary("go-libp2p/gossipsub-bin", percent_of_nodes=50), - ] - case "all-jvm": - return [ - Binary( - "jvm-libp2p/build/install/jvm-libp2p-gossip/bin/jvm-libp2p-gossip", - percent_of_nodes=100, - ) - ] - case "jvm-and-go": - return [ - Binary( - "jvm-libp2p/build/install/jvm-libp2p-gossip/bin/jvm-libp2p-gossip", - percent_of_nodes=50, - ), - Binary("go-libp2p/gossipsub-bin", percent_of_nodes=50), - ] - case "jvm-and-rust": - return [ - Binary( - "jvm-libp2p/build/install/jvm-libp2p-gossip/bin/jvm-libp2p-gossip", - percent_of_nodes=50, - ), - Binary( - "rust-libp2p/target/debug/rust-libp2p-gossip", percent_of_nodes=50 - ), - ] - case "all-three": - return [ - Binary("go-libp2p/gossipsub-bin", percent_of_nodes=34), - Binary( - "rust-libp2p/target/debug/rust-libp2p-gossip", percent_of_nodes=33 - ), - Binary( - "jvm-libp2p/build/install/jvm-libp2p-gossip/bin/jvm-libp2p-gossip", - percent_of_nodes=33, - ), - ] - raise ValueError(f"Unknown preset name: {preset_name}") +IMPLEMENTATIONS: Dict[str, str] = { + "go": "go-libp2p/gossipsub-bin", + # Always use debug rust. We don't measure compute performance here. + "rust": "rust-libp2p/target/debug/rust-libp2p-gossip", + "nim": "nim-libp2p/gossipsub-bin", + "jvm": "jvm-libp2p/build/install/jvm-libp2p-gossip/bin/jvm-libp2p-gossip", +} + + +def composition(impls: List[str]) -> List[Binary]: + if not impls: + raise ValueError("composition requires at least one implementation") + for name in impls: + if name not in IMPLEMENTATIONS: + raise ValueError( + f"Unknown implementation '{name}'. " + f"Known: {sorted(IMPLEMENTATIONS)}" + ) + + # Split 100% as evenly as possible. First `leftover` impls get +1 (e.g. 3 impls -> 34/33/33). + base, leftover = divmod(100, len(impls)) + percents = [base + 1] * leftover + [base] * (len(impls) - leftover) + + return [ + Binary(IMPLEMENTATIONS[name], percent_of_nodes=pct) + for name, pct in zip(impls, percents) + ] def random_network_mesh( diff --git a/gossipsub-interop/nim-libp2p/.gitkeep b/gossipsub-interop/nim-libp2p/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/gossipsub-interop/run.py b/gossipsub-interop/run.py index 98adddc7b..90dfe111b 100644 --- a/gossipsub-interop/run.py +++ b/gossipsub-interop/run.py @@ -28,7 +28,15 @@ def main(): parser.add_argument( "--scenario", type=str, required=False, default="subnet-blob-msg" ) - parser.add_argument("--composition", type=str, required=False, default="all-go") + parser.add_argument( + "--composition", + type=str, + nargs="+", + required=False, + default=["go"], + help="One or more implementation names (e.g. go rust nim jvm). " + "Nodes are split evenly across them.", + ) parser.add_argument("--output_dir", type=str, required=False) args = parser.parse_args() @@ -48,7 +56,8 @@ def main(): import datetime timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S") - args.output_dir = f"{args.scenario}-{args.node_count}-{args.composition}-{ + composition_label = "-".join(args.composition) + args.output_dir = f"{args.scenario}-{args.node_count}-{composition_label}-{ args.seed }-{timestamp}-{git_describe}.data"