Test gossipsub interop 4

.github/workflows/gossipsub-interop-pr.yml

@@ -27,6 +27,11 @@ jobs:
       - name: Set up Rust
         uses: dtolnay/rust-toolchain@stable
 
+      - name: Set up Nim
+        run: |
+          curl https://nim-lang.org/choosenim/init.sh -sSf | sh -s -- -y
+          echo "$HOME/.nimble/bin" >> $GITHUB_PATH
+
       - name: Install uv
         run: |
           curl -LsSf https://astral.sh/uv/install.sh | sh
@@ -60,3 +65,12 @@ jobs:
       - name: Run gossipsub interop tests
         working-directory: gossipsub-interop
         run: make test
+        continue-on-error: true
+
+      - name: Upload Shadow output
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: shadow-output
+          path: gossipsub-interop/shadow-outputs/
+          retention-days: 5

gossipsub-interop/.gitignore

@@ -21,3 +21,4 @@ backup
 timelines
 experiment-results/**
 __pycache__
+nim-libp2p-src/

gossipsub-interop/Makefile

@@ -1,71 +1,122 @@
 # Default target
 all: binaries
 
-binaries:
+NIM_LIBP2P_COMMIT = 22ab0402cd86f88fba0103f7946311d93e2566cd
+nim-libp2p/gossipsub-bin:
+	@echo "Building nim-libp2p from commit: $(NIM_LIBP2P_COMMIT)"
+	# Clone nim-libp2p (if not already present) and build gossipsub binary
+	@if [ ! -d "nim-libp2p-src" ]; then \
+		git clone https://github.com/vacp2p/nim-libp2p.git nim-libp2p-src && \
+		cd nim-libp2p-src && git checkout $(NIM_LIBP2P_COMMIT); \
+	fi
+	@mkdir -p nim-libp2p
+	cd nim-libp2p-src && nimble install_pinned
+	cd nim-libp2p-src && nim c -d:release \
+		--skipProjCfg --skipParentCfg \
+		--NimblePath:./nimbledeps/pkgs2 \
+		-p:. --mm:refc \
+		-d:chronicles_colors=None \
+		-d:chronicles_log_level=TRACE \
+		-d:chronicles_default_output_device=stderr \
+		--threads:on \
+		-o:$(CURDIR)/nim-libp2p/gossipsub-bin \
+		./interop/gossipsub/peer.nim
+
+binaries: nim-libp2p/gossipsub-bin
 	cd go-libp2p && go build -linkshared -o gossipsub-bin
 	cd rust-libp2p && cargo build
 	cd jvm-libp2p && ./gradlew installDist
 
 # Clean all generated shadow simulation files
 clean:
 	rm -rf shadow-outputs || true
+	rm -rf nim-libp2p-src || true
+	rm -f nim-libp2p/gossipsub-bin || true
 	rm latest
 	rm plots/* || true
 
 test: test-partial-messages test-subnet-blob
 
 test-partial-messages:
 	# Testing partial messages
-	@echo "Testing partial messages"
-	@uv run run.py --node_count 32 --composition "rust-and-go" --scenario "partial-messages" && uv run checks/partial_messages.py latest --count 1
+	@echo "Testing partial messages (rust-and-go)"
+	@uv run run.py --node_count 32 --composition rust go --scenario "partial-messages" && uv run checks/partial_messages.py latest --count 1
 
-	@echo "Testing partial messages chain"
-	@uv run run.py --node_count 8 --composition "rust-and-go" --scenario "partial-messages-chain" && uv run checks/partial_messages.py latest --count 16
+	@echo "Testing partial messages (nim-and-go)"
+	@uv run run.py --node_count 32 --composition nim go --scenario "partial-messages" && uv run checks/partial_messages.py latest --count 1
 
-	@echo "Testing fanout"
-	uv run run.py --node_count 8 --composition "rust-and-go" --scenario "partial-messages-fanout" && uv run checks/partial_messages.py latest/
-	uv run run.py --node_count 8 --seed 1 --composition "rust-and-go" --scenario "partial-messages-fanout" && uv run checks/partial_messages.py latest/
-	uv run run.py --node_count 8 --seed 2 --composition "rust-and-go" --scenario "partial-messages-fanout" && uv run checks/partial_messages.py latest/
-	uv run run.py --node_count 8 --seed 3 --composition "rust-and-go" --scenario "partial-messages-fanout" && uv run checks/partial_messages.py latest/
+	@echo "Testing partial messages (nim-and-rust)"
+	@uv run run.py --node_count 32 --composition nim rust --scenario "partial-messages" && uv run checks/partial_messages.py latest --count 1
+
+	@echo "Testing partial messages chain (rust-and-go)"
+	@uv run run.py --node_count 8 --composition rust go --scenario "partial-messages-chain" && uv run checks/partial_messages.py latest --count 16
+
+	@echo "Testing partial messages chain (nim-and-go)"
+	@uv run run.py --node_count 8 --composition nim go --scenario "partial-messages-chain" && uv run checks/partial_messages.py latest --count 16
+
+	@echo "Testing partial messages chain (nim-and-rust)"
+	@uv run run.py --node_count 8 --composition nim rust --scenario "partial-messages-chain" && uv run checks/partial_messages.py latest --count 16
+
+	@echo "Testing fanout (rust-and-go)"
+	uv run run.py --node_count 8 --composition rust go --scenario "partial-messages-fanout" && uv run checks/partial_messages.py latest/
+	uv run run.py --node_count 8 --seed 1 --composition rust go --scenario "partial-messages-fanout" && uv run checks/partial_messages.py latest/
+	uv run run.py --node_count 8 --seed 2 --composition rust go --scenario "partial-messages-fanout" && uv run checks/partial_messages.py latest/
+	uv run run.py --node_count 8 --seed 3 --composition rust go --scenario "partial-messages-fanout" && uv run checks/partial_messages.py latest/
+
+	@echo "Testing fanout (nim-and-go)"
+	uv run run.py --node_count 8 --composition nim go --scenario "partial-messages-fanout" && uv run checks/partial_messages.py latest/
+	uv run run.py --node_count 8 --seed 1 --composition nim go --scenario "partial-messages-fanout" && uv run checks/partial_messages.py latest/
+	uv run run.py --node_count 8 --seed 2 --composition nim go --scenario "partial-messages-fanout" && uv run checks/partial_messages.py latest/
+	uv run run.py --node_count 8 --seed 3 --composition nim go --scenario "partial-messages-fanout" && uv run checks/partial_messages.py latest/
+
+	@echo "Testing fanout (nim-and-rust)"
+	uv run run.py --node_count 8 --composition nim rust --scenario "partial-messages-fanout" && uv run checks/partial_messages.py latest/
+	uv run run.py --node_count 8 --seed 1 --composition nim rust --scenario "partial-messages-fanout" && uv run checks/partial_messages.py latest/
+	uv run run.py --node_count 8 --seed 2 --composition nim rust --scenario "partial-messages-fanout" && uv run checks/partial_messages.py latest/
+	uv run run.py --node_count 8 --seed 3 --composition nim rust --scenario "partial-messages-fanout" && uv run checks/partial_messages.py latest/
 
 test-subnet-blob:
 	# Testing subnet blob scenario
 	@echo "Testing subnet blob messages"
 	@echo "Testing single implementations"
-	uv run run.py --node_count 32 --composition "all-go" && uv run checks/subnet_blob_msg.py latest/
-	uv run run.py --node_count 32 --composition "all-rust" && uv run checks/subnet_blob_msg.py latest/
-	uv run run.py --node_count 32 --composition "all-jvm" && uv run checks/subnet_blob_msg.py latest/
+	uv run run.py --node_count 32 --composition go && uv run checks/subnet_blob_msg.py latest/
+	uv run run.py --node_count 32 --composition rust && uv run checks/subnet_blob_msg.py latest/
+	uv run run.py --node_count 32 --composition jvm && uv run checks/subnet_blob_msg.py latest/
+	uv run run.py --node_count 32 --composition nim && uv run checks/subnet_blob_msg.py latest/
 
 	@echo "Testing impl pairs"
-	uv run run.py --node_count 32 --composition "rust-and-go" && uv run checks/subnet_blob_msg.py latest/
-	uv run run.py --node_count 32 --composition "jvm-and-go" && uv run checks/subnet_blob_msg.py latest/
-	uv run run.py --node_count 32 --composition "jvm-and-rust" && uv run checks/subnet_blob_msg.py latest/
+	uv run run.py --node_count 32 --composition rust go && uv run checks/subnet_blob_msg.py latest/
+	uv run run.py --node_count 32 --composition jvm go && uv run checks/subnet_blob_msg.py latest/
+	uv run run.py --node_count 32 --composition jvm rust && uv run checks/subnet_blob_msg.py latest/
+	uv run run.py --node_count 32 --composition nim go && uv run checks/subnet_blob_msg.py latest/
+	uv run run.py --node_count 32 --composition nim rust && uv run checks/subnet_blob_msg.py latest/
+	uv run run.py --node_count 32 --composition nim jvm && uv run checks/subnet_blob_msg.py latest/
 
 	@echo "Testing all"
-	uv run run.py --node_count 32 --composition "all-three" && uv run checks/subnet_blob_msg.py latest/
+	uv run run.py --node_count 32 --composition go rust jvm nim && uv run checks/subnet_blob_msg.py latest/
 
 test-go:
 	# Testing partial messages
 	@echo "Testing partial messages"
-	@uv run run.py --node_count 8 --composition "all-go" --scenario "partial-messages" && uv run checks/partial_messages.py latest --count 1
+	@uv run run.py --node_count 8 --composition go --scenario "partial-messages" && uv run checks/partial_messages.py latest --count 1
 
 	@echo "Testing partial messages chain"
-	@uv run run.py --node_count 8 --composition "all-go" --scenario "partial-messages-chain" && uv run checks/partial_messages.py latest --count 16
+	@uv run run.py --node_count 8 --composition go --scenario "partial-messages-chain" && uv run checks/partial_messages.py latest --count 16
 
 	@echo "Testing fanout"
-	@uv run run.py --node_count 2 --composition "all-go" --scenario "partial-messages-fanout" && uv run checks/partial_messages.py latest/
+	@uv run run.py --node_count 2 --composition go --scenario "partial-messages-fanout" && uv run checks/partial_messages.py latest/
 
 
 test-rust-only:
 	# Testing partial messages
 	@echo "Testing partial messages"
-	@uv run run.py --node_count 8 --composition "all-rust" --scenario "partial-messages" && uv run checks/partial_messages.py latest --count 1
+	@uv run run.py --node_count 8 --composition rust --scenario "partial-messages" && uv run checks/partial_messages.py latest --count 1
 
 	@echo "Testing partial messages chain"
-	@uv run run.py --node_count 8 --composition "all-rust" --scenario "partial-messages-chain" && uv run checks/partial_messages.py latest --count 16
+	@uv run run.py --node_count 8 --composition rust --scenario "partial-messages-chain" && uv run checks/partial_messages.py latest --count 16
 
 	@echo "Testing fanout"
-	@uv run run.py --node_count 2 --composition "all-rust" --scenario "partial-messages-fanout" && uv run checks/partial_messages.py latest/
+	@uv run run.py --node_count 2 --composition rust --scenario "partial-messages-fanout" && uv run checks/partial_messages.py latest/
 
 
 

gossipsub-interop/README.md

@@ -50,7 +50,7 @@ nodes with default GossipSub parameters and sending large messages to a network
 of 700 nodes:
 
 ```bash
-uv run run.py --node_count 700 --composition "rust-and-go" --scenario "subnet-blob-msg"
+uv run run.py --node_count 700 --composition rust go --scenario "subnet-blob-msg"
 ```
 
 The definitions of the experiment, composition, and scenarios are defined in `experiment.py`.
@@ -59,7 +59,7 @@ After running an experiment all the results and configuration needed to
 reproduce the test are saved in an output folder which, by default, is named by
 the specific scenario, node count, and composition. For the above
 example, the output folder is
-`subnet-blob-msg-700-rust-and-go.data`. This output folder contains the following files:
+`subnet-blob-msg-700-rust-go.data`. This output folder contains the following files:
 
 - shadow.yaml: The Shadow config defining the binaries and network.
 - graph.gml: The graph of the network links for Shadow.
@@ -74,14 +74,14 @@ To build the implementation reference `./test-specs/implementation.md`.
 
 After implementing it, make sure to add build commands in the Makefile's `binaries` recipe.
 
-Finally, add it to the `composition` function in `experiment.py`.
+Finally, add an entry to the `IMPLEMENTATIONS` dict in `experiment.py` mapping a short name (e.g. `nim`) to the binary path. It then becomes available to `--composition` automatically.
 
 ## Examples
 
 Minimal test of partial messages
 
 ```bash
-uv run run.py --node_count 2 --composition "all-go" --scenario "partial-messages" && uv run checks/partial_messages.py latest/
+uv run run.py --node_count 2 --composition go --scenario "partial-messages" && uv run checks/partial_messages.py latest/
 ```
 
 That command runs the shadow simulation and then verifies the stdout logs have the expected message.

gossipsub-interop/experiment.py

@@ -340,61 +340,33 @@ def scenario(
     return ExperimentParams(script=instructions)
 
 
-def composition(preset_name: str) -> List[Binary]:
-    match preset_name:
-        case "all-go":
-            return [Binary("go-libp2p/gossipsub-bin", percent_of_nodes=100)]
-        case "all-rust":
-            # Always use debug. We don't measure compute performance here.
-            return [
-                Binary(
-                    "rust-libp2p/target/debug/rust-libp2p-gossip", percent_of_nodes=100
-                )
-            ]
-        case "rust-and-go":
-            return [
-                Binary(
-                    "rust-libp2p/target/debug/rust-libp2p-gossip", percent_of_nodes=50
-                ),
-                Binary("go-libp2p/gossipsub-bin", percent_of_nodes=50),
-            ]
-        case "all-jvm":
-            return [
-                Binary(
-                    "jvm-libp2p/build/install/jvm-libp2p-gossip/bin/jvm-libp2p-gossip",
-                    percent_of_nodes=100,
-                )
-            ]
-        case "jvm-and-go":
-            return [
-                Binary(
-                    "jvm-libp2p/build/install/jvm-libp2p-gossip/bin/jvm-libp2p-gossip",
-                    percent_of_nodes=50,
-                ),
-                Binary("go-libp2p/gossipsub-bin", percent_of_nodes=50),
-            ]
-        case "jvm-and-rust":
-            return [
-                Binary(
-                    "jvm-libp2p/build/install/jvm-libp2p-gossip/bin/jvm-libp2p-gossip",
-                    percent_of_nodes=50,
-                ),
-                Binary(
-                    "rust-libp2p/target/debug/rust-libp2p-gossip", percent_of_nodes=50
-                ),
-            ]
-        case "all-three":
-            return [
-                Binary("go-libp2p/gossipsub-bin", percent_of_nodes=34),
-                Binary(
-                    "rust-libp2p/target/debug/rust-libp2p-gossip", percent_of_nodes=33
-                ),
-                Binary(
-                    "jvm-libp2p/build/install/jvm-libp2p-gossip/bin/jvm-libp2p-gossip",
-                    percent_of_nodes=33,
-                ),
-            ]
-    raise ValueError(f"Unknown preset name: {preset_name}")
+IMPLEMENTATIONS: Dict[str, str] = {
+    "go": "go-libp2p/gossipsub-bin",
+    # Always use debug rust. We don't measure compute performance here.
+    "rust": "rust-libp2p/target/debug/rust-libp2p-gossip",
+    "nim": "nim-libp2p/gossipsub-bin",
+    "jvm": "jvm-libp2p/build/install/jvm-libp2p-gossip/bin/jvm-libp2p-gossip",
+}
+
+
+def composition(impls: List[str]) -> List[Binary]:
+    if not impls:
+        raise ValueError("composition requires at least one implementation")
+    for name in impls:
+        if name not in IMPLEMENTATIONS:
+            raise ValueError(
+                f"Unknown implementation '{name}'. "
+                f"Known: {sorted(IMPLEMENTATIONS)}"
+            )
+
+    # Split 100% as evenly as possible. First `leftover` impls get +1 (e.g. 3 impls -> 34/33/33).
+    base, leftover = divmod(100, len(impls))
+    percents = [base + 1] * leftover + [base] * (len(impls) - leftover)
+
+    return [
+        Binary(IMPLEMENTATIONS[name], percent_of_nodes=pct)
+        for name, pct in zip(impls, percents)
+    ]
 
 
 def random_network_mesh(

gossipsub-interop/run.py

@@ -28,7 +28,15 @@ def main():
     parser.add_argument(
         "--scenario", type=str, required=False, default="subnet-blob-msg"
     )
-    parser.add_argument("--composition", type=str, required=False, default="all-go")
+    parser.add_argument(
+        "--composition",
+        type=str,
+        nargs="+",
+        required=False,
+        default=["go"],
+        help="One or more implementation names (e.g. go rust nim jvm). "
+        "Nodes are split evenly across them.",
+    )
     parser.add_argument("--output_dir", type=str, required=False)
     args = parser.parse_args()
 
@@ -48,7 +56,8 @@ def main():
         import datetime
 
         timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
-        args.output_dir = f"{args.scenario}-{args.node_count}-{args.composition}-{
+        composition_label = "-".join(args.composition)
+        args.output_dir = f"{args.scenario}-{args.node_count}-{composition_label}-{
             args.seed
         }-{timestamp}-{git_describe}.data"