From 4635f300b81cd44bd8501b58016a3c81f5a5b13d Mon Sep 17 00:00:00 2001 From: Ruy Adorno Date: Fri, 5 Sep 2025 22:48:50 -0400 Subject: [PATCH] add registries benchmarks --- .github/workflows/benchmark.yaml | 8 +- registry-configs/verdaccio/config.yaml | 24 ++++ registry-configs/vsr/vlt.json | 7 ++ scripts/benchmark.sh | 6 + scripts/process-results.sh | 2 +- scripts/registry-helpers.sh | 134 +++++++++++++++++++++ scripts/setup.sh | 47 +++++++- scripts/variations/registry-local-cache.sh | 93 ++++++++++++++ scripts/variations/registry-proxied.sh | 92 ++++++++++++++ 9 files changed, 408 insertions(+), 5 deletions(-) create mode 100644 registry-configs/verdaccio/config.yaml create mode 100644 registry-configs/vsr/vlt.json create mode 100755 scripts/registry-helpers.sh create mode 100755 scripts/variations/registry-local-cache.sh create mode 100755 scripts/variations/registry-proxied.sh diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index 11c4fcfab..f9bed5bda 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -8,7 +8,7 @@ on: default: '["next", "astro", "vue", "svelte"]' variations: description: 'The benchmark variations to run' - default: '["clean", "node_modules", "cache", "cache+node_modules", "cache+lockfile", "cache+lockfile+node_modules", "lockfile", "lockfile+node_modules"]' + default: '["clean", "node_modules", "cache", "cache+node_modules", "cache+lockfile", "cache+lockfile+node_modules", "lockfile", "lockfile+node_modules", "registry-local-cache", "registry-proxied"]' binaries: description: 'The binaries to run the benchmarks on' default: '"npm,yarn,berry,pnpm,vlt,bun,deno,nx,turbo,node"' @@ -34,7 +34,7 @@ jobs: strategy: matrix: fixture: ${{ fromJson(inputs.fixtures || '["next", "astro", "vue", "svelte"]') }} - variation: ${{ fromJson(inputs.variations || '["clean", "node_modules", "cache", "cache+node_modules", "cache+lockfile", "cache+lockfile+node_modules", "lockfile", "lockfile+node_modules"]') }} + variation: ${{ fromJson(inputs.variations || '["clean", "node_modules", "cache", "cache+node_modules", "cache+lockfile", "cache+lockfile+node_modules", "lockfile", "lockfile+node_modules", "registry-local-cache", "registry-proxied"]') }} include: - variation: "run" fixture: "run" @@ -115,7 +115,9 @@ jobs: "lockfile", "lockfile+node_modules", "node_modules", - "run" + "run", + "registry-local-cache", + "registry-proxied" ]; // Helper functions for statistical calculations diff --git a/registry-configs/verdaccio/config.yaml b/registry-configs/verdaccio/config.yaml new file mode 100644 index 000000000..7c9376dca --- /dev/null +++ b/registry-configs/verdaccio/config.yaml @@ -0,0 +1,24 @@ +storage: ./storage +auth: + htpasswd: + file: ./htpasswd +uplinks: + npmjs: + url: https://registry.npmjs.org/ +packages: + '@*/*': + access: $all + publish: $authenticated + proxy: npmjs + '**': + access: $all + publish: $authenticated + proxy: npmjs +server: + keepAliveTimeout: 60 + bodyParser: + limit: 100mb +logs: + - type: stdout + format: pretty + level: warn diff --git a/registry-configs/vsr/vlt.json b/registry-configs/vsr/vlt.json new file mode 100644 index 000000000..8ba3cedb1 --- /dev/null +++ b/registry-configs/vsr/vlt.json @@ -0,0 +1,7 @@ +{ + "registry": { + "upstream": "https://registry.npmjs.org/", + "cache": "./cache", + "port": 1337 + } +} diff --git a/scripts/benchmark.sh b/scripts/benchmark.sh index 950534d93..691c006dc 100644 --- a/scripts/benchmark.sh +++ b/scripts/benchmark.sh @@ -47,6 +47,12 @@ case "$2" in run) bash ../../scripts/variations/run.sh "../../scripts" "../../results" "run" "run" ;; + registry-local-cache) + bash ../../scripts/variations/registry-local-cache.sh "../../scripts" "../../results" "$1" "$2" + ;; + registry-proxied) + bash ../../scripts/variations/registry-proxied.sh "../../scripts" "../../results" "$1" "$2" + ;; *) echo "Error: Unknown install variation '$2'" exit 1 diff --git a/scripts/process-results.sh b/scripts/process-results.sh index 27a41fff5..f89122d8d 100644 --- a/scripts/process-results.sh +++ b/scripts/process-results.sh @@ -53,7 +53,7 @@ echo "Processing results..." # Process variations results for fixture in next astro svelte vue; do - for variation in cache cache+lockfile cache+lockfile+node_modules cache+node_modules clean lockfile lockfile+node_modules node_modules run; do + for variation in cache cache+lockfile cache+lockfile+node_modules cache+node_modules clean lockfile lockfile+node_modules node_modules run registry-local-cache registry-proxied; do if [ -f "results/results-$fixture-$variation/benchmarks.json" ]; then print_summary "results/results-$fixture-$variation/benchmarks.json" "$fixture" "$variation" cp "results/results-$fixture-$variation/benchmarks.json" "results/$DATE/$fixture-$variation.json" diff --git a/scripts/registry-helpers.sh b/scripts/registry-helpers.sh new file mode 100755 index 000000000..65397bab0 --- /dev/null +++ b/scripts/registry-helpers.sh @@ -0,0 +1,134 @@ +#!/bin/bash +# Registry helper functions + +# Registry ports +VSR_PORT=1337 +VERDACCIO_PORT=4873 + +# Start registry function +start_registry() { + local registry_type=$1 + local output_folder=$2 + + # Kill any existing registry processes + stop_registry "$registry_type" + sleep 2 + + if [ "$registry_type" = "vsr" ]; then + echo "Starting VSR registry on port $VSR_PORT..." + cd "$(dirname "$0")/../registry-configs/vsr" + npx @vltpkg/vsr --port $VSR_PORT --config ./vlt.json > "$output_folder/vsr.log" 2>&1 & + echo $! > /tmp/vsr.pid + cd - > /dev/null + + # Wait for VSR to start + local retries=0 + while ! curl -s "http://localhost:$VSR_PORT/-/ping" > /dev/null 2>&1; do + sleep 1 + retries=$((retries + 1)) + if [ $retries -gt 10 ]; then + echo "Error: VSR failed to start" + return 1 + fi + done + + echo "http://localhost:$VSR_PORT" + else + echo "Starting Verdaccio registry on port $VERDACCIO_PORT..." + cd "$(dirname "$0")/../registry-configs/verdaccio" + verdaccio --config ./config.yaml > "$output_folder/verdaccio.log" 2>&1 & + echo $! > /tmp/verdaccio.pid + cd - > /dev/null + + # Wait for Verdaccio to start + local retries=0 + while ! curl -s "http://localhost:$VERDACCIO_PORT/-/ping" > /dev/null 2>&1; do + sleep 1 + retries=$((retries + 1)) + if [ $retries -gt 10 ]; then + echo "Error: Verdaccio failed to start" + return 1 + fi + done + + echo "http://localhost:$VERDACCIO_PORT" + fi +} + +# Stop registry function +stop_registry() { + local registry_type=$1 + + if [ "$registry_type" = "vsr" ]; then + if [ -f /tmp/vsr.pid ]; then + kill $(cat /tmp/vsr.pid) 2>/dev/null || true + rm -f /tmp/vsr.pid + fi + pkill -f "vsr --port $VSR_PORT" || true + else + if [ -f /tmp/verdaccio.pid ]; then + kill $(cat /tmp/verdaccio.pid) 2>/dev/null || true + rm -f /tmp/verdaccio.pid + fi + pkill -f "verdaccio --config" || true + fi +} + +# Clean registry cache function +clean_registry_cache() { + local registry_type=$1 + + if [ "$registry_type" = "vsr" ]; then + rm -rf "$(dirname "$0")/../registry-configs/vsr/cache" || true + else + rm -rf "$(dirname "$0")/../registry-configs/verdaccio/storage" || true + fi +} + +# Stop all registries +stop_all_registries() { + stop_registry "vsr" + stop_registry "verdaccio" +} + +# Main execution logic +if [ $# -eq 0 ]; then + echo "Usage: $0 [args]" + echo "Commands:" + echo " start - Start a registry" + echo " stop - Stop a registry" + echo " stop_all - Stop all registries" + echo " clean - Clean registry cache" + exit 1 +fi + +case "$1" in + start) + if [ $# -lt 3 ]; then + echo "Usage: $0 start " + exit 1 + fi + start_registry "$2" "$3" + ;; + stop) + if [ $# -lt 2 ]; then + echo "Usage: $0 stop " + exit 1 + fi + stop_registry "$2" + ;; + stop_all) + stop_all_registries + ;; + clean) + if [ $# -lt 2 ]; then + echo "Usage: $0 clean " + exit 1 + fi + clean_registry_cache "$2" + ;; + *) + echo "Unknown command: $1" + exit 1 + ;; +esac diff --git a/scripts/setup.sh b/scripts/setup.sh index f86179057..06e3416ac 100644 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -32,7 +32,7 @@ echo "hyperfine: $HYPERFINE_VERSION" # Install Node.js package managers and tools echo "Installing package managers and tools..." -npm install -g npm@latest corepack@latest vlt@latest bun@latest deno@latest nx@latest turbo@latest +npm install -g npm@latest corepack@latest vlt@latest bun@latest deno@latest nx@latest turbo@latest @vltpkg/vsr@latest verdaccio@latest # Configure Package Managers echo "Configuring package managers..." @@ -83,4 +83,49 @@ echo "{ \"node\": \"$NODE_VERSION\" }" > ./results/versions.json +# Create registry configuration directories +echo "Creating registry configuration directories..." +mkdir -p ./registry-configs/vsr +mkdir -p ./registry-configs/verdaccio + +# Create Verdaccio configuration +cat > ./registry-configs/verdaccio/config.yaml << 'EOF' +storage: ./storage +auth: + htpasswd: + file: ./htpasswd +uplinks: + npmjs: + url: https://registry.npmjs.org/ +packages: + '@*/*': + access: $all + publish: $authenticated + proxy: npmjs + '**': + access: $all + publish: $authenticated + proxy: npmjs +server: + keepAliveTimeout: 60 + bodyParser: + limit: 100mb +logs: + - type: stdout + format: pretty + level: warn +EOF + +# Create VSR configuration +cat > ./registry-configs/vsr/vlt.json << 'EOF' +{ + "registry": { + "upstream": "https://registry.npmjs.org/", + "cache": "./cache", + "port": 1337 + } +} +EOF + +echo "Registry configurations created successfully!" echo "Setup completed successfully!" diff --git a/scripts/variations/registry-local-cache.sh b/scripts/variations/registry-local-cache.sh new file mode 100755 index 000000000..61e041ac0 --- /dev/null +++ b/scripts/variations/registry-local-cache.sh @@ -0,0 +1,93 @@ +#!/bin/bash +# Exit on error +set -Eeuxo pipefail + +# Load common variables +source "$1/variations/common.sh" + +# Update command strings to use registry configuration +update_commands_for_registry() { + BENCH_COMMAND_NPM="npm install --registry=$BENCH_REGISTRY_URL --no-audit --no-fund --silent >> $BENCH_OUTPUT_FOLDER/npm-output-\${HYPERFINE_ITERATION}.log 2>&1" + BENCH_COMMAND_YARN="corepack yarn@1 install --registry $BENCH_REGISTRY_URL --silent > $BENCH_OUTPUT_FOLDER/yarn-output-\${HYPERFINE_ITERATION}.log 2>&1" + BENCH_COMMAND_BERRY="echo \"$BENCH_COMMAND_BERRY_PRE\" > .yarnrc.yml; echo \"npmRegistryServer: $BENCH_REGISTRY_URL\" >> .yarnrc.yml; corepack yarn@latest install > $BENCH_OUTPUT_FOLDER/berry-output-\${HYPERFINE_ITERATION}.log 2>&1" + BENCH_COMMAND_PNPM="corepack pnpm@latest install --registry=$BENCH_REGISTRY_URL --silent > $BENCH_OUTPUT_FOLDER/pnpm-output-\${HYPERFINE_ITERATION}.log 2>&1" + BENCH_COMMAND_VLT="vlt install --registry=$BENCH_REGISTRY_URL --view=silent > $BENCH_OUTPUT_FOLDER/vlt-output-\${HYPERFINE_ITERATION}.log 2>&1" + # Bun and Deno don't support custom registries in the same way + BENCH_COMMAND_BUN="" + BENCH_COMMAND_DENO="" + BENCH_INCLUDE_BUN="" + BENCH_INCLUDE_DENO="" +} + +# Pre-warm registry function +warm_registry() { + local registry_type=$1 + + echo "Pre-warming $registry_type registry..." + + # Save current directory + local current_dir=$(pwd) + + # Use npm to install packages and warm the cache + npm install --registry="$BENCH_REGISTRY_URL" --silent > "$BENCH_OUTPUT_FOLDER/warm-$registry_type.log" 2>&1 || true + + # Clean up after warming + bash "$BENCH_SCRIPTS/clean-helpers.sh" clean_node_modules clean_lockfiles + + cd "$current_dir" +} + +# Function to run benchmarks for a specific registry +run_registry_benchmark() { + local registry_type=$1 + + # Stop any existing registries + bash "$BENCH_SCRIPTS/registry-helpers.sh" stop_all + + # Clean the registry cache first + bash "$BENCH_SCRIPTS/registry-helpers.sh" clean "$registry_type" + + # Start the registry + BENCH_REGISTRY_URL=$(bash "$BENCH_SCRIPTS/registry-helpers.sh" start "$registry_type" "$BENCH_OUTPUT_FOLDER") + export BENCH_REGISTRY_URL + + echo "Registry started at: $BENCH_REGISTRY_URL" + + # Update commands with registry URL + update_commands_for_registry + + # Pre-warm the registry + warm_registry "$registry_type" + + # Run the benchmark suite + echo "Running benchmarks with $registry_type registry (pre-warmed)..." + hyperfine --ignore-failure \ + --time-unit=millisecond \ + --export-json="$BENCH_OUTPUT_FOLDER/benchmarks-$registry_type.json" \ + --warmup="$BENCH_WARMUP" \ + --runs="$BENCH_RUNS" \ + --prepare="sleep 1; bash $BENCH_SCRIPTS/clean-helpers.sh clean_all" \ + --conclude="sleep 1; bash $BENCH_SCRIPTS/package-count.sh $BENCH_OUTPUT_FOLDER; bash $BENCH_SCRIPTS/clean-helpers.sh clean_all" \ + --cleanup="bash $BENCH_SCRIPTS/clean-helpers.sh clean_all" \ + ${BENCH_INCLUDE_NPM:+--command-name="npm-$registry_type" "$BENCH_COMMAND_NPM"} \ + ${BENCH_INCLUDE_YARN:+--command-name="yarn-$registry_type" "$BENCH_COMMAND_YARN"} \ + ${BENCH_INCLUDE_BERRY:+--command-name="berry-$registry_type" "$BENCH_COMMAND_BERRY"} \ + ${BENCH_INCLUDE_PNPM:+--command-name="pnpm-$registry_type" "$BENCH_COMMAND_PNPM"} \ + ${BENCH_INCLUDE_VLT:+--command-name="vlt-$registry_type" "$BENCH_COMMAND_VLT"} + + # Stop the registry + bash "$BENCH_SCRIPTS/registry-helpers.sh" stop "$registry_type" +} + +# Run benchmarks for both registries +run_registry_benchmark "vsr" +run_registry_benchmark "verdaccio" + +# Merge results +echo "Merging benchmark results..." +jq -s '.[0] * {results: (.[0].results + .[1].results)}' \ + "$BENCH_OUTPUT_FOLDER/benchmarks-vsr.json" \ + "$BENCH_OUTPUT_FOLDER/benchmarks-verdaccio.json" \ + > "$BENCH_OUTPUT_FOLDER/benchmarks.json" + +collect_package_count \ No newline at end of file diff --git a/scripts/variations/registry-proxied.sh b/scripts/variations/registry-proxied.sh new file mode 100755 index 000000000..6171eabe1 --- /dev/null +++ b/scripts/variations/registry-proxied.sh @@ -0,0 +1,92 @@ +#!/bin/bash +# Exit on error +set -Eeuxo pipefail + +# Load common variables +source "$1/variations/common.sh" + +# Update command strings to use registry configuration +update_commands_for_registry() { + BENCH_COMMAND_NPM="npm install --registry=$BENCH_REGISTRY_URL --no-audit --no-fund --silent >> $BENCH_OUTPUT_FOLDER/npm-output-\${HYPERFINE_ITERATION}.log 2>&1" + BENCH_COMMAND_YARN="corepack yarn@1 install --registry $BENCH_REGISTRY_URL --silent > $BENCH_OUTPUT_FOLDER/yarn-output-\${HYPERFINE_ITERATION}.log 2>&1" + BENCH_COMMAND_BERRY="echo \"$BENCH_COMMAND_BERRY_PRE\" > .yarnrc.yml; echo \"npmRegistryServer: $BENCH_REGISTRY_URL\" >> .yarnrc.yml; corepack yarn@latest install > $BENCH_OUTPUT_FOLDER/berry-output-\${HYPERFINE_ITERATION}.log 2>&1" + BENCH_COMMAND_PNPM="corepack pnpm@latest install --registry=$BENCH_REGISTRY_URL --silent > $BENCH_OUTPUT_FOLDER/pnpm-output-\${HYPERFINE_ITERATION}.log 2>&1" + BENCH_COMMAND_VLT="vlt install --registry=$BENCH_REGISTRY_URL --view=silent > $BENCH_OUTPUT_FOLDER/vlt-output-\${HYPERFINE_ITERATION}.log 2>&1" + # Bun and Deno don't support custom registries in the same way + BENCH_COMMAND_BUN="" + BENCH_COMMAND_DENO="" + BENCH_INCLUDE_BUN="" + BENCH_INCLUDE_DENO="" +} + +# Function to run benchmarks for a specific registry +run_registry_benchmark() { + local registry_type=$1 + + # Stop any existing registries + bash "$BENCH_SCRIPTS/registry-helpers.sh" stop_all + + # Clean the registry cache + bash "$BENCH_SCRIPTS/registry-helpers.sh" clean "$registry_type" + + # Start the registry + BENCH_REGISTRY_URL=$(bash "$BENCH_SCRIPTS/registry-helpers.sh" start "$registry_type" "$BENCH_OUTPUT_FOLDER") + export BENCH_REGISTRY_URL + + echo "Registry started at: $BENCH_REGISTRY_URL" + + # Update commands with registry URL + update_commands_for_registry + + # Create a wrapper script for cleaning registry cache between runs + cat > "$BENCH_OUTPUT_FOLDER/clean-registry-$registry_type.sh" << EOF +#!/bin/bash +set -e +# Clean package manager artifacts +bash "$BENCH_SCRIPTS/clean-helpers.sh" clean_all + +# Stop and restart registry with clean cache +bash "$BENCH_SCRIPTS/registry-helpers.sh" stop "$registry_type" +bash "$BENCH_SCRIPTS/registry-helpers.sh" clean "$registry_type" +BENCH_REGISTRY_URL=\$(bash "$BENCH_SCRIPTS/registry-helpers.sh" start "$registry_type" "$BENCH_OUTPUT_FOLDER") + +# Small delay to ensure registry is ready +sleep 2 +EOF + chmod +x "$BENCH_OUTPUT_FOLDER/clean-registry-$registry_type.sh" + + # Run the benchmark suite + echo "Running benchmarks with $registry_type registry (proxied)..." + hyperfine --ignore-failure \ + --time-unit=millisecond \ + --export-json="$BENCH_OUTPUT_FOLDER/benchmarks-$registry_type.json" \ + --warmup="$BENCH_WARMUP" \ + --runs="$BENCH_RUNS" \ + --prepare="bash $BENCH_OUTPUT_FOLDER/clean-registry-$registry_type.sh" \ + --conclude="sleep 1; bash $BENCH_SCRIPTS/package-count.sh $BENCH_OUTPUT_FOLDER" \ + --cleanup="bash $BENCH_SCRIPTS/clean-helpers.sh clean_all; bash $BENCH_SCRIPTS/registry-helpers.sh stop $registry_type" \ + ${BENCH_INCLUDE_NPM:+--command-name="npm-$registry_type" "$BENCH_COMMAND_NPM"} \ + ${BENCH_INCLUDE_YARN:+--command-name="yarn-$registry_type" "$BENCH_COMMAND_YARN"} \ + ${BENCH_INCLUDE_BERRY:+--command-name="berry-$registry_type" "$BENCH_COMMAND_BERRY"} \ + ${BENCH_INCLUDE_PNPM:+--command-name="pnpm-$registry_type" "$BENCH_COMMAND_PNPM"} \ + ${BENCH_INCLUDE_VLT:+--command-name="vlt-$registry_type" "$BENCH_COMMAND_VLT"} + + # Stop the registry + bash "$BENCH_SCRIPTS/registry-helpers.sh" stop "$registry_type" + + # Clean up wrapper script + rm -f "$BENCH_OUTPUT_FOLDER/clean-registry-$registry_type.sh" +} + +# Run benchmarks for both registries +run_registry_benchmark "vsr" +run_registry_benchmark "verdaccio" + +# Merge results +echo "Merging benchmark results..." +jq -s '.[0] * {results: (.[0].results + .[1].results)}' \ + "$BENCH_OUTPUT_FOLDER/benchmarks-vsr.json" \ + "$BENCH_OUTPUT_FOLDER/benchmarks-verdaccio.json" \ + > "$BENCH_OUTPUT_FOLDER/benchmarks.json" + +collect_package_count \ No newline at end of file