From df58ed9419d3333a3393ba2b39ff276971c9c832 Mon Sep 17 00:00:00 2001 From: Adam Wildavsky Date: Sun, 28 Jun 2026 11:25:10 -0500 Subject: [PATCH 01/28] benchmark.sh: add --branch to build the compare binary from a git branch Co-authored-by: Cursor --- benchmark.sh | 100 +++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 94 insertions(+), 6 deletions(-) diff --git a/benchmark.sh b/benchmark.sh index e4f2a605..2f002beb 100755 --- a/benchmark.sh +++ b/benchmark.sh @@ -12,6 +12,7 @@ # ./benchmark.sh --build # ./benchmark.sh -- -n 8 -r # ./benchmark.sh --build --compare /path/to/other/dtest +# ./benchmark.sh --branch develop -- -n 8 # ./benchmark.sh --compare /path/to/other/dtest --epsilon 1 # ./benchmark.sh --repeats 5 -- -n 4 # REPEATS=3 ./benchmark.sh @@ -19,6 +20,7 @@ # Environment: # BRANCH Path to branch dtest (default: bazel-bin in this repo) # COMPARE Optional second dtest binary for comparison +# (or use --branch NAME to build the compare binary from a git branch) # HANDS_DIR Directory containing list*.txt files (default: ./hands) # REPEATS Runs per combination per binary (default: 1) # MAX_DEALS Include list10^n.txt files with 10^n <= N (default: 100) @@ -38,8 +40,31 @@ DETAILS="${DETAILS:-0}" EPSILON="${EPSILON:-0.5}" BUILD=0 REVERSE=0 +COMPARE_BRANCH="" DTEST_EXTRA=() +# Cleanup state (set later). The EXIT trap restores the original git branch if +# --branch switched away, and removes temp files. +RESULTS="" +ORIG_BRANCH="" +COMPARE_TMP="" + +cleanup() { + if [[ -n "$ORIG_BRANCH" ]]; then + local cur + cur="$(git -C "$ROOT" symbolic-ref --quiet --short HEAD 2>/dev/null \ + || git -C "$ROOT" rev-parse HEAD 2>/dev/null || true)" + if [[ -n "$cur" && "$cur" != "$ORIG_BRANCH" ]]; then + echo "Restoring git branch '$ORIG_BRANCH'..." >&2 + git -C "$ROOT" checkout "$ORIG_BRANCH" >/dev/null 2>&1 || true + fi + fi + [[ -n "$COMPARE_TMP" ]] && rm -f "$COMPARE_TMP" + [[ -n "$RESULTS" ]] && rm -f "$RESULTS" + return 0 +} +trap cleanup EXIT + SOLVERS=(solve calc) usage() { @@ -55,7 +80,9 @@ Options: --max-deals N Include list10^n.txt files with 10^n <= N (default: 100; env: MAX_DEALS) (alias: --max_deals) --build Build branch dtest only (bazel build //library/tests:dtest) - --branch PATH Branch dtest binary (default: $BRANCH) + --branch NAME Git branch to compare against: check it out, build dtest, save the + binary as the compare binary, restore the current branch, rebuild, + then run. Mutually exclusive with --compare. Requires a clean tree. --compare PATH Optional second dtest binary (summary; transient progress on tty) --details With --compare, keep per-run timing rows in final output --epsilon PCT With --compare, treat timings within PCT% as equal (default: 0.5; env: EPSILON) @@ -71,6 +98,8 @@ Examples: ./benchmark.sh --build ./benchmark.sh -- -n 8 ./benchmark.sh --repeats 3 -- -n 4 -r + ./benchmark.sh --branch develop + ./benchmark.sh --branch develop --repeats 3 -- -n 8 ./benchmark.sh --compare /path/to/dtest ./benchmark.sh --compare /path/to/dtest --details ./benchmark.sh --compare /path/to/dtest --epsilon 1 @@ -93,7 +122,7 @@ while [[ $# -gt 0 ]]; do ;; --branch) shift - BRANCH="${1:?missing value for --branch}" + COMPARE_BRANCH="${1:?missing value for --branch}" shift ;; --compare) @@ -151,11 +180,66 @@ if ! [[ "$EPSILON" =~ ^[0-9]+(\.[0-9]+)?$ ]]; then exit 1 fi -if [[ "$REVERSE" == "1" && -z "${COMPARE:-}" ]]; then - echo "error: --reverse requires --compare" >&2 +if [[ "$REVERSE" == "1" && -z "${COMPARE:-}" && -z "$COMPARE_BRANCH" ]]; then + echo "error: --reverse requires --compare or --branch" >&2 exit 1 fi +if [[ -n "$COMPARE_BRANCH" && -n "${COMPARE:-}" ]]; then + echo "error: --branch and --compare are mutually exclusive" >&2 + exit 1 +fi + +build_compare_from_branch() { + local dtest_rel="bazel-bin/library/tests/dtest" + + if ! git -C "$ROOT" rev-parse --is-inside-work-tree >/dev/null 2>&1; then + echo "error: --branch requires a git work tree at $ROOT" >&2 + exit 1 + fi + + if ! git -C "$ROOT" rev-parse --verify --quiet "$COMPARE_BRANCH" >/dev/null; then + echo "error: --branch: unknown git ref '$COMPARE_BRANCH'" >&2 + exit 1 + fi + + if [[ -n "$(git -C "$ROOT" status --porcelain --untracked-files=no)" ]]; then + echo "error: tracked changes present; commit or stash before using --branch" >&2 + exit 1 + fi + + ORIG_BRANCH="$(git -C "$ROOT" symbolic-ref --quiet --short HEAD 2>/dev/null \ + || git -C "$ROOT" rev-parse HEAD)" + COMPARE_TMP="$(mktemp "${TMPDIR:-/tmp}/dds-dtest-compare.XXXXXX")" + + if [[ "$DRY_RUN" == "1" ]]; then + echo "DRY_RUN: git -C $ROOT checkout $COMPARE_BRANCH" >&2 + echo "DRY_RUN: (cd $ROOT && bazel build //library/tests:dtest)" >&2 + echo "DRY_RUN: cp -L $ROOT/$dtest_rel $COMPARE_TMP" >&2 + echo "DRY_RUN: git -C $ROOT checkout $ORIG_BRANCH" >&2 + echo "DRY_RUN: (cd $ROOT && bazel build //library/tests:dtest)" >&2 + COMPARE="$COMPARE_TMP" + return 0 + fi + + echo "Building compare binary from '$COMPARE_BRANCH' (current: '$ORIG_BRANCH')..." >&2 + git -C "$ROOT" checkout "$COMPARE_BRANCH" + (cd "$ROOT" && bazel build //library/tests:dtest) + cp -L "$ROOT/$dtest_rel" "$COMPARE_TMP" + chmod +x "$COMPARE_TMP" + + echo "Restoring '$ORIG_BRANCH' and rebuilding..." >&2 + git -C "$ROOT" checkout "$ORIG_BRANCH" + (cd "$ROOT" && bazel build //library/tests:dtest) + + COMPARE="$COMPARE_TMP" +} + +if [[ -n "$COMPARE_BRANCH" ]]; then + build_compare_from_branch + BUILD=0 # build already done as part of the branch workflow +fi + select_hand_files() { is_power_of_10() { local n="$1" @@ -241,7 +325,7 @@ if git -C "$ROOT" rev-parse --is-inside-work-tree >/dev/null 2>&1; then fi RESULTS="$(mktemp "${TMPDIR:-/tmp}/dds-benchmark.XXXXXX")" -trap 'rm -f "$RESULTS"' EXIT +# Removal handled by the cleanup() EXIT trap installed near the top. parse_dtest_output() { awk ' @@ -334,7 +418,11 @@ echo "DDS dtest benchmark" echo "===================" printf "%-12s %s\n" "branch:" "$BRANCH" if [[ -n "${COMPARE:-}" ]]; then - printf "%-12s %s\n" "compare:" "$COMPARE" + if [[ -n "$COMPARE_BRANCH" ]]; then + printf "%-12s %s\n" "compare:" "branch '$COMPARE_BRANCH' ($COMPARE)" + else + printf "%-12s %s\n" "compare:" "$COMPARE" + fi if [[ "$DETAILS" == "1" ]]; then printf "%-12s %s\n" "details:" "on" elif [[ -t 1 ]]; then From 2799ad1fb1659b1993e1353c2dfe364dffc5589c Mon Sep 17 00:00:00 2001 From: Adam Wildavsky Date: Sun, 28 Jun 2026 11:30:19 -0500 Subject: [PATCH 02/28] benchmark.sh: show branch names in summary header columns Co-authored-by: Cursor --- benchmark.sh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/benchmark.sh b/benchmark.sh index 2f002beb..8865606d 100755 --- a/benchmark.sh +++ b/benchmark.sh @@ -502,10 +502,22 @@ clear_transient_progress if [[ -n "${COMPARE:-}" && "$DRY_RUN" != "1" ]]; then echo + # Column headers default to the generic labels, but show the actual branch + # names when known: the current git branch for the branch binary, and the + # --branch name for the compare binary. Truncated to the 12-char column. + cmp_label="compare_avg" + if [[ -n "$COMPARE_BRANCH" ]]; then + cmp_label="${COMPARE_BRANCH:0:12}" + fi + br_label="branch_avg" + if [[ -n "$git_branch" && "$git_branch" != "unknown" ]]; then + br_label="${git_branch:0:12}" + fi + echo "Summary (branch vs compare, avg user ms)" echo "==============================================================================" printf "%-6s %-13s %12s %12s %10s %-15s\n" \ - "solver" "file" "compare_avg" "branch_avg" "cmp/branch" "note" + "solver" "file" "$cmp_label" "$br_label" "cmp/branch" "note" printf "%-6s %-13s %12s %12s %10s %-15s\n" \ "------" "-------------" "------------" "------------" "----------" "---------------" From 8a21d9be216cdc452427f648a23656d31c254b84 Mon Sep 17 00:00:00 2001 From: Adam Wildavsky Date: Sun, 28 Jun 2026 11:44:51 -0500 Subject: [PATCH 03/28] benchmark.sh: allow --branch twice to compare two branches Co-authored-by: Cursor --- benchmark.sh | 139 +++++++++++++++++++++++++++++++++++---------------- 1 file changed, 96 insertions(+), 43 deletions(-) diff --git a/benchmark.sh b/benchmark.sh index 8865606d..b2f514c4 100755 --- a/benchmark.sh +++ b/benchmark.sh @@ -40,7 +40,7 @@ DETAILS="${DETAILS:-0}" EPSILON="${EPSILON:-0.5}" BUILD=0 REVERSE=0 -COMPARE_BRANCH="" +BRANCH_NAMES=() DTEST_EXTRA=() # Cleanup state (set later). The EXIT trap restores the original git branch if @@ -48,6 +48,7 @@ DTEST_EXTRA=() RESULTS="" ORIG_BRANCH="" COMPARE_TMP="" +BRANCH_TMP="" cleanup() { if [[ -n "$ORIG_BRANCH" ]]; then @@ -60,6 +61,7 @@ cleanup() { fi fi [[ -n "$COMPARE_TMP" ]] && rm -f "$COMPARE_TMP" + [[ -n "$BRANCH_TMP" ]] && rm -f "$BRANCH_TMP" [[ -n "$RESULTS" ]] && rm -f "$RESULTS" return 0 } @@ -80,9 +82,11 @@ Options: --max-deals N Include list10^n.txt files with 10^n <= N (default: 100; env: MAX_DEALS) (alias: --max_deals) --build Build branch dtest only (bazel build //library/tests:dtest) - --branch NAME Git branch to compare against: check it out, build dtest, save the - binary as the compare binary, restore the current branch, rebuild, - then run. Mutually exclusive with --compare. Requires a clean tree. + --branch NAME Git branch to build and compare. Once: compare the current branch + against NAME. Twice (--branch A --branch B): compare A vs B and + ignore the current branch. Each branch is checked out, dtest is + built and its binary saved; the original branch is then restored. + Mutually exclusive with --compare. Requires a clean tree. --compare PATH Optional second dtest binary (summary; transient progress on tty) --details With --compare, keep per-run timing rows in final output --epsilon PCT With --compare, treat timings within PCT% as equal (default: 0.5; env: EPSILON) @@ -99,6 +103,7 @@ Examples: ./benchmark.sh -- -n 8 ./benchmark.sh --repeats 3 -- -n 4 -r ./benchmark.sh --branch develop + ./benchmark.sh --branch develop --branch opus-two-percent ./benchmark.sh --branch develop --repeats 3 -- -n 8 ./benchmark.sh --compare /path/to/dtest ./benchmark.sh --compare /path/to/dtest --details @@ -122,7 +127,7 @@ while [[ $# -gt 0 ]]; do ;; --branch) shift - COMPARE_BRANCH="${1:?missing value for --branch}" + BRANCH_NAMES+=("${1:?missing value for --branch}") shift ;; --compare) @@ -180,29 +185,54 @@ if ! [[ "$EPSILON" =~ ^[0-9]+(\.[0-9]+)?$ ]]; then exit 1 fi -if [[ "$REVERSE" == "1" && -z "${COMPARE:-}" && -z "$COMPARE_BRANCH" ]]; then +num_branches=${#BRANCH_NAMES[@]} + +if [[ "$REVERSE" == "1" && -z "${COMPARE:-}" && "$num_branches" -eq 0 ]]; then echo "error: --reverse requires --compare or --branch" >&2 exit 1 fi -if [[ -n "$COMPARE_BRANCH" && -n "${COMPARE:-}" ]]; then +if [[ "$num_branches" -gt 0 && -n "${COMPARE:-}" ]]; then echo "error: --branch and --compare are mutually exclusive" >&2 exit 1 fi -build_compare_from_branch() { +if [[ "$num_branches" -gt 2 ]]; then + echo "error: --branch may be given at most twice (got $num_branches)" >&2 + exit 1 +fi + +# Check out $1, build dtest, and copy the binary to $2. +build_branch_binary() { + local name="$1" dest="$2" local dtest_rel="bazel-bin/library/tests/dtest" + if [[ "$DRY_RUN" == "1" ]]; then + echo "DRY_RUN: git -C $ROOT checkout $name" >&2 + echo "DRY_RUN: (cd $ROOT && bazel build //library/tests:dtest)" >&2 + echo "DRY_RUN: cp -L $ROOT/$dtest_rel $dest" >&2 + return 0 + fi + echo "Building dtest from '$name'..." >&2 + git -C "$ROOT" checkout "$name" + (cd "$ROOT" && bazel build //library/tests:dtest) + cp -L "$ROOT/$dtest_rel" "$dest" + chmod +x "$dest" +} +# With one --branch, compare the current branch against the named branch. +# With two, compare the two named branches and ignore the current branch. +setup_branches() { if ! git -C "$ROOT" rev-parse --is-inside-work-tree >/dev/null 2>&1; then echo "error: --branch requires a git work tree at $ROOT" >&2 exit 1 fi - - if ! git -C "$ROOT" rev-parse --verify --quiet "$COMPARE_BRANCH" >/dev/null; then - echo "error: --branch: unknown git ref '$COMPARE_BRANCH'" >&2 - exit 1 - fi - + local name + for name in "${BRANCH_NAMES[@]}"; do + if ! git -C "$ROOT" rev-parse --verify --quiet "$name" >/dev/null; then + echo "error: --branch: unknown git ref '$name'" >&2 + exit 1 + fi + done if [[ -n "$(git -C "$ROOT" status --porcelain --untracked-files=no)" ]]; then echo "error: tracked changes present; commit or stash before using --branch" >&2 exit 1 @@ -210,33 +240,39 @@ build_compare_from_branch() { ORIG_BRANCH="$(git -C "$ROOT" symbolic-ref --quiet --short HEAD 2>/dev/null \ || git -C "$ROOT" rev-parse HEAD)" - COMPARE_TMP="$(mktemp "${TMPDIR:-/tmp}/dds-dtest-compare.XXXXXX")" - if [[ "$DRY_RUN" == "1" ]]; then - echo "DRY_RUN: git -C $ROOT checkout $COMPARE_BRANCH" >&2 - echo "DRY_RUN: (cd $ROOT && bazel build //library/tests:dtest)" >&2 - echo "DRY_RUN: cp -L $ROOT/$dtest_rel $COMPARE_TMP" >&2 - echo "DRY_RUN: git -C $ROOT checkout $ORIG_BRANCH" >&2 - echo "DRY_RUN: (cd $ROOT && bazel build //library/tests:dtest)" >&2 + if [[ "$num_branches" -eq 1 ]]; then + COMPARE_TMP="$(mktemp "${TMPDIR:-/tmp}/dds-dtest-compare.XXXXXX")" + build_branch_binary "${BRANCH_NAMES[0]}" "$COMPARE_TMP" + # Restore the current branch and rebuild it as the branch binary. + if [[ "$DRY_RUN" == "1" ]]; then + echo "DRY_RUN: git -C $ROOT checkout $ORIG_BRANCH" >&2 + echo "DRY_RUN: (cd $ROOT && bazel build //library/tests:dtest)" >&2 + else + echo "Restoring '$ORIG_BRANCH' and rebuilding..." >&2 + git -C "$ROOT" checkout "$ORIG_BRANCH" + (cd "$ROOT" && bazel build //library/tests:dtest) + fi + COMPARE="$COMPARE_TMP" + else + # Two branches: build both, ignore the current branch's binary. + BRANCH_TMP="$(mktemp "${TMPDIR:-/tmp}/dds-dtest-branch.XXXXXX")" + COMPARE_TMP="$(mktemp "${TMPDIR:-/tmp}/dds-dtest-compare.XXXXXX")" + build_branch_binary "${BRANCH_NAMES[0]}" "$BRANCH_TMP" + build_branch_binary "${BRANCH_NAMES[1]}" "$COMPARE_TMP" + if [[ "$DRY_RUN" == "1" ]]; then + echo "DRY_RUN: git -C $ROOT checkout $ORIG_BRANCH" >&2 + else + echo "Restoring '$ORIG_BRANCH'..." >&2 + git -C "$ROOT" checkout "$ORIG_BRANCH" + fi + BRANCH="$BRANCH_TMP" COMPARE="$COMPARE_TMP" - return 0 fi - - echo "Building compare binary from '$COMPARE_BRANCH' (current: '$ORIG_BRANCH')..." >&2 - git -C "$ROOT" checkout "$COMPARE_BRANCH" - (cd "$ROOT" && bazel build //library/tests:dtest) - cp -L "$ROOT/$dtest_rel" "$COMPARE_TMP" - chmod +x "$COMPARE_TMP" - - echo "Restoring '$ORIG_BRANCH' and rebuilding..." >&2 - git -C "$ROOT" checkout "$ORIG_BRANCH" - (cd "$ROOT" && bazel build //library/tests:dtest) - - COMPARE="$COMPARE_TMP" } -if [[ -n "$COMPARE_BRANCH" ]]; then - build_compare_from_branch +if [[ "$num_branches" -gt 0 ]]; then + setup_branches BUILD=0 # build already done as part of the branch workflow fi @@ -416,10 +452,25 @@ clear_transient_progress() { echo "DDS dtest benchmark" echo "===================" -printf "%-12s %s\n" "branch:" "$BRANCH" +# Branch names backing each binary, when --branch was used. With one --branch +# the branch binary is the current checkout; with two it is the first name. +branch_branch_name="" +compare_branch_name="" +if [[ "$num_branches" -eq 1 ]]; then + compare_branch_name="${BRANCH_NAMES[0]}" +elif [[ "$num_branches" -eq 2 ]]; then + branch_branch_name="${BRANCH_NAMES[0]}" + compare_branch_name="${BRANCH_NAMES[1]}" +fi + +if [[ -n "$branch_branch_name" ]]; then + printf "%-12s %s\n" "branch:" "branch '$branch_branch_name' ($BRANCH)" +else + printf "%-12s %s\n" "branch:" "$BRANCH" +fi if [[ -n "${COMPARE:-}" ]]; then - if [[ -n "$COMPARE_BRANCH" ]]; then - printf "%-12s %s\n" "compare:" "branch '$COMPARE_BRANCH' ($COMPARE)" + if [[ -n "$compare_branch_name" ]]; then + printf "%-12s %s\n" "compare:" "branch '$compare_branch_name' ($COMPARE)" else printf "%-12s %s\n" "compare:" "$COMPARE" fi @@ -506,15 +557,17 @@ if [[ -n "${COMPARE:-}" && "$DRY_RUN" != "1" ]]; then # names when known: the current git branch for the branch binary, and the # --branch name for the compare binary. Truncated to the 12-char column. cmp_label="compare_avg" - if [[ -n "$COMPARE_BRANCH" ]]; then - cmp_label="${COMPARE_BRANCH:0:12}" + if [[ -n "$compare_branch_name" ]]; then + cmp_label="${compare_branch_name:0:12}" fi br_label="branch_avg" - if [[ -n "$git_branch" && "$git_branch" != "unknown" ]]; then + if [[ -n "$branch_branch_name" ]]; then + br_label="${branch_branch_name:0:12}" + elif [[ -n "$git_branch" && "$git_branch" != "unknown" ]]; then br_label="${git_branch:0:12}" fi - echo "Summary (branch vs compare, avg user ms)" + echo "Summary (avg user ms)" echo "==============================================================================" printf "%-6s %-13s %12s %12s %10s %-15s\n" \ "solver" "file" "$cmp_label" "$br_label" "cmp/branch" "note" From 1af48f119ceea7ac2a5815d72212a320b2f11552 Mon Sep 17 00:00:00 2001 From: Adam Wildavsky Date: Sun, 28 Jun 2026 11:50:25 -0500 Subject: [PATCH 04/28] benchmark.sh: use branch names in summary note column Co-authored-by: Cursor --- benchmark.sh | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/benchmark.sh b/benchmark.sh index b2f514c4..6651462b 100755 --- a/benchmark.sh +++ b/benchmark.sh @@ -567,6 +567,19 @@ if [[ -n "${COMPARE:-}" && "$DRY_RUN" != "1" ]]; then br_label="${git_branch:0:12}" fi + # Names used in the "note" column (" faster"); fall back to the generic + # "branch"/"compare" when no branch name is known. + note_branch="branch" + if [[ -n "$branch_branch_name" ]]; then + note_branch="$branch_branch_name" + elif [[ -n "$compare_branch_name" && -n "$git_branch" && "$git_branch" != "unknown" ]]; then + note_branch="$git_branch" + fi + note_compare="compare" + if [[ -n "$compare_branch_name" ]]; then + note_compare="$compare_branch_name" + fi + echo "Summary (avg user ms)" echo "==============================================================================" printf "%-6s %-13s %12s %12s %10s %-15s\n" \ @@ -574,7 +587,8 @@ if [[ -n "${COMPARE:-}" && "$DRY_RUN" != "1" ]]; then printf "%-6s %-13s %12s %12s %10s %-15s\n" \ "------" "-------------" "------------" "------------" "----------" "---------------" - awk -F'\t' -v files="${FILES[*]}" -v epsilon_pct="$EPSILON" ' + awk -F'\t' -v files="${FILES[*]}" -v epsilon_pct="$EPSILON" \ + -v note_branch="$note_branch" -v note_compare="$note_compare" ' function within_epsilon(a, b, eps, hi, lo) { eps = epsilon_pct / 100 if (a > b) { hi = a; lo = b } else { hi = b; lo = a } @@ -607,9 +621,9 @@ if [[ -n "${COMPARE:-}" && "$DRY_RUN" != "1" ]]; then if (within_epsilon(u1, u2)) { note = "equal" } else if (cmp_branch >= 1) { - note = "branch faster" + note = note_branch " faster" } else { - note = "compare faster" + note = note_compare " faster" } sp = sprintf("%9.2fx", cmp_branch) printf "%-6s %-13s %12.2f %12.2f %10s %-15s\n", From ac3ef3a4ba226622340a5b53ee8280c9d848e5fc Mon Sep 17 00:00:00 2001 From: Adam Wildavsky Date: Sun, 28 Jun 2026 11:55:45 -0500 Subject: [PATCH 05/28] benchmark.sh: hide transient per-run rows when repeats > 1 Co-authored-by: Cursor --- benchmark.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/benchmark.sh b/benchmark.sh index 6651462b..162ab534 100755 --- a/benchmark.sh +++ b/benchmark.sh @@ -500,7 +500,11 @@ echo show_run_lines=1 TRANSIENT_PROGRESS=0 -if [[ -n "${COMPARE:-}" && "$DETAILS" != "1" ]]; then +# Per-run rows are detail: hide them from the final output (transient on a tty, +# suppressed otherwise) when comparing or when repeating, unless --details asks +# to keep them. With repeats > 1 the per-run rows are intermediate samples, so +# they are treated as transient too. +if [[ "$DETAILS" != "1" ]] && { [[ -n "${COMPARE:-}" ]] || (( REPEATS > 1 )); }; then show_run_lines=0 if [[ -t 1 ]]; then TRANSIENT_PROGRESS=1 From 7be136a84b968a7e5fd2ccfaba3b4f9a64967e87 Mon Sep 17 00:00:00 2001 From: Adam Wildavsky Date: Sun, 28 Jun 2026 14:23:13 -0500 Subject: [PATCH 06/28] benchmark.sh: add per-branch elapsed totals row with ratio and note Co-authored-by: Cursor --- benchmark.sh | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/benchmark.sh b/benchmark.sh index 162ab534..09fe292f 100755 --- a/benchmark.sh +++ b/benchmark.sh @@ -398,8 +398,11 @@ run_dtest() { return 0 fi + # Wrap with `time -p` so we can capture wall-clock elapsed for this run. Its + # "real/user/sys" lines go to the merged output but do not collide with the + # dtest lines parse_dtest_output looks for. local out - if ! out="$("${cmd[@]}" 2>&1)"; then + if ! out="$(/usr/bin/time -p "${cmd[@]}" 2>&1)"; then echo "error: dtest failed: ${cmd[*]}" >&2 echo "$out" >&2 exit 1 @@ -411,7 +414,10 @@ run_dtest() { if [[ "$parsed_user" == "NA" || "$parsed_sys" == "NA" ]]; then echo "warning: incomplete dtest timing output: ${cmd[*]}" >&2 fi - echo "$parsed" + local wall + wall="$(awk '/^real[[:space:]]/ { print $2; exit }' <<<"$out")" + [[ -z "$wall" ]] && wall="NA" + echo "$parsed $wall" } progress_lines=0 @@ -539,14 +545,14 @@ for solver in "${SOLVERS[@]}"; do continue fi - read -r user sys avg ratio < <(run_dtest "$bin" "$solver" "$hands") + read -r user sys avg ratio wall < <(run_dtest "$bin" "$solver" "$hands") if [[ "$show_run_lines" == "1" || "$TRANSIENT_PROGRESS" == "1" ]]; then print_run_row "$solver" "$file" "$ver" "$user" "$sys" "$avg" "$ratio" "$run_label" fi - printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n" \ - "$solver" "$file" "$ver" "$rep" "$user" "$sys" "$avg" "$ratio" \ + printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n" \ + "$solver" "$file" "$ver" "$rep" "$user" "$sys" "$avg" "$ratio" "$wall" \ >>"$RESULTS" done done @@ -603,9 +609,11 @@ if [[ -n "${COMPARE:-}" && "$DRY_RUN" != "1" ]]; then if ($3 == "compare") { s2[base] += $7 c2[base]++ + if ($9 != "NA") tw2 += $9 # total wall-clock elapsed, compare } else if ($3 == "branch") { s1[base] += $7 c1[base]++ + if ($9 != "NA") tw1 += $9 # total wall-clock elapsed, branch } } END { @@ -634,6 +642,22 @@ if [[ -n "${COMPARE:-}" && "$DRY_RUN" != "1" ]]; then solvers[si], filearr[fi], u2, u1, sp, note } } + + # Total elapsed (wall-clock seconds) summed per binary across all runs. + printf "%-6s %-13s %12s %12s %10s %-15s\n", + "------", "-------------", "------------", "------------", "----------", "---------------" + if (tw2 > 0 && tw1 > 0) { + tnote = "" + if (within_epsilon(tw1, tw2)) tnote = "equal" + else if (tw2 / tw1 >= 1) tnote = note_branch " faster" + else tnote = note_compare " faster" + tsp = sprintf("%9.2fx", tw2 / tw1) + printf "%-6s %-13s %12.2f %12.2f %10s %-15s\n", + "TOTAL", "elapsed (s)", tw2, tw1, tsp, tnote + } else { + printf "%-6s %-13s %12.2f %12.2f %10s %-15s\n", + "TOTAL", "elapsed (s)", tw2, tw1, "", "" + } } ' "$RESULTS" fi From 367b7253cc063b9617016ddb26ff447e228f7893 Mon Sep 17 00:00:00 2001 From: Adam Wildavsky Date: Sun, 28 Jun 2026 15:08:41 -0500 Subject: [PATCH 07/28] benchmark.sh: allow --branch with --compare to set branch binary Combining --branch NAME with --compare PATH now builds NAME as the branch binary and compares it against PATH, ignoring the current branch dtest. Co-authored-by: Cursor --- benchmark.sh | 49 +++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 39 insertions(+), 10 deletions(-) diff --git a/benchmark.sh b/benchmark.sh index 09fe292f..dfa8bcc7 100755 --- a/benchmark.sh +++ b/benchmark.sh @@ -41,6 +41,7 @@ EPSILON="${EPSILON:-0.5}" BUILD=0 REVERSE=0 BRANCH_NAMES=() +COMPARE_GIVEN=0 DTEST_EXTRA=() # Cleanup state (set later). The EXIT trap restores the original git branch if @@ -84,10 +85,12 @@ Options: --build Build branch dtest only (bazel build //library/tests:dtest) --branch NAME Git branch to build and compare. Once: compare the current branch against NAME. Twice (--branch A --branch B): compare A vs B and - ignore the current branch. Each branch is checked out, dtest is - built and its binary saved; the original branch is then restored. - Mutually exclusive with --compare. Requires a clean tree. - --compare PATH Optional second dtest binary (summary; transient progress on tty) + ignore the current branch. With --compare PATH: build NAME as the + branch binary and compare it against PATH, ignoring the current + branch. Each branch is checked out, dtest is built and its binary + saved; the original branch is then restored. Requires a clean tree. + --compare PATH Second dtest binary (summary; transient progress on tty). May be + combined with a single --branch NAME (NAME backs the branch binary). --details With --compare, keep per-run timing rows in final output --epsilon PCT With --compare, treat timings within PCT% as equal (default: 0.5; env: EPSILON) --reverse With --compare, run compare before branch each repeat (default: branch first) @@ -104,6 +107,7 @@ Examples: ./benchmark.sh --repeats 3 -- -n 4 -r ./benchmark.sh --branch develop ./benchmark.sh --branch develop --branch opus-two-percent + ./benchmark.sh --branch opus-two-percent --compare /path/to/dtest ./benchmark.sh --branch develop --repeats 3 -- -n 8 ./benchmark.sh --compare /path/to/dtest ./benchmark.sh --compare /path/to/dtest --details @@ -133,6 +137,7 @@ while [[ $# -gt 0 ]]; do --compare) shift COMPARE="${1:?missing value for --compare}" + COMPARE_GIVEN=1 shift ;; --max-deals|--max_deals|-max-deals|-max_deals) @@ -185,6 +190,11 @@ if ! [[ "$EPSILON" =~ ^[0-9]+(\.[0-9]+)?$ ]]; then exit 1 fi +# A compare binary supplied via the COMPARE env var behaves like --compare PATH. +if [[ "$COMPARE_GIVEN" == "0" && -n "${COMPARE:-}" ]]; then + COMPARE_GIVEN=1 +fi + num_branches=${#BRANCH_NAMES[@]} if [[ "$REVERSE" == "1" && -z "${COMPARE:-}" && "$num_branches" -eq 0 ]]; then @@ -192,8 +202,8 @@ if [[ "$REVERSE" == "1" && -z "${COMPARE:-}" && "$num_branches" -eq 0 ]]; then exit 1 fi -if [[ "$num_branches" -gt 0 && -n "${COMPARE:-}" ]]; then - echo "error: --branch and --compare are mutually exclusive" >&2 +if [[ "$num_branches" -gt 0 && "$COMPARE_GIVEN" == "1" && "$num_branches" -ne 1 ]]; then + echo "error: --compare accepts exactly one --branch (got $num_branches)" >&2 exit 1 fi @@ -219,8 +229,12 @@ build_branch_binary() { chmod +x "$dest" } -# With one --branch, compare the current branch against the named branch. -# With two, compare the two named branches and ignore the current branch. +# Branch-mode binary selection: +# --branch NAME : branch = current checkout, compare = build(NAME) +# --branch NAME --compare PATH : branch = build(NAME), compare = PATH +# (the current branch's dtest is ignored) +# --branch A --branch B : branch = build(A), compare = build(B) +# (the current branch's dtest is ignored) setup_branches() { if ! git -C "$ROOT" rev-parse --is-inside-work-tree >/dev/null 2>&1; then echo "error: --branch requires a git work tree at $ROOT" >&2 @@ -241,7 +255,19 @@ setup_branches() { ORIG_BRANCH="$(git -C "$ROOT" symbolic-ref --quiet --short HEAD 2>/dev/null \ || git -C "$ROOT" rev-parse HEAD)" - if [[ "$num_branches" -eq 1 ]]; then + if [[ "$COMPARE_GIVEN" == "1" ]]; then + # --branch NAME --compare PATH: build NAME as the branch binary and keep the + # user-supplied compare path. The current branch's dtest is not used. + BRANCH_TMP="$(mktemp "${TMPDIR:-/tmp}/dds-dtest-branch.XXXXXX")" + build_branch_binary "${BRANCH_NAMES[0]}" "$BRANCH_TMP" + if [[ "$DRY_RUN" == "1" ]]; then + echo "DRY_RUN: git -C $ROOT checkout $ORIG_BRANCH" >&2 + else + echo "Restoring '$ORIG_BRANCH'..." >&2 + git -C "$ROOT" checkout "$ORIG_BRANCH" + fi + BRANCH="$BRANCH_TMP" + elif [[ "$num_branches" -eq 1 ]]; then COMPARE_TMP="$(mktemp "${TMPDIR:-/tmp}/dds-dtest-compare.XXXXXX")" build_branch_binary "${BRANCH_NAMES[0]}" "$COMPARE_TMP" # Restore the current branch and rebuild it as the branch binary. @@ -462,7 +488,10 @@ echo "===================" # the branch binary is the current checkout; with two it is the first name. branch_branch_name="" compare_branch_name="" -if [[ "$num_branches" -eq 1 ]]; then +if [[ "$num_branches" -eq 1 && "$COMPARE_GIVEN" == "1" ]]; then + # --branch NAME --compare PATH: NAME backs the branch binary; compare is a path. + branch_branch_name="${BRANCH_NAMES[0]}" +elif [[ "$num_branches" -eq 1 ]]; then compare_branch_name="${BRANCH_NAMES[0]}" elif [[ "$num_branches" -eq 2 ]]; then branch_branch_name="${BRANCH_NAMES[0]}" From cd052656466e136dbe86fad431423a19848df2ad Mon Sep 17 00:00:00 2001 From: Adam Wildavsky Date: Sun, 28 Jun 2026 16:26:24 -0500 Subject: [PATCH 08/28] Reject duplicate --repeats in benchmark.sh A second --repeats silently overwrote the first; error out instead. Co-authored-by: Cursor --- benchmark.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/benchmark.sh b/benchmark.sh index dfa8bcc7..c3a942d9 100755 --- a/benchmark.sh +++ b/benchmark.sh @@ -42,6 +42,7 @@ BUILD=0 REVERSE=0 BRANCH_NAMES=() COMPARE_GIVEN=0 +REPEATS_GIVEN=0 DTEST_EXTRA=() # Cleanup state (set later). The EXIT trap restores the original git branch if @@ -125,6 +126,11 @@ while [[ $# -gt 0 ]]; do exit 0 ;; --repeats) + if (( REPEATS_GIVEN )); then + echo "error: --repeats may be given only once" >&2 + exit 1 + fi + REPEATS_GIVEN=1 shift REPEATS="${1:?missing value for --repeats}" shift From 1dbbed6fc446c3c68b660ecee18cc6c5cda874b1 Mon Sep 17 00:00:00 2001 From: Adam Wildavsky Date: Sun, 28 Jun 2026 16:32:20 -0500 Subject: [PATCH 09/28] Always show benchmark summary; gate detail rows on --details The summary was only printed in --compare mode, so a single binary with --repeats > 1 cleared its transient per-run rows and showed nothing. Add a single-binary summary (avg user ms per solver/file plus total elapsed) for the non-compare case so a summary is always shown. Also show the per-run detail table only with --details. Without it the rows are transient progress on a tty and suppressed otherwise. Co-authored-by: Cursor --- benchmark.sh | 57 ++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 46 insertions(+), 11 deletions(-) diff --git a/benchmark.sh b/benchmark.sh index c3a942d9..a43af64d 100755 --- a/benchmark.sh +++ b/benchmark.sh @@ -539,17 +539,16 @@ if ((${#DTEST_EXTRA[@]} > 0)); then fi echo -show_run_lines=1 +show_run_lines=0 TRANSIENT_PROGRESS=0 -# Per-run rows are detail: hide them from the final output (transient on a tty, -# suppressed otherwise) when comparing or when repeating, unless --details asks -# to keep them. With repeats > 1 the per-run rows are intermediate samples, so -# they are treated as transient too. -if [[ "$DETAILS" != "1" ]] && { [[ -n "${COMPARE:-}" ]] || (( REPEATS > 1 )); }; then - show_run_lines=0 - if [[ -t 1 ]]; then - TRANSIENT_PROGRESS=1 - fi +# Per-run rows are detail: keep them in the final output only with --details. +# Otherwise show them transiently as progress on a tty (cleared before the +# summary) and suppress them entirely when output is not a tty. The summary is +# always shown regardless. +if [[ "$DETAILS" == "1" ]]; then + show_run_lines=1 +elif [[ -t 1 ]]; then + TRANSIENT_PROGRESS=1 fi if [[ "$DRY_RUN" != "1" && ( "$show_run_lines" == "1" || "$TRANSIENT_PROGRESS" == "1" ) ]]; then @@ -597,7 +596,7 @@ done clear_transient_progress if [[ -n "${COMPARE:-}" && "$DRY_RUN" != "1" ]]; then - echo + echo # compare summary (two binaries) # Column headers default to the generic labels, but show the actual branch # names when known: the current git branch for the branch binary, and the # --branch name for the compare binary. Truncated to the 12-char column. @@ -695,6 +694,42 @@ if [[ -n "${COMPARE:-}" && "$DRY_RUN" != "1" ]]; then } } ' "$RESULTS" +elif [[ "$DRY_RUN" != "1" ]]; then + echo # single-binary summary (no --compare) + br_label="branch_avg" + if [[ -n "$git_branch" && "$git_branch" != "unknown" ]]; then + br_label="${git_branch:0:12}" + fi + + echo "Summary (avg user ms)" + echo "============================================================" + printf "%-6s %-13s %12s %6s\n" "solver" "file" "$br_label" "runs" + printf "%-6s %-13s %12s %6s\n" \ + "------" "-------------" "------------" "------" + + awk -F'\t' -v files="${FILES[*]}" ' + $3 == "branch" { + base = $1 SUBSEP $2 + s[base] += $7 + c[base]++ + if ($9 != "NA") tw += $9 # total wall-clock elapsed + } + END { + split("solve calc", solvers, " ") + nfiles = split(files, filearr, " ") + for (si = 1; si <= 2; si++) { + for (fi = 1; fi <= nfiles; fi++) { + base = solvers[si] SUBSEP filearr[fi] + if (!(base in c)) continue + printf "%-6s %-13s %12.2f %6d\n", + solvers[si], filearr[fi], s[base] / c[base], c[base] + } + } + printf "%-6s %-13s %12s %6s\n", + "------", "-------------", "------------", "------" + printf "%-6s %-13s %12.2f %6s\n", "TOTAL", "elapsed (s)", tw, "" + } + ' "$RESULTS" fi echo From 8ae75ff90ffbf300a266528b571fbbbba9d097c5 Mon Sep 17 00:00:00 2001 From: Adam Wildavsky Date: Sun, 28 Jun 2026 16:47:08 -0500 Subject: [PATCH 10/28] Suppress per-run detail rows entirely without --details The detail rows were shown as transient progress on a tty and then erased with cursor-up escapes. With many repeats the rows scroll off-screen, so the erase cannot reach them and they persist. Show per-run rows only with --details and drop the transient progress table; the summary is always shown. Co-authored-by: Cursor --- benchmark.sh | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/benchmark.sh b/benchmark.sh index a43af64d..c0bae34a 100755 --- a/benchmark.sh +++ b/benchmark.sh @@ -541,14 +541,11 @@ echo show_run_lines=0 TRANSIENT_PROGRESS=0 -# Per-run rows are detail: keep them in the final output only with --details. -# Otherwise show them transiently as progress on a tty (cleared before the -# summary) and suppress them entirely when output is not a tty. The summary is -# always shown regardless. +# Per-run rows are detail: show them only with --details. Without it they are +# suppressed entirely (no transient progress table, which could otherwise scroll +# off-screen and leave residue). The summary is always shown regardless. if [[ "$DETAILS" == "1" ]]; then show_run_lines=1 -elif [[ -t 1 ]]; then - TRANSIENT_PROGRESS=1 fi if [[ "$DRY_RUN" != "1" && ( "$show_run_lines" == "1" || "$TRANSIENT_PROGRESS" == "1" ) ]]; then From c1ba9d67e1164749d9f39588fe91777c145058b9 Mon Sep 17 00:00:00 2001 From: Adam Wildavsky Date: Sun, 28 Jun 2026 17:17:13 -0500 Subject: [PATCH 11/28] Hide build output unless --details git checkout + bazel output is build noise. Capture it to a log and surface it only on failure (or with --details); the short "Building..." labels remain as progress markers. ANSI save/restore (DECSC/DECRC) cannot erase this output reliably: bazel drives its own cursor save/restore for its progress display, clobbering the saved position, so a later restore+clear erases nothing. Capturing to a log is robust and works whether or not stderr is a tty. Co-authored-by: Cursor --- benchmark.sh | 59 ++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 41 insertions(+), 18 deletions(-) diff --git a/benchmark.sh b/benchmark.sh index c0bae34a..6a7e0eb4 100755 --- a/benchmark.sh +++ b/benchmark.sh @@ -2,10 +2,11 @@ # Benchmark dtest performance on one or two binaries. # # Runs all combinations of solver (solve, calc) and hand file -# (list100/1000/…/1), largest files first. With --compare, prints summary only -# unless --details; without --compare, prints per-run rows. With --compare and a -# tty, per-run rows appear during the run then are cleared before the summary. -# Does not pass dtest options unless given after "--" (see below). +# (list100/1000/…/1), largest files first. Always prints a summary. Per-run +# timing rows and build (git/bazel) output are shown only with --details; +# otherwise build output is captured to a log (surfaced only on failure) and +# per-run rows are suppressed. Does not pass dtest options unless given after +# "--" (see below). # # Usage: # ./benchmark.sh @@ -25,7 +26,7 @@ # REPEATS Runs per combination per binary (default: 1) # MAX_DEALS Include list10^n.txt files with 10^n <= N (default: 100) # DRY_RUN If 1, print commands only -# DETAILS If 1 with --compare, keep per-run rows in output (default: transient on tty) +# DETAILS If 1, keep per-run rows and build output (default: 0, summary only) # EPSILON With --compare, max % diff to treat branch/compare as equal (default: 0.5) set -euo pipefail @@ -51,6 +52,7 @@ RESULTS="" ORIG_BRANCH="" COMPARE_TMP="" BRANCH_TMP="" +BUILD_LOG="" cleanup() { if [[ -n "$ORIG_BRANCH" ]]; then @@ -64,6 +66,7 @@ cleanup() { fi [[ -n "$COMPARE_TMP" ]] && rm -f "$COMPARE_TMP" [[ -n "$BRANCH_TMP" ]] && rm -f "$BRANCH_TMP" + [[ -n "$BUILD_LOG" ]] && rm -f "$BUILD_LOG" [[ -n "$RESULTS" ]] && rm -f "$RESULTS" return 0 } @@ -75,8 +78,8 @@ usage() { cat <"$BUILD_LOG" 2>&1; then + cat "$BUILD_LOG" >&2 + return 1 + fi +} + build_branch_binary() { local name="$1" dest="$2" local dtest_rel="bazel-bin/library/tests/dtest" @@ -229,8 +256,7 @@ build_branch_binary() { return 0 fi echo "Building dtest from '$name'..." >&2 - git -C "$ROOT" checkout "$name" - (cd "$ROOT" && bazel build //library/tests:dtest) + run_build checkout_and_build "$name" cp -L "$ROOT/$dtest_rel" "$dest" chmod +x "$dest" } @@ -270,7 +296,7 @@ setup_branches() { echo "DRY_RUN: git -C $ROOT checkout $ORIG_BRANCH" >&2 else echo "Restoring '$ORIG_BRANCH'..." >&2 - git -C "$ROOT" checkout "$ORIG_BRANCH" + run_build git -C "$ROOT" checkout "$ORIG_BRANCH" fi BRANCH="$BRANCH_TMP" elif [[ "$num_branches" -eq 1 ]]; then @@ -282,8 +308,7 @@ setup_branches() { echo "DRY_RUN: (cd $ROOT && bazel build //library/tests:dtest)" >&2 else echo "Restoring '$ORIG_BRANCH' and rebuilding..." >&2 - git -C "$ROOT" checkout "$ORIG_BRANCH" - (cd "$ROOT" && bazel build //library/tests:dtest) + run_build checkout_and_build "$ORIG_BRANCH" fi COMPARE="$COMPARE_TMP" else @@ -296,7 +321,7 @@ setup_branches() { echo "DRY_RUN: git -C $ROOT checkout $ORIG_BRANCH" >&2 else echo "Restoring '$ORIG_BRANCH'..." >&2 - git -C "$ROOT" checkout "$ORIG_BRANCH" + run_build git -C "$ROOT" checkout "$ORIG_BRANCH" fi BRANCH="$BRANCH_TMP" COMPARE="$COMPARE_TMP" @@ -353,7 +378,7 @@ if [[ "$BUILD" == "1" ]]; then echo "DRY_RUN: (cd $ROOT && bazel build //library/tests:dtest)" >&2 else echo "Building //library/tests:dtest..." >&2 - (cd "$ROOT" && bazel build //library/tests:dtest) + run_build bazel_dtest fi fi @@ -516,9 +541,7 @@ if [[ -n "${COMPARE:-}" ]]; then printf "%-12s %s\n" "compare:" "$COMPARE" fi if [[ "$DETAILS" == "1" ]]; then - printf "%-12s %s\n" "details:" "on" - elif [[ -t 1 ]]; then - printf "%-12s %s\n" "details:" "transient (cleared before summary)" + printf "%-12s %s\n" "details:" "on (per-run rows + build output)" else printf "%-12s %s\n" "details:" "off (summary only)" fi From 1e8f5945571a2e70276239df81fc2ae310134a27 Mon Sep 17 00:00:00 2001 From: Adam Wildavsky Date: Sun, 28 Jun 2026 17:35:19 -0500 Subject: [PATCH 12/28] Allow --branch . to mean the current branch Resolve a "." branch argument to the current branch (or HEAD when detached) before ref validation, so it can be compared without naming it explicitly. Co-authored-by: Cursor --- benchmark.sh | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/benchmark.sh b/benchmark.sh index 6a7e0eb4..9550e19b 100755 --- a/benchmark.sh +++ b/benchmark.sh @@ -87,12 +87,13 @@ Options: --max-deals N Include list10^n.txt files with 10^n <= N (default: 100; env: MAX_DEALS) (alias: --max_deals) --build Build branch dtest only (bazel build //library/tests:dtest) - --branch NAME Git branch to build and compare. Once: compare the current branch - against NAME. Twice (--branch A --branch B): compare A vs B and - ignore the current branch. With --compare PATH: build NAME as the - branch binary and compare it against PATH, ignoring the current - branch. Each branch is checked out, dtest is built and its binary - saved; the original branch is then restored. Requires a clean tree. + --branch NAME Git branch to build and compare ("." means the current branch). + Once: compare the current branch against NAME. Twice (--branch A + --branch B): compare A vs B and ignore the current branch. With + --compare PATH: build NAME as the branch binary and compare it + against PATH, ignoring the current branch. Each branch is checked + out, dtest is built and its binary saved; the original branch is + then restored. Requires a clean tree. --compare PATH Second dtest binary (summary; transient progress on tty). May be combined with a single --branch NAME (NAME backs the branch binary). --details Keep per-run timing rows and build (git/bazel) output @@ -272,6 +273,18 @@ setup_branches() { echo "error: --branch requires a git work tree at $ROOT" >&2 exit 1 fi + + ORIG_BRANCH="$(git -C "$ROOT" symbolic-ref --quiet --short HEAD 2>/dev/null \ + || git -C "$ROOT" rev-parse HEAD)" + + # "." is shorthand for the current branch. + local i + for i in "${!BRANCH_NAMES[@]}"; do + if [[ "${BRANCH_NAMES[$i]}" == "." ]]; then + BRANCH_NAMES[$i]="$ORIG_BRANCH" + fi + done + local name for name in "${BRANCH_NAMES[@]}"; do if ! git -C "$ROOT" rev-parse --verify --quiet "$name" >/dev/null; then @@ -284,9 +297,6 @@ setup_branches() { exit 1 fi - ORIG_BRANCH="$(git -C "$ROOT" symbolic-ref --quiet --short HEAD 2>/dev/null \ - || git -C "$ROOT" rev-parse HEAD)" - if [[ "$COMPARE_GIVEN" == "1" ]]; then # --branch NAME --compare PATH: build NAME as the branch binary and keep the # user-supplied compare path. The current branch's dtest is not used. From 14e30fd8210285fd785cfd66d47abda20696b720 Mon Sep 17 00:00:00 2001 From: Adam Wildavsky Date: Sun, 28 Jun 2026 17:52:03 -0500 Subject: [PATCH 13/28] Show live per-run rows during runs, hidden before the summary Display the script's per-run timing rows as progress while the benchmark runs. On a tty without --details they are shown on the alternate screen and discarded when we switch back to the main screen just before the summary, so they never clutter the final output regardless of how much scrolls. With --details they stay on the main screen; off a tty they are suppressed (summary only). dtest's own output is captured, not shown. Replaces the old transient per-run rows, whose ANSI line-clearing could not erase content that had scrolled off-screen. Co-authored-by: Cursor --- benchmark.sh | 64 +++++++++++++++++++++++++++------------------------- 1 file changed, 33 insertions(+), 31 deletions(-) diff --git a/benchmark.sh b/benchmark.sh index 9550e19b..eea85f82 100755 --- a/benchmark.sh +++ b/benchmark.sh @@ -55,6 +55,12 @@ BRANCH_TMP="" BUILD_LOG="" cleanup() { + # Leave the alternate screen first so any restore/error messages and the shell + # prompt land on the normal screen. + if [[ "${ALT_SCREEN_ACTIVE:-0}" == "1" ]]; then + printf '\033[?1049l' >/dev/tty 2>/dev/null || true + ALT_SCREEN_ACTIVE=0 + fi if [[ -n "$ORIG_BRANCH" ]]; then local cur cur="$(git -C "$ROOT" symbolic-ref --quiet --short HEAD 2>/dev/null \ @@ -468,6 +474,7 @@ run_dtest() { # Wrap with `time -p` so we can capture wall-clock elapsed for this run. Its # "real/user/sys" lines go to the merged output but do not collide with the # dtest lines parse_dtest_output looks for. + # local out if ! out="$(/usr/bin/time -p "${cmd[@]}" 2>&1)"; then echo "error: dtest failed: ${cmd[*]}" >&2 @@ -487,8 +494,6 @@ run_dtest() { echo "$parsed $wall" } -progress_lines=0 -TRANSIENT_PROGRESS=0 show_run_lines=1 print_run_table_header() { @@ -496,31 +501,11 @@ print_run_table_header() { "solver" "file" "ver" "user_ms" "sys_ms" "avg_user" "ratio" "run" printf "%-6s %-13s %7s %8s %8s %10s %6s %s\n" \ "------" "-------------" "-------" "--------" "--------" "----------" "------" "---" - if [[ "$TRANSIENT_PROGRESS" == "1" ]]; then - progress_lines=$((progress_lines + 2)) - fi } print_run_row() { printf "%-6s %-13s %7s %8s %8s %10s %6s %s\n" \ "$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8" - if [[ "$TRANSIENT_PROGRESS" == "1" ]]; then - progress_lines=$((progress_lines + 1)) - fi -} - -clear_transient_progress() { - if [[ "$TRANSIENT_PROGRESS" != "1" || $progress_lines -le 0 ]]; then - return - fi - # Cursor rests on a blank line below the last row; erase it, then each table line. - # Do not move up after clearing the topmost line (would hit the header above). - printf '\033[2K' - local i - for (( i = 0; i < progress_lines; i++ )); do - printf '\033[1A\033[2K' - done - progress_lines=0 } echo "DDS dtest benchmark" @@ -572,16 +557,28 @@ if ((${#DTEST_EXTRA[@]} > 0)); then fi echo +# The per-run rows are the script's live progress. With --details they are kept +# in the final output. Without --details, on a tty, they are shown on the +# alternate screen so the user sees progress, then discarded when we switch back +# to the main screen just before the summary; off a tty they are suppressed +# (summary only), since there is nothing to hide them. show_run_lines=0 -TRANSIENT_PROGRESS=0 -# Per-run rows are detail: show them only with --details. Without it they are -# suppressed entirely (no transient progress table, which could otherwise scroll -# off-screen and leave residue). The summary is always shown regardless. -if [[ "$DETAILS" == "1" ]]; then - show_run_lines=1 +ALT_SCREEN=0 +if [[ "$DRY_RUN" != "1" ]]; then + if [[ "$DETAILS" == "1" ]]; then + show_run_lines=1 + elif [[ -t 1 ]]; then + show_run_lines=1 + ALT_SCREEN=1 + fi fi -if [[ "$DRY_RUN" != "1" && ( "$show_run_lines" == "1" || "$TRANSIENT_PROGRESS" == "1" ) ]]; then +if [[ "$ALT_SCREEN" == "1" ]]; then + printf '\033[?1049h\033[H\033[2J' >/dev/tty # enter alt screen, home, clear + ALT_SCREEN_ACTIVE=1 +fi + +if [[ "$DRY_RUN" != "1" && "$show_run_lines" == "1" ]]; then print_run_table_header fi @@ -611,7 +608,7 @@ for solver in "${SOLVERS[@]}"; do read -r user sys avg ratio wall < <(run_dtest "$bin" "$solver" "$hands") - if [[ "$show_run_lines" == "1" || "$TRANSIENT_PROGRESS" == "1" ]]; then + if [[ "$show_run_lines" == "1" ]]; then print_run_row "$solver" "$file" "$ver" "$user" "$sys" "$avg" "$ratio" "$run_label" fi @@ -623,7 +620,12 @@ for solver in "${SOLVERS[@]}"; do done done -clear_transient_progress +# Return to the normal screen, discarding the live dtest output, before the +# summary so the final output is just the header and the summary. +if [[ "$ALT_SCREEN" == "1" ]]; then + printf '\033[?1049l' >/dev/tty + ALT_SCREEN_ACTIVE=0 +fi if [[ -n "${COMPARE:-}" && "$DRY_RUN" != "1" ]]; then echo # compare summary (two binaries) From badd81f8a5488d259f5c25182574ddcc37ec1004 Mon Sep 17 00:00:00 2001 From: Adam Wildavsky Date: Sun, 28 Jun 2026 22:45:11 -0500 Subject: [PATCH 14/28] Treat untracked files as non-clean in --branch check The cleanliness check for --branch used --untracked-files=no, so untracked files passed the check but could still block git checkout ("would be overwritten"). Include untracked files so the check matches the documented "clean tree" requirement. Co-authored-by: Cursor --- benchmark.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/benchmark.sh b/benchmark.sh index eea85f82..3df030e2 100755 --- a/benchmark.sh +++ b/benchmark.sh @@ -298,8 +298,10 @@ setup_branches() { exit 1 fi done - if [[ -n "$(git -C "$ROOT" status --porcelain --untracked-files=no)" ]]; then - echo "error: tracked changes present; commit or stash before using --branch" >&2 + # Untracked files can also block a checkout ("would be overwritten"), so treat + # any working tree change (tracked or untracked) as non-clean. + if [[ -n "$(git -C "$ROOT" status --porcelain --untracked-files=normal)" ]]; then + echo "error: working tree not clean; commit, stash, or remove changes (tracked or untracked) before using --branch" >&2 exit 1 fi From affe33c8511442a943501c23982348896fd2d9cf Mon Sep 17 00:00:00 2001 From: Adam Wildavsky Date: Mon, 29 Jun 2026 04:46:03 +0100 Subject: [PATCH 15/28] Use `command time -p` for portability Hard-coding /usr/bin/time reduces portability (some environments only provide time as a shell keyword or at a different path). Using command time -p keeps the portable format while avoiding an absolute path dependency. Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- benchmark.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark.sh b/benchmark.sh index 3df030e2..22d136a9 100755 --- a/benchmark.sh +++ b/benchmark.sh @@ -478,7 +478,7 @@ run_dtest() { # dtest lines parse_dtest_output looks for. # local out - if ! out="$(/usr/bin/time -p "${cmd[@]}" 2>&1)"; then + if ! out="$(command time -p "${cmd[@]}" 2>&1)"; then echo "error: dtest failed: ${cmd[*]}" >&2 echo "$out" >&2 exit 1 From 057e37c7855108784eed96d5637912cb4e086e1d Mon Sep 17 00:00:00 2001 From: Adam Wildavsky Date: Mon, 29 Jun 2026 09:41:44 -0500 Subject: [PATCH 16/28] Show branch names in per-run details rows The --details per-run table showed the generic "branch"/"compare" labels in the ver column. Use the backing branch name when known (current git branch for the branch binary, --branch name for the compare binary), falling back to the generic labels. Widen the ver column to fit. Co-authored-by: Cursor --- benchmark.sh | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/benchmark.sh b/benchmark.sh index 22d136a9..539696ca 100755 --- a/benchmark.sh +++ b/benchmark.sh @@ -499,14 +499,14 @@ run_dtest() { show_run_lines=1 print_run_table_header() { - printf "%-6s %-13s %7s %8s %8s %10s %6s %s\n" \ + printf "%-6s %-13s %-12s %8s %8s %10s %6s %s\n" \ "solver" "file" "ver" "user_ms" "sys_ms" "avg_user" "ratio" "run" - printf "%-6s %-13s %7s %8s %8s %10s %6s %s\n" \ - "------" "-------------" "-------" "--------" "--------" "----------" "------" "---" + printf "%-6s %-13s %-12s %8s %8s %10s %6s %s\n" \ + "------" "-------------" "------------" "--------" "--------" "----------" "------" "---" } print_run_row() { - printf "%-6s %-13s %7s %8s %8s %10s %6s %s\n" \ + printf "%-6s %-13s %-12s %8s %8s %10s %6s %s\n" \ "$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8" } @@ -526,6 +526,20 @@ elif [[ "$num_branches" -eq 2 ]]; then compare_branch_name="${BRANCH_NAMES[1]}" fi +# Labels for the per-run details "ver" column: prefer the backing branch name +# when known (the current git branch for the branch binary, the --branch name +# for the compare binary), falling back to the generic "branch"/"compare". +branch_ver_label="branch" +if [[ -n "$branch_branch_name" ]]; then + branch_ver_label="$branch_branch_name" +elif [[ -n "$git_branch" && "$git_branch" != "unknown" ]]; then + branch_ver_label="$git_branch" +fi +compare_ver_label="compare" +if [[ -n "$compare_branch_name" ]]; then + compare_ver_label="$compare_branch_name" +fi + if [[ -n "$branch_branch_name" ]]; then printf "%-12s %s\n" "branch:" "branch '$branch_branch_name' ($BRANCH)" else @@ -611,7 +625,13 @@ for solver in "${SOLVERS[@]}"; do read -r user sys avg ratio wall < <(run_dtest "$bin" "$solver" "$hands") if [[ "$show_run_lines" == "1" ]]; then - print_run_row "$solver" "$file" "$ver" "$user" "$sys" "$avg" "$ratio" "$run_label" + if [[ "$ver" == "compare" ]]; then + ver_disp="$compare_ver_label" + else + ver_disp="$branch_ver_label" + fi + print_run_row "$solver" "$file" "${ver_disp:0:12}" \ + "$user" "$sys" "$avg" "$ratio" "$run_label" fi printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n" \ From 8085ba8f0ca55518b92e9765c9f23b99a8b14ef3 Mon Sep 17 00:00:00 2001 From: Adam Wildavsky Date: Mon, 29 Jun 2026 16:43:02 -0500 Subject: [PATCH 17/28] benchmark.sh: anchor ROOT to the current directory Use the working directory the script is invoked from instead of the script location, so it targets whatever DDS checkout you run it in. Add an early guard that fails fast when the current directory is not a DDS checkout root (no MODULE.bazel / library/tests/BUILD.bazel). Co-authored-by: Cursor --- benchmark.sh | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/benchmark.sh b/benchmark.sh index 539696ca..87fb355e 100755 --- a/benchmark.sh +++ b/benchmark.sh @@ -19,7 +19,7 @@ # REPEATS=3 ./benchmark.sh # # Environment: -# BRANCH Path to branch dtest (default: bazel-bin in this repo) +# BRANCH Path to branch dtest (default: bazel-bin under the current dir) # COMPARE Optional second dtest binary for comparison # (or use --branch NAME to build the compare binary from a git branch) # HANDS_DIR Directory containing list*.txt files (default: ./hands) @@ -31,7 +31,17 @@ set -euo pipefail -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# Operate on the current working directory (the repo you invoke this from), +# not the directory the script happens to live in. +ROOT="$(pwd)" + +# Fail fast if ROOT is not the root of a DDS checkout: the git/bazel operations +# and the default branch-binary and hands paths are all relative to it. +if [[ ! -f "$ROOT/MODULE.bazel" || ! -f "$ROOT/library/tests/BUILD.bazel" ]]; then + echo "error: '$ROOT' is not a DDS checkout root" >&2 + echo " cd to the root of the dds repository before running benchmark.sh" >&2 + exit 1 +fi BRANCH="${BRANCH:-$ROOT/bazel-bin/library/tests/dtest}" HANDS_DIR="${HANDS_DIR:-$ROOT/hands}" REPEATS="${REPEATS:-1}" From 0d2652da794ff82cffe54b298fce3a48582a7b04 Mon Sep 17 00:00:00 2001 From: Adam Wildavsky Date: Mon, 29 Jun 2026 22:01:38 -0500 Subject: [PATCH 18/28] benchmark.sh: support any number of --branch/--compare binaries Both flags are now repeatable and can be combined; binaries are benchmarked in specification order with the first as baseline. With one spec the current checkout is still included as the baseline; with two or more explicit specs the checkout is not auto-included. The summary shows one average column per binary, adding a ratio and faster/slower note only for a two-binary comparison; with three or more binaries the note (and ratio) are dropped. Co-authored-by: Cursor --- benchmark.sh | 551 +++++++++++++++++++++++++-------------------------- 1 file changed, 267 insertions(+), 284 deletions(-) diff --git a/benchmark.sh b/benchmark.sh index 87fb355e..59bd8dbb 100755 --- a/benchmark.sh +++ b/benchmark.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Benchmark dtest performance on one or two binaries. +# Benchmark dtest performance across one or more binaries. # # Runs all combinations of solver (solve, calc) and hand file # (list100/1000/…/1), largest files first. Always prints a summary. Per-run @@ -19,15 +19,14 @@ # REPEATS=3 ./benchmark.sh # # Environment: -# BRANCH Path to branch dtest (default: bazel-bin under the current dir) -# COMPARE Optional second dtest binary for comparison -# (or use --branch NAME to build the compare binary from a git branch) +# BRANCH Path to the baseline dtest (default: bazel-bin under the current dir) +# COMPARE Optional extra dtest binary to benchmark (like a trailing --compare) # HANDS_DIR Directory containing list*.txt files (default: ./hands) # REPEATS Runs per combination per binary (default: 1) # MAX_DEALS Include list10^n.txt files with 10^n <= N (default: 100) # DRY_RUN If 1, print commands only # DETAILS If 1, keep per-run rows and build output (default: 0, summary only) -# EPSILON With --compare, max % diff to treat branch/compare as equal (default: 0.5) +# EPSILON For a two-binary comparison, max % diff treated as equal (default: 0.5) set -euo pipefail @@ -51,8 +50,11 @@ DETAILS="${DETAILS:-0}" EPSILON="${EPSILON:-0.5}" BUILD=0 REVERSE=0 -BRANCH_NAMES=() -COMPARE_GIVEN=0 +# Ordered list of binaries to benchmark, captured in command-line order. Each +# --branch/--compare appends one spec; the first spec is the baseline. +SPEC_KINDS=() # parallel: "branch" (git ref) or "compare" (path to a binary) +SPEC_VALS=() +CLI_COMPARE_GIVEN=0 REPEATS_GIVEN=0 DTEST_EXTRA=() @@ -60,8 +62,7 @@ DTEST_EXTRA=() # --branch switched away, and removes temp files. RESULTS="" ORIG_BRANCH="" -COMPARE_TMP="" -BRANCH_TMP="" +TMP_BINS=() BUILD_LOG="" cleanup() { @@ -80,8 +81,9 @@ cleanup() { git -C "$ROOT" checkout "$ORIG_BRANCH" >/dev/null 2>&1 || true fi fi - [[ -n "$COMPARE_TMP" ]] && rm -f "$COMPARE_TMP" - [[ -n "$BRANCH_TMP" ]] && rm -f "$BRANCH_TMP" + if ((${#TMP_BINS[@]} > 0)); then + rm -f "${TMP_BINS[@]}" + fi [[ -n "$BUILD_LOG" ]] && rm -f "$BUILD_LOG" [[ -n "$RESULTS" ]] && rm -f "$RESULTS" return 0 @@ -103,21 +105,24 @@ Options: --max-deals N Include list10^n.txt files with 10^n <= N (default: 100; env: MAX_DEALS) (alias: --max_deals) --build Build branch dtest only (bazel build //library/tests:dtest) - --branch NAME Git branch to build and compare ("." means the current branch). - Once: compare the current branch against NAME. Twice (--branch A - --branch B): compare A vs B and ignore the current branch. With - --compare PATH: build NAME as the branch binary and compare it - against PATH, ignoring the current branch. Each branch is checked - out, dtest is built and its binary saved; the original branch is - then restored. Requires a clean tree. - --compare PATH Second dtest binary (summary; transient progress on tty). May be - combined with a single --branch NAME (NAME backs the branch binary). + --branch NAME Git branch to build and benchmark ("." means the current branch). + Repeatable. Each named branch is checked out, dtest is built and + its binary saved, then the original branch is restored (requires a + clean tree). + --compare PATH Path to a prebuilt dtest binary to benchmark. Repeatable. --details Keep per-run timing rows and build (git/bazel) output - --epsilon PCT With --compare, treat timings within PCT% as equal (default: 0.5; env: EPSILON) - --reverse With --compare, run compare before branch each repeat (default: branch first) + --epsilon PCT For a two-binary comparison, treat timings within PCT% as equal + (default: 0.5; env: EPSILON) + --reverse Reverse the per-repeat dispatch order of the binaries -- End benchmark options; remaining args are passed to dtest (e.g. -- -n 8 -r for 8 threads and slow-board report) +--branch and --compare may be given any number of times and combined freely; the +binaries are benchmarked in the order specified and the first is the baseline. With +exactly one spec, the current checkout is also benchmarked as the baseline. With two +binaries the summary adds a ratio and a "faster" note; with three or more it shows +only the per-binary averages (no note). + Environment: BRANCH, COMPARE, HANDS_DIR, REPEATS, MAX_DEALS, DRY_RUN, DETAILS, EPSILON @@ -128,6 +133,7 @@ Examples: ./benchmark.sh --repeats 3 -- -n 4 -r ./benchmark.sh --branch develop ./benchmark.sh --branch develop --branch opus-two-percent + ./benchmark.sh --branch develop --branch opus-two-percent --branch fastest ./benchmark.sh --branch opus-two-percent --compare /path/to/dtest ./benchmark.sh --branch develop --repeats 3 -- -n 8 ./benchmark.sh --compare /path/to/dtest @@ -157,13 +163,15 @@ while [[ $# -gt 0 ]]; do ;; --branch) shift - BRANCH_NAMES+=("${1:?missing value for --branch}") + SPEC_KINDS+=("branch") + SPEC_VALS+=("${1:?missing value for --branch}") shift ;; --compare) shift - COMPARE="${1:?missing value for --compare}" - COMPARE_GIVEN=1 + SPEC_KINDS+=("compare") + SPEC_VALS+=("${1:?missing value for --compare}") + CLI_COMPARE_GIVEN=1 shift ;; --max-deals|--max_deals|-max-deals|-max_deals) @@ -216,25 +224,23 @@ if ! [[ "$EPSILON" =~ ^[0-9]+(\.[0-9]+)?$ ]]; then exit 1 fi -# A compare binary supplied via the COMPARE env var behaves like --compare PATH. -if [[ "$COMPARE_GIVEN" == "0" && -n "${COMPARE:-}" ]]; then - COMPARE_GIVEN=1 +# A compare binary supplied via the COMPARE env var behaves like a trailing +# --compare PATH, unless --compare was already given on the command line. +if [[ "$CLI_COMPARE_GIVEN" == "0" && -n "${COMPARE:-}" ]]; then + SPEC_KINDS+=("compare") + SPEC_VALS+=("$COMPARE") fi -num_branches=${#BRANCH_NAMES[@]} - -if [[ "$REVERSE" == "1" && -z "${COMPARE:-}" && "$num_branches" -eq 0 ]]; then - echo "error: --reverse requires --compare or --branch" >&2 - exit 1 -fi - -if [[ "$num_branches" -gt 0 && "$COMPARE_GIVEN" == "1" && "$num_branches" -ne 1 ]]; then - echo "error: --compare accepts exactly one --branch (got $num_branches)" >&2 - exit 1 +nspecs=${#SPEC_KINDS[@]} +nbranch=0 +if (( nspecs > 0 )); then + for k in "${SPEC_KINDS[@]}"; do + [[ "$k" == "branch" ]] && nbranch=$((nbranch + 1)) + done fi -if [[ "$num_branches" -gt 2 ]]; then - echo "error: --branch may be given at most twice (got $num_branches)" >&2 +if [[ "$REVERSE" == "1" && "$nspecs" -eq 0 ]]; then + echo "error: --reverse requires at least one --branch or --compare" >&2 exit 1 fi @@ -278,33 +284,44 @@ build_branch_binary() { chmod +x "$dest" } -# Branch-mode binary selection: -# --branch NAME : branch = current checkout, compare = build(NAME) -# --branch NAME --compare PATH : branch = build(NAME), compare = PATH -# (the current branch's dtest is ignored) -# --branch A --branch B : branch = build(A), compare = build(B) -# (the current branch's dtest is ignored) -setup_branches() { +# Display label for a compare binary given by path: its basename, or the raw +# path if basename is somehow empty. +label_for_path() { + local b + b="$(basename -- "$1")" + [[ -n "$b" ]] && printf '%s' "$b" || printf '%s' "$1" +} + +# Label for the current checkout's binary: the git branch, else "branch". +current_label() { + if [[ -n "$git_branch" && "$git_branch" != "unknown" ]]; then + printf '%s' "$git_branch" + else + printf 'branch' + fi +} + +# Validate the git work tree and branch refs before any checkout. Only called +# when at least one --branch spec is present. +git_prep_for_branches() { if ! git -C "$ROOT" rev-parse --is-inside-work-tree >/dev/null 2>&1; then echo "error: --branch requires a git work tree at $ROOT" >&2 exit 1 fi - ORIG_BRANCH="$(git -C "$ROOT" symbolic-ref --quiet --short HEAD 2>/dev/null \ || git -C "$ROOT" rev-parse HEAD)" - # "." is shorthand for the current branch. local i - for i in "${!BRANCH_NAMES[@]}"; do - if [[ "${BRANCH_NAMES[$i]}" == "." ]]; then - BRANCH_NAMES[$i]="$ORIG_BRANCH" + # "." is shorthand for the current branch. + for i in "${!SPEC_VALS[@]}"; do + if [[ "${SPEC_KINDS[$i]}" == "branch" && "${SPEC_VALS[$i]}" == "." ]]; then + SPEC_VALS[$i]="$ORIG_BRANCH" fi done - - local name - for name in "${BRANCH_NAMES[@]}"; do - if ! git -C "$ROOT" rev-parse --verify --quiet "$name" >/dev/null; then - echo "error: --branch: unknown git ref '$name'" >&2 + for i in "${!SPEC_VALS[@]}"; do + if [[ "${SPEC_KINDS[$i]}" == "branch" ]] \ + && ! git -C "$ROOT" rev-parse --verify --quiet "${SPEC_VALS[$i]}" >/dev/null; then + echo "error: --branch: unknown git ref '${SPEC_VALS[$i]}'" >&2 exit 1 fi done @@ -314,50 +331,103 @@ setup_branches() { echo "error: working tree not clean; commit, stash, or remove changes (tracked or untracked) before using --branch" >&2 exit 1 fi +} - if [[ "$COMPARE_GIVEN" == "1" ]]; then - # --branch NAME --compare PATH: build NAME as the branch binary and keep the - # user-supplied compare path. The current branch's dtest is not used. - BRANCH_TMP="$(mktemp "${TMPDIR:-/tmp}/dds-dtest-branch.XXXXXX")" - build_branch_binary "${BRANCH_NAMES[0]}" "$BRANCH_TMP" - if [[ "$DRY_RUN" == "1" ]]; then - echo "DRY_RUN: git -C $ROOT checkout $ORIG_BRANCH" >&2 - else - echo "Restoring '$ORIG_BRANCH'..." >&2 - run_build git -C "$ROOT" checkout "$ORIG_BRANCH" - fi - BRANCH="$BRANCH_TMP" - elif [[ "$num_branches" -eq 1 ]]; then - COMPARE_TMP="$(mktemp "${TMPDIR:-/tmp}/dds-dtest-compare.XXXXXX")" - build_branch_binary "${BRANCH_NAMES[0]}" "$COMPARE_TMP" - # Restore the current branch and rebuild it as the branch binary. - if [[ "$DRY_RUN" == "1" ]]; then - echo "DRY_RUN: git -C $ROOT checkout $ORIG_BRANCH" >&2 +# Restore the original branch. Pass 1 to also rebuild dtest (needed only when +# the current checkout's bazel-bin binary is used as a baseline). +restore_branch() { + local rebuild="$1" + if [[ "$DRY_RUN" == "1" ]]; then + echo "DRY_RUN: git -C $ROOT checkout $ORIG_BRANCH" >&2 + [[ "$rebuild" == "1" ]] && \ echo "DRY_RUN: (cd $ROOT && bazel build //library/tests:dtest)" >&2 + return 0 + fi + if [[ "$rebuild" == "1" ]]; then + echo "Restoring '$ORIG_BRANCH' and rebuilding..." >&2 + run_build checkout_and_build "$ORIG_BRANCH" + else + echo "Restoring '$ORIG_BRANCH'..." >&2 + run_build git -C "$ROOT" checkout "$ORIG_BRANCH" + fi +} + +new_tmp_bin() { + local t + t="$(mktemp "${TMPDIR:-/tmp}/dds-dtest-bin.XXXXXX")" + TMP_BINS+=("$t") + printf '%s' "$t" +} + +# Build the ordered list of binaries to benchmark into BIN_LABELS / BIN_PATHS. +# Binary-set selection: +# (no spec) : one binary, the current checkout +# one spec : current checkout (baseline) + that spec +# two or more specs : exactly those specs in order; current ignored +# In all cases the first binary is the baseline. --branch specs are built from +# git; --compare specs are existing binary paths. +build_binaries() { + BIN_LABELS=() + BIN_PATHS=() + + if (( nspecs == 0 )); then + BIN_PATHS=("$BRANCH") + BIN_LABELS=("$(current_label)") + return 0 + fi + + if (( nbranch > 0 )); then + git_prep_for_branches + fi + + local kind val t + if (( nspecs == 1 )); then + # Current checkout is the baseline; the single spec is the second binary. + BIN_PATHS=("$BRANCH") + BIN_LABELS=("$(current_label)") + kind="${SPEC_KINDS[0]}" + val="${SPEC_VALS[0]}" + if [[ "$kind" == "branch" ]]; then + t="$(new_tmp_bin)" + build_branch_binary "$val" "$t" + restore_branch 1 # rebuild current as the baseline binary + BIN_PATHS+=("$t") + BIN_LABELS+=("$val") else - echo "Restoring '$ORIG_BRANCH' and rebuilding..." >&2 - run_build checkout_and_build "$ORIG_BRANCH" + BIN_PATHS+=("$val") + BIN_LABELS+=("$(label_for_path "$val")") fi - COMPARE="$COMPARE_TMP" - else - # Two branches: build both, ignore the current branch's binary. - BRANCH_TMP="$(mktemp "${TMPDIR:-/tmp}/dds-dtest-branch.XXXXXX")" - COMPARE_TMP="$(mktemp "${TMPDIR:-/tmp}/dds-dtest-compare.XXXXXX")" - build_branch_binary "${BRANCH_NAMES[0]}" "$BRANCH_TMP" - build_branch_binary "${BRANCH_NAMES[1]}" "$COMPARE_TMP" - if [[ "$DRY_RUN" == "1" ]]; then - echo "DRY_RUN: git -C $ROOT checkout $ORIG_BRANCH" >&2 + return 0 + fi + + # Two or more specs: benchmark exactly those, in order; ignore the checkout. + local i + for i in "${!SPEC_KINDS[@]}"; do + kind="${SPEC_KINDS[$i]}" + val="${SPEC_VALS[$i]}" + if [[ "$kind" == "branch" ]]; then + t="$(new_tmp_bin)" + build_branch_binary "$val" "$t" + BIN_PATHS+=("$t") + BIN_LABELS+=("$val") else - echo "Restoring '$ORIG_BRANCH'..." >&2 - run_build git -C "$ROOT" checkout "$ORIG_BRANCH" + BIN_PATHS+=("$val") + BIN_LABELS+=("$(label_for_path "$val")") fi - BRANCH="$BRANCH_TMP" - COMPARE="$COMPARE_TMP" + done + if (( nbranch > 0 )); then + restore_branch 0 fi } -if [[ "$num_branches" -gt 0 ]]; then - setup_branches +git_branch="unknown" +if git -C "$ROOT" rev-parse --is-inside-work-tree >/dev/null 2>&1; then + git_branch="$(git -C "$ROOT" rev-parse --abbrev-ref HEAD 2>/dev/null || echo unknown)" +fi + +build_binaries +num_bins=${#BIN_PATHS[@]} +if (( nbranch > 0 )); then BUILD=0 # build already done as part of the branch workflow fi @@ -411,27 +481,29 @@ if [[ "$BUILD" == "1" ]]; then fi if [[ "$DRY_RUN" != "1" ]]; then - if [[ ! -x "$BRANCH" ]]; then - echo "error: branch binary not found or not executable: $BRANCH" >&2 - echo "hint: bazel build //library/tests:dtest" >&2 - exit 1 - fi - - if [[ -n "${COMPARE:-}" && ! -x "$COMPARE" ]]; then - echo "error: compare binary not found or not executable: $COMPARE" >&2 - exit 1 - fi + for i in "${!BIN_PATHS[@]}"; do + if [[ ! -x "${BIN_PATHS[$i]}" ]]; then + echo "error: binary not found or not executable: ${BIN_PATHS[$i]} (${BIN_LABELS[$i]})" >&2 + (( i == 0 )) && echo "hint: bazel build //library/tests:dtest" >&2 + exit 1 + fi + done fi -BIN_PAIRS=("branch:$BRANCH") -if [[ -n "${COMPARE:-}" ]]; then - if [[ "$REVERSE" == "1" ]]; then - BIN_PAIRS=("compare:$COMPARE" "branch:$BRANCH") - else - BIN_PAIRS=("branch:$BRANCH" "compare:$COMPARE") - fi +# Dispatch order of the binaries within each (solver, file, repeat). The +# baseline (index 0) leads by default; --reverse flips it. The summary keys on +# the stored binary index, so order never affects the reported results. +RUN_ORDER=() +for (( i = 0; i < num_bins; i++ )); do + RUN_ORDER+=("$i") +done +if [[ "$REVERSE" == "1" ]]; then + rev=() + for (( i = num_bins - 1; i >= 0; i-- )); do + rev+=("${RUN_ORDER[$i]}") + done + RUN_ORDER=("${rev[@]}") fi -num_bins=${#BIN_PAIRS[@]} for f in "${FILES[@]}"; do if [[ ! -f "$HANDS_DIR/$f" ]]; then @@ -440,11 +512,6 @@ for f in "${FILES[@]}"; do fi done -git_branch="unknown" -if git -C "$ROOT" rev-parse --is-inside-work-tree >/dev/null 2>&1; then - git_branch="$(git -C "$ROOT" rev-parse --abbrev-ref HEAD 2>/dev/null || echo unknown)" -fi - RESULTS="$(mktemp "${TMPDIR:-/tmp}/dds-benchmark.XXXXXX")" # Removal handled by the cleanup() EXIT trap installed near the top. @@ -522,56 +589,28 @@ print_run_row() { echo "DDS dtest benchmark" echo "===================" -# Branch names backing each binary, when --branch was used. With one --branch -# the branch binary is the current checkout; with two it is the first name. -branch_branch_name="" -compare_branch_name="" -if [[ "$num_branches" -eq 1 && "$COMPARE_GIVEN" == "1" ]]; then - # --branch NAME --compare PATH: NAME backs the branch binary; compare is a path. - branch_branch_name="${BRANCH_NAMES[0]}" -elif [[ "$num_branches" -eq 1 ]]; then - compare_branch_name="${BRANCH_NAMES[0]}" -elif [[ "$num_branches" -eq 2 ]]; then - branch_branch_name="${BRANCH_NAMES[0]}" - compare_branch_name="${BRANCH_NAMES[1]}" -fi - -# Labels for the per-run details "ver" column: prefer the backing branch name -# when known (the current git branch for the branch binary, the --branch name -# for the compare binary), falling back to the generic "branch"/"compare". -branch_ver_label="branch" -if [[ -n "$branch_branch_name" ]]; then - branch_ver_label="$branch_branch_name" -elif [[ -n "$git_branch" && "$git_branch" != "unknown" ]]; then - branch_ver_label="$git_branch" -fi -compare_ver_label="compare" -if [[ -n "$compare_branch_name" ]]; then - compare_ver_label="$compare_branch_name" -fi - -if [[ -n "$branch_branch_name" ]]; then - printf "%-12s %s\n" "branch:" "branch '$branch_branch_name' ($BRANCH)" -else - printf "%-12s %s\n" "branch:" "$BRANCH" -fi -if [[ -n "${COMPARE:-}" ]]; then - if [[ -n "$compare_branch_name" ]]; then - printf "%-12s %s\n" "compare:" "branch '$compare_branch_name' ($COMPARE)" - else - printf "%-12s %s\n" "compare:" "$COMPARE" - fi +# One line per binary, baseline first. The label is the branch name (--branch), +# the binary's basename (--compare), or the current git branch (checkout). +for i in "${!BIN_PATHS[@]}"; do + tag="binary $((i + 1)):" + (( i == 0 )) && tag="baseline:" + printf "%-12s %s\n" "$tag" "${BIN_LABELS[$i]} (${BIN_PATHS[$i]})" +done +if (( num_bins >= 2 )); then if [[ "$DETAILS" == "1" ]]; then printf "%-12s %s\n" "details:" "on (per-run rows + build output)" else printf "%-12s %s\n" "details:" "off (summary only)" fi - if [[ "$REVERSE" == "1" ]]; then - printf "%-12s %s\n" "run order:" "interleaved compare, branch" - else - printf "%-12s %s\n" "run order:" "interleaved branch, compare" + order_str="" + for idx in "${RUN_ORDER[@]}"; do + order_str+="${order_str:+, }${BIN_LABELS[$idx]}" + done + printf "%-12s %s\n" "run order:" "interleaved $order_str" + # The note column (and thus epsilon) only applies to a two-binary comparison. + if (( num_bins == 2 )); then + printf "%-12s %s\n" "epsilon:" "${EPSILON}%" fi - printf "%-12s %s\n" "epsilon:" "${EPSILON}%" fi printf "%-12s %s\n" "hands dir:" "$HANDS_DIR" printf "%-12s %s\n" "max_deals:" "$MAX_DEALS" @@ -622,9 +661,8 @@ for solver in "${SOLVERS[@]}"; do run_label="1/1" fi - for pair in "${BIN_PAIRS[@]}"; do - ver="${pair%%:*}" - bin="${pair#*:}" + for idx in "${RUN_ORDER[@]}"; do + bin="${BIN_PATHS[$idx]}" run_no=$((run_no + 1)) if [[ "$DRY_RUN" == "1" ]]; then @@ -635,17 +673,13 @@ for solver in "${SOLVERS[@]}"; do read -r user sys avg ratio wall < <(run_dtest "$bin" "$solver" "$hands") if [[ "$show_run_lines" == "1" ]]; then - if [[ "$ver" == "compare" ]]; then - ver_disp="$compare_ver_label" - else - ver_disp="$branch_ver_label" - fi - print_run_row "$solver" "$file" "${ver_disp:0:12}" \ + print_run_row "$solver" "$file" "${BIN_LABELS[$idx]:0:12}" \ "$user" "$sys" "$avg" "$ratio" "$run_label" fi + # Column 3 is the binary index; the summary keys on it. printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n" \ - "$solver" "$file" "$ver" "$rep" "$user" "$sys" "$avg" "$ratio" "$wall" \ + "$solver" "$file" "$idx" "$rep" "$user" "$sys" "$avg" "$ratio" "$wall" \ >>"$RESULTS" done done @@ -659,139 +693,88 @@ if [[ "$ALT_SCREEN" == "1" ]]; then ALT_SCREEN_ACTIVE=0 fi -if [[ -n "${COMPARE:-}" && "$DRY_RUN" != "1" ]]; then - echo # compare summary (two binaries) - # Column headers default to the generic labels, but show the actual branch - # names when known: the current git branch for the branch binary, and the - # --branch name for the compare binary. Truncated to the 12-char column. - cmp_label="compare_avg" - if [[ -n "$compare_branch_name" ]]; then - cmp_label="${compare_branch_name:0:12}" - fi - br_label="branch_avg" - if [[ -n "$branch_branch_name" ]]; then - br_label="${branch_branch_name:0:12}" - elif [[ -n "$git_branch" && "$git_branch" != "unknown" ]]; then - br_label="${git_branch:0:12}" - fi - - # Names used in the "note" column (" faster"); fall back to the generic - # "branch"/"compare" when no branch name is known. - note_branch="branch" - if [[ -n "$branch_branch_name" ]]; then - note_branch="$branch_branch_name" - elif [[ -n "$compare_branch_name" && -n "$git_branch" && "$git_branch" != "unknown" ]]; then - note_branch="$git_branch" - fi - note_compare="compare" - if [[ -n "$compare_branch_name" ]]; then - note_compare="$compare_branch_name" - fi - +if [[ "$DRY_RUN" != "1" ]]; then + echo echo "Summary (avg user ms)" echo "==============================================================================" - printf "%-6s %-13s %12s %12s %10s %-15s\n" \ - "solver" "file" "$cmp_label" "$br_label" "cmp/branch" "note" - printf "%-6s %-13s %12s %12s %10s %-15s\n" \ - "------" "-------------" "------------" "------------" "----------" "---------------" - - awk -F'\t' -v files="${FILES[*]}" -v epsilon_pct="$EPSILON" \ - -v note_branch="$note_branch" -v note_compare="$note_compare" ' - function within_epsilon(a, b, eps, hi, lo) { - eps = epsilon_pct / 100 + + # One avg column per binary (baseline first). A ratio + note column are added + # only for a two-binary comparison; with three or more binaries the note has + # no single meaning and is dropped, leaving just the per-binary averages. + # Labels go through the environment (ENVIRON) rather than -v: BSD awk rejects + # newlines in a -v value, and labels are newline-joined. + labels_joined="$(printf '%s\n' "${BIN_LABELS[@]}")" + + LABELS="$labels_joined" awk -F'\t' -v nb="$num_bins" \ + -v files="${FILES[*]}" -v eps="$EPSILON" ' + function within_epsilon(a, b, e, hi, lo) { + e = eps / 100 if (a > b) { hi = a; lo = b } else { hi = b; lo = a } - return (hi <= 0 || (hi - lo) / hi <= eps) + return (hi <= 0 || (hi - lo) / hi <= e) } + function L(b) { return substr(lab[b + 1], 1, 12) } # truncated, for columns + function Lf(b) { return lab[b + 1] } # full, for notes + function pdash( b) { + printf "%-6s %-13s", "------", "-------------" + for (b = 0; b < nb; b++) printf " %12s", "------------" + if (nb == 2) printf " %10s %-15s", "----------", "---------------" + printf "\n" + } + BEGIN { split(ENVIRON["LABELS"], lab, "\n") } # lab[1..nb]; trailing empty ignored { base = $1 SUBSEP $2 - if ($3 == "compare") { - s2[base] += $7 - c2[base]++ - if ($9 != "NA") tw2 += $9 # total wall-clock elapsed, compare - } else if ($3 == "branch") { - s1[base] += $7 - c1[base]++ - if ($9 != "NA") tw1 += $9 # total wall-clock elapsed, branch - } + b = $3 + 0 + s[base, b] += $7 + c[base, b]++ + if ($9 != "NA") tw[b] += $9 # total wall-clock elapsed, per binary } END { - split("solve calc", solvers, " ") - nfiles = split(files, filearr, " ") + printf "%-6s %-13s", "solver", "file" + for (b = 0; b < nb; b++) printf " %12s", L(b) + if (nb == 2) printf " %10s %-15s", "ratio", "note" + printf "\n" + pdash() + split("solve calc", solvers, " ") + nfiles = split(files, farr, " ") for (si = 1; si <= 2; si++) { for (fi = 1; fi <= nfiles; fi++) { - base = solvers[si] SUBSEP filearr[fi] - if (!(base in c2) || !(base in c1)) continue - # Every member of s1, c1, s2, and c2 should be positive. - # If not, it will be due to rounding to zero. To fix, update - # TestTimer.cpp to accumulate microseconds rather than milliseconds. - u2 = s2[base] / c2[base] - u1 = s1[base] / c1[base] - cmp_branch = u2 / u1 - if (within_epsilon(u1, u2)) { - note = "equal" - } else if (cmp_branch >= 1) { - note = note_branch " faster" - } else { - note = note_compare " faster" + base = solvers[si] SUBSEP farr[fi] + ok = 1 + for (b = 0; b < nb; b++) if (!((base, b) in c)) ok = 0 + if (!ok) continue + # Every average should be positive; a zero is rounding and means + # TestTimer.cpp should accumulate microseconds, not milliseconds. + printf "%-6s %-13s", solvers[si], farr[fi] + for (b = 0; b < nb; b++) { u[b] = s[base, b] / c[base, b]; printf " %12.2f", u[b] } + if (nb == 2) { + r = u[1] / u[0] + if (within_epsilon(u[0], u[1])) note = "equal" + else if (r >= 1) note = Lf(0) " faster" + else note = Lf(1) " faster" + printf " %10s %-15s", sprintf("%.2fx", r), note } - sp = sprintf("%9.2fx", cmp_branch) - printf "%-6s %-13s %12.2f %12.2f %10s %-15s\n", - solvers[si], filearr[fi], u2, u1, sp, note + printf "\n" } } + pdash() # Total elapsed (wall-clock seconds) summed per binary across all runs. - printf "%-6s %-13s %12s %12s %10s %-15s\n", - "------", "-------------", "------------", "------------", "----------", "---------------" - if (tw2 > 0 && tw1 > 0) { - tnote = "" - if (within_epsilon(tw1, tw2)) tnote = "equal" - else if (tw2 / tw1 >= 1) tnote = note_branch " faster" - else tnote = note_compare " faster" - tsp = sprintf("%9.2fx", tw2 / tw1) - printf "%-6s %-13s %12.2f %12.2f %10s %-15s\n", - "TOTAL", "elapsed (s)", tw2, tw1, tsp, tnote - } else { - printf "%-6s %-13s %12.2f %12.2f %10s %-15s\n", - "TOTAL", "elapsed (s)", tw2, tw1, "", "" - } - } - ' "$RESULTS" -elif [[ "$DRY_RUN" != "1" ]]; then - echo # single-binary summary (no --compare) - br_label="branch_avg" - if [[ -n "$git_branch" && "$git_branch" != "unknown" ]]; then - br_label="${git_branch:0:12}" - fi - - echo "Summary (avg user ms)" - echo "============================================================" - printf "%-6s %-13s %12s %6s\n" "solver" "file" "$br_label" "runs" - printf "%-6s %-13s %12s %6s\n" \ - "------" "-------------" "------------" "------" - - awk -F'\t' -v files="${FILES[*]}" ' - $3 == "branch" { - base = $1 SUBSEP $2 - s[base] += $7 - c[base]++ - if ($9 != "NA") tw += $9 # total wall-clock elapsed - } - END { - split("solve calc", solvers, " ") - nfiles = split(files, filearr, " ") - for (si = 1; si <= 2; si++) { - for (fi = 1; fi <= nfiles; fi++) { - base = solvers[si] SUBSEP filearr[fi] - if (!(base in c)) continue - printf "%-6s %-13s %12.2f %6d\n", - solvers[si], filearr[fi], s[base] / c[base], c[base] + printf "%-6s %-13s", "TOTAL", "elapsed (s)" + allpos = 1 + for (b = 0; b < nb; b++) { printf " %12.2f", tw[b] + 0; if (!(tw[b] > 0)) allpos = 0 } + if (nb == 2) { + if (allpos) { + r = tw[1] / tw[0] + if (within_epsilon(tw[0], tw[1])) tnote = "equal" + else if (r >= 1) tnote = Lf(0) " faster" + else tnote = Lf(1) " faster" + printf " %10s %-15s", sprintf("%.2fx", r), tnote + } else { + printf " %10s %-15s", "", "" } } - printf "%-6s %-13s %12s %6s\n", - "------", "-------------", "------------", "------" - printf "%-6s %-13s %12.2f %6s\n", "TOTAL", "elapsed (s)", tw, "" + printf "\n" } ' "$RESULTS" fi From fbcb54e6377e35e0f4428a70ce0f6b3b5661d31f Mon Sep 17 00:00:00 2001 From: Adam Wildavsky Date: Tue, 30 Jun 2026 22:49:57 -0500 Subject: [PATCH 19/28] benchmark.sh: only benchmark the current checkout when no spec is given Previously a lone --branch or --compare was compared against the current checkout. Now the checkout is included only when neither flag is specified; one or more specs benchmark exactly those binaries (first is baseline). --reverse now requires at least two binaries. Co-authored-by: Cursor --- benchmark.sh | 42 +++++++++++------------------------------- 1 file changed, 11 insertions(+), 31 deletions(-) diff --git a/benchmark.sh b/benchmark.sh index 59bd8dbb..13421499 100755 --- a/benchmark.sh +++ b/benchmark.sh @@ -118,8 +118,8 @@ Options: (e.g. -- -n 8 -r for 8 threads and slow-board report) --branch and --compare may be given any number of times and combined freely; the -binaries are benchmarked in the order specified and the first is the baseline. With -exactly one spec, the current checkout is also benchmarked as the baseline. With two +binaries are benchmarked in the order specified and the first is the baseline. The +current checkout is benchmarked by default only when neither flag is given. With two binaries the summary adds a ratio and a "faster" note; with three or more it shows only the per-binary averages (no note). @@ -239,8 +239,8 @@ if (( nspecs > 0 )); then done fi -if [[ "$REVERSE" == "1" && "$nspecs" -eq 0 ]]; then - echo "error: --reverse requires at least one --branch or --compare" >&2 +if [[ "$REVERSE" == "1" && "$nspecs" -lt 2 ]]; then + echo "error: --reverse requires at least two binaries (two or more --branch/--compare)" >&2 exit 1 fi @@ -361,11 +361,10 @@ new_tmp_bin() { # Build the ordered list of binaries to benchmark into BIN_LABELS / BIN_PATHS. # Binary-set selection: -# (no spec) : one binary, the current checkout -# one spec : current checkout (baseline) + that spec -# two or more specs : exactly those specs in order; current ignored -# In all cases the first binary is the baseline. --branch specs are built from -# git; --compare specs are existing binary paths. +# (no spec) : one binary, the current checkout (the default) +# one or more specs : exactly those specs, in order; the checkout is ignored +# The first binary is the baseline. --branch specs are built from git; --compare +# specs are existing binary paths. build_binaries() { BIN_LABELS=() BIN_PATHS=() @@ -380,28 +379,9 @@ build_binaries() { git_prep_for_branches fi - local kind val t - if (( nspecs == 1 )); then - # Current checkout is the baseline; the single spec is the second binary. - BIN_PATHS=("$BRANCH") - BIN_LABELS=("$(current_label)") - kind="${SPEC_KINDS[0]}" - val="${SPEC_VALS[0]}" - if [[ "$kind" == "branch" ]]; then - t="$(new_tmp_bin)" - build_branch_binary "$val" "$t" - restore_branch 1 # rebuild current as the baseline binary - BIN_PATHS+=("$t") - BIN_LABELS+=("$val") - else - BIN_PATHS+=("$val") - BIN_LABELS+=("$(label_for_path "$val")") - fi - return 0 - fi - - # Two or more specs: benchmark exactly those, in order; ignore the checkout. - local i + # Benchmark exactly the specified binaries, in order; the current checkout is + # only used when no spec is given at all. + local i kind val t for i in "${!SPEC_KINDS[@]}"; do kind="${SPEC_KINDS[$i]}" val="${SPEC_VALS[$i]}" From 373255facbdce8b5ae10dc04e10ae8363ae2b5cb Mon Sep 17 00:00:00 2001 From: Adam Wildavsky Date: Fri, 3 Jul 2026 18:47:30 -0400 Subject: [PATCH 20/28] Updated usage comment. --- benchmark.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/benchmark.sh b/benchmark.sh index 13421499..18bd0aae 100755 --- a/benchmark.sh +++ b/benchmark.sh @@ -3,9 +3,8 @@ # # Runs all combinations of solver (solve, calc) and hand file # (list100/1000/…/1), largest files first. Always prints a summary. Per-run -# timing rows and build (git/bazel) output are shown only with --details; -# otherwise build output is captured to a log (surfaced only on failure) and -# per-run rows are suppressed. Does not pass dtest options unless given after +# timing rows and build (git/bazel) output are shown transiently, then hidden +# unless the --details flag is set. Passes dtest options if given after # "--" (see below). # # Usage: From f923ef02d218fd804dcc21b5e8d0566b0662accf Mon Sep 17 00:00:00 2001 From: Adam Wildavsky Date: Fri, 3 Jul 2026 19:22:37 -0400 Subject: [PATCH 21/28] benchmark.sh: use Bash time keyword instead of external time binary Avoid depending on /usr/bin/time by capturing wall-clock elapsed with TIMEFORMAT=%R on a side channel while dtest output stays separate. Co-authored-by: Cursor --- benchmark.sh | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/benchmark.sh b/benchmark.sh index 18bd0aae..c250e1f8 100755 --- a/benchmark.sh +++ b/benchmark.sh @@ -60,6 +60,7 @@ DTEST_EXTRA=() # Cleanup state (set later). The EXIT trap restores the original git branch if # --branch switched away, and removes temp files. RESULTS="" +TIME_FILE="" ORIG_BRANCH="" TMP_BINS=() BUILD_LOG="" @@ -85,6 +86,7 @@ cleanup() { fi [[ -n "$BUILD_LOG" ]] && rm -f "$BUILD_LOG" [[ -n "$RESULTS" ]] && rm -f "$RESULTS" + [[ -n "$TIME_FILE" ]] && rm -f "$TIME_FILE" return 0 } trap cleanup EXIT @@ -492,6 +494,7 @@ for f in "${FILES[@]}"; do done RESULTS="$(mktemp "${TMPDIR:-/tmp}/dds-benchmark.XXXXXX")" +TIME_FILE="$(mktemp "${TMPDIR:-/tmp}/dds-benchmark-time.XXXXXX")" # Removal handled by the cleanup() EXIT trap installed near the top. parse_dtest_output() { @@ -529,12 +532,11 @@ run_dtest() { return 0 fi - # Wrap with `time -p` so we can capture wall-clock elapsed for this run. Its - # "real/user/sys" lines go to the merged output but do not collide with the - # dtest lines parse_dtest_output looks for. - # - local out - if ! out="$(command time -p "${cmd[@]}" 2>&1)"; then + # Bash time keyword (TIMEFORMAT=%R) captures wall-clock seconds on a side + # channel; dtest stdout/stderr stay separate for parse_dtest_output. + local out wall + local TIMEFORMAT='%R' + if ! { time out="$("${cmd[@]}" 2>&1)"; } 2>"$TIME_FILE"; then echo "error: dtest failed: ${cmd[*]}" >&2 echo "$out" >&2 exit 1 @@ -546,8 +548,7 @@ run_dtest() { if [[ "$parsed_user" == "NA" || "$parsed_sys" == "NA" ]]; then echo "warning: incomplete dtest timing output: ${cmd[*]}" >&2 fi - local wall - wall="$(awk '/^real[[:space:]]/ { print $2; exit }' <<<"$out")" + wall="$(<"$TIME_FILE")" [[ -z "$wall" ]] && wall="NA" echo "$parsed $wall" } From 556e719ee5a857c22a9037abfc974a1505769161 Mon Sep 17 00:00:00 2001 From: Adam Wildavsky Date: Fri, 3 Jul 2026 19:25:50 -0400 Subject: [PATCH 22/28] benchmark.sh: write alternate-screen escapes to stdout Avoid /dev/tty, which can fail without a controlling terminal and abort the script under set -e; this path only runs when stdout is already a tty. Co-authored-by: Cursor --- benchmark.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmark.sh b/benchmark.sh index c250e1f8..b4fa61ed 100755 --- a/benchmark.sh +++ b/benchmark.sh @@ -619,7 +619,7 @@ if [[ "$DRY_RUN" != "1" ]]; then fi if [[ "$ALT_SCREEN" == "1" ]]; then - printf '\033[?1049h\033[H\033[2J' >/dev/tty # enter alt screen, home, clear + printf '\033[?1049h\033[H\033[2J' # enter alt screen, home, clear ALT_SCREEN_ACTIVE=1 fi @@ -669,7 +669,7 @@ done # Return to the normal screen, discarding the live dtest output, before the # summary so the final output is just the header and the summary. if [[ "$ALT_SCREEN" == "1" ]]; then - printf '\033[?1049l' >/dev/tty + printf '\033[?1049l' ALT_SCREEN_ACTIVE=0 fi From b817bf7b397c1831427ecc1063e436f3c935c355 Mon Sep 17 00:00:00 2001 From: Adam Wildavsky Date: Fri, 3 Jul 2026 19:36:29 -0400 Subject: [PATCH 23/28] benchmark.sh: rename per-run "ver" column to branch/binary/label Choose the header from how binaries were specified so multi-branch runs are not mislabeled. Co-authored-by: Cursor --- benchmark.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/benchmark.sh b/benchmark.sh index b4fa61ed..c54697ef 100755 --- a/benchmark.sh +++ b/benchmark.sh @@ -408,6 +408,13 @@ fi build_binaries num_bins=${#BIN_PATHS[@]} +if (( nspecs == 0 || nbranch == nspecs )); then + RUN_LABEL_COL="branch" +elif (( nbranch == 0 )); then + RUN_LABEL_COL="binary" +else + RUN_LABEL_COL="label" +fi if (( nbranch > 0 )); then BUILD=0 # build already done as part of the branch workflow fi @@ -557,7 +564,7 @@ show_run_lines=1 print_run_table_header() { printf "%-6s %-13s %-12s %8s %8s %10s %6s %s\n" \ - "solver" "file" "ver" "user_ms" "sys_ms" "avg_user" "ratio" "run" + "solver" "file" "$RUN_LABEL_COL" "user_ms" "sys_ms" "avg_user" "ratio" "run" printf "%-6s %-13s %-12s %8s %8s %10s %6s %s\n" \ "------" "-------------" "------------" "--------" "--------" "----------" "------" "---" } From c58253b86cbaf9d2ba38d864c42f8fc041608b48 Mon Sep 17 00:00:00 2001 From: Adam Wildavsky Date: Fri, 3 Jul 2026 20:02:02 -0400 Subject: [PATCH 24/28] benchmark.sh: harden --branch refs against git option parsing Reject refs starting with '-' and pass '--' to git checkout and rev-parse so valid refnames cannot be mistaken for flags. Co-authored-by: Cursor --- benchmark.sh | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/benchmark.sh b/benchmark.sh index c54697ef..49e43b28 100755 --- a/benchmark.sh +++ b/benchmark.sh @@ -78,7 +78,7 @@ cleanup() { || git -C "$ROOT" rev-parse HEAD 2>/dev/null || true)" if [[ -n "$cur" && "$cur" != "$ORIG_BRANCH" ]]; then echo "Restoring git branch '$ORIG_BRANCH'..." >&2 - git -C "$ROOT" checkout "$ORIG_BRANCH" >/dev/null 2>&1 || true + git -C "$ROOT" checkout -- "$ORIG_BRANCH" >/dev/null 2>&1 || true fi fi if ((${#TMP_BINS[@]} > 0)); then @@ -164,8 +164,13 @@ while [[ $# -gt 0 ]]; do ;; --branch) shift + val="${1:?missing value for --branch}" + if [[ "$val" == -* ]]; then + echo "error: --branch ref must not start with '-': $val" >&2 + exit 1 + fi SPEC_KINDS+=("branch") - SPEC_VALS+=("${1:?missing value for --branch}") + SPEC_VALS+=("$val") shift ;; --compare) @@ -254,7 +259,7 @@ fi # a non-tty here and emits plain, line-based output. The short "Building..." # labels are kept as progress markers. bazel_dtest() { ( cd "$ROOT" && bazel build //library/tests:dtest ); } -checkout_and_build() { git -C "$ROOT" checkout "$1" && bazel_dtest; } +checkout_and_build() { git -C "$ROOT" checkout -- "$1" && bazel_dtest; } run_build() { if [[ "$DETAILS" == "1" ]]; then @@ -321,7 +326,7 @@ git_prep_for_branches() { done for i in "${!SPEC_VALS[@]}"; do if [[ "${SPEC_KINDS[$i]}" == "branch" ]] \ - && ! git -C "$ROOT" rev-parse --verify --quiet "${SPEC_VALS[$i]}" >/dev/null; then + && ! git -C "$ROOT" rev-parse --verify --quiet -- "${SPEC_VALS[$i]}" >/dev/null; then echo "error: --branch: unknown git ref '${SPEC_VALS[$i]}'" >&2 exit 1 fi @@ -349,7 +354,7 @@ restore_branch() { run_build checkout_and_build "$ORIG_BRANCH" else echo "Restoring '$ORIG_BRANCH'..." >&2 - run_build git -C "$ROOT" checkout "$ORIG_BRANCH" + run_build git -C "$ROOT" checkout -- "$ORIG_BRANCH" fi } From b473a03329aebca997e25b19edf0b0f8d488c51f Mon Sep 17 00:00:00 2001 From: Adam Wildavsky Date: Sat, 4 Jul 2026 12:57:08 -0400 Subject: [PATCH 25/28] benchmark.sh: fix branch checkout broken by checkout -- git checkout -- treats the argument as a pathspec, not a ref, so --branch builds never switched branches. Keep rejecting refs starting with '-' and the rev-parse -- guard from the prior commit. Co-authored-by: Cursor --- benchmark.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/benchmark.sh b/benchmark.sh index 49e43b28..c698131d 100755 --- a/benchmark.sh +++ b/benchmark.sh @@ -78,7 +78,7 @@ cleanup() { || git -C "$ROOT" rev-parse HEAD 2>/dev/null || true)" if [[ -n "$cur" && "$cur" != "$ORIG_BRANCH" ]]; then echo "Restoring git branch '$ORIG_BRANCH'..." >&2 - git -C "$ROOT" checkout -- "$ORIG_BRANCH" >/dev/null 2>&1 || true + git -C "$ROOT" checkout "$ORIG_BRANCH" >/dev/null 2>&1 || true fi fi if ((${#TMP_BINS[@]} > 0)); then @@ -259,7 +259,7 @@ fi # a non-tty here and emits plain, line-based output. The short "Building..." # labels are kept as progress markers. bazel_dtest() { ( cd "$ROOT" && bazel build //library/tests:dtest ); } -checkout_and_build() { git -C "$ROOT" checkout -- "$1" && bazel_dtest; } +checkout_and_build() { git -C "$ROOT" checkout "$1" && bazel_dtest; } run_build() { if [[ "$DETAILS" == "1" ]]; then @@ -354,7 +354,7 @@ restore_branch() { run_build checkout_and_build "$ORIG_BRANCH" else echo "Restoring '$ORIG_BRANCH'..." >&2 - run_build git -C "$ROOT" checkout -- "$ORIG_BRANCH" + run_build git -C "$ROOT" checkout "$ORIG_BRANCH" fi } From 83d519905c6280afd4bdfb0314f0286b9d2d4a1c Mon Sep 17 00:00:00 2001 From: Adam Wildavsky Date: Sat, 4 Jul 2026 13:02:30 -0400 Subject: [PATCH 26/28] Updated comment --- benchmark.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/benchmark.sh b/benchmark.sh index c698131d..7bb67e97 100755 --- a/benchmark.sh +++ b/benchmark.sh @@ -735,6 +735,7 @@ if [[ "$DRY_RUN" != "1" ]]; then ok = 1 for (b = 0; b < nb; b++) if (!((base, b) in c)) ok = 0 if (!ok) continue + # We intentionally do not test for averages that round to 0. # Every average should be positive; a zero is rounding and means # TestTimer.cpp should accumulate microseconds, not milliseconds. printf "%-6s %-13s", solvers[si], farr[fi] From eb7847dfabf507240308657dc1d3fa2d64c0d9fc Mon Sep 17 00:00:00 2001 From: Adam Wildavsky Date: Sat, 4 Jul 2026 13:10:30 -0400 Subject: [PATCH 27/28] benchmark.sh: fix branch ref validation broken by rev-parse -- git rev-parse treats arguments after -- as paths, so valid branch names were rejected. Parse-time rejection of refs starting with '-' is enough. Co-authored-by: Cursor --- benchmark.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark.sh b/benchmark.sh index 7bb67e97..6eb6de0b 100755 --- a/benchmark.sh +++ b/benchmark.sh @@ -326,7 +326,7 @@ git_prep_for_branches() { done for i in "${!SPEC_VALS[@]}"; do if [[ "${SPEC_KINDS[$i]}" == "branch" ]] \ - && ! git -C "$ROOT" rev-parse --verify --quiet -- "${SPEC_VALS[$i]}" >/dev/null; then + && ! git -C "$ROOT" rev-parse --verify --quiet "${SPEC_VALS[$i]}" >/dev/null; then echo "error: --branch: unknown git ref '${SPEC_VALS[$i]}'" >&2 exit 1 fi From dd038c766b2db1d05ab67972cc659bcef375e388 Mon Sep 17 00:00:00 2001 From: Adam Wildavsky Date: Sat, 4 Jul 2026 13:13:12 -0400 Subject: [PATCH 28/28] Updated comment --- benchmark.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark.sh b/benchmark.sh index 6eb6de0b..4169d7cc 100755 --- a/benchmark.sh +++ b/benchmark.sh @@ -105,7 +105,7 @@ Options: --repeats N Runs per combination per binary (default: 1; env: REPEATS) --max-deals N Include list10^n.txt files with 10^n <= N (default: 100; env: MAX_DEALS) (alias: --max_deals) - --build Build branch dtest only (bazel build //library/tests:dtest) + --build Build dtest for the current checkout (bazel build //library/tests:dtest) --branch NAME Git branch to build and benchmark ("." means the current branch). Repeatable. Each named branch is checked out, dtest is built and its binary saved, then the original branch is restored (requires a