diff --git a/benchmark.sh b/benchmark.sh index e4f2a605..4169d7cc 100755 --- a/benchmark.sh +++ b/benchmark.sh @@ -1,34 +1,45 @@ #!/usr/bin/env bash -# Benchmark dtest performance on one or two binaries. +# Benchmark dtest performance across one or more binaries. # # Runs all combinations of solver (solve, calc) and hand file -# (list100/1000/…/1), largest files first. With --compare, prints summary only -# unless --details; without --compare, prints per-run rows. With --compare and a -# tty, per-run rows appear during the run then are cleared before the summary. -# Does not pass dtest options unless given after "--" (see below). +# (list100/1000/…/1), largest files first. Always prints a summary. Per-run +# timing rows and build (git/bazel) output are shown transiently, then hidden +# unless the --details flag is set. Passes dtest options if given after +# "--" (see below). # # Usage: # ./benchmark.sh # ./benchmark.sh --build # ./benchmark.sh -- -n 8 -r # ./benchmark.sh --build --compare /path/to/other/dtest +# ./benchmark.sh --branch develop -- -n 8 # ./benchmark.sh --compare /path/to/other/dtest --epsilon 1 # ./benchmark.sh --repeats 5 -- -n 4 # REPEATS=3 ./benchmark.sh # # Environment: -# BRANCH Path to branch dtest (default: bazel-bin in this repo) -# COMPARE Optional second dtest binary for comparison +# BRANCH Path to the baseline dtest (default: bazel-bin under the current dir) +# COMPARE Optional extra dtest binary to benchmark (like a trailing --compare) # HANDS_DIR Directory containing list*.txt files (default: ./hands) # REPEATS Runs per combination per binary (default: 1) # MAX_DEALS Include list10^n.txt files with 10^n <= N (default: 100) # DRY_RUN If 1, print commands only -# DETAILS If 1 with --compare, keep per-run rows in output (default: transient on tty) -# EPSILON With --compare, max % diff to treat branch/compare as equal (default: 0.5) +# DETAILS If 1, keep per-run rows and build output (default: 0, summary only) +# EPSILON For a two-binary comparison, max % diff treated as equal (default: 0.5) set -euo pipefail -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# Operate on the current working directory (the repo you invoke this from), +# not the directory the script happens to live in. +ROOT="$(pwd)" + +# Fail fast if ROOT is not the root of a DDS checkout: the git/bazel operations +# and the default branch-binary and hands paths are all relative to it. +if [[ ! -f "$ROOT/MODULE.bazel" || ! -f "$ROOT/library/tests/BUILD.bazel" ]]; then + echo "error: '$ROOT' is not a DDS checkout root" >&2 + echo " cd to the root of the dds repository before running benchmark.sh" >&2 + exit 1 +fi BRANCH="${BRANCH:-$ROOT/bazel-bin/library/tests/dtest}" HANDS_DIR="${HANDS_DIR:-$ROOT/hands}" REPEATS="${REPEATS:-1}" @@ -38,31 +49,81 @@ DETAILS="${DETAILS:-0}" EPSILON="${EPSILON:-0.5}" BUILD=0 REVERSE=0 +# Ordered list of binaries to benchmark, captured in command-line order. Each +# --branch/--compare appends one spec; the first spec is the baseline. +SPEC_KINDS=() # parallel: "branch" (git ref) or "compare" (path to a binary) +SPEC_VALS=() +CLI_COMPARE_GIVEN=0 +REPEATS_GIVEN=0 DTEST_EXTRA=() +# Cleanup state (set later). The EXIT trap restores the original git branch if +# --branch switched away, and removes temp files. +RESULTS="" +TIME_FILE="" +ORIG_BRANCH="" +TMP_BINS=() +BUILD_LOG="" + +cleanup() { + # Leave the alternate screen first so any restore/error messages and the shell + # prompt land on the normal screen. + if [[ "${ALT_SCREEN_ACTIVE:-0}" == "1" ]]; then + printf '\033[?1049l' >/dev/tty 2>/dev/null || true + ALT_SCREEN_ACTIVE=0 + fi + if [[ -n "$ORIG_BRANCH" ]]; then + local cur + cur="$(git -C "$ROOT" symbolic-ref --quiet --short HEAD 2>/dev/null \ + || git -C "$ROOT" rev-parse HEAD 2>/dev/null || true)" + if [[ -n "$cur" && "$cur" != "$ORIG_BRANCH" ]]; then + echo "Restoring git branch '$ORIG_BRANCH'..." >&2 + git -C "$ROOT" checkout "$ORIG_BRANCH" >/dev/null 2>&1 || true + fi + fi + if ((${#TMP_BINS[@]} > 0)); then + rm -f "${TMP_BINS[@]}" + fi + [[ -n "$BUILD_LOG" ]] && rm -f "$BUILD_LOG" + [[ -n "$RESULTS" ]] && rm -f "$RESULTS" + [[ -n "$TIME_FILE" ]] && rm -f "$TIME_FILE" + return 0 +} +trap cleanup EXIT + SOLVERS=(solve calc) usage() { cat <&2 + exit 1 + fi + REPEATS_GIVEN=1 shift REPEATS="${1:?missing value for --repeats}" shift ;; --branch) shift - BRANCH="${1:?missing value for --branch}" + val="${1:?missing value for --branch}" + if [[ "$val" == -* ]]; then + echo "error: --branch ref must not start with '-': $val" >&2 + exit 1 + fi + SPEC_KINDS+=("branch") + SPEC_VALS+=("$val") shift ;; --compare) shift - COMPARE="${1:?missing value for --compare}" + SPEC_KINDS+=("compare") + SPEC_VALS+=("${1:?missing value for --compare}") + CLI_COMPARE_GIVEN=1 shift ;; --max-deals|--max_deals|-max-deals|-max_deals) @@ -151,11 +230,200 @@ if ! [[ "$EPSILON" =~ ^[0-9]+(\.[0-9]+)?$ ]]; then exit 1 fi -if [[ "$REVERSE" == "1" && -z "${COMPARE:-}" ]]; then - echo "error: --reverse requires --compare" >&2 +# A compare binary supplied via the COMPARE env var behaves like a trailing +# --compare PATH, unless --compare was already given on the command line. +if [[ "$CLI_COMPARE_GIVEN" == "0" && -n "${COMPARE:-}" ]]; then + SPEC_KINDS+=("compare") + SPEC_VALS+=("$COMPARE") +fi + +nspecs=${#SPEC_KINDS[@]} +nbranch=0 +if (( nspecs > 0 )); then + for k in "${SPEC_KINDS[@]}"; do + [[ "$k" == "branch" ]] && nbranch=$((nbranch + 1)) + done +fi + +if [[ "$REVERSE" == "1" && "$nspecs" -lt 2 ]]; then + echo "error: --reverse requires at least two binaries (two or more --branch/--compare)" >&2 exit 1 fi +# Check out $1, build dtest, and copy the binary to $2. +# Build output (git checkout + bazel) is noise unless --details was given. +# Trying to show it live and erase it afterward with ANSI does not work: bazel +# drives its own cursor save/restore for its progress display, clobbering any +# saved position, so a restore+clear erases nothing. Instead, capture the +# output to a log and surface it only on failure (or with --details). bazel sees +# a non-tty here and emits plain, line-based output. The short "Building..." +# labels are kept as progress markers. +bazel_dtest() { ( cd "$ROOT" && bazel build //library/tests:dtest ); } +checkout_and_build() { git -C "$ROOT" checkout "$1" && bazel_dtest; } + +run_build() { + if [[ "$DETAILS" == "1" ]]; then + "$@" + return + fi + if [[ -z "$BUILD_LOG" ]]; then + BUILD_LOG="$(mktemp "${TMPDIR:-/tmp}/dds-dtest-build.XXXXXX")" + fi + if ! "$@" >"$BUILD_LOG" 2>&1; then + cat "$BUILD_LOG" >&2 + return 1 + fi +} + +build_branch_binary() { + local name="$1" dest="$2" + local dtest_rel="bazel-bin/library/tests/dtest" + if [[ "$DRY_RUN" == "1" ]]; then + echo "DRY_RUN: git -C $ROOT checkout $name" >&2 + echo "DRY_RUN: (cd $ROOT && bazel build //library/tests:dtest)" >&2 + echo "DRY_RUN: cp -L $ROOT/$dtest_rel $dest" >&2 + return 0 + fi + echo "Building dtest from '$name'..." >&2 + run_build checkout_and_build "$name" + cp -L "$ROOT/$dtest_rel" "$dest" + chmod +x "$dest" +} + +# Display label for a compare binary given by path: its basename, or the raw +# path if basename is somehow empty. +label_for_path() { + local b + b="$(basename -- "$1")" + [[ -n "$b" ]] && printf '%s' "$b" || printf '%s' "$1" +} + +# Label for the current checkout's binary: the git branch, else "branch". +current_label() { + if [[ -n "$git_branch" && "$git_branch" != "unknown" ]]; then + printf '%s' "$git_branch" + else + printf 'branch' + fi +} + +# Validate the git work tree and branch refs before any checkout. Only called +# when at least one --branch spec is present. +git_prep_for_branches() { + if ! git -C "$ROOT" rev-parse --is-inside-work-tree >/dev/null 2>&1; then + echo "error: --branch requires a git work tree at $ROOT" >&2 + exit 1 + fi + ORIG_BRANCH="$(git -C "$ROOT" symbolic-ref --quiet --short HEAD 2>/dev/null \ + || git -C "$ROOT" rev-parse HEAD)" + + local i + # "." is shorthand for the current branch. + for i in "${!SPEC_VALS[@]}"; do + if [[ "${SPEC_KINDS[$i]}" == "branch" && "${SPEC_VALS[$i]}" == "." ]]; then + SPEC_VALS[$i]="$ORIG_BRANCH" + fi + done + for i in "${!SPEC_VALS[@]}"; do + if [[ "${SPEC_KINDS[$i]}" == "branch" ]] \ + && ! git -C "$ROOT" rev-parse --verify --quiet "${SPEC_VALS[$i]}" >/dev/null; then + echo "error: --branch: unknown git ref '${SPEC_VALS[$i]}'" >&2 + exit 1 + fi + done + # Untracked files can also block a checkout ("would be overwritten"), so treat + # any working tree change (tracked or untracked) as non-clean. + if [[ -n "$(git -C "$ROOT" status --porcelain --untracked-files=normal)" ]]; then + echo "error: working tree not clean; commit, stash, or remove changes (tracked or untracked) before using --branch" >&2 + exit 1 + fi +} + +# Restore the original branch. Pass 1 to also rebuild dtest (needed only when +# the current checkout's bazel-bin binary is used as a baseline). +restore_branch() { + local rebuild="$1" + if [[ "$DRY_RUN" == "1" ]]; then + echo "DRY_RUN: git -C $ROOT checkout $ORIG_BRANCH" >&2 + [[ "$rebuild" == "1" ]] && \ + echo "DRY_RUN: (cd $ROOT && bazel build //library/tests:dtest)" >&2 + return 0 + fi + if [[ "$rebuild" == "1" ]]; then + echo "Restoring '$ORIG_BRANCH' and rebuilding..." >&2 + run_build checkout_and_build "$ORIG_BRANCH" + else + echo "Restoring '$ORIG_BRANCH'..." >&2 + run_build git -C "$ROOT" checkout "$ORIG_BRANCH" + fi +} + +new_tmp_bin() { + local t + t="$(mktemp "${TMPDIR:-/tmp}/dds-dtest-bin.XXXXXX")" + TMP_BINS+=("$t") + printf '%s' "$t" +} + +# Build the ordered list of binaries to benchmark into BIN_LABELS / BIN_PATHS. +# Binary-set selection: +# (no spec) : one binary, the current checkout (the default) +# one or more specs : exactly those specs, in order; the checkout is ignored +# The first binary is the baseline. --branch specs are built from git; --compare +# specs are existing binary paths. +build_binaries() { + BIN_LABELS=() + BIN_PATHS=() + + if (( nspecs == 0 )); then + BIN_PATHS=("$BRANCH") + BIN_LABELS=("$(current_label)") + return 0 + fi + + if (( nbranch > 0 )); then + git_prep_for_branches + fi + + # Benchmark exactly the specified binaries, in order; the current checkout is + # only used when no spec is given at all. + local i kind val t + for i in "${!SPEC_KINDS[@]}"; do + kind="${SPEC_KINDS[$i]}" + val="${SPEC_VALS[$i]}" + if [[ "$kind" == "branch" ]]; then + t="$(new_tmp_bin)" + build_branch_binary "$val" "$t" + BIN_PATHS+=("$t") + BIN_LABELS+=("$val") + else + BIN_PATHS+=("$val") + BIN_LABELS+=("$(label_for_path "$val")") + fi + done + if (( nbranch > 0 )); then + restore_branch 0 + fi +} + +git_branch="unknown" +if git -C "$ROOT" rev-parse --is-inside-work-tree >/dev/null 2>&1; then + git_branch="$(git -C "$ROOT" rev-parse --abbrev-ref HEAD 2>/dev/null || echo unknown)" +fi + +build_binaries +num_bins=${#BIN_PATHS[@]} +if (( nspecs == 0 || nbranch == nspecs )); then + RUN_LABEL_COL="branch" +elif (( nbranch == 0 )); then + RUN_LABEL_COL="binary" +else + RUN_LABEL_COL="label" +fi +if (( nbranch > 0 )); then + BUILD=0 # build already done as part of the branch workflow +fi + select_hand_files() { is_power_of_10() { local n="$1" @@ -201,32 +469,34 @@ if [[ "$BUILD" == "1" ]]; then echo "DRY_RUN: (cd $ROOT && bazel build //library/tests:dtest)" >&2 else echo "Building //library/tests:dtest..." >&2 - (cd "$ROOT" && bazel build //library/tests:dtest) + run_build bazel_dtest fi fi if [[ "$DRY_RUN" != "1" ]]; then - if [[ ! -x "$BRANCH" ]]; then - echo "error: branch binary not found or not executable: $BRANCH" >&2 - echo "hint: bazel build //library/tests:dtest" >&2 - exit 1 - fi - - if [[ -n "${COMPARE:-}" && ! -x "$COMPARE" ]]; then - echo "error: compare binary not found or not executable: $COMPARE" >&2 - exit 1 - fi + for i in "${!BIN_PATHS[@]}"; do + if [[ ! -x "${BIN_PATHS[$i]}" ]]; then + echo "error: binary not found or not executable: ${BIN_PATHS[$i]} (${BIN_LABELS[$i]})" >&2 + (( i == 0 )) && echo "hint: bazel build //library/tests:dtest" >&2 + exit 1 + fi + done fi -BIN_PAIRS=("branch:$BRANCH") -if [[ -n "${COMPARE:-}" ]]; then - if [[ "$REVERSE" == "1" ]]; then - BIN_PAIRS=("compare:$COMPARE" "branch:$BRANCH") - else - BIN_PAIRS=("branch:$BRANCH" "compare:$COMPARE") - fi +# Dispatch order of the binaries within each (solver, file, repeat). The +# baseline (index 0) leads by default; --reverse flips it. The summary keys on +# the stored binary index, so order never affects the reported results. +RUN_ORDER=() +for (( i = 0; i < num_bins; i++ )); do + RUN_ORDER+=("$i") +done +if [[ "$REVERSE" == "1" ]]; then + rev=() + for (( i = num_bins - 1; i >= 0; i-- )); do + rev+=("${RUN_ORDER[$i]}") + done + RUN_ORDER=("${rev[@]}") fi -num_bins=${#BIN_PAIRS[@]} for f in "${FILES[@]}"; do if [[ ! -f "$HANDS_DIR/$f" ]]; then @@ -235,13 +505,9 @@ for f in "${FILES[@]}"; do fi done -git_branch="unknown" -if git -C "$ROOT" rev-parse --is-inside-work-tree >/dev/null 2>&1; then - git_branch="$(git -C "$ROOT" rev-parse --abbrev-ref HEAD 2>/dev/null || echo unknown)" -fi - RESULTS="$(mktemp "${TMPDIR:-/tmp}/dds-benchmark.XXXXXX")" -trap 'rm -f "$RESULTS"' EXIT +TIME_FILE="$(mktemp "${TMPDIR:-/tmp}/dds-benchmark-time.XXXXXX")" +# Removal handled by the cleanup() EXIT trap installed near the top. parse_dtest_output() { awk ' @@ -278,8 +544,11 @@ run_dtest() { return 0 fi - local out - if ! out="$("${cmd[@]}" 2>&1)"; then + # Bash time keyword (TIMEFORMAT=%R) captures wall-clock seconds on a side + # channel; dtest stdout/stderr stay separate for parse_dtest_output. + local out wall + local TIMEFORMAT='%R' + if ! { time out="$("${cmd[@]}" 2>&1)"; } 2>"$TIME_FILE"; then echo "error: dtest failed: ${cmd[*]}" >&2 echo "$out" >&2 exit 1 @@ -291,63 +560,49 @@ run_dtest() { if [[ "$parsed_user" == "NA" || "$parsed_sys" == "NA" ]]; then echo "warning: incomplete dtest timing output: ${cmd[*]}" >&2 fi - echo "$parsed" + wall="$(<"$TIME_FILE")" + [[ -z "$wall" ]] && wall="NA" + echo "$parsed $wall" } -progress_lines=0 -TRANSIENT_PROGRESS=0 show_run_lines=1 print_run_table_header() { - printf "%-6s %-13s %7s %8s %8s %10s %6s %s\n" \ - "solver" "file" "ver" "user_ms" "sys_ms" "avg_user" "ratio" "run" - printf "%-6s %-13s %7s %8s %8s %10s %6s %s\n" \ - "------" "-------------" "-------" "--------" "--------" "----------" "------" "---" - if [[ "$TRANSIENT_PROGRESS" == "1" ]]; then - progress_lines=$((progress_lines + 2)) - fi + printf "%-6s %-13s %-12s %8s %8s %10s %6s %s\n" \ + "solver" "file" "$RUN_LABEL_COL" "user_ms" "sys_ms" "avg_user" "ratio" "run" + printf "%-6s %-13s %-12s %8s %8s %10s %6s %s\n" \ + "------" "-------------" "------------" "--------" "--------" "----------" "------" "---" } print_run_row() { - printf "%-6s %-13s %7s %8s %8s %10s %6s %s\n" \ + printf "%-6s %-13s %-12s %8s %8s %10s %6s %s\n" \ "$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8" - if [[ "$TRANSIENT_PROGRESS" == "1" ]]; then - progress_lines=$((progress_lines + 1)) - fi -} - -clear_transient_progress() { - if [[ "$TRANSIENT_PROGRESS" != "1" || $progress_lines -le 0 ]]; then - return - fi - # Cursor rests on a blank line below the last row; erase it, then each table line. - # Do not move up after clearing the topmost line (would hit the header above). - printf '\033[2K' - local i - for (( i = 0; i < progress_lines; i++ )); do - printf '\033[1A\033[2K' - done - progress_lines=0 } echo "DDS dtest benchmark" echo "===================" -printf "%-12s %s\n" "branch:" "$BRANCH" -if [[ -n "${COMPARE:-}" ]]; then - printf "%-12s %s\n" "compare:" "$COMPARE" +# One line per binary, baseline first. The label is the branch name (--branch), +# the binary's basename (--compare), or the current git branch (checkout). +for i in "${!BIN_PATHS[@]}"; do + tag="binary $((i + 1)):" + (( i == 0 )) && tag="baseline:" + printf "%-12s %s\n" "$tag" "${BIN_LABELS[$i]} (${BIN_PATHS[$i]})" +done +if (( num_bins >= 2 )); then if [[ "$DETAILS" == "1" ]]; then - printf "%-12s %s\n" "details:" "on" - elif [[ -t 1 ]]; then - printf "%-12s %s\n" "details:" "transient (cleared before summary)" + printf "%-12s %s\n" "details:" "on (per-run rows + build output)" else printf "%-12s %s\n" "details:" "off (summary only)" fi - if [[ "$REVERSE" == "1" ]]; then - printf "%-12s %s\n" "run order:" "interleaved compare, branch" - else - printf "%-12s %s\n" "run order:" "interleaved branch, compare" + order_str="" + for idx in "${RUN_ORDER[@]}"; do + order_str+="${order_str:+, }${BIN_LABELS[$idx]}" + done + printf "%-12s %s\n" "run order:" "interleaved $order_str" + # The note column (and thus epsilon) only applies to a two-binary comparison. + if (( num_bins == 2 )); then + printf "%-12s %s\n" "epsilon:" "${EPSILON}%" fi - printf "%-12s %s\n" "epsilon:" "${EPSILON}%" fi printf "%-12s %s\n" "hands dir:" "$HANDS_DIR" printf "%-12s %s\n" "max_deals:" "$MAX_DEALS" @@ -359,16 +614,28 @@ if ((${#DTEST_EXTRA[@]} > 0)); then fi echo -show_run_lines=1 -TRANSIENT_PROGRESS=0 -if [[ -n "${COMPARE:-}" && "$DETAILS" != "1" ]]; then - show_run_lines=0 - if [[ -t 1 ]]; then - TRANSIENT_PROGRESS=1 +# The per-run rows are the script's live progress. With --details they are kept +# in the final output. Without --details, on a tty, they are shown on the +# alternate screen so the user sees progress, then discarded when we switch back +# to the main screen just before the summary; off a tty they are suppressed +# (summary only), since there is nothing to hide them. +show_run_lines=0 +ALT_SCREEN=0 +if [[ "$DRY_RUN" != "1" ]]; then + if [[ "$DETAILS" == "1" ]]; then + show_run_lines=1 + elif [[ -t 1 ]]; then + show_run_lines=1 + ALT_SCREEN=1 fi fi -if [[ "$DRY_RUN" != "1" && ( "$show_run_lines" == "1" || "$TRANSIENT_PROGRESS" == "1" ) ]]; then +if [[ "$ALT_SCREEN" == "1" ]]; then + printf '\033[?1049h\033[H\033[2J' # enter alt screen, home, clear + ALT_SCREEN_ACTIVE=1 +fi + +if [[ "$DRY_RUN" != "1" && "$show_run_lines" == "1" ]]; then print_run_table_header fi @@ -386,9 +653,8 @@ for solver in "${SOLVERS[@]}"; do run_label="1/1" fi - for pair in "${BIN_PAIRS[@]}"; do - ver="${pair%%:*}" - bin="${pair#*:}" + for idx in "${RUN_ORDER[@]}"; do + bin="${BIN_PATHS[$idx]}" run_no=$((run_no + 1)) if [[ "$DRY_RUN" == "1" ]]; then @@ -396,73 +662,112 @@ for solver in "${SOLVERS[@]}"; do continue fi - read -r user sys avg ratio < <(run_dtest "$bin" "$solver" "$hands") + read -r user sys avg ratio wall < <(run_dtest "$bin" "$solver" "$hands") - if [[ "$show_run_lines" == "1" || "$TRANSIENT_PROGRESS" == "1" ]]; then - print_run_row "$solver" "$file" "$ver" "$user" "$sys" "$avg" "$ratio" "$run_label" + if [[ "$show_run_lines" == "1" ]]; then + print_run_row "$solver" "$file" "${BIN_LABELS[$idx]:0:12}" \ + "$user" "$sys" "$avg" "$ratio" "$run_label" fi - printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n" \ - "$solver" "$file" "$ver" "$rep" "$user" "$sys" "$avg" "$ratio" \ + # Column 3 is the binary index; the summary keys on it. + printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n" \ + "$solver" "$file" "$idx" "$rep" "$user" "$sys" "$avg" "$ratio" "$wall" \ >>"$RESULTS" done done done done -clear_transient_progress +# Return to the normal screen, discarding the live dtest output, before the +# summary so the final output is just the header and the summary. +if [[ "$ALT_SCREEN" == "1" ]]; then + printf '\033[?1049l' + ALT_SCREEN_ACTIVE=0 +fi -if [[ -n "${COMPARE:-}" && "$DRY_RUN" != "1" ]]; then +if [[ "$DRY_RUN" != "1" ]]; then echo - echo "Summary (branch vs compare, avg user ms)" + echo "Summary (avg user ms)" echo "==============================================================================" - printf "%-6s %-13s %12s %12s %10s %-15s\n" \ - "solver" "file" "compare_avg" "branch_avg" "cmp/branch" "note" - printf "%-6s %-13s %12s %12s %10s %-15s\n" \ - "------" "-------------" "------------" "------------" "----------" "---------------" - - awk -F'\t' -v files="${FILES[*]}" -v epsilon_pct="$EPSILON" ' - function within_epsilon(a, b, eps, hi, lo) { - eps = epsilon_pct / 100 + + # One avg column per binary (baseline first). A ratio + note column are added + # only for a two-binary comparison; with three or more binaries the note has + # no single meaning and is dropped, leaving just the per-binary averages. + # Labels go through the environment (ENVIRON) rather than -v: BSD awk rejects + # newlines in a -v value, and labels are newline-joined. + labels_joined="$(printf '%s\n' "${BIN_LABELS[@]}")" + + LABELS="$labels_joined" awk -F'\t' -v nb="$num_bins" \ + -v files="${FILES[*]}" -v eps="$EPSILON" ' + function within_epsilon(a, b, e, hi, lo) { + e = eps / 100 if (a > b) { hi = a; lo = b } else { hi = b; lo = a } - return (hi <= 0 || (hi - lo) / hi <= eps) + return (hi <= 0 || (hi - lo) / hi <= e) } + function L(b) { return substr(lab[b + 1], 1, 12) } # truncated, for columns + function Lf(b) { return lab[b + 1] } # full, for notes + function pdash( b) { + printf "%-6s %-13s", "------", "-------------" + for (b = 0; b < nb; b++) printf " %12s", "------------" + if (nb == 2) printf " %10s %-15s", "----------", "---------------" + printf "\n" + } + BEGIN { split(ENVIRON["LABELS"], lab, "\n") } # lab[1..nb]; trailing empty ignored { base = $1 SUBSEP $2 - if ($3 == "compare") { - s2[base] += $7 - c2[base]++ - } else if ($3 == "branch") { - s1[base] += $7 - c1[base]++ - } + b = $3 + 0 + s[base, b] += $7 + c[base, b]++ + if ($9 != "NA") tw[b] += $9 # total wall-clock elapsed, per binary } END { - split("solve calc", solvers, " ") - nfiles = split(files, filearr, " ") + printf "%-6s %-13s", "solver", "file" + for (b = 0; b < nb; b++) printf " %12s", L(b) + if (nb == 2) printf " %10s %-15s", "ratio", "note" + printf "\n" + pdash() + split("solve calc", solvers, " ") + nfiles = split(files, farr, " ") for (si = 1; si <= 2; si++) { for (fi = 1; fi <= nfiles; fi++) { - base = solvers[si] SUBSEP filearr[fi] - if (!(base in c2) || !(base in c1)) continue - # Every member of s1, c1, s2, and c2 should be positive. - # If not, it will be due to rounding to zero. To fix, update - # TestTimer.cpp to accumulate microseconds rather than milliseconds. - u2 = s2[base] / c2[base] - u1 = s1[base] / c1[base] - cmp_branch = u2 / u1 - if (within_epsilon(u1, u2)) { - note = "equal" - } else if (cmp_branch >= 1) { - note = "branch faster" - } else { - note = "compare faster" + base = solvers[si] SUBSEP farr[fi] + ok = 1 + for (b = 0; b < nb; b++) if (!((base, b) in c)) ok = 0 + if (!ok) continue + # We intentionally do not test for averages that round to 0. + # Every average should be positive; a zero is rounding and means + # TestTimer.cpp should accumulate microseconds, not milliseconds. + printf "%-6s %-13s", solvers[si], farr[fi] + for (b = 0; b < nb; b++) { u[b] = s[base, b] / c[base, b]; printf " %12.2f", u[b] } + if (nb == 2) { + r = u[1] / u[0] + if (within_epsilon(u[0], u[1])) note = "equal" + else if (r >= 1) note = Lf(0) " faster" + else note = Lf(1) " faster" + printf " %10s %-15s", sprintf("%.2fx", r), note } - sp = sprintf("%9.2fx", cmp_branch) - printf "%-6s %-13s %12.2f %12.2f %10s %-15s\n", - solvers[si], filearr[fi], u2, u1, sp, note + printf "\n" + } + } + + pdash() + # Total elapsed (wall-clock seconds) summed per binary across all runs. + printf "%-6s %-13s", "TOTAL", "elapsed (s)" + allpos = 1 + for (b = 0; b < nb; b++) { printf " %12.2f", tw[b] + 0; if (!(tw[b] > 0)) allpos = 0 } + if (nb == 2) { + if (allpos) { + r = tw[1] / tw[0] + if (within_epsilon(tw[0], tw[1])) tnote = "equal" + else if (r >= 1) tnote = Lf(0) " faster" + else tnote = Lf(1) " faster" + printf " %10s %-15s", sprintf("%.2fx", r), tnote + } else { + printf " %10s %-15s", "", "" } } + printf "\n" } ' "$RESULTS" fi