Skip to content

Commit d43ae0d

Browse files
mblandmbland
authored
Clean test_{runner,scalafmt_helper}, add diagnostic env vars, and improve test_cross_build performance (#1646)
* Clean test_{runner,scalafmt_helper}, add env vars Did a little cleaning up while trying to get protobuf to stop recompiling so often. Didn't impact that problem, but good cleanups regardless. Also added two environment variables to make diagnosing test failures (or slowdowns) easier: - Defining `RULES_SCALA_TEST_ONLY`will cause only one specific test case to run, with full Bazel output. This is helpful for pinpointing and working on a specific test without hunting for it or recreating its commands. - Defining `RULES_SCALA_TEST_VERBOSE` will cause all tests to emit full Bazel output. This is more useful when running all the tests from a specific script from `test/shell/test_*.sh`. Also, today I learned that `SECONDS` is a special Bash variable. Making it `local` screwed it up and showed all tests as taking zero seconds. * Don't clean test_cross_build, shutdown at end This speeds up the `test_cross_build.sh` tests substantially, particularly between runs. Here are the times for two runs before this change; the first is after running `bazel clean` and `bazel shutdown` in the `test_cross_build` directory: ```txt $ /usr/bin/time ./test_cross_build.sh running test test_cross_build Test "test_cross_build" successful (73 sec) running test test_scalafmt Test "test_scalafmt" successful (94 sec) 167.01 real 0.37 user 0.50 sys $ /usr/bin/time ./test_cross_build.sh running test test_cross_build Test "test_cross_build" successful (8 sec) running test test_scalafmt Test "test_scalafmt" successful (92 sec) 99.96 real 0.35 user 0.48 sys ``` Here are the times for two equivalent runs after this change: ```txt $ /usr/bin/time ./test_cross_build.sh running test test_cross_build Test "test_cross_build" successful (71 sec) running test test_scalafmt Test "test_scalafmt" successful (6 sec) 77.50 real 0.33 user 0.44 sys $ /usr/bin/time ./test_cross_build.sh running test test_cross_build Test "test_cross_build" successful (5 sec) running test test_scalafmt Test "test_scalafmt" successful (2 sec) 7.21 real 0.26 user 0.38 sys ``` The effect is even more dramatic when setting compiler flags in `.bazelrc`, as I've tried while building protobuf-v28.3 under Bazel 6.5.0.
1 parent f758956 commit d43ae0d

File tree

3 files changed

+64
-35
lines changed

3 files changed

+64
-35
lines changed

‎test/shell/test_cross_build.sh‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,11 @@ cd test_cross_build
99

1010
function test_cross_build() {
1111
bazel test //...
12-
bazel clean
13-
bazel shutdown;
1412
}
1513

1614
function test_scalafmt() {
15+
bazel build //scalafmt/...
16+
1717
run_formatting scalafmt binary2 binary2
1818
run_formatting scalafmt binary3 binary3
1919
run_formatting scalafmt library2 library2
@@ -24,3 +24,10 @@ function test_scalafmt() {
2424

2525
$runner test_cross_build
2626
$runner test_scalafmt
27+
28+
# `bazel shutdown` used to be in `test_cross_build`, after a `bazel clean`.
29+
# However, the protobuf library tends to rebuild frequently. Not cleaning and
30+
# postponing the shutdown to the end of the script helps avoid rebuilding as
31+
# often, speeding up the tests. It also potentially speeds up debugging by
32+
# preserving the workspace state when a test fails.
33+
bazel shutdown

‎test/shell/test_runner.sh‎

Lines changed: 32 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,10 @@ run_test_ci() {
1313
echo "running test $TEST_ARG"
1414
eval $TEST_ARG &>$log_file &
1515
local test_pid=$!
16+
1617
SECONDS=0
17-
TIMOUT=${TEST_TIMEOUT-60}
18-
test_pulse_printer $! $TIMOUT $TEST_ARG &
18+
test_pulse_printer "$test_pid" "${TEST_TIMEOUT:-60}" $TEST_ARG &
19+
1920
local pulse_printer_pid=$!
2021
local result
2122

@@ -25,7 +26,7 @@ run_test_ci() {
2526
kill $pulse_printer_pid && wait $pulse_printer_pid 2>/dev/null || true
2627
} || return 1
2728

28-
DURATION=$SECONDS
29+
local DURATION=$SECONDS
2930
if [ $result -eq 0 ]; then
3031
echo -e "\n${GREEN}Test \"$TEST_ARG\" successful ($DURATION sec) $NC"
3132
else
@@ -38,9 +39,9 @@ run_test_ci() {
3839

3940
test_pulse_printer() {
4041
# makes sure something is printed to stdout while test is running
41-
local test_pid=$1
42+
local test_pid="$1"
4243
shift
43-
local timeout=$1 # in minutes
44+
local timeout="$1" # in minutes
4445
shift
4546
local count=0
4647

@@ -60,24 +61,42 @@ test_pulse_printer() {
6061
run_test_local() {
6162
# runs the tests locally
6263
set +e
63-
SECONDS=0
64-
TEST_ARG=$@
64+
local TEST_ARG=$@
65+
local RES=''
66+
67+
# This allows us to run a single test case with full Bazel output without
68+
# having to search for it and recreate its command line.
69+
if [[ -n "$RULES_SCALA_TEST_ONLY" &&
70+
"$TEST_ARG" != "$RULES_SCALA_TEST_ONLY" ]]; then
71+
return
72+
fi
73+
6574
echo "running test $TEST_ARG"
66-
RES=$($TEST_ARG 2>&1)
67-
RESPONSE_CODE=$?
68-
DURATION=$SECONDS
75+
SECONDS=0
76+
77+
if [[ -n "$RULES_SCALA_TEST_VERBOSE" || -n "$RULES_SCALA_TEST_ONLY" ]]; then
78+
$TEST_ARG
79+
else
80+
RES="$($TEST_ARG 2>&1)"
81+
fi
82+
83+
local RESPONSE_CODE="$?"
84+
local DURATION="$SECONDS"
85+
6986
if [ $RESPONSE_CODE -eq 0 ]; then
7087
echo -e "${GREEN} Test \"$TEST_ARG\" successful ($DURATION sec) $NC"
7188
else
72-
echo -e "\nLog:\n"
73-
echo "$RES"
89+
if [[ -n "$RES" ]]; then
90+
echo -e "\nLog:\n"
91+
echo "$RES"
92+
fi
7493
echo -e "${RED} Test \"$TEST_ARG\" failed $NC ($DURATION sec) $NC"
7594
exit $RESPONSE_CODE
7695
fi
7796
}
7897

7998
get_test_runner() {
80-
test_env=$1
99+
local test_env="$1"
81100
if [[ "${test_env}" != "ci" && "${test_env}" != "local" ]]; then
82101
echo -e "${RED}test_env must be either 'local' or 'ci'"
83102
exit 1

‎test/shell/test_scalafmt_helper.sh‎

Lines changed: 23 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,57 +1,60 @@
11
backup_unformatted() {
2-
FILE_PATH=$1
3-
FILENAME=$2
4-
cp $FILE_PATH/unformatted/unformatted-$FILENAME.scala $FILE_PATH/unformatted/unformatted-$FILENAME.backup.scala
2+
local FILE_PATH="$1"
3+
local FILENAME="$2"
4+
cp "$FILE_PATH/unformatted/unformatted-$FILENAME.scala" \
5+
"$FILE_PATH/unformatted/unformatted-$FILENAME.backup.scala"
56
}
67

78
restore_unformatted_before_exit() {
8-
FILE_PATH=$1
9-
FILENAME=$2
10-
cp $FILE_PATH/unformatted/unformatted-$FILENAME.backup.scala $FILE_PATH/unformatted/unformatted-$FILENAME.scala
11-
rm -f $FILE_PATH/unformatted/unformatted-$FILENAME.backup.scala
9+
local FILE_PATH="$1"
10+
local FILENAME="$2"
11+
cp "$FILE_PATH/unformatted/unformatted-$FILENAME.backup.scala" \
12+
"$FILE_PATH/unformatted/unformatted-$FILENAME.scala"
13+
rm -f "$FILE_PATH/unformatted/unformatted-$FILENAME.backup.scala"
1214
}
1315

1416
run_formatting() {
1517
set +e
1618

17-
PACKAGE_DIR=$1
18-
RULE_TYPE=$2
19-
FILENAME=$3
19+
local PACKAGE_DIR="$1"
20+
local RULE_TYPE="$2"
21+
local FILENAME="$3"
2022

2123
#on windows scalafmt targets need to be run using bash.
2224
#TODO: improve the scalafmt funcitonality so we don't need to use the run_under mechanism
23-
local run_under=""
25+
local bazel_run=('bazel' 'run')
2426
if is_windows; then
25-
run_under="--run_under=bash"
27+
bazel_run+=('--run_under=bash')
2628
fi
2729

28-
bazel run //$PACKAGE_DIR:formatted-$RULE_TYPE.format-test $run_under
30+
"${bazel_run[@]}" "//$PACKAGE_DIR:formatted-$RULE_TYPE.format-test"
2931
if [ $? -ne 0 ]; then
3032
echo -e "${RED} formatted-$RULE_TYPE.format-test should be a formatted target. $NC"
3133
exit 1
3234
fi
3335

34-
bazel run //$PACKAGE_DIR:unformatted-$RULE_TYPE.format-test $run_under
36+
"${bazel_run[@]}" "//$PACKAGE_DIR:unformatted-$RULE_TYPE.format-test"
3537
if [ $? -eq 0 ]; then
3638
echo -e "${RED} unformatted-$RULE_TYPE.format-test should be an unformatted target. $NC"
3739
exit 1
3840
fi
3941

40-
backup_unformatted $PACKAGE_DIR $FILENAME
42+
backup_unformatted "$PACKAGE_DIR" "$FILENAME"
4143
# format unformatted*.scala
4244

43-
bazel run //$PACKAGE_DIR:unformatted-$RULE_TYPE.format $run_under
45+
"${bazel_run[@]}" "//$PACKAGE_DIR:unformatted-$RULE_TYPE.format"
4446
if [ $? -ne 0 ]; then
4547
echo -e "${RED} unformatted-$RULE_TYPE.format should run formatting. $NC"
46-
restore_unformatted_before_exit $PACKAGE_DIR $FILENAME
48+
restore_unformatted_before_exit "$PACKAGE_DIR" "$FILENAME"
4749
exit 1
4850
fi
4951

50-
diff $FILE_PATH/unformatted/unformatted-$FILENAME.scala $FILE_PATH/formatted/formatted-$FILENAME.scala
52+
diff "$PACKAGE_DIR/unformatted/unformatted-$FILENAME.scala" \
53+
"$PACKAGE_DIR/formatted/formatted-$FILENAME.scala"
5154
if [ $? -ne 0 ]; then
5255
echo -e "${RED} unformatted-$FILENAME.scala should be the same as formatted-$FILENAME.scala after formatting. $NC"
53-
restore_unformatted_before_exit $PACKAGE_DIR $FILENAME
56+
restore_unformatted_before_exit "$PACKAGE_DIR" "$FILENAME"
5457
exit 1
5558
fi
56-
restore_unformatted_before_exit $FILE_PATH $FILENAME
59+
restore_unformatted_before_exit "$PACKAGE_DIR" "$FILENAME"
5760
}

0 commit comments

Comments
 (0)