Merge remote-tracking branch 'origin/dap/drafts/dropshot-update' into dap/dropshot-builder

Author: davepacheco

.github/buildomat/build-and-test.sh

@@ -4,8 +4,14 @@ set -o errexit
 set -o pipefail
 set -o xtrace
 
-# Color the output for easier readability.
-export CARGO_TERM_COLOR=always
+#
+# Set up our PATH for the test suite.
+#
+
+# shellcheck source=/dev/null
+source ./env.sh
+# shellcheck source=/dev/null
+source .github/buildomat/ci-env.sh
 
 target_os=$1
 
@@ -30,14 +36,26 @@ OUTPUT_DIR='/work'
 echo "tests will store non-ephemeral output in $OUTPUT_DIR" >&2
 mkdir -p "$OUTPUT_DIR"
 
-#
-# Set up our PATH for the test suite.
-#
-source ./env.sh
+
 
 banner prerequisites
 ptime -m bash ./tools/install_builder_prerequisites.sh -y
 
+# Do some test runs of the `ls-apis` command.
+#
+# This may require cloning some dependent private repos.  We do this before the
+# main battery of tests because the GitHub tokens required for this only last
+# for an hour so we want to get this done early.
+#
+# (TODO: This makes the build timings we record inaccurate.)
+banner ls-apis
+(
+    source ./tools/include/force-git-over-https.sh;
+    ptime -m cargo xtask ls-apis apis &&
+        ptime -m cargo xtask ls-apis deployment-units &&
+        ptime -m cargo xtask ls-apis servers
+)
+
 #
 # We build with:
 #
@@ -79,19 +97,6 @@ export RUSTC_BOOTSTRAP=1
 # We report build progress to stderr, and the "--timings=json" output goes to stdout.
 ptime -m cargo build -Z unstable-options --timings=json --workspace --tests --locked --verbose 1> "$OUTPUT_DIR/crate-build-timings.json"
 
-# Do some test runs of the `ls-apis` command.
-#
-# This may require cloning some dependent private repos.  We do this before the
-# main battery of tests because the GitHub tokens required for this only last
-# for an hour so we want to get this done early.
-banner ls-apis
-(
-    source ./tools/include/force-git-over-https.sh;
-    ptime -m cargo xtask ls-apis apis &&
-        ptime -m cargo xtask ls-apis deployment-units &&
-        ptime -m cargo xtask ls-apis servers
-)
-
 #
 # We apply our own timeout to ensure that we get a normal failure on timeout
 # rather than a buildomat timeout.  See oxidecomputer/buildomat#8.

.github/buildomat/ci-env.sh

@@ -0,0 +1,6 @@
+# Setup shared across Buildomat CI builds.
+#
+# This file contains environment variables shared across Buildomat CI jobs.
+
+# Color the output for easier readability.
+export CARGO_TERM_COLOR=always

.github/buildomat/jobs/a4x2-deploy.sh

@@ -21,6 +21,9 @@ set -o errexit
 set -o pipefail
 set -o xtrace
 
+# shellcheck source=/dev/null
+source .github/buildomat/ci-env.sh
+
 pfexec mkdir -p /out
 pfexec chown "$UID" /out
 

.github/buildomat/jobs/a4x2-prepare.sh

@@ -22,7 +22,10 @@
 #: ]
 #: enable = false
 
+# shellcheck source=/dev/null
 source ./env.sh
+# shellcheck source=/dev/null
+source .github/buildomat/ci-env.sh
 
 set -o errexit
 set -o pipefail
@@ -91,4 +94,3 @@ for x in ce cr1 cr2 g0 g1 g2 g3 tools omicron-common; do
     tar -czf cargo-bay-$x.tgz cargo-bay/$x
     mv cargo-bay-$x.tgz /out/
 done
-

.github/buildomat/jobs/check-features.sh

@@ -14,6 +14,9 @@ set -o errexit
 set -o pipefail
 set -o xtrace
 
+# shellcheck source=/dev/null
+source .github/buildomat/ci-env.sh
+
 cargo --version
 rustc --version
 

.github/buildomat/jobs/clippy.sh

@@ -16,13 +16,17 @@ set -o errexit
 set -o pipefail
 set -o xtrace
 
-cargo --version
-rustc --version
-
 #
 # Set up our PATH for use with this workspace.
 #
+
+# shellcheck source=/dev/null
 source ./env.sh
+# shellcheck source=/dev/null
+source .github/buildomat/ci-env.sh
+
+cargo --version
+rustc --version
 
 banner prerequisites
 ptime -m bash ./tools/install_builder_prerequisites.sh -y

.github/buildomat/jobs/deploy.sh

@@ -13,7 +13,8 @@
 #:  "%/pool/ext/*/crypt/debug/global/oxide-sled-agent:default.log.*",
 #:  "%/pool/ext/*/crypt/debug/oxz_*/oxide-*.log.*",
 #:  "%/pool/ext/*/crypt/debug/oxz_*/system-illumos-*.log.*",
-#:  "!/pool/ext/*/crypt/debug/oxz_propolis-server_*/*.log.*"
+#:  "!/pool/ext/*/crypt/debug/oxz_propolis-server_*/*.log.*",
+#:  "/tmp/kstat/*.kstat"
 #: ]
 #: skip_clone = true
 #:
@@ -32,6 +33,10 @@ _exit_trap() {
 	local status=$?
 	set +o errexit
 
+	if [[ "x$OPTE_COMMIT" != "x" ]]; then
+		pfexec cp /tmp/opteadm /opt/oxide/opte/bin/opteadm
+	fi
+
 	#
 	# Stop cron in all zones (to stop logadm log rotation)
 	#
@@ -65,12 +70,16 @@ _exit_trap() {
 
 	PORTS=$(pfexec /opt/oxide/opte/bin/opteadm list-ports | tail +2 | awk '{ print $1; }')
 	for p in $PORTS; do
+		pfexec /opt/oxide/opte/bin/opteadm dump-uft -p $p
 		LAYERS=$(pfexec /opt/oxide/opte/bin/opteadm list-layers -p $p | tail +2 | awk '{ print $1; }')
 		for l in $LAYERS; do
 			pfexec /opt/oxide/opte/bin/opteadm dump-layer -p $p $l
 		done
 	done
 
+	mkdir -p /tmp/kstat
+	pfexec kstat -p xde: > /tmp/kstat/xde.kstat
+
 	pfexec zfs list
 	pfexec zpool list
 	pfexec fmdump -eVp
@@ -88,9 +97,30 @@ _exit_trap() {
 
 	for z in $(zoneadm list -n | grep oxz_ntp); do
 		banner "${z/oxz_/}"
-		pfexec zlogin "$z" chronyc tracking
-		pfexec zlogin "$z" chronyc sources
+		pfexec zlogin "$z" chronyc -n tracking
+		pfexec zlogin "$z" chronyc -n sources -a
 		pfexec zlogin "$z" cat /etc/inet/chrony.conf
+		pfexec zlogin "$z" ping -sn oxide.computer 56 1
+		pfexec zlogin "$z" ping -sn 1.1.1.1 56 1
+		pfexec zlogin "$z" /usr/sbin/dig 0.pool.ntp.org @1.1.1.1
+		pfexec zlogin "$z" getent hosts time.cloudfare.com
+
+		# Attempt to get chrony to do some time sync from the CLI with
+		# messages being written to the terminal and with debugging
+		# enabled if the chrony package was built with that option.
+		# Since chronyd on the CLI needs to use the ports that the
+		# service will be using, stop it first (with -s to wait for it
+		# to exit).
+		pfexec /usr/sbin/svcadm -z "$z" disable -s oxide/ntp
+		# Run in dry-run one-shot mode (-Q)
+		pfexec zlogin "$z" /usr/sbin/chronyd -t 10 -ddQ
+		# Run in one-shot mode (-q) -- attempt to set the clock
+		pfexec zlogin "$z" /usr/sbin/chronyd -t 10 -ddq
+		# Run in one-shot mode (-q) but override the configuration
+		# to talk to an explicit external service. This command line is
+		# similar to that used by the pre-flight NTP checks.
+		pfexec zlogin "$z" /usr/sbin/chronyd -t 10 -ddq \
+		    "'pool time.cloudflare.com iburst maxdelay 0.1'"
 	done
 
 	pfexec zlogin sidecar_softnpu cat /var/log/softnpu.log
@@ -104,6 +134,20 @@ z_swadm () {
 	pfexec zlogin oxz_switch /opt/oxide/dendrite/bin/swadm $@
 }
 
+# only set this if you want to override the version of opte/xde installed by the
+# install_opte.sh script
+OPTE_COMMIT="f3002b356da7d0e4ca15beb66a5566a92919baaa"
+if [[ "x$OPTE_COMMIT" != "x" ]]; then
+	curl  -sSfOL https://buildomat.eng.oxide.computer/public/file/oxidecomputer/opte/module/$OPTE_COMMIT/xde
+	pfexec rem_drv xde || true
+	pfexec mv xde /kernel/drv/amd64/xde
+	pfexec add_drv xde || true
+	curl  -sSfOL https://buildomat.eng.oxide.computer/public/file/oxidecomputer/opte/release/$OPTE_COMMIT/opteadm
+	chmod +x opteadm
+	cp opteadm /tmp/opteadm
+	pfexec mv opteadm /opt/oxide/opte/bin/opteadm
+fi
+
 #
 # XXX work around 14537 (UFS should not allow directories to be unlinked) which
 # is probably not yet fixed in xde branch?  Once the xde branch merges from
@@ -150,6 +194,9 @@ cd /opt/oxide/work
 
 ptime -m tar xvzf /input/package/work/package.tar.gz
 
+# shellcheck source=/dev/null
+source .github/buildomat/ci-env.sh
+
 # Ask buildomat for the range of extra addresses that we're allowed to use, and
 # break them up into the ranges we need.
 
@@ -293,7 +340,7 @@ E2E_TLS_CERT="/opt/oxide/sled-agent/pkg/initial-tls-cert.pem"
 #
 pfexec mkdir -p /usr/oxide
 pfexec curl -sSfL -o /usr/oxide/oxide \
-	http://catacomb.eng.oxide.computer:12346/oxide-v0.1.0
+	http://catacomb.eng.oxide.computer:12346/oxide-v0.1.1
 pfexec chmod +x /usr/oxide/oxide
 
 curl -sSfL -o debian-11-genericcloud-amd64.raw \
@@ -371,7 +418,9 @@ done
 
 /usr/oxide/oxide --resolve "$OXIDE_RESOLVE" --cacert "$E2E_TLS_CERT" \
 	project create --name images --description "some images"
-/usr/oxide/oxide --resolve "$OXIDE_RESOLVE" --cacert "$E2E_TLS_CERT" \
+/usr/oxide/oxide \
+    --resolve "$OXIDE_RESOLVE" \
+    --cacert "$E2E_TLS_CERT" \
 	disk import \
 	--path debian-11-genericcloud-amd64.raw \
 	--disk debian11-boot \
@@ -381,7 +430,8 @@ done
 	--image debian11 \
 	--image-description "debian 11 original base image" \
 	--image-os debian \
-	--image-version "11"
+	--image-version "11" \
+	--parallelism 1
 /usr/oxide/oxide --resolve "$OXIDE_RESOLVE" --cacert "$E2E_TLS_CERT" \
 	image promote --project images --image debian11
 

.github/buildomat/jobs/omicron-common.sh

@@ -14,6 +14,9 @@ set -o errexit
 set -o pipefail
 set -o xtrace
 
+# shellcheck source=/dev/null
+source .github/buildomat/ci-env.sh
+
 cargo --version
 rustc --version
 

.github/buildomat/jobs/package.sh

@@ -13,6 +13,9 @@ set -o errexit
 set -o pipefail
 set -o xtrace
 
+# shellcheck source=/dev/null
+source .github/buildomat/ci-env.sh
+
 cargo --version
 rustc --version
 
@@ -48,6 +51,7 @@ mkdir tests
 # deployment phases of buildomat.
 
 files=(
+	.github/buildomat/ci-env.sh
 	out/target/test
 	out/npuzone/*
 	package-manifest.toml

.github/buildomat/jobs/tuf-repo.sh

@@ -47,9 +47,17 @@ set -o errexit
 set -o pipefail
 set -o xtrace
 
+# shellcheck source=/dev/null
+source .github/buildomat/ci-env.sh
+
 cargo --version
 rustc --version
 
+# Before we do _anything_, quickly check that Cargo.lock is properly locked.
+# Most of our tools (including releng!) eventually call `cargo xtask`, which
+# runs without `--locked` and will update the lockfile.
+cargo tree --locked >/dev/null
+
 ptime -m ./tools/install_builder_prerequisites.sh -yp
 source ./tools/include/force-git-over-https.sh