many: run nested tests for UC22

Backported from #153

Author: alfonsosanchezbeato

.github/workflows/tests.yaml

@@ -23,11 +23,16 @@ jobs:
           mkdir "${{ github.workspace }}"
       - uses: actions/checkout@v2
 
-      - name: Build snap
+      - name: x86 build
         run: |
           spread -artifacts=./artifacts google-nested:tests/spread/build/
           find ./artifacts -type f -name "*.artifact" -exec cp {} "${{ github.workspace }}" \;
       
+      - name: arm64 build
+        run: |
+          spread-arm -artifacts=./artifacts google-nested-arm:tests/spread/build/
+          find ./artifacts -type f -name "*.artifact" -exec cp {} "${{ github.workspace }}" \;
+
       - uses: actions/upload-artifact@v2
         with:
           name: core-snap
@@ -94,6 +99,9 @@ jobs:
       - name: Build image
         working-directory: '${{ github.workspace }}'
         run: |
+          PROJECT_PATH=$PWD/core-base
+          # For os.query and others
+          PATH=$PATH:$PROJECT_PATH/tests/lib/external/snapd-testing-tools/tools/
           . "core-base/tests/lib/prepare-utils.sh"
 
           echo "************* INSTALLING DEPS *************"

spread.yaml

@@ -3,7 +3,7 @@ project: core22
 environment:
   SETUPDIR: /home/core22
   PROJECT_PATH: $SETUPDIR
-  PATH: $PATH:$PROJECT_PATH/tests/bin
+  PATH: $PATH:$PROJECT_PATH/tests/bin:$PROJECT_PATH/tests/lib/external/snapd-testing-tools/tools/
   TESTSLIB: $PROJECT_PATH/tests/lib
   SNAP_BRANCH: "edge" # stable/edge/beta
   UC_VERSION: 22
@@ -16,13 +16,26 @@ backends:
     type: google
     key: '$(HOST: echo "$SPREAD_GOOGLE_KEY")'
     location: snapd-spread/us-east1-b
-    plan: n2-standard-2
+    plan: n2-standard-4
     halt-timeout: 2h
     systems:
       - ubuntu-22.04-64:
+          workers: 4
           image: ubuntu-2204-64-virt-enabled
           storage: 20G
 
+  google-nested-arm:
+    type: google
+    key: '$(HOST: echo "$SPREAD_GOOGLE_KEY")'
+    location: snapd-spread/us-central1-a
+    plan: t2a-standard-4
+    halt-timeout: 2h
+    systems:
+      - ubuntu-22.04-arm-64:
+          workers: 4
+          image: ubuntu-2204-arm-64-virt-enabled
+          storage: 25G
+
   qemu-nested:
     type: qemu
     # TODO:UC22: uc22 needs 2G or grub will not loopback the kernel snap
@@ -83,7 +96,7 @@ prepare: |
   unset REBOOT
 
 warn-timeout: 40m
-kill-timeout: 50m
+kill-timeout: 60m
 
 suites:
   tests/spread/build/:

tests/lib/nested.sh

@@ -11,7 +11,7 @@ execute_remote(){
 
 wait_for_ssh(){
     local service_name="$1"
-    retry=800
+    retry=1800
     wait=1
     while ! execute_remote true; do
         if ! systemctl is-active "$service_name"; then
@@ -54,7 +54,7 @@ cleanup_nested_core_vm(){
         # remove the swtpm
         # TODO: we could just remove/reset the swtpm instead of removing the snap 
         # wholesale
-        snap remove swtpm-mvo
+        snap remove test-snapd-swtpm
     fi
 
     # delete the image file
@@ -74,13 +74,10 @@ start_nested_core_vm_unit(){
     cp "${SETUPDIR}/pc.img" "${IMAGE_FILE}"
 
     # use only 2G of RAM for qemu-nested
-    if [ "${SPREAD_BACKEND}" = "google-nested" ]; then
-        # Do not enable SMP on GCE as it will cause boot issues. There is most likely
-        # a bug in the combination of the kernel version used in GCE images, combined with
-        # a new qemu version (v6) and OVMF
-        # TODO try again to enable more cores in the future to see if it is fixed
+    if [ "${SPREAD_BACKEND}" = "google-nested" ] ||
+           [ "${SPREAD_BACKEND}" = "google-nested-arm" ]; then
         PARAM_MEM="-m 4096"
-        PARAM_SMP="-smp 1"
+        PARAM_SMP="-smp 4"
     elif [ "${SPREAD_BACKEND}" = "lxd-nested" ]; then
         PARAM_MEM="-m 4096"
         PARAM_SMP="-smp 2"
@@ -103,40 +100,55 @@ start_nested_core_vm_unit(){
     PARAM_SERIAL="-serial file:${WORK_DIR}/serial.log"
     PARAM_TPM=""
 
-    ATTR_KVM=""
-    if [ "$ENABLE_KVM" = "true" ]; then
-        ATTR_KVM=",accel=kvm"
-        # CPU can be defined just when kvm is enabled
-        PARAM_CPU="-cpu host"
-    fi
-
     # TODO: enable ms key booting for i.e. nightly edge jobs ?
-    OVMF_CODE=""
-    OVMF_VARS=""
+    VMF_CODE=""
+    VMF_VARS=""
     if [ "${ENABLE_SECURE_BOOT:-false}" = "true" ]; then
-        OVMF_CODE=".secboot"
+        VMF_CODE=".ms"
     fi
     if [ "${ENABLE_OVMF_SNAKEOIL:-false}" = "true" ]; then
-        OVMF_VARS=".snakeoil"
+        VMF_VARS=".snakeoil"
     fi
 
     mkdir -p "${WORK_DIR}/image/"
-    cp -f "/usr/share/OVMF/OVMF_VARS${OVMF_VARS}.fd" "${WORK_DIR}/image/OVMF_VARS${OVMF_VARS}.fd"
-    PARAM_BIOS="-drive file=/usr/share/OVMF/OVMF_CODE${OVMF_CODE}.fd,if=pflash,format=raw,unit=0,readonly=on -drive file=${WORK_DIR}/image/OVMF_VARS${OVMF_VARS}.fd,if=pflash,format=raw"
-    PARAM_MACHINE="-machine q35${ATTR_KVM} -global ICH9-LPC.disable_s3=1"
+    if os.query is-pc-amd64; then
+        ATTR_KVM=""
+        if [ "$ENABLE_KVM" = "true" ]; then
+            ATTR_KVM=",accel=kvm"
+            # CPU can be defined just when kvm is enabled
+            PARAM_CPU="-cpu host"
+        fi
+        QEMU_BIN=qemu-system-x86_64
+        PARAM_MACHINE="-machine q35${ATTR_KVM} -global ICH9-LPC.disable_s3=1"
+        PARAM_BIOS="-drive file=/usr/share/OVMF/OVMF_CODE${VMF_CODE}.fd,if=pflash,format=raw,unit=0,readonly=on -drive file=${WORK_DIR}/image/OVMF_VARS${VMF_VARS}.fd,if=pflash,format=raw"
+        TPM_DEVICE=tpm-tis
+        cp -f "/usr/share/OVMF/OVMF_VARS${VMF_VARS}.fd" "${WORK_DIR}/image/OVMF_VARS${VMF_VARS}.fd"
+    elif os.query is-arm64; then
+        # Assume arm64
+        # Unfortunately gce does not offer kvm enabled arm64 VMs
+        PARAM_CPU="-cpu cortex-a57"
+        QEMU_BIN=qemu-system-aarch64
+        PARAM_MACHINE="-machine virt"
+        PARAM_BIOS="-drive file=/usr/share/AAVMF/AAVMF_CODE${VMF_CODE}.fd,if=pflash,format=raw,unit=0,readonly=on -drive file=${WORK_DIR}/image/AAVMF_VARS${VMF_VARS}.fd,if=pflash,format=raw"
+        TPM_DEVICE=tpm-tis-device
+        cp -f "/usr/share/AAVMF/AAVMF_VARS${VMF_VARS}.fd" "${WORK_DIR}/image/AAVMF_VARS${VMF_VARS}.fd"
+    else
+        printf "ERROR: unsupported archtecture\n"
+        exit 1
+    fi
 
-    # Unfortunately the swtpm-mvo snap does not work correctly in lxd container. It's not possible
+    # Unfortunately the test-snapd-swtpm snap does not work correctly in lxd container. It's not possible
     # for the socket to come up due to being containerized.
     if [ "${ENABLE_TPM:-false}" = "true" ]; then
-        TPMSOCK_PATH="/var/snap/swtpm-mvo/current/swtpm-sock"
+        TPMSOCK_PATH="/var/snap/test-snapd-swtpm/current/swtpm-sock"
         if [ "${SPREAD_BACKEND}" = "lxd-nested" ]; then
             mkdir -p /tmp/qtpm
             swtpm socket --tpmstate dir=/tmp/qtpm --ctrl type=unixio,path=/tmp/qtpm/sock --tpm2 -d -t
             TPMSOCK_PATH="/tmp/qtpm/sock"
-        elif ! snap list swtpm-mvo > /dev/null; then
-            snap install swtpm-mvo --beta
+        elif ! snap list test-snapd-swtpm > /dev/null; then
+            snap install test-snapd-swtpm --beta
             retry=60
-            while ! test -S /var/snap/swtpm-mvo/current/swtpm-sock; do
+            while ! test -S /var/snap/test-snapd-swtpm/current/swtpm-sock; do
                 retry=$(( retry - 1 ))
                 if [ $retry -le 0 ]; then
                     echo "Timed out waiting for the swtpm socket. Aborting!"
@@ -145,14 +157,15 @@ start_nested_core_vm_unit(){
                 sleep 1
             done
         fi
-        PARAM_TPM="-chardev socket,id=chrtpm,path=${TPMSOCK_PATH} -tpmdev emulator,id=tpm0,chardev=chrtpm -device tpm-tis,tpmdev=tpm0"
+        PARAM_TPM="-chardev socket,id=chrtpm,path=${TPMSOCK_PATH} -tpmdev emulator,id=tpm0,chardev=chrtpm -device $TPM_DEVICE,tpmdev=tpm0"
     fi
 
     PARAM_IMAGE="-drive file=${IMAGE_FILE},cache=none,format=raw,id=disk1,if=none -device virtio-blk-pci,drive=disk1,bootindex=1"
 
     SVC_NAME="nested-vm-$(systemd-escape "${SPREAD_JOB:-unknown}")"
+    # shellcheck disable=SC2086
     if ! systemd-run --service-type=simple --unit="${SVC_NAME}" -- \
-                qemu-system-x86_64 \
+         "$QEMU_BIN" \
                 ${PARAM_SMP} \
                 ${PARAM_CPU} \
                 ${PARAM_MEM} \

tests/lib/prepare-utils.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 set -e
-set -x 
+set -x
 
 SSH_PORT=${SSH_PORT:-8022}
 MON_PORT=${MON_PORT:-8888}
@@ -114,9 +114,20 @@ start_snapd_core_vm() {
     nested_wait_for_snap_command
 }
 
+get_arch() {
+    if os.query is-pc-amd64; then
+        printf amd64
+    elif os.query is-arm64; then
+        printf arm64
+    else
+        printf "ERROR: unsupported archtecture\n"
+        exit 1
+    fi
+}
+
 get_core_snap_name() {
     printf -v date '%(%Y%m%d)T' -1
-    echo "core22_${date}_amd64.snap"
+    echo "core22_${date}_$(get_arch).snap"
 }
 
 install_core22_deps() {
@@ -141,7 +152,7 @@ download_core22_snaps() {
     local snap_branch="$1"
 
     # get the model
-    curl -o ubuntu-core-amd64-dangerous.model https://raw.githubusercontent.com/snapcore/models/master/ubuntu-core-22-amd64-dangerous.model
+    curl -o ubuntu-core-dangerous.model https://raw.githubusercontent.com/snapcore/models/master/ubuntu-core-22-$(get_arch)-dangerous.model
 
     # download neccessary images
     snap download pc-kernel --channel=22/${snap_branch} --basename=upstream-pc-kernel
@@ -209,5 +220,5 @@ build_core22_image() {
         --snap upstream-snapd.snap \
         --snap upstream-pc-kernel.snap \
         --snap upstream-pc-gadget.snap \
-        ubuntu-core-amd64-dangerous.model
+        ubuntu-core-dangerous.model
 }

tests/spread/main/basic/task.yaml

@@ -4,7 +4,7 @@ prepare: |
   # for various utilities
   . "$TESTSLIB/nested.sh"
 
-  # Start the nested UC20 VM
+  # Start the nested UC VM
   start_nested_core_vm_unit
 
   # At this point we are able to SSH to the nested VM, so things probably worked