engineering: Azure Linux 4.0 (AZL4) full enablement stack

.gitattributes

@@ -0,0 +1,22 @@
+* text=auto eol=lf
+
+# Anything that gets executed inside an image must keep LF endings; CRLF
+# on shebang lines breaks the interpreter lookup with `bad interpreter:
+# /bin/bash^M`.
+*.sh text eol=lf
+*.py text eol=lf
+*.service text eol=lf
+*.network text eol=lf
+*.yaml text eol=lf
+*.yml text eol=lf
+
+# Binary artifacts — never normalize.
+*.vhdx binary
+*.cosi binary
+*.qcow2 binary
+*.iso binary
+*.raw binary
+*.png binary
+*.jpg binary
+*.zst binary
+*.patch text eol=lf

.gitignore

@@ -366,4 +366,7 @@ vendor/
 
 # Virtdeploy files
 /tools/vm-netlaunch.yaml
-/tools/virt-deploy-metadata.json
+/tools/virt-deploy-metadata.json
+# AZL4 trident binary baked into test image (built locally)
+tests/images/trident-vm-testimage/base/trident-bin/
+tests/images/trident-vm-testimage/base/osmodifier-bin/

.pipelines/templates/e2e-template.yml

@@ -224,6 +224,44 @@ stages:
       micVersion: ${{ parameters.micVersion }}
       dependsOnStage: ${{ parameters.baseImageArtifactStage }}
 
+  # Build the AZL4 test image (pinned-MIC path).
+  #
+  # TODO(azl4-release): Drop the bespoke build-image-azl4.yml call once AZL4
+  # has feed-published base VHDXes, RPMs, and a released MIC container.
+  # Then this can be a plain build-image.yml call with an azureLinuxVersion
+  # parameter, matching the other testimage stages.
+  #
+  # Gating mirrors the AzL installer ISO below so AZL4 build runs in every
+  # stage type that gates a trunk merge. Previously this only ran on
+  # pr-e2e / ci / pr-e2e-azure, which silently skipped AZL4 in
+  # azl-validation / full-validation — exactly the stage you'd want it.
+  - ${{ if or(eq(parameters.stageType, 'pr-e2e'), eq(parameters.stageType, 'ci'), eq(parameters.stageType, 'pr-e2e-azure'), eq(parameters.stageType, 'azl-validation'), eq(parameters.stageType, 'full-validation')) }}:
+      - template: stages/build_image/build-image-azl4.yml
+        parameters:
+          imageName: trident-vm-grub-testimage-azl4
+          dependsOnStage: ${{ parameters.baseImageArtifactStage }}
+
+      # AZL4 base qcow2 — boot point for the VM offline-init / rollback
+      # path. Same build template as the COSI above; output_format
+      # differs (QCOW2 vs COSI) per the testimages.py registration.
+      - template: stages/build_image/build-image-azl4.yml
+        parameters:
+          imageName: trident-vm-grub-testimage-azl4-base
+          dependsOnStage: ${{ parameters.baseImageArtifactStage }}
+
+      # AZL4 BM-simulated netlaunch test. Uses the AZL3 MOS installer ISO
+      # (built by TridentTestImg_trident_installer below) plus the AZL4
+      # COSI built above. Trident runs from the live MOS environment and
+      # installs the AZL4 COSI onto a fresh virtdeploy VM disk. This is
+      # the same flow we proved out manually on karhu-ubuntu.
+      - template: stages/testing_vm/netlaunch-testing-azl4.yml
+
+      # AZL4 VM offline-init rollback test. The base qcow2 already has
+      # trident's datastore populated by its first-boot offline-init
+      # oneshot, so storm-trident can drive A/B update + rollback against
+      # the AZL4 COSI without the MOS bridge.
+      - template: stages/testing_rollback/vm-testing-azl4.yml
+
   # Build AzL installer ISO (attended and unattended)
   - ${{ if or(eq(parameters.stageType, 'pr-e2e'), eq(parameters.stageType, 'ci'), eq(parameters.stageType, 'pr-e2e-azure'), eq(parameters.stageType, 'azl-validation')) }}:
       - template: stages/azl_installer/azl-installer.yml

.pipelines/templates/stages/build_image/build-image-azl4.yml

@@ -0,0 +1,79 @@
+# AZL4 variant of build-image.yml.
+#
+# Forked from build-image.yml on 2026-05-13. Calls build-image-template-azl4.yml
+# (which uses MCR MIC container + blob-sourced base VHDX) instead of the
+# external test-images repo template.
+#
+# TODO(azl4-merge-back): Merge this back into build-image.yml with an
+# `azureLinuxVersion` parameter switch once AZL4 has feed-published base VHDXes
+# and RPMs.
+
+parameters:
+  - name: imageName
+    type: string
+
+  - name: clones
+    displayName: "Number of clones to generate"
+    type: number
+    default: 2
+
+  - name: dependsOnTrident
+    type: boolean
+    default: true
+
+  - name: dependsOnStage
+    type: string
+    default: ""
+
+stages:
+  - stage: TridentTestImg_${{ replace(parameters.imageName, '-', '_') }}
+    displayName: Build ${{ parameters.imageName }}
+    ${{ if parameters.dependsOnTrident }}:
+      dependsOn:
+        # AZL4 doesn't have RPM publication so we depend on the
+        # trident-binaries artifact (which the GetTridentBinaries stage
+        # produces and copies to artifacts/binaries/trident).
+        - GetTridentBinaries_rpms_amd64
+        # PrepareSSHKeys produces the shared 'ssh-keys' artifact.
+        # build-image-template-azl4.yml stages it into the testimage
+        # tree so qcow2 + cosi builds share the same SSH keypair,
+        # which lets storm-trident SSH into both A/B sides after
+        # update.
+        - PrepareSSHKeys
+        - ${{ if ne(parameters.dependsOnStage, '') }}:
+          - ${{ parameters.dependsOnStage }}
+    ${{ elseif ne(parameters.dependsOnStage, '') }}:
+      dependsOn:
+        - PrepareSSHKeys
+        - ${{ parameters.dependsOnStage }}
+
+    jobs:
+      - job: BuildTridentTestImgAzl4
+        displayName: Build (AZL4 MIC)
+        # Pinned MIC container build adds ~5 min cold-cache. Bump the timeout
+        # accordingly. TODO(azl4-release): lower back to 20 min once we use a
+        # released MIC container.
+        timeoutInMinutes: 30
+        pool:
+          type: linux
+
+        variables:
+          ob_outputDirectory: /tmp/output
+          ob_artifactBaseName: ${{ parameters.imageName }}
+
+        steps:
+          - template: ../common_tasks/checkout_trident.yml
+
+          - task: DownloadPipelineArtifact@2
+            inputs:
+              buildType: current
+              artifactName: trident-binaries
+              targetPath: "$(Build.ArtifactStagingDirectory)/trident-binaries"
+            displayName: Download Trident binaries
+            condition: eq('${{ parameters.dependsOnTrident }}', true)
+
+          - template: build-image-template-azl4.yml
+            parameters:
+              tridentSourceDirectory: $(TRIDENT_SOURCE_DIR)
+              imageName: ${{ parameters.imageName }}
+              clones: ${{ parameters.clones }}

.pipelines/templates/stages/build_image/build-image-template-azl4.yml

@@ -0,0 +1,167 @@
+# AZL4 variant of build-image-template.yml.
+#
+# Forked from build-image-template.yml on 2026-05-13. The AZL3 path pulls the
+# base VHDX from the AzureLinuxArtifacts ADO feed and the Trident RPM from the
+# trident-binaries pipeline artifact, then runs `testimages.py build`. None of
+# that works for AZL4 today because:
+#
+#   1. There is no AzureLinuxArtifacts feed entry for AZL4 base VHDX. We
+#      download from the AZL preview gallery's backing storage account
+#      (azlpubdev2mruiyvi/images-dev) instead. See the BlobImageManifest
+#      registration in tests/images/testimages.py.
+#
+#   2. There is no Trident RPM for AZL4. The binary is baked in via
+#      additionalFiles in tests/images/trident-vm-testimage/base/updateimg-grub-azl4.yaml.
+#
+# TODO(azl4-merge-back): When AZL4 has feed-published base VHDXes and RPMs,
+# fold this template back into build-image-template.yml by adding a
+# `azureLinuxVersion: "4.0"` branch.
+
+parameters:
+  - name: tridentSourceDirectory
+    type: string
+
+  - name: imageName
+    type: string
+
+  - name: clones
+    type: number
+    default: 1
+    displayName: Number of clones to create
+
+  # The AZL4 base VHDX is sourced from the Azure Linux preview gallery's
+  # backing storage account. The pipeline service connection at
+  # $(BLOB_SERVICE_CONNECTION) must have `Storage Blob Data Reader` on
+  # this account. See tests/images/SERVICE-CONNECTION-RUNBOOK.md.
+  - name: blobStorageAccount
+    type: string
+    default: "azlpubdev2mruiyvi"
+
+  - name: blobContainer
+    type: string
+    default: "images-dev"
+
+  - name: blobSubscription
+    type: string
+    # Subscription where the storage account lives. The SC's default
+    # subscription may differ — we explicitly set context before download.
+    default: "e4ab81f8-030f-4593-a8f2-3ea2c7630a19"
+
+  - name: blobServiceConnection
+    type: string
+    # NB: this must be a service connection that exists in the ADO project.
+    # Trident infra needs to create it manually (Karhu can't); see the PR-5
+    # follow-up validation report for the runbook.
+    default: "trident-azl4-blob-reader"
+
+  - name: micContainerTag
+    type: string
+    default: "imagecustomizer:1.4.0-1"
+
+steps:
+  - template: ../common_tasks/avoid-pypi-usage.yml
+
+  - template: common/sfi-enforce-isolation-with-etc-hosts.yaml@platform-pipelines
+
+  # Stage the Trident binary that gets baked into the COSI via additionalFiles.
+  # The trident-binaries artifact comes from the same upstream Trident build
+  # stage the AZL3 path uses; we just copy the binary rather than installing
+  # an RPM.
+  #
+  # TODO(azl4-rpm): replace this binary copy with an RPM install once the
+  # trident-service RPM is packaged for AZL4 (same TODO as in
+  # tests/images/testimages.py registration).
+  - bash: |
+      set -euxo pipefail
+      TRIDENT_BIN_SRC="$(Build.ArtifactStagingDirectory)/trident-binaries"
+      TRIDENT_BIN_DEST="${{ parameters.tridentSourceDirectory }}/tests/images/trident-vm-testimage/base/trident-bin"
+
+      if [ ! -f "$TRIDENT_BIN_SRC/trident" ]; then
+        echo "trident binary not found at $TRIDENT_BIN_SRC/trident"
+        echo "Available artifacts:"
+        find "$TRIDENT_BIN_SRC" -type f 2>/dev/null | head -20 || true
+        exit 1
+      fi
+
+      mkdir -p "$TRIDENT_BIN_DEST"
+      cp "$TRIDENT_BIN_SRC/trident" "$TRIDENT_BIN_DEST/trident"
+      chmod +x "$TRIDENT_BIN_DEST/trident"
+      file "$TRIDENT_BIN_DEST/trident"
+    displayName: "Stage Trident binary into testimage tree"
+    workingDirectory: ${{ parameters.tridentSourceDirectory }}
+
+  # Pull the released MIC container from MCR. AZL4 support is included
+  # in imagecustomizer >= 1.4.0. Tag it locally so testimages.py can
+  # reference it by short name.
+  - bash: |
+      set -euxo pipefail
+      docker pull "mcr.microsoft.com/azurelinux/${{ parameters.micContainerTag }}"
+      docker tag "mcr.microsoft.com/azurelinux/${{ parameters.micContainerTag }}" "${{ parameters.micContainerTag }}"
+    displayName: "Pull MIC container from MCR"
+
+  # Stage the pipeline-wide SSH key into the testimage tree before
+  # MIC runs. testimages.py's generate_ssh_keys() generates a new
+  # keypair UNLESS files/id_rsa.pub already exists at the source path
+  # — in which case it reuses it. By dropping the shared key from the
+  # PrepareSSHKeys artifact here, both the qcow2 base build and the
+  # COSI build end up with the same key baked into testuser's
+  # authorized_keys, so storm-trident's A/B update test can SSH into
+  # both A-side and B-side after the update reboot.
+  #
+  # The matching private key lives at ssh-keys/id_rsa from the
+  # PrepareSSHKeys stage. storm-trident's rollback stage picks it up
+  # the same way for AZL3 builds.
+  - task: DownloadPipelineArtifact@2
+    displayName: "Download shared SSH keys"
+    inputs:
+      buildType: current
+      artifactName: "ssh-keys"
+      targetPath: "$(Build.ArtifactStagingDirectory)/ssh-keys"
+
+  - bash: |
+      set -euxo pipefail
+      SSH_PUB_SRC="$(Build.ArtifactStagingDirectory)/ssh-keys/id_rsa.pub"
+      SSH_PUB_DEST="${{ parameters.tridentSourceDirectory }}/tests/images/trident-vm-testimage/base/files/id_rsa.pub"
+      if [ ! -f "$SSH_PUB_SRC" ]; then
+        echo "shared SSH public key not found at $SSH_PUB_SRC"
+        find "$(Build.ArtifactStagingDirectory)/ssh-keys" -type f
+        exit 1
+      fi
+      cp "$SSH_PUB_SRC" "$SSH_PUB_DEST"
+      echo "Staged shared SSH public key:"
+      cat "$SSH_PUB_DEST"
+    displayName: "Stage shared SSH key into testimage tree"
+    workingDirectory: ${{ parameters.tridentSourceDirectory }}
+
+  # Download the AZL4 base VHDX from the preview gallery's backing storage.
+  # Authenticates via the federated identity attached to the service
+  # connection — no storage keys handled here.
+  #
+  # The SC's default subscription (Polar_ImageTools_Staging) differs from
+  # the storage account's subscription (ControlTower_Test). We must switch
+  # context so `az storage blob list` resolves the account correctly.
+  - task: AzureCLI@2
+    displayName: "Download AZL4 base VHDX from blob"
+    inputs:
+      azureSubscription: ${{ parameters.blobServiceConnection }}
+      scriptType: bash
+      scriptLocation: inlineScript
+      workingDirectory: ${{ parameters.tridentSourceDirectory }}
+      inlineScript: |
+        set -euxo pipefail
+        az account set --subscription "${{ parameters.blobSubscription }}"
+        python3 ./tests/images/testimages.py download-image azl4_qemu_guest \
+            --blob-storage-account "${{ parameters.blobStorageAccount }}" \
+            --blob-container "${{ parameters.blobContainer }}"
+        ls -la artifacts/azl4_qemu_guest.vhdx
+
+  - bash: |
+      set -euxo pipefail
+      python3 ./tests/images/testimages.py build \
+          "${{ parameters.imageName }}" \
+          --container "${{ parameters.micContainerTag }}" \
+          --output-dir "$(ob_outputDirectory)" \
+          --no-download \
+          --clones ${{ parameters.clones }}
+    displayName: "Build ${{ parameters.imageName }}"
+    workingDirectory: ${{ parameters.tridentSourceDirectory }}