From 0856719e189084490a3b610989e39954186815cd Mon Sep 17 00:00:00 2001 From: dotnet-docker-bot <60522487+dotnet-docker-bot@users.noreply.github.com> Date: Wed, 3 Dec 2025 08:04:31 -0800 Subject: [PATCH 1/3] Update common Docker engineering infrastructure with latest --- eng/docker-tools/Dockerfile.WithRepo | 6 + eng/docker-tools/Dockerfile.syft | 16 + eng/docker-tools/Get-BaseImageStatus.ps1 | 33 ++ eng/docker-tools/Get-ImageBuilder.ps1 | 13 + eng/docker-tools/Get-ImageNameVars.ps1 | 12 + eng/docker-tools/Install-DotNetSdk.ps1 | 64 ++++ eng/docker-tools/Invoke-CleanupDocker.ps1 | 20 + eng/docker-tools/Invoke-ImageBuilder.ps1 | 105 ++++++ eng/docker-tools/Invoke-WithRetry.ps1 | 41 +++ eng/docker-tools/Pull-Image.ps1 | 18 + eng/docker-tools/Retain-Build.ps1 | 43 +++ eng/docker-tools/build.ps1 | 76 ++++ eng/docker-tools/readme.md | 28 ++ eng/docker-tools/templates/1es-official.yml | 70 ++++ eng/docker-tools/templates/1es-unofficial.yml | 79 ++++ eng/docker-tools/templates/1es.yml | 75 ++++ .../templates/jobs/build-images.yml | 131 +++++++ .../templates/jobs/cg-build-projects.yml | 58 +++ .../jobs/copy-base-images-staging.yml | 29 ++ .../templates/jobs/copy-base-images.yml | 39 ++ .../templates/jobs/generate-matrix.yml | 72 ++++ .../templates/jobs/post-build.yml | 87 +++++ eng/docker-tools/templates/jobs/publish.yml | 312 ++++++++++++++++ .../jobs/test-images-linux-client.yml | 30 ++ .../jobs/test-images-windows-client.yml | 25 ++ .../templates/stages/build-and-test.yml | 343 ++++++++++++++++++ .../stages/dotnet/build-and-test.yml | 133 +++++++ .../stages/dotnet/build-test-publish-repo.yml | 74 ++++ .../stages/dotnet/publish-config-nonprod.yml | 104 ++++++ .../stages/dotnet/publish-config-prod.yml | 104 ++++++ .../templates/stages/dotnet/publish.yml | 56 +++ eng/docker-tools/templates/stages/publish.yml | 80 ++++ .../stages/setup-service-connections.yml | 38 ++ .../templates/steps/annotate-eol-digests.yml | 48 +++ .../templates/steps/clean-acr-images.yml | 33 ++ .../templates/steps/cleanup-docker-linux.yml | 15 + .../steps/cleanup-docker-windows.yml | 18 + .../templates/steps/copy-base-images.yml | 52 +++ .../steps/download-build-artifact.yml | 40 ++ .../templates/steps/init-common.yml | 9 + .../templates/steps/init-docker-linux.yml | 90 +++++ .../templates/steps/init-docker-windows.yml | 48 +++ .../steps/init-matrix-build-publish.yml | 78 ++++ .../templates/steps/parse-test-arg-arrays.yml | 15 + .../templates/steps/publish-artifact.yml | 28 ++ .../templates/steps/publish-readmes.yml | 29 ++ .../templates/steps/retain-build.yml | 9 + .../templates/steps/run-imagebuilder.yml | 76 ++++ .../templates/steps/run-pwsh-with-auth.yml | 39 ++ .../templates/steps/set-dry-run.yml | 30 ++ .../steps/set-image-info-path-var.yml | 19 + .../steps/test-images-linux-client.yml | 107 ++++++ .../steps/test-images-windows-client.yml | 66 ++++ .../templates/steps/validate-branch.yml | 52 +++ .../steps/wait-for-mcr-doc-ingestion.yml | 21 ++ .../steps/wait-for-mcr-image-ingestion.yml | 37 ++ .../templates/task-prefix-decorator.yml | 63 ++++ .../templates/variables/common-paths.yml | 5 + .../templates/variables/common.yml | 79 ++++ .../templates/variables/docker-images.yml | 7 + .../variables/dotnet/build-test-publish.yml | 48 +++ .../templates/variables/dotnet/common.yml | 63 ++++ .../variables/dotnet/secrets-unofficial.yml | 5 + .../templates/variables/dotnet/secrets.yml | 17 + .../templates/variables/sdl-pool.yml | 9 + 65 files changed, 3669 insertions(+) create mode 100644 eng/docker-tools/Dockerfile.WithRepo create mode 100644 eng/docker-tools/Dockerfile.syft create mode 100644 eng/docker-tools/Get-BaseImageStatus.ps1 create mode 100644 eng/docker-tools/Get-ImageBuilder.ps1 create mode 100644 eng/docker-tools/Get-ImageNameVars.ps1 create mode 100644 eng/docker-tools/Install-DotNetSdk.ps1 create mode 100644 eng/docker-tools/Invoke-CleanupDocker.ps1 create mode 100644 eng/docker-tools/Invoke-ImageBuilder.ps1 create mode 100644 eng/docker-tools/Invoke-WithRetry.ps1 create mode 100644 eng/docker-tools/Pull-Image.ps1 create mode 100644 eng/docker-tools/Retain-Build.ps1 create mode 100644 eng/docker-tools/build.ps1 create mode 100644 eng/docker-tools/readme.md create mode 100644 eng/docker-tools/templates/1es-official.yml create mode 100644 eng/docker-tools/templates/1es-unofficial.yml create mode 100644 eng/docker-tools/templates/1es.yml create mode 100644 eng/docker-tools/templates/jobs/build-images.yml create mode 100644 eng/docker-tools/templates/jobs/cg-build-projects.yml create mode 100644 eng/docker-tools/templates/jobs/copy-base-images-staging.yml create mode 100644 eng/docker-tools/templates/jobs/copy-base-images.yml create mode 100644 eng/docker-tools/templates/jobs/generate-matrix.yml create mode 100644 eng/docker-tools/templates/jobs/post-build.yml create mode 100644 eng/docker-tools/templates/jobs/publish.yml create mode 100644 eng/docker-tools/templates/jobs/test-images-linux-client.yml create mode 100644 eng/docker-tools/templates/jobs/test-images-windows-client.yml create mode 100644 eng/docker-tools/templates/stages/build-and-test.yml create mode 100644 eng/docker-tools/templates/stages/dotnet/build-and-test.yml create mode 100644 eng/docker-tools/templates/stages/dotnet/build-test-publish-repo.yml create mode 100644 eng/docker-tools/templates/stages/dotnet/publish-config-nonprod.yml create mode 100644 eng/docker-tools/templates/stages/dotnet/publish-config-prod.yml create mode 100644 eng/docker-tools/templates/stages/dotnet/publish.yml create mode 100644 eng/docker-tools/templates/stages/publish.yml create mode 100644 eng/docker-tools/templates/stages/setup-service-connections.yml create mode 100644 eng/docker-tools/templates/steps/annotate-eol-digests.yml create mode 100644 eng/docker-tools/templates/steps/clean-acr-images.yml create mode 100644 eng/docker-tools/templates/steps/cleanup-docker-linux.yml create mode 100644 eng/docker-tools/templates/steps/cleanup-docker-windows.yml create mode 100644 eng/docker-tools/templates/steps/copy-base-images.yml create mode 100644 eng/docker-tools/templates/steps/download-build-artifact.yml create mode 100644 eng/docker-tools/templates/steps/init-common.yml create mode 100644 eng/docker-tools/templates/steps/init-docker-linux.yml create mode 100644 eng/docker-tools/templates/steps/init-docker-windows.yml create mode 100644 eng/docker-tools/templates/steps/init-matrix-build-publish.yml create mode 100644 eng/docker-tools/templates/steps/parse-test-arg-arrays.yml create mode 100644 eng/docker-tools/templates/steps/publish-artifact.yml create mode 100644 eng/docker-tools/templates/steps/publish-readmes.yml create mode 100644 eng/docker-tools/templates/steps/retain-build.yml create mode 100644 eng/docker-tools/templates/steps/run-imagebuilder.yml create mode 100644 eng/docker-tools/templates/steps/run-pwsh-with-auth.yml create mode 100644 eng/docker-tools/templates/steps/set-dry-run.yml create mode 100644 eng/docker-tools/templates/steps/set-image-info-path-var.yml create mode 100644 eng/docker-tools/templates/steps/test-images-linux-client.yml create mode 100644 eng/docker-tools/templates/steps/test-images-windows-client.yml create mode 100644 eng/docker-tools/templates/steps/validate-branch.yml create mode 100644 eng/docker-tools/templates/steps/wait-for-mcr-doc-ingestion.yml create mode 100644 eng/docker-tools/templates/steps/wait-for-mcr-image-ingestion.yml create mode 100644 eng/docker-tools/templates/task-prefix-decorator.yml create mode 100644 eng/docker-tools/templates/variables/common-paths.yml create mode 100644 eng/docker-tools/templates/variables/common.yml create mode 100644 eng/docker-tools/templates/variables/docker-images.yml create mode 100644 eng/docker-tools/templates/variables/dotnet/build-test-publish.yml create mode 100644 eng/docker-tools/templates/variables/dotnet/common.yml create mode 100644 eng/docker-tools/templates/variables/dotnet/secrets-unofficial.yml create mode 100644 eng/docker-tools/templates/variables/dotnet/secrets.yml create mode 100644 eng/docker-tools/templates/variables/sdl-pool.yml diff --git a/eng/docker-tools/Dockerfile.WithRepo b/eng/docker-tools/Dockerfile.WithRepo new file mode 100644 index 000000000..d1126b3d1 --- /dev/null +++ b/eng/docker-tools/Dockerfile.WithRepo @@ -0,0 +1,6 @@ +# Use this Dockerfile to create an ImageBuilder image +ARG IMAGE +FROM $IMAGE + +WORKDIR /repo +COPY . . diff --git a/eng/docker-tools/Dockerfile.syft b/eng/docker-tools/Dockerfile.syft new file mode 100644 index 000000000..2e564e2ae --- /dev/null +++ b/eng/docker-tools/Dockerfile.syft @@ -0,0 +1,16 @@ +ARG SYFT_IMAGE_NAME +ARG TARGET_IMAGE_NAME + +FROM ${SYFT_IMAGE_NAME} AS syft +FROM ${TARGET_IMAGE_NAME} AS scan-image + +FROM syft AS run-scan +ARG TARGET_IMAGE_NAME +ENV SYFT_CHECK_FOR_APP_UPDATE=0 \ + SYFT_SOURCE_NAME=${TARGET_IMAGE_NAME} +USER root +RUN --mount=from=scan-image,source=/,target=/rootfs \ + ["/syft", "scan", "/rootfs/", "--select-catalogers", "image", "--output", "spdx-json=/manifest.spdx.json"] + +FROM scratch AS output +COPY --from=run-scan /manifest.spdx.json /manifest.spdx.json diff --git a/eng/docker-tools/Get-BaseImageStatus.ps1 b/eng/docker-tools/Get-BaseImageStatus.ps1 new file mode 100644 index 000000000..6b2546265 --- /dev/null +++ b/eng/docker-tools/Get-BaseImageStatus.ps1 @@ -0,0 +1,33 @@ +#!/usr/bin/env pwsh + +<# +.SYNOPSIS +Outputs the status of external base images referenced in the Dockerfiles. +#> +[cmdletbinding()] +param( + # Path to the manifest file to use + [string] + $Manifest = "manifest.json", + + # Architecture to filter Dockerfiles to + [string] + $Architecture = "*", + + # A value indicating whether to run the script continously + [switch] + $Continuous, + + # Number of seconds to wait between each iteration + [int] + $ContinuousDelay = 10 +) + +Set-StrictMode -Version Latest + +$imageBuilderArgs = "getBaseImageStatus --manifest $Manifest --architecture $Architecture" +if ($Continuous) { + $imageBuilderArgs += " --continuous --continuous-delay $ContinuousDelay" +} + +& "$PSScriptRoot/Invoke-ImageBuilder.ps1" -ImageBuilderArgs $imageBuilderArgs diff --git a/eng/docker-tools/Get-ImageBuilder.ps1 b/eng/docker-tools/Get-ImageBuilder.ps1 new file mode 100644 index 000000000..3eb35240b --- /dev/null +++ b/eng/docker-tools/Get-ImageBuilder.ps1 @@ -0,0 +1,13 @@ +#!/usr/bin/env pwsh + +# Load common image names +$imageNameVars = & $PSScriptRoot/Get-ImageNameVars.ps1 +foreach ($varName in $imageNameVars.Keys) { + Set-Variable -Name $varName -Value $imageNameVars[$varName] -Scope Global +} + +& docker inspect ${imageNames.imagebuilderName} | Out-Null +if (-not $?) { + Write-Output "Pulling" + & $PSScriptRoot/Invoke-WithRetry.ps1 "docker pull ${imageNames.imagebuilderName}" +} diff --git a/eng/docker-tools/Get-ImageNameVars.ps1 b/eng/docker-tools/Get-ImageNameVars.ps1 new file mode 100644 index 000000000..0b255c925 --- /dev/null +++ b/eng/docker-tools/Get-ImageNameVars.ps1 @@ -0,0 +1,12 @@ +# Returns a hashtable of variable name-to-value mapping representing the image name variables +# used by the common build infrastructure. + +$vars = @{} +Get-Content $PSScriptRoot/templates/variables/docker-images.yml | + Where-Object { $_.Trim().Length -gt 0 -and $_.Trim() -notlike 'variables:' -and $_.Trim() -notlike '# *' } | + ForEach-Object { + $parts = $_.Split(':', 2) + $vars[$parts[0].Trim()] = $parts[1].Trim() + } + +return $vars diff --git a/eng/docker-tools/Install-DotNetSdk.ps1 b/eng/docker-tools/Install-DotNetSdk.ps1 new file mode 100644 index 000000000..114faa260 --- /dev/null +++ b/eng/docker-tools/Install-DotNetSdk.ps1 @@ -0,0 +1,64 @@ +#!/usr/bin/env pwsh +# +# Copyright (c) .NET Foundation and contributors. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for full license information. +# + +<# +.SYNOPSIS +Install the .NET Core SDK at the specified path. + +.PARAMETER InstallPath +The path where the .NET Core SDK is to be installed. + +.PARAMETER Channel +The version of the .NET Core SDK to be installed. + +#> +[cmdletbinding()] +param( + [string] + $InstallPath, + [string] + $Channel = "9.0" +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +if (!(Test-Path "$InstallPath")) { + mkdir "$InstallPath" | Out-Null +} + +$IsRunningOnUnix = $PSVersionTable.contains("Platform") -and $PSVersionTable.Platform -eq "Unix" +if ($IsRunningOnUnix) { + $DotnetInstallScript = "dotnet-install.sh" +} +else { + $DotnetInstallScript = "dotnet-install.ps1" +} + +$DotnetInstallScriptPath = Join-Path -Path $InstallPath -ChildPath $DotnetInstallScript + +if (!(Test-Path $DotnetInstallScriptPath)) { + [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; + & "$PSScriptRoot/Invoke-WithRetry.ps1" "Invoke-WebRequest 'https://builds.dotnet.microsoft.com/dotnet/scripts/v1/$DotnetInstallScript' -OutFile $DotnetInstallScriptPath" +} + +$DotnetChannel = $Channel + +$InstallFailed = $false +if ($IsRunningOnUnix) { + & chmod +x $DotnetInstallScriptPath + & "$PSScriptRoot/Invoke-WithRetry.ps1" "$DotnetInstallScriptPath --channel $DotnetChannel --install-dir $InstallPath" -Retries 5 + $InstallFailed = ($LASTEXITCODE -ne 0) +} +else { + & "$PSScriptRoot/Invoke-WithRetry.ps1" "$DotnetInstallScriptPath -Channel $DotnetChannel -InstallDir $InstallPath" -Retries 5 + $InstallFailed = (-not $?) +} + +# See https://github.com/NuGet/NuGet.Client/pull/4259 +$Env:NUGET_EXPERIMENTAL_CHAIN_BUILD_RETRY_POLICY = "6,1500" + +if ($InstallFailed) { throw "Failed to install the .NET Core SDK" } diff --git a/eng/docker-tools/Invoke-CleanupDocker.ps1 b/eng/docker-tools/Invoke-CleanupDocker.ps1 new file mode 100644 index 000000000..ad637872c --- /dev/null +++ b/eng/docker-tools/Invoke-CleanupDocker.ps1 @@ -0,0 +1,20 @@ +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +docker ps -a -q | ForEach-Object { docker rm -f $_ } + +docker volume prune -f + +# Preserve the tagged Windows base images and the common eng infra images (e.g. ImageBuilder) +# to avoid the expense of having to repull continuously. +$imageNameVars = & $PSScriptRoot/Get-ImageNameVars.ps1 + +docker images --format "{{.Repository}}:{{.Tag}} {{.ID}}" | + Where-Object { + $localImage = $_ + $localImage.Contains(": ")` + -Or -Not ($localImage.StartsWith("mcr.microsoft.com/windows")` + -Or ($imageNameVars.Values.Where({ $localImage.StartsWith($_) }, 'First').Count -gt 0)) } | + ForEach-Object { $_.Split(' ', [System.StringSplitOptions]::RemoveEmptyEntries)[1] } | + Select-Object -Unique | + ForEach-Object { docker rmi -f $_ } diff --git a/eng/docker-tools/Invoke-ImageBuilder.ps1 b/eng/docker-tools/Invoke-ImageBuilder.ps1 new file mode 100644 index 000000000..4cec5ba2b --- /dev/null +++ b/eng/docker-tools/Invoke-ImageBuilder.ps1 @@ -0,0 +1,105 @@ +#!/usr/bin/env pwsh + +<# +.SYNOPSIS +Executes ImageBuilder with the specified args. + +.PARAMETER ImageBuilderArgs +The args to pass to ImageBuilder. + +.PARAMETER ReuseImageBuilderImage +Indicates that a previously built ImageBuilder image is presumed to exist locally and that +it should be used for this execution of the script. This allows some optimization when +multiple calls are being made to this script that don't require a fresh image (i.e. the +repo contents in the image don't need to be or should not be updated with each call to +this script). + +.PARAMETER OnCommandExecuted +A ScriptBlock that will be invoked after the ImageBuilder command has been executed. +This allows the caller to execute extra logic in the context of the ImageBuilder while +its container is still running. +The ScriptBlock is passed the following argument values: + 1. Container name +#> +[cmdletbinding()] +param( + [string] + $ImageBuilderArgs, + + [switch] + $ReuseImageBuilderImage, + + [scriptblock] + $OnCommandExecuted +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +function Log { + param ([string] $Message) + + Write-Output $Message +} + +function Exec { + param ([string] $Cmd) + + Log "Executing: '$Cmd'" + Invoke-Expression $Cmd + if ($LASTEXITCODE -ne 0) { + $host.SetShouldExit($LASTEXITCODE) + exit $LASTEXITCODE + throw "Failed: '$Cmd'" + } +} + +$imageBuilderContainerName = "ImageBuilder-$(Get-Date -Format yyyyMMddhhmmss)" +$containerCreated = $false + +pushd $PSScriptRoot/../../ +try { + $activeOS = docker version -f "{{ .Server.Os }}" + if ($activeOS -eq "linux") { + # On Linux, ImageBuilder is run within a container. + $imageBuilderImageName = "microsoft-dotnet-imagebuilder-withrepo" + if ($ReuseImageBuilderImage -ne $True) { + & ./eng/docker-tools/Get-ImageBuilder.ps1 + Exec ("docker build -t $imageBuilderImageName --build-arg " ` + + "IMAGE=${imageNames.imageBuilderName} -f eng/docker-tools/Dockerfile.WithRepo .") + } + + $imageBuilderCmd = "docker run --name $imageBuilderContainerName -v /var/run/docker.sock:/var/run/docker.sock $imageBuilderImageName" + $containerCreated = $true + } + else { + # On Windows, ImageBuilder is run locally due to limitations with running Docker client within a container. + # Remove when https://github.com/dotnet/docker-tools/issues/159 is resolved + $imageBuilderFolder = ".Microsoft.DotNet.ImageBuilder" + $imageBuilderCmd = [System.IO.Path]::Combine($imageBuilderFolder, "Microsoft.DotNet.ImageBuilder.exe") + if (-not (Test-Path -Path "$imageBuilderCmd" -PathType Leaf)) { + & ./eng/docker-tools/Get-ImageBuilder.ps1 + Exec "docker create --name $imageBuilderContainerName ${imageNames.imageBuilderName}" + $containerCreated = $true + if (Test-Path -Path $imageBuilderFolder) + { + Remove-Item -Recurse -Force -Path $imageBuilderFolder + } + + Exec "docker cp ${imageBuilderContainerName}:/image-builder $imageBuilderFolder" + } + } + + Exec "$imageBuilderCmd $ImageBuilderArgs" + + if ($OnCommandExecuted) { + Invoke-Command $OnCommandExecuted -ArgumentList $imageBuilderContainerName + } +} +finally { + if ($containerCreated) { + Exec "docker container rm -f $imageBuilderContainerName" + } + + popd +} diff --git a/eng/docker-tools/Invoke-WithRetry.ps1 b/eng/docker-tools/Invoke-WithRetry.ps1 new file mode 100644 index 000000000..0aa2c2737 --- /dev/null +++ b/eng/docker-tools/Invoke-WithRetry.ps1 @@ -0,0 +1,41 @@ +#!/usr/bin/env pwsh + +# Executes a command and retries if it fails. +[cmdletbinding()] +param ( + [Parameter(Mandatory = $true)][string]$Cmd, + [int]$Retries = 2, + [int]$WaitFactor = 6 + ) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +$count = 0 +$completed = $false + +Write-Output "Executing '$Cmd'" + +while (-not $completed) { + try { + Invoke-Expression $Cmd + if (-not $(Test-Path variable:LASTEXITCODE) -or $LASTEXITCODE -eq 0) { + $completed = $true + continue + } + } + catch { + } + + $count++ + + if ($count -lt $Retries) { + $wait = [Math]::Pow($WaitFactor, $count - 1) + Write-Output "Retry $count/$Retries, retrying in $wait seconds..." + Start-Sleep $wait + } + else { + Write-Output "Retry $count/$Retries, no more retries left." + throw "Failed to execute '$Cmd'" + } +} diff --git a/eng/docker-tools/Pull-Image.ps1 b/eng/docker-tools/Pull-Image.ps1 new file mode 100644 index 000000000..2d0a82cf5 --- /dev/null +++ b/eng/docker-tools/Pull-Image.ps1 @@ -0,0 +1,18 @@ +#!/usr/bin/env pwsh + +[cmdletbinding()] +param( + [Parameter(Mandatory = $true, Position = 0)] + [string]$Image, + + [Parameter(Mandatory = $false)] + [int]$Retries = 2, + + [Parameter(Mandatory = $false)] + [int]$WaitFactor = 6 +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +& "$PSScriptRoot/Invoke-WithRetry.ps1" "docker pull $Image" -Retries $Retries -WaitFactor $WaitFactor diff --git a/eng/docker-tools/Retain-Build.ps1 b/eng/docker-tools/Retain-Build.ps1 new file mode 100644 index 000000000..7f836e121 --- /dev/null +++ b/eng/docker-tools/Retain-Build.ps1 @@ -0,0 +1,43 @@ +# Adapted from https://github.com/dotnet/arcade/blob/main/eng/docker-tools/retain-build.ps1 +Param( + [Parameter(Mandatory = $true)][int] $BuildId, + [Parameter(Mandatory = $true)][string] $AzdoOrgUri, + [Parameter(Mandatory = $true)][string] $AzdoProject, + [Parameter(Mandatory = $true)][string] $Token +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version 2.0 + +function Get-AzDOHeaders( + [string] $Token) { + $base64AuthInfo = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(":${Token}")) + $headers = @{"Authorization" = "Basic $base64AuthInfo" } + return $headers +} + +function Update-BuildRetention( + [string] $AzdoOrgUri, + [string] $AzdoProject, + [int] $BuildId, + [string] $Token) { + $headers = Get-AzDOHeaders -Token $Token + $requestBody = "{ + `"keepForever`": `"true`" + }" + + $requestUri = "${AzdoOrgUri}/${AzdoProject}/_apis/build/builds/${BuildId}?api-version=6.0" + Write-Host "Attempting to retain build using the following URI: ${requestUri} ..." + + try { + Invoke-RestMethod -Uri $requestUri -Method Patch -Body $requestBody -Header $headers -contentType "application/json" + Write-Host "Updated retention settings for build ${BuildId}." + } + catch { + Write-Host "##[error] Failed to update retention settings for build: $($_.Exception.Response.StatusDescription)" + exit 1 + } +} + +Update-BuildRetention -AzdoOrgUri $AzdoOrgUri -AzdoProject $AzdoProject -BuildId $BuildId -Token $Token +exit 0 diff --git a/eng/docker-tools/build.ps1 b/eng/docker-tools/build.ps1 new file mode 100644 index 000000000..13c0806fb --- /dev/null +++ b/eng/docker-tools/build.ps1 @@ -0,0 +1,76 @@ +#!/usr/bin/env pwsh + +<# + .SYNOPSIS + Builds the Dockerfiles +#> + +[cmdletbinding()] +param( + # Product versions to filter by + [string[]]$Version = "*", + + # Names of OS to filter by + [string[]]$OS, + + # Type of architecture to filter by + [string]$Architecture, + + # Additional custom path filters + [string[]]$Paths, + + # Path to manifest file + [string]$Manifest = "manifest.json", + + # Additional args to pass to ImageBuilder + [string]$OptionalImageBuilderArgs +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +function Log { + param ([string] $Message) + + Write-Output $Message +} + +function Exec { + param ([string] $Cmd) + + Log "Executing: '$Cmd'" + Invoke-Expression $Cmd + if ($LASTEXITCODE -ne 0) { + throw "Failed: '$Cmd'" + } +} + +pushd $PSScriptRoot/../.. +try { + $args = $OptionalImageBuilderArgs + + if ($Version) { + $args += ($Version | foreach { ' --version "{0}"' -f $_ }) + } + + if ($OS) { + $args += ($OS | foreach { ' --os-version "{0}"' -f $_ }) + } + + if ($Architecture) { + $args += ' --architecture "{0}"' -f $Architecture + } + + if ($Paths) { + $args += ($Paths | foreach { ' --path "{0}"' -f $_ }) + } + + if ($Manifest) { + $args += ' --manifest "{0}"' -f $Manifest + } + + ./eng/docker-tools/Invoke-ImageBuilder.ps1 "build $args" +} +finally { + popd +} diff --git a/eng/docker-tools/readme.md b/eng/docker-tools/readme.md new file mode 100644 index 000000000..c0fe1f2eb --- /dev/null +++ b/eng/docker-tools/readme.md @@ -0,0 +1,28 @@ +# Don't touch this folder + + uuuuuuuuuuuuuuuuuuuu + u" uuuuuuuuuuuuuuuuuu "u + u" u$$$$$$$$$$$$$$$$$$$$u "u + u" u$$$$$$$$$$$$$$$$$$$$$$$$u "u + u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u + u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u + u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u + $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $ + $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $ + $ $$$" ... "$... ...$" ... "$$$ ... "$$$ $ + $ $$$u `"$$$$$$$ $$$ $$$$$ $$ $$$ $$$ $ + $ $$$$$$uu "$$$$ $$$ $$$$$ $$ """ u$$$ $ + $ $$$""$$$ $$$$ $$$u "$$$" u$$ $$$$$$$$ $ + $ $$$$....,$$$$$..$$$$$....,$$$$..$$$$$$$$ $ + $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $ + "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u" + "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u" + "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u" + "u "$$$$$$$$$$$$$$$$$$$$$$$$" u" + "u "$$$$$$$$$$$$$$$$$$$$" u" + "u """""""""""""""""" u" + """""""""""""""""""" + +!!! Changes made in this directory are subject to being overwritten by automation !!! + +The files in this directory are shared by all .NET Docker repos. If you need to make changes to these files, open an issue or submit a pull request in https://github.com/dotnet/docker-tools. \ No newline at end of file diff --git a/eng/docker-tools/templates/1es-official.yml b/eng/docker-tools/templates/1es-official.yml new file mode 100644 index 000000000..ebf8fcd70 --- /dev/null +++ b/eng/docker-tools/templates/1es-official.yml @@ -0,0 +1,70 @@ +# When extending this template, pipelines using a repository resource containing versions files for image caching must +# do the following: +# +# - Do not rely on any source code from the versions repo so as to not circumvent SDL and CG guidelines +# - The versions repo resource must be named `VersionsRepo` to avoid SDL scans +# - The versions repo must be checked out to `$(Build.SourcesDirectory)/versions` to avoid CG scans +# +# If the pipeline is not using a separate repository resource, ensure that there is no source code checked out in +# `$(Build.SourcesDirectory)/versions`, as it will not be scanned. +# +# The `cgDryRun` parameter will run CG but not submit the results, for testing purposes. + +parameters: +- name: cgDryRun + type: boolean + default: false +- name: stages + type: stageList + default: [] +- name: serviceConnections + type: object + default: [] +- name: pool + type: object + default: + name: $(default1ESInternalPoolName) + image: $(default1ESInternalPoolImage) + os: linux +- name: sourceAnalysisPool + type: object + default: + name: $(defaultSourceAnalysisPoolName) + image: $(defaultSourceAnalysisPoolImage) + os: windows + +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + +extends: + template: /eng/docker-tools/templates/task-prefix-decorator.yml@self + parameters: + baseTemplate: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + templateParameters: + pool: ${{ parameters.pool }} + sdl: + binskim: + enabled: true + componentgovernance: + ignoreDirectories: $(Build.SourcesDirectory)/versions + whatIf: ${{ parameters.cgDryRun }} + showAlertLink: true + policheck: + enabled: true + sourceRepositoriesToScan: + exclude: + - repository: VersionsRepo + sourceAnalysisPool: ${{ parameters.sourceAnalysisPool }} + tsa: + enabled: true + stages: + - ${{ if gt(length(parameters.serviceConnections), 0) }}: + - template: /eng/docker-tools/templates/stages/setup-service-connections.yml@self + parameters: + pool: ${{ parameters.pool }} + serviceConnections: ${{ parameters.serviceConnections }} + - ${{ parameters.stages }} diff --git a/eng/docker-tools/templates/1es-unofficial.yml b/eng/docker-tools/templates/1es-unofficial.yml new file mode 100644 index 000000000..bc584cd38 --- /dev/null +++ b/eng/docker-tools/templates/1es-unofficial.yml @@ -0,0 +1,79 @@ +# This unofficial template will always run CG in "what if" mode, which will not submit results to the CG. SDL tools may +# also be disabled for testing purposes. +# +# When extending this template, pipelines using a repository resource containing versions files for image caching must +# do the following: +# +# - Do not rely on any source code from the versions repo so as to not circumvent SDL and CG guidelines +# - The versions repo resource must be named `InternalVersionsRepo` or `PublicVersionsRepo` to avoid SDL scans +# - The versions repo must be checked out to `$(Build.SourcesDirectory)/versions` to avoid CG scans +# +# If the pipeline is not using a separate repository resource, ensure that there is no source code checked out in +# `$(Build.SourcesDirectory)/versions`, as it will not be scanned. + +parameters: +- name: disableSDL + type: boolean + default: false + displayName: Disable SDL +- name: stages + type: stageList + default: [] + # 1ES Pipeline Template parameters +- name: serviceConnections + type: object + default: [] +- name: pool + type: object + default: + name: $(default1ESInternalPoolName) + image: $(default1ESInternalPoolImage) + os: linux +- name: sourceAnalysisPool + type: object + default: + name: $(defaultSourceAnalysisPoolName) + image: $(defaultSourceAnalysisPoolImage) + os: windows + +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + +extends: + template: /eng/docker-tools/templates/task-prefix-decorator.yml@self + parameters: + # Use a unique task prefix for unofficial pipelines + taskPrefix: "🟦" + baseTemplate: v1/1ES.Unofficial.PipelineTemplate.yml@1ESPipelineTemplates + templateParameters: + pool: ${{ parameters.pool }} + sdl: + binskim: + enabled: true + componentgovernance: + ignoreDirectories: $(Build.SourcesDirectory)/versions + whatIf: true + showAlertLink: true + enableAllTools: ${{ not(parameters.disableSDL) }} + policheck: + enabled: true + sbom: + enabled: true + sourceRepositoriesToScan: + exclude: + - repository: InternalVersionsRepo + - repository: PublicVersionsRepo + sourceAnalysisPool: ${{ parameters.sourceAnalysisPool }} + tsa: + enabled: true + stages: + - ${{ if gt(length(parameters.serviceConnections), 0) }}: + - template: /eng/docker-tools/templates/stages/setup-service-connections.yml@self + parameters: + pool: ${{ parameters.pool }} + serviceConnections: ${{ parameters.serviceConnections }} + - ${{ parameters.stages }} diff --git a/eng/docker-tools/templates/1es.yml b/eng/docker-tools/templates/1es.yml new file mode 100644 index 000000000..5fde4a73d --- /dev/null +++ b/eng/docker-tools/templates/1es.yml @@ -0,0 +1,75 @@ +# When extending this template, pipelines using a repository resource containing versions files for image caching must +# do the following: +# +# - Do not rely on any source code from the versions repo so as to not circumvent SDL and CG guidelines +# - The versions repo resource must be named `VersionsRepo` to avoid SDL scans +# - The versions repo must be checked out to `$(Build.SourcesDirectory)/versions` to avoid CG scans +# +# If the pipeline is not using a separate repository resource, ensure that there is no source code checked out in +# `$(Build.SourcesDirectory)/versions`, as it will not be scanned. + +parameters: +- name: stages + type: stageList + default: [] +# List of repositories that will be excluded from SDL scanning. This should +# only be used when including other repos without building their source code. +# E.g. for the dotnet/versions repo. +- name: reposToExcludeFromScanning + type: object + default: [] +# The pool that will be used for initializing service connections. +- name: pool + type: object + default: + name: $(default1ESInternalPoolName) + image: $(default1ESInternalPoolImage) + os: linux +# The pool that will be used for SDL jobs. +- name: sourceAnalysisPool + type: object + default: + name: $(defaultSourceAnalysisPoolName) + image: $(defaultSourceAnalysisPoolImage) + os: windows +# Container image SBOMs are generated manually during the build job. 1ESPT's +# automatic SBOM generation only adds unnecessary steps and artifacts to +# builds. SBOM is not needed for JSON outputs. If a pipeline outputs binary +# artifacts that ship to customers, then set this parameter to true. +- name: enableSbom + type: boolean + default: false + +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + +extends: + template: /eng/docker-tools/templates/task-prefix-decorator.yml@self + parameters: + baseTemplate: v1/1ES.${{ iif(contains(variables['Build.DefinitionName'], '-official'), 'Official', 'Unofficial') }}.PipelineTemplate.yml@1ESPipelineTemplates + templateParameters: + pool: ${{ parameters.pool }} + sdl: + sbom: + enabled: ${{ parameters.enableSbom }} + binskim: + enabled: true + componentgovernance: + ignoreDirectories: $(Build.SourcesDirectory)/versions + showAlertLink: true + policheck: + enabled: true + ${{ if ne(length(parameters.reposToExcludeFromScanning), 0) }}: + sourceRepositoriesToScan: + exclude: + - ${{ each repo in parameters.reposToExcludeFromScanning }}: + - repository: ${{ repo }} + sourceAnalysisPool: ${{ parameters.sourceAnalysisPool }} + tsa: + enabled: true + stages: + - ${{ parameters.stages }} diff --git a/eng/docker-tools/templates/jobs/build-images.yml b/eng/docker-tools/templates/jobs/build-images.yml new file mode 100644 index 000000000..1f7f9b566 --- /dev/null +++ b/eng/docker-tools/templates/jobs/build-images.yml @@ -0,0 +1,131 @@ +parameters: + name: null + pool: {} + matrix: {} + dockerClientOS: null + buildJobTimeout: 60 + commonInitStepsForMatrixAndBuild: [] + customInitSteps: [] + publishConfig: null + noCache: false + internalProjectName: null + publicProjectName: null + storageAccountServiceConnection: null + +jobs: +- job: ${{ parameters.name }} + condition: and(${{ parameters.matrix }}, not(canceled()), in(dependencies.PreBuildValidation.result, 'Succeeded', 'SucceededWithIssues', 'Skipped')) + dependsOn: + - PreBuildValidation + - CopyBaseImages + - GenerateBuildMatrix + pool: ${{ parameters.pool }} + strategy: + matrix: $[ ${{ parameters.matrix }} ] + timeoutInMinutes: ${{ parameters.buildJobTimeout }} + variables: + imageBuilderDockerRunExtraOptions: $(build.imageBuilderDockerRunExtraOptions) + sbomDirectory: $(Build.ArtifactStagingDirectory)/sbom + imageInfoHostDir: $(Build.ArtifactStagingDirectory)/imageInfo + imageInfoContainerDir: $(artifactsPath)/imageInfo + steps: + - ${{ parameters.commonInitStepsForMatrixAndBuild }} + - template: /eng/docker-tools/templates/jobs/${{ format('../steps/init-docker-{0}.yml', parameters.dockerClientOS) }}@self + parameters: + cleanupDocker: true + - ${{ parameters.customInitSteps }} + - template: /eng/docker-tools/templates/steps/set-image-info-path-var.yml@self + parameters: + publicSourceBranch: $(publicSourceBranch) + - powershell: echo "##vso[task.setvariable variable=imageBuilderBuildArgs]" + condition: eq(variables.imageBuilderBuildArgs, '') + displayName: Initialize Image Builder Build Args + - powershell: | + New-Item -Path $(imageInfoHostDir) -ItemType Directory -Force + + # Reference the existing imageBuilderBuildArgs variable as an environment variable rather than injecting it directly + # with the $(imageBuilderBuildArgs) syntax. This is to avoid issues where the string may contain single quotes $ chars + # which really mess up assigning to a variable. It would require assigning the string with single quotes but also needing + # to escape the single quotes that are in the string which would need to be done outside the context of PowerShell. Since + # all we need is for that value to be in a PowerShell variable, we can get that by the fact that AzDO automatically creates + # the environment variable for us. + $imageBuilderBuildArgs = "$env:IMAGEBUILDERBUILDARGS $env:IMAGEBUILDER_QUEUEARGS --image-info-output-path $(imageInfoContainerDir)/$(legName)-image-info.json $(commonMatrixAndBuildOptions)" + if ($env:SYSTEM_TEAMPROJECT -eq "${{ parameters.internalProjectName }}" -and $env:BUILD_REASON -ne "PullRequest") { + $imageBuilderBuildArgs = "$imageBuilderBuildArgs --repo-prefix ${{ parameters.publishConfig.buildAcr.repoPrefix }} --push" + } + + # If the pipeline isn't configured to disable the cache and a build variable hasn't been set to disable the cache + if ("$(pipelineDisabledCache)" -ne "true" -and "${{ parameters.noCache }}" -ne "true") { + $imageBuilderBuildArgs = "$imageBuilderBuildArgs --image-info-source-path $(versionsBasePath)$(imageInfoVersionsPath)" + } + + echo "imageBuilderBuildArgs: $imageBuilderBuildArgs" + echo "##vso[task.setvariable variable=imageBuilderBuildArgs]$imageBuilderBuildArgs" + displayName: Set Image Builder Build Args + - template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self + parameters: + name: BuildImages + displayName: Build Images + serviceConnections: + # "name" here refers to the argument name, not the service connection name. + # It should probably be changed to "argName". + - name: acr + id: ${{ parameters.publishConfig.buildAcr.serviceConnection.id }} + tenantId: ${{ parameters.publishConfig.buildAcr.serviceConnection.tenantId }} + clientId: ${{ parameters.publishConfig.buildAcr.serviceConnection.clientId }} + - ${{ if parameters.storageAccountServiceConnection }}: + - name: storage + id: ${{ parameters.storageAccountServiceConnection.id }} + tenantId: ${{ parameters.storageAccountServiceConnection.tenantId }} + clientId: ${{ parameters.storageAccountServiceConnection.clientId }} + internalProjectName: ${{ parameters.internalProjectName }} + dockerClientOS: ${{ parameters.dockerClientOS }} + args: >- + build + --manifest $(manifest) + $(imageBuilderPaths) + $(osVersions) + --os-type $(osType) + --architecture $(architecture) + --retry + --digests-out-var 'builtImages' + --acr-subscription '${{ parameters.publishConfig.buildAcr.subscription }}' + --acr-resource-group '${{ parameters.publishConfig.buildAcr.resourceGroup }}' + $(manifestVariables) + $(imageBuilderBuildArgs) + - template: /eng/docker-tools/templates/steps/publish-artifact.yml@self + parameters: + path: $(imageInfoHostDir) + artifactName: $(legName)-image-info-$(System.JobAttempt) + displayName: Publish Image Info File Artifact + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.dockerClientOS, 'linux')) }}: + - powershell: | + $images = "$(BuildImages.builtImages)" + if (-not $images) { return 0 } + $syftImageName = "${{ parameters.publishConfig.publicMirrorAcr.server }}/$(imageNames.syft)" + & $(engDockerToolsPath)/Pull-Image.ps1 $syftImageName + $images -Split ',' | ForEach-Object { + echo "Generating SBOM for $_"; + $targetImageName = "$_"; + $formattedImageName = $targetImageName.Replace('${{ parameters.publishConfig.buildAcr.server }}/${{ parameters.publishConfig.buildAcr.repoPrefix }}', "").Replace('/', '_').Replace(':', '_'); + $sbomChildDir = "$(sbomDirectory)/$formattedImageName"; + New-Item -Type Directory -Path $sbomChildDir > $null; + docker build --output=$sbomChildDir -f $(engDockerToolsPath)/Dockerfile.syft --build-arg SYFT_IMAGE_NAME=$syftImageName --build-arg TARGET_IMAGE_NAME=$targetImageName -t syft-sbom $(engDockerToolsPath); + } + displayName: Generate SBOMs + condition: and(succeeded(), ne(variables['BuildImages.builtImages'], '')) + - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + - template: /eng/docker-tools/templates/jobs/${{ format('../steps/test-images-{0}-client.yml', parameters.dockerClientOS) }}@self + parameters: + condition: ne(variables.testScriptPath, '') + - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.dockerClientOS, 'linux')) }}: + - template: /eng/docker-tools/templates/steps/publish-artifact.yml@self + parameters: + path: $(sbomDirectory) + artifactName: $(legName)-sboms + displayName: Publish SBOM + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + condition: ne(variables['BuildImages.builtImages'], '') diff --git a/eng/docker-tools/templates/jobs/cg-build-projects.yml b/eng/docker-tools/templates/jobs/cg-build-projects.yml new file mode 100644 index 000000000..845a6719c --- /dev/null +++ b/eng/docker-tools/templates/jobs/cg-build-projects.yml @@ -0,0 +1,58 @@ +# This job builds all projects in the repository. It is intended to be used for CG purposes. +# The 1ES CG step does not scan artifacts that are built within Dockerfiles therefore they +# need to be built outside of Dockerfiles. +parameters: +# Setting cgDryRun will run CG but not submit the results +- name: cgDryRun + type: boolean + default: false + displayName: CG Dry Run +# See https://learn.microsoft.com/en-us/dotnet/core/tools/dotnet-install-script#options for possible Channel values +- name: dotnetVersionChannel + type: string + default: '9.0' + displayName: .NET Version + +jobs: +- job: BuildProjects + displayName: Build Projects + pool: + name: $(default1ESInternalPoolName) + image: $(default1ESInternalPoolImage) + os: linux + steps: + - powershell: > + ./eng/docker-tools/Install-DotNetSdk.ps1 -Channel ${{ parameters.dotnetVersionChannel }} -InstallPath "/usr/share/.dotnet" + displayName: Run Dotnet Install Script + - script: > + find . -name '*.csproj' | grep $(cgBuildGrepArgs) | xargs -n 1 /usr/share/.dotnet/dotnet build + displayName: Build Projects + + # Component Detection is only automatically run on production branches. + # To run Component Detection on non-production branches, the task must be manually injected. + - ${{ if eq(parameters.cgDryRun, true) }}: + - powershell: | + Write-Host "##vso[build.updatebuildnumber]$env:BUILD_BUILDNUMBER (Dry run)" + Write-Host "##vso[build.addbuildtag]dry-run" + + if ("$(officialBranches)".Split(',').Contains("$(Build.SourceBranch)")) + { + Write-Host "##vso[task.logissue type=error]Cannot run a CG dry-run build from an official branch ($(officialBranches))." + Write-Host "##vso[task.logissue type=error]Run the pipeline again from a different branch to avoid registering scan results." + exit 1 + } + displayName: Update Build Number + - task: ComponentGovernanceComponentDetection@0 + displayName: Component Detection (manually injected) + inputs: + # Running CG with `whatIf: true` or `scanType: LogOnly` outputs a list of detected components, but doesn't show + # which components would trigger an alert on a production build. + # As long as the build isn't ran on a tracked branch (typically main or nightly), it's OK to submit components + # to CG for analysis. Only the results for tracked branches matter. + scanType: Register + whatIf: false + alertWarningLevel: Low + failOnAlert: false + ignoreDirectories: $(Build.SourcesDirectory)/versions + showAlertLink: true + timeoutInMinutes: 10 diff --git a/eng/docker-tools/templates/jobs/copy-base-images-staging.yml b/eng/docker-tools/templates/jobs/copy-base-images-staging.yml new file mode 100644 index 000000000..20ce1f4da --- /dev/null +++ b/eng/docker-tools/templates/jobs/copy-base-images-staging.yml @@ -0,0 +1,29 @@ +parameters: +- name: name + type: string + default: null +- name: pool + type: object + default: {} +- name: publishConfig + type: object + default: null +- name: customInitSteps + type: stepList + default: [] +- name: additionalOptions + type: string + default: '' +- name: continueOnError + type: string + default: false + +jobs: +- template: /eng/docker-tools/templates/jobs/copy-base-images.yml@self + parameters: + name: ${{ parameters.name }} + pool: ${{ parameters.pool }} + customInitSteps: ${{ parameters.customInitSteps }} + additionalOptions: ${{ parameters.additionalOptions }} + acr: ${{ parameters.publishConfig.internalMirrorAcr }} + repoPrefix: ${{ parameters.publishConfig.internalMirrorAcr.repoPrefix }} diff --git a/eng/docker-tools/templates/jobs/copy-base-images.yml b/eng/docker-tools/templates/jobs/copy-base-images.yml new file mode 100644 index 000000000..ffc5aee15 --- /dev/null +++ b/eng/docker-tools/templates/jobs/copy-base-images.yml @@ -0,0 +1,39 @@ +parameters: +- name: name + type: string + default: null +- name: pool + type: object + default: {} +- name: acr + type: object + default: null +- name: repoPrefix + type: string + default: null +- name: customInitSteps + type: stepList + default: [] +- name: additionalOptions + type: string + default: '' +- name: continueOnError + type: string + default: false +- name: forceDryRun + type: boolean + default: false + +jobs: +- job: ${{ parameters.name }} + pool: ${{ parameters.pool }} + steps: + - template: /eng/docker-tools/templates/steps/init-docker-linux.yml@self + - ${{ parameters.customInitSteps }} + - template: /eng/docker-tools/templates/steps/copy-base-images.yml@self + parameters: + acr: ${{ parameters.acr }} + repoPrefix: ${{ parameters.repoPrefix }} + additionalOptions: ${{ parameters.additionalOptions }} + continueOnError: ${{ parameters.continueOnError }} + forceDryRun: ${{ parameters.forceDryRun }} diff --git a/eng/docker-tools/templates/jobs/generate-matrix.yml b/eng/docker-tools/templates/jobs/generate-matrix.yml new file mode 100644 index 000000000..606dec57f --- /dev/null +++ b/eng/docker-tools/templates/jobs/generate-matrix.yml @@ -0,0 +1,72 @@ +parameters: + matrixType: null + name: null + pool: {} + customBuildLegGroupArgs: "" + isTestStage: false + internalProjectName: null + noCache: false + publishConfig: null + customInitSteps: [] + commonInitStepsForMatrixAndBuild: [] + sourceBuildPipelineRunId: "" + +jobs: +- job: ${{ parameters.name }} + pool: ${{ parameters.pool }} + steps: + - ${{ parameters.commonInitStepsForMatrixAndBuild }} + - template: /eng/docker-tools/templates/steps/retain-build.yml@self + - template: /eng/docker-tools/templates/steps/init-docker-linux.yml@self + - ${{ parameters.customInitSteps }} + - template: /eng/docker-tools/templates/steps/validate-branch.yml@self + parameters: + publishConfig: ${{ parameters.publishConfig }} + internalProjectName: ${{ parameters.internalProjectName }} + - template: /eng/docker-tools/templates/steps/set-image-info-path-var.yml + parameters: + publicSourceBranch: $(publicSourceBranch) + - ${{ if eq(parameters.isTestStage, true) }}: + - template: /eng/docker-tools/templates/steps/download-build-artifact.yml@self + parameters: + targetPath: $(Build.ArtifactStagingDirectory) + artifactName: image-info + pipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + - powershell: | + $additionalGenerateBuildMatrixOptions = "$(additionalGenerateBuildMatrixOptions)" + + if ("${{ parameters.isTestStage}}" -eq "true") { + $additionalGenerateBuildMatrixOptions = "$additionalGenerateBuildMatrixOptions --image-info $(artifactsPath)/image-info.json" + } + elseif ("$(pipelineDisabledCache)" -ne "true" -and "${{ parameters.noCache }}" -ne "true" -and "$(trimCachedImagesForMatrix)" -eq "true") { + # If the pipeline isn't configured to disable the cache and a build variable hasn't been set to disable the cache + $additionalGenerateBuildMatrixOptions = "$additionalGenerateBuildMatrixOptions --image-info $(versionsBasePath)$(imageInfoVersionsPath) --trim-cached-images" + } + + echo "##vso[task.setvariable variable=additionalGenerateBuildMatrixOptions]$additionalGenerateBuildMatrixOptions" + displayName: Set GenerateBuildMatrix Variables + - script: > + echo "##vso[task.setvariable variable=generateBuildMatrixCommand] + generateBuildMatrix + --manifest $(manifest) + --type ${{ parameters.matrixType }} + --os-type '*' + --architecture '*' + --product-version-components $(productVersionComponents) + ${{ parameters.customBuildLegGroupArgs }} + $(imageBuilder.pathArgs) + $(manifestVariables) + $(commonMatrixAndBuildOptions) + $(additionalGenerateBuildMatrixOptions)" + displayName: Set GenerateBuildMatrix Command + - template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self + parameters: + name: matrix + displayName: Generate ${{ parameters.matrixType }} Matrix + serviceConnections: + - name: acr + tenantId: $(build.serviceConnection.tenantId) + clientId: $(build.serviceConnection.clientId) + id: $(build.serviceConnection.id) + internalProjectName: internal + args: $(generateBuildMatrixCommand) diff --git a/eng/docker-tools/templates/jobs/post-build.yml b/eng/docker-tools/templates/jobs/post-build.yml new file mode 100644 index 000000000..33b91ec03 --- /dev/null +++ b/eng/docker-tools/templates/jobs/post-build.yml @@ -0,0 +1,87 @@ +parameters: + pool: {} + internalProjectName: null + publicProjectName: null + +jobs: +- job: Build + pool: ${{ parameters.pool }} + variables: + imageInfosSubDir: "/image-infos" + imageInfosHostDir: "$(Build.ArtifactStagingDirectory)$(imageInfosSubDir)" + imageInfosContainerDir: "$(artifactsPath)$(imageInfosSubDir)" + imageInfosOutputSubDir: "/output" + sbomOutputDir: "$(Build.ArtifactStagingDirectory)/sbom" + steps: + - template: /eng/docker-tools/templates/steps/init-docker-linux.yml@self + - template: /eng/docker-tools/templates/steps/download-build-artifact.yml@self + parameters: + targetPath: $(Build.ArtifactStagingDirectory) + # This can fail if no build jobs ran to produce any artifacts + continueOnError: true + - powershell: | + # Move all image-info artifacts to their own directory + New-Item -ItemType Directory -Path $(imageInfosHostDir) + Get-ChildItem -Directory -Filter "*-image-info-*" $(Build.ArtifactStagingDirectory) | + Move-Item -Verbose -Destination $(imageInfosHostDir) + displayName: Collect Image Info Files + - powershell: | + # Move the contents of all the SBOM artifact directories to a single location + New-Item -ItemType Directory -Path $(sbomOutputDir) + Get-ChildItem -Directory -Filter "*-sboms" $(Build.ArtifactStagingDirectory) | + ForEach-Object { + Get-ChildItem $_ -Directory | Move-Item -Force -Verbose -Destination $(sbomOutputDir) + } + displayName: Consolidate SBOMs to Single Directory + - powershell: | + # Deletes the artifacts from all the unsuccessful jobs + Get-ChildItem $(imageInfosHostDir) -Directory | + ForEach-Object { + [pscustomobject]@{ + # Parse the artifact name to separate the base of the name from the job attempt number + BaseName = $_.Name.Substring(0, $_.Name.LastIndexOf('-')); + JobAttempt = $_.Name.Substring($_.Name.LastIndexOf('-') + 1) + FullName = $_.FullName + } + } | + Group-Object BaseName | + # Delete all but the last artifact from each base name + ForEach-Object { + $_.Group | + Sort-Object JobAttempt | + Select-Object -ExpandProperty FullName -SkipLast 1 | + Remove-Item -Recurse -Force + } + displayName: Prune Publish Artifacts + - powershell: | + $imageInfoFiles = Get-ChildItem "$(imageInfosHostDir)" + if ($imageInfoFiles.Count -eq 0) { + echo "No image info files found." + echo "##vso[task.setvariable variable=noImageInfos;isOutput=true]true" + exit 0 + } + + New-Item -ItemType Directory -Path $(imageInfosHostDir)$(imageInfosOutputSubDir) -Force + $(runImageBuilderCmd) mergeImageInfo ` + --manifest $(manifest) ` + $(imageInfosContainerDir) ` + $(imageInfosContainerDir)$(imageInfosOutputSubDir)/image-info.json ` + $(manifestVariables) + name: MergeImageInfoFiles + displayName: Merge Image Info Files + - template: /eng/docker-tools/templates/steps/publish-artifact.yml@self + parameters: + condition: and(succeeded(), ne(variables['MergeImageInfoFiles.noImageInfos'], 'true')) + path: $(sbomOutputDir) + artifactName: sboms + displayName: Publish SBOM Artifact + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + - template: /eng/docker-tools/templates/steps/publish-artifact.yml@self + parameters: + condition: and(succeeded(), ne(variables['MergeImageInfoFiles.noImageInfos'], 'true')) + path: $(imageInfosHostDir)$(imageInfosOutputSubDir) + artifactName: image-info + displayName: Publish Image Info File Artifact + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} diff --git a/eng/docker-tools/templates/jobs/publish.yml b/eng/docker-tools/templates/jobs/publish.yml new file mode 100644 index 000000000..cb9b454ac --- /dev/null +++ b/eng/docker-tools/templates/jobs/publish.yml @@ -0,0 +1,312 @@ +parameters: + pool: {} + internalProjectName: null + publishConfig: null + customInitSteps: [] + customPublishVariables: [] + sourceBuildPipelineDefinitionId: "" + sourceBuildPipelineRunId: "" + versionsRepoRef: null + versionsRepoPath: "" + # When true, overrides the commit SHA in merged image info files to use the current repository commit. + # This ensures that updated images reference the correct commit in their commitUrl properties. + overrideImageInfoCommit: false + +jobs: +- job: Publish + pool: ${{ parameters.pool }} + timeoutInMinutes: 90 + + variables: + - name: imageBuilder.commonCmdArgs + value: >- + --manifest '$(manifest)' + --registry-override '${{ parameters.publishConfig.publishAcr.server }}' + $(manifestVariables) + $(imageBuilder.queueArgs) + - name: publishNotificationRepoName + value: $(Build.Repository.Name) + - name: branchName + ${{ if startsWith(variables['Build.SourceBranch'], 'refs/heads/') }}: + value: $[ replace(variables['Build.SourceBranch'], 'refs/heads/', '') ] + ${{ if startsWith(variables['Build.SourceBranch'], 'refs/pull/') }}: + value: $[ replace(variables['System.PullRequest.SourceBranch'], 'refs/heads/', '') ] + - name: imageInfoHostDir + value: $(Build.ArtifactStagingDirectory)/imageInfo + - name: imageInfoContainerDir + value: $(artifactsPath)/imageInfo + - name: sourceBuildIdOutputDir + value: $(Build.ArtifactStagingDirectory)/sourceBuildId + - name: commitOverrideArg + ${{ if eq(parameters.overrideImageInfoCommit, true) }}: + value: --commit-override $(Build.SourceVersion) + ${{ else }}: + value: '' + - ${{ parameters.customPublishVariables }} + + steps: + - template: /eng/docker-tools/templates/steps/init-matrix-build-publish.yml@self + parameters: + publishConfig: ${{ parameters.publishConfig }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} + + - template: /eng/docker-tools/templates/steps/retain-build.yml@self + + - template: /eng/docker-tools/templates/steps/init-docker-linux.yml@self + + - pwsh: | + $azdoOrgName = Split-Path -Leaf $Env:SYSTEM_COLLECTIONURI + echo "##vso[task.setvariable variable=azdoOrgName]$azdoOrgName" + $versionsRepoRoot = "$(Pipeline.Workspace)/s/${{ parameters.versionsRepoPath }}" + echo "##vso[task.setvariable variable=versionsRepoRoot]$versionsRepoRoot" + displayName: Set Publish Variables + + - ${{ parameters.customInitSteps }} + + - template: /eng/docker-tools/templates/steps/validate-branch.yml@self + parameters: + internalProjectName: ${{ parameters.internalProjectName }} + + - template: /eng/docker-tools/templates/steps/download-build-artifact.yml@self + parameters: + targetPath: $(imageInfoHostDir) + artifactName: image-info + piplineDefinitionId: ${{ parameters.sourceBuildPipelineDefinitionId }} + pipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + # This can fail in scenarios where no build jobs have run to produce any artifacts + continueOnError: true + + - template: /eng/docker-tools/templates/steps/set-image-info-path-var.yml@self + parameters: + publicSourceBranch: $(publicSourceBranch) + + - template: /eng/docker-tools/templates/steps/set-dry-run.yml@self + parameters: + publishConfig: ${{ parameters.publishConfig }} + + - script: echo "##vso[task.setvariable variable=imageQueueTime]$(date --rfc-2822)" + displayName: Set Publish Variables + + - script: > + $(runImageBuilderCmd) trimUnchangedPlatforms + '$(imageInfoContainerDir)/image-info.json' + displayName: Trim Unchanged Images + + - template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self + parameters: + displayName: Copy Images + serviceConnections: + - name: acr + id: ${{ parameters.publishConfig.publishAcr.serviceConnection.id }} + tenantId: ${{ parameters.publishConfig.publishAcr.serviceConnection.tenantId }} + clientId: ${{ parameters.publishConfig.publishAcr.serviceConnection.clientId }} + internalProjectName: ${{ parameters.internalProjectName }} + args: >- + copyAcrImages + '${{ parameters.publishConfig.buildAcr.subscription }}' + '${{ parameters.publishConfig.buildAcr.resourceGroup }}' + '${{ parameters.publishConfig.buildAcr.repoPrefix }}' + '${{ parameters.publishConfig.buildAcr.server }}' + --os-type '*' + --architecture '*' + --repo-prefix '${{ parameters.publishConfig.publishAcr.repoPrefix }}' + --image-info '$(imageInfoContainerDir)/image-info.json' + $(dryRunArg) + $(imageBuilder.pathArgs) + $(imageBuilder.commonCmdArgs) + + - template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self + parameters: + displayName: Publish Manifest + serviceConnections: + - name: acr + id: ${{ parameters.publishConfig.publishAcr.serviceConnection.id }} + tenantId: ${{ parameters.publishConfig.publishAcr.serviceConnection.tenantId }} + clientId: ${{ parameters.publishConfig.publishAcr.serviceConnection.clientId }} + internalProjectName: ${{ parameters.internalProjectName }} + dockerClientOS: ${{ parameters.dockerClientOS }} + args: >- + publishManifest + '$(imageInfoContainerDir)/image-info.json' + --repo-prefix '${{ parameters.publishConfig.publishAcr.repoPrefix }}' + --os-type '*' + --architecture '*' + $(dryRunArg) + $(imageBuilder.pathArgs) + $(imageBuilder.commonCmdArgs) + + - template: /eng/docker-tools/templates/steps/publish-artifact.yml@self + parameters: + path: $(imageInfoHostDir) + artifactName: image-info-final-$(System.JobAttempt) + displayName: Publish Image Info File Artifact + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + + - template: /eng/docker-tools/templates/steps/wait-for-mcr-image-ingestion.yml@self + parameters: + publishConfig: ${{ parameters.publishConfig }} + imageInfoPath: '$(imageinfoContainerDir)/image-info.json' + minQueueTime: $(imageQueueTime) + dryRunArg: $(dryRunArg) + condition: succeeded() + + - template: /eng/docker-tools/templates/steps/publish-readmes.yml@self + parameters: + dryRunArg: $(dryRunArg) + condition: and(succeeded(), eq(variables['publishReadme'], 'true')) + + - script: mkdir -p $(Build.ArtifactStagingDirectory)/eol-annotation-data + displayName: Create EOL Annotation Data Directory + + - script: |- + cd $(versionsRepoRoot) + git pull origin $(gitHubVersionsRepoInfo.branch) + condition: and(succeeded(), eq(variables['publishImageInfo'], 'true')) + displayName: Pull Latest Changes from Versions Repo + + - script: >- + cp $(versionsRepoRoot)/$(gitHubImageInfoVersionsPath) $(imageInfoHostDir)/full-image-info-orig.json + condition: and(succeeded(), eq(variables['publishImageInfo'], 'true')) + displayName: Copy Latest Image Info from Versions Repo + + - script: > + $(runImageBuilderCmd) mergeImageInfo + $(imageInfoContainerDir) + $(imageInfoContainerDir)/full-image-info-new.json + $(manifestVariables) + $(dryRunArg) + --manifest $(manifest) + --publish + --initial-image-info-path $(imageInfoContainerDir)/full-image-info-orig.json + $(commitOverrideArg) + condition: and(succeeded(), eq(variables['publishImageInfo'], 'true')) + displayName: Merge Image Info + + - template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self + parameters: + displayName: Ingest Kusto Image Info + serviceConnections: + - name: kusto + id: $(kusto.serviceConnection.id) + tenantId: $(kusto.serviceConnection.tenantId) + clientId: $(kusto.serviceConnection.clientId) + internalProjectName: ${{ parameters.internalProjectName }} + condition: and(succeeded(), eq(variables['ingestKustoImageInfo'], 'true')) + args: >- + ingestKustoImageInfo + '$(imageInfoContainerDir)/image-info.json' + '$(kusto.cluster)' + '$(kusto.database)' + '$(kusto.imageTable)' + '$(kusto.layerTable)' + --os-type '*' + --architecture '*' + $(dryRunArg) + $(imageBuilder.commonCmdArgs) + + - template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self + parameters: + displayName: Generate EOL Annotation Data + serviceConnections: + - name: acr + id: ${{ parameters.publishConfig.publishAcr.serviceConnection.id }} + tenantId: ${{ parameters.publishConfig.publishAcr.serviceConnection.tenantId }} + clientId: ${{ parameters.publishConfig.publishAcr.serviceConnection.clientId }} + internalProjectName: internal + condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true')) + args: >- + generateEolAnnotationDataForPublish + '${{ parameters.publishConfig.publishAcr.server }}' + '${{ parameters.publishConfig.publishAcr.repoPrefix }}' + '$(artifactsPath)/eol-annotation-data/eol-annotation-data.json' + '$(imageInfoContainerDir)/full-image-info-orig.json' + '$(imageInfoContainerDir)/full-image-info-new.json' + $(generateEolAnnotationDataExtraOptions) + $(dryRunArg) + + - template: /eng/docker-tools/templates/steps/publish-artifact.yml@self + parameters: + path: $(Build.ArtifactStagingDirectory)/eol-annotation-data + artifactName: eol-annotation-data-$(System.JobAttempt) + displayName: Publish EOL Annotation Data Artifact + internalProjectName: internal + publicProjectName: public + condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true')) + + - template: /eng/docker-tools/templates/steps/annotate-eol-digests.yml@self + parameters: + acr: ${{ parameters.publishConfig.publishAcr }} + dataFile: $(artifactsPath)/eol-annotation-data/eol-annotation-data.json + + - script: > + $(runImageBuilderCmd) publishImageInfo + '$(imageInfoContainerDir)/full-image-info-new.json' + '$(gitHubVersionsRepoInfo.userName)' + '$(gitHubVersionsRepoInfo.email)' + $(gitHubVersionsRepoInfo.authArgs) + --git-owner '$(gitHubVersionsRepoInfo.org)' + --git-repo '$(gitHubVersionsRepoInfo.repo)' + --git-branch '$(gitHubVersionsRepoInfo.branch)' + --git-path '$(gitHubImageInfoVersionsPath)' + $(dryRunArg) + $(imageBuilder.commonCmdArgs) + condition: and(succeeded(), eq(variables['publishImageInfo'], 'true')) + displayName: Publish Image Info + + # Task displayNames names are hardcoded to reference the task prefix used by 1ES official + # pipelines in eng/docker-tools/templates/1es-official.yml. + # + # These will fail if they are dependend on by an unofficial pipeline since they use a unique task + # prefix compared to official pipelines (see eng/docker-tools/templates/1es-unofficial.yml). This is + # acceptable because unofficial pipelines should not publish images. + # + # https://github.com/dotnet/docker-tools/issues/1698 tracks making this command no longer depend + # on individual step displayNames. + - script: > + $(runImageBuilderCmd) postPublishNotification + '$(publishNotificationRepoName)' + '$(branchName)' + '$(imageInfoContainerDir)/image-info.json' + $(Build.BuildId) + '$(System.AccessToken)' + '$(azdoOrgName)' + '$(System.TeamProject)' + $(gitHubNotificationsRepoInfo.authArgs) + '$(gitHubNotificationsRepoInfo.org)' + '$(gitHubNotificationsRepoInfo.repo)' + --repo-prefix '${{ parameters.publishConfig.publishAcr.repoPrefix }}' + --task "🟪 Copy Images" + --task "🟪 Publish Manifest" + --task "🟪 Wait for Image Ingestion" + --task "🟪 Publish Readmes" + --task "🟪 Wait for MCR Doc Ingestion" + --task "🟪 Publish Image Info" + --task "🟪 Ingest Kusto Image Info" + --task "🟪 Generate EOL Annotation Data" + --task "🟪 Annotate EOL Images (${{ parameters.publishConfig.publishAcr.server }})" + --task "🟪 Wait for Annotation Ingestion (${{ parameters.publishConfig.publishAcr.server }})" + $(dryRunArg) + $(imageBuilder.commonCmdArgs) + displayName: Post Publish Notification + condition: and(always(), eq(variables['publishNotificationsEnabled'], 'true')) + + - powershell: | + # Default to current build number if parameter was not overridden + $buildId = "${{ parameters.sourceBuildPipelineRunId }}" + if ($buildId -eq "") { + $buildId = "$(Build.BuildNumber)" + } + + New-Item -ItemType Directory -Path $(sourceBuildIdOutputDir) + Set-Content -Path $(sourceBuildIdOutputDir)/source-build-id.txt -Value "$buildId" + condition: succeeded() + displayName: Write Source Build ID to File + + - template: /eng/docker-tools/templates/steps/publish-artifact.yml@self + parameters: + path: $(sourceBuildIdOutputDir) + artifactName: source-build-id + displayName: Publish Source Build ID Artifact + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} diff --git a/eng/docker-tools/templates/jobs/test-images-linux-client.yml b/eng/docker-tools/templates/jobs/test-images-linux-client.yml new file mode 100644 index 000000000..44e9e2fb7 --- /dev/null +++ b/eng/docker-tools/templates/jobs/test-images-linux-client.yml @@ -0,0 +1,30 @@ +parameters: + name: null + pool: {} + matrix: {} + testJobTimeout: 60 + preBuildValidation: false + internalProjectName: null + publishConfig: null + customInitSteps: [] + sourceBuildPipelineRunId: "" + +jobs: +- job: ${{ parameters.name }} + ${{ if eq(parameters.preBuildValidation, 'false') }}: + condition: and(succeeded(), ${{ parameters.matrix }}) + dependsOn: GenerateTestMatrix + strategy: + matrix: $[ ${{ parameters.matrix }} ] + ${{ if eq(parameters.preBuildValidation, 'true') }}: + condition: and(succeeded(), ne(variables.testScriptPath, '')) + pool: ${{ parameters.pool }} + timeoutInMinutes: ${{ parameters.testJobTimeout }} + steps: + - template: /eng/docker-tools/templates/steps/test-images-linux-client.yml@self + parameters: + preBuildValidation: ${{ parameters.preBuildValidation }} + internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} + customInitSteps: ${{ parameters.customInitSteps }} + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} diff --git a/eng/docker-tools/templates/jobs/test-images-windows-client.yml b/eng/docker-tools/templates/jobs/test-images-windows-client.yml new file mode 100644 index 000000000..390f01da6 --- /dev/null +++ b/eng/docker-tools/templates/jobs/test-images-windows-client.yml @@ -0,0 +1,25 @@ +parameters: + name: null + pool: {} + matrix: {} + testJobTimeout: 60 + internalProjectName: null + publishConfig: null + customInitSteps: [] + sourceBuildPipelineRunId: "" + +jobs: +- job: ${{ parameters.name }} + condition: and(succeeded(), ${{ parameters.matrix }}) + dependsOn: GenerateTestMatrix + pool: ${{ parameters.pool }} + strategy: + matrix: $[ ${{ parameters.matrix }} ] + timeoutInMinutes: ${{ parameters.testJobTimeout }} + steps: + - template: /eng/docker-tools/templates/steps/test-images-windows-client.yml@self + parameters: + internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} + customInitSteps: ${{ parameters.customInitSteps }} + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} diff --git a/eng/docker-tools/templates/stages/build-and-test.yml b/eng/docker-tools/templates/stages/build-and-test.yml new file mode 100644 index 000000000..b9df61a4c --- /dev/null +++ b/eng/docker-tools/templates/stages/build-and-test.yml @@ -0,0 +1,343 @@ +parameters: + buildMatrixType: platformDependencyGraph + testMatrixType: platformVersionedOs + buildMatrixCustomBuildLegGroupArgs: "" + testMatrixCustomBuildLegGroupArgs: "" + customCopyBaseImagesInitSteps: [] + customGenerateMatrixInitSteps: [] + customBuildInitSteps: [] + customTestInitSteps: [] + sourceBuildPipelineRunId: "" + + linuxAmdBuildJobTimeout: 60 + linuxArmBuildJobTimeout: 60 + windowsAmdBuildJobTimeout: 60 + + linuxAmdTestJobTimeout: 60 + linuxArmTestJobTimeout: 60 + windowsAmdTestJobTimeout: 60 + + noCache: false + publishConfig: null + + internalProjectName: null + publicProjectName: null + + versionsRepoRef: "" + + storageAccountServiceConnection: null + + linuxAmd64Pool: + vmImage: $(defaultLinuxAmd64PoolImage) + linuxArm32Pool: + vmImage: $(defaultLinuxArm32PoolImage) + linuxArm64Pool: + vmImage: $(defaultLinuxArm64PoolImage) + windows2016Pool: + vmImage: $(defaultWindows2016PoolImage) + windows1809Pool: + vmImage: $(defaultWindows1809PoolImage) + windows2022Pool: + vmImage: $(defaultWindows2022PoolImage) + windows2025Pool: + vmImage: $(defaultWindows2025PoolImage) + + +################################################################################ +# Build Images +################################################################################ +stages: +- stage: Build + condition: and(succeeded(), contains(variables['stages'], 'build')) + dependsOn: [] + jobs: + + - template: /eng/docker-tools/templates/jobs/test-images-linux-client.yml@self + parameters: + name: PreBuildValidation + pool: ${{ parameters.linuxAmd64Pool }} + testJobTimeout: ${{ parameters.linuxAmdTestJobTimeout }} + preBuildValidation: true + internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} + customInitSteps: + - ${{ parameters.customTestInitSteps }} + # These variables are normally set by the matrix. Since this test job is not generated + # by a matrix, we need to set them manually. They can be set to empty values since their + # values aren't actually used for the pre-build tests. + - powershell: | + echo "##vso[task.setvariable variable=productVersion]" + echo "##vso[task.setvariable variable=imageBuilderPaths]" + echo "##vso[task.setvariable variable=osVersions]" + echo "##vso[task.setvariable variable=architecture]" + displayName: Initialize Test Variables + + - template: /eng/docker-tools/templates/jobs/copy-base-images-staging.yml@self + parameters: + name: CopyBaseImages + publishConfig: ${{ parameters.publishConfig }} + pool: ${{ parameters.linuxAmd64Pool }} + additionalOptions: "--manifest '$(manifest)' $(imageBuilder.pathArgs) $(manifestVariables)" + customInitSteps: ${{ parameters.customCopyBaseImagesInitSteps }} + + - template: /eng/docker-tools/templates/jobs/generate-matrix.yml@self + parameters: + matrixType: ${{ parameters.buildMatrixType }} + name: GenerateBuildMatrix + pool: ${{ parameters.linuxAmd64Pool }} + customBuildLegGroupArgs: ${{ parameters.buildMatrixCustomBuildLegGroupArgs }} + internalProjectName: ${{ parameters.internalProjectName }} + noCache: ${{ parameters.noCache }} + publishConfig: ${{ parameters.publishConfig }} + customInitSteps: ${{ parameters.customGenerateMatrixInitSteps }} + commonInitStepsForMatrixAndBuild: + - template: /eng/docker-tools/templates/steps/init-matrix-build-publish.yml@self + parameters: + publishConfig: ${{ parameters.publishConfig }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} + + - template: /eng/docker-tools/templates/jobs/build-images.yml@self + parameters: + name: Linux_amd64 + pool: ${{ parameters.linuxAmd64Pool }} + matrix: dependencies.GenerateBuildMatrix.outputs['matrix.LinuxAmd64'] + dockerClientOS: linux + buildJobTimeout: ${{ parameters.linuxAmdBuildJobTimeout }} + commonInitStepsForMatrixAndBuild: + - template: /eng/docker-tools/templates/steps/init-matrix-build-publish.yml@self + parameters: + publishConfig: ${{ parameters.publishConfig }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} + customInitSteps: ${{ parameters.customBuildInitSteps }} + noCache: ${{ parameters.noCache }} + publishConfig: ${{ parameters.publishConfig }} + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + storageAccountServiceConnection: ${{ parameters.storageAccountServiceConnection }} + - template: /eng/docker-tools/templates/jobs/build-images.yml@self + parameters: + name: Linux_arm64 + pool: ${{ parameters.linuxArm64Pool }} + matrix: dependencies.GenerateBuildMatrix.outputs['matrix.LinuxArm64'] + dockerClientOS: linux + buildJobTimeout: ${{ parameters.linuxArmBuildJobTimeout }} + commonInitStepsForMatrixAndBuild: + - template: /eng/docker-tools/templates/steps/init-matrix-build-publish.yml@self + parameters: + publishConfig: ${{ parameters.publishConfig }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} + customInitSteps: ${{ parameters.customBuildInitSteps }} + noCache: ${{ parameters.noCache }} + publishConfig: ${{ parameters.publishConfig }} + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + storageAccountServiceConnection: ${{ parameters.storageAccountServiceConnection }} + - template: /eng/docker-tools/templates/jobs/build-images.yml@self + parameters: + name: Linux_arm32 + pool: ${{ parameters.linuxArm32Pool }} + matrix: dependencies.GenerateBuildMatrix.outputs['matrix.LinuxArm32'] + dockerClientOS: linux + buildJobTimeout: ${{ parameters.linuxArmBuildJobTimeout }} + commonInitStepsForMatrixAndBuild: + - template: /eng/docker-tools/templates/steps/init-matrix-build-publish.yml@self + parameters: + publishConfig: ${{ parameters.publishConfig }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} + customInitSteps: ${{ parameters.customBuildInitSteps }} + noCache: ${{ parameters.noCache }} + publishConfig: ${{ parameters.publishConfig }} + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + storageAccountServiceConnection: ${{ parameters.storageAccountServiceConnection }} + - template: /eng/docker-tools/templates/jobs/build-images.yml@self + parameters: + name: Windows1809_amd64 + pool: ${{ parameters.windows1809Pool }} + matrix: dependencies.GenerateBuildMatrix.outputs['matrix.Windows1809Amd64'] + dockerClientOS: windows + buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }} + commonInitStepsForMatrixAndBuild: + - template: /eng/docker-tools/templates/steps/init-matrix-build-publish.yml@self + parameters: + publishConfig: ${{ parameters.publishConfig }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} + customInitSteps: ${{ parameters.customBuildInitSteps }} + noCache: ${{ parameters.noCache }} + publishConfig: ${{ parameters.publishConfig }} + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + storageAccountServiceConnection: ${{ parameters.storageAccountServiceConnection }} + - template: /eng/docker-tools/templates/jobs/build-images.yml@self + parameters: + name: Windows2022_amd64 + pool: ${{ parameters.windows2022Pool }} + matrix: dependencies.GenerateBuildMatrix.outputs['matrix.WindowsLtsc2022Amd64'] + dockerClientOS: windows + buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }} + commonInitStepsForMatrixAndBuild: + - template: /eng/docker-tools/templates/steps/init-matrix-build-publish.yml@self + parameters: + publishConfig: ${{ parameters.publishConfig }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} + customInitSteps: ${{ parameters.customBuildInitSteps }} + noCache: ${{ parameters.noCache }} + publishConfig: ${{ parameters.publishConfig }} + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + storageAccountServiceConnection: ${{ parameters.storageAccountServiceConnection }} + - template: /eng/docker-tools/templates/jobs/build-images.yml@self + parameters: + name: Windows2025_amd64 + pool: ${{ parameters.windows2025Pool }} + matrix: dependencies.GenerateBuildMatrix.outputs['matrix.WindowsLtsc2025Amd64'] + dockerClientOS: windows + buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }} + commonInitStepsForMatrixAndBuild: + - template: /eng/docker-tools/templates/steps/init-matrix-build-publish.yml@self + parameters: + publishConfig: ${{ parameters.publishConfig }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} + customInitSteps: ${{ parameters.customBuildInitSteps }} + noCache: ${{ parameters.noCache }} + publishConfig: ${{ parameters.publishConfig }} + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} + storageAccountServiceConnection: ${{ parameters.storageAccountServiceConnection }} + - template: /eng/docker-tools/templates/jobs/build-images.yml@self + parameters: + name: WindowsLtsc2016_amd64 + pool: ${{ parameters.windows2016Pool }} + matrix: dependencies.GenerateBuildMatrix.outputs['matrix.WindowsLtsc2016Amd64'] + dockerClientOS: windows + buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }} + commonInitStepsForMatrixAndBuild: + - template: /eng/docker-tools/templates/steps/init-matrix-build-publish.yml@self + parameters: + publishConfig: ${{ parameters.publishConfig }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} + customInitSteps: ${{ parameters.customBuildInitSteps }} + noCache: ${{ parameters.noCache }} + publishConfig: ${{ parameters.publishConfig }} + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + storageAccountServiceConnection: ${{ parameters.storageAccountServiceConnection }} + +################################################################################ +# Post-Build +################################################################################ +- stage: Post_Build + dependsOn: Build + condition: and(succeeded(), contains(variables['stages'], 'build')) + jobs: + - template: /eng/docker-tools/templates/jobs/post-build.yml@self + parameters: + pool: ${{ parameters.linuxAmd64Pool }} + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + +################################################################################ +# Test Images +################################################################################ +- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: + - stage: Test + dependsOn: Post_Build + condition: " + and( + ne(stageDependencies.Post_Build.outputs['Build.MergeImageInfoFiles.noImageInfos'], 'true'), + ne(variables['testScriptPath'], ''), + and( + contains(variables['stages'], 'test'), + or( + and( + succeeded(), + contains(variables['stages'], 'build')), + not(contains(variables['stages'], 'build')))))" + jobs: + - template: /eng/docker-tools/templates/jobs/generate-matrix.yml@self + parameters: + matrixType: ${{ parameters.testMatrixType }} + name: GenerateTestMatrix + pool: ${{ parameters.linuxAmd64Pool }} + customBuildLegGroupArgs: ${{ parameters.testMatrixCustomBuildLegGroupArgs }} + isTestStage: true + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + customInitSteps: ${{ parameters.customGenerateMatrixInitSteps }} + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + commonInitStepsForMatrixAndBuild: + - template: /eng/docker-tools/templates/steps/init-matrix-build-publish.yml@self + parameters: + versionsRepoRef: ${{ parameters.versionsRepoRef }} + publishConfig: ${{ parameters.publishConfig }} + - template: /eng/docker-tools/templates/jobs/test-images-linux-client.yml@self + parameters: + name: Linux_amd64 + pool: ${{ parameters.linuxAmd64Pool }} + matrix: dependencies.GenerateTestMatrix.outputs['matrix.LinuxAmd64'] + testJobTimeout: ${{ parameters.linuxAmdTestJobTimeout }} + internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} + customInitSteps: ${{ parameters.customTestInitSteps }} + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + - template: /eng/docker-tools/templates/jobs/test-images-linux-client.yml@self + parameters: + name: Linux_arm64 + pool: ${{ parameters.linuxArm64Pool }} + matrix: dependencies.GenerateTestMatrix.outputs['matrix.LinuxArm64'] + testJobTimeout: ${{ parameters.linuxArmTestJobTimeout }} + internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} + customInitSteps: ${{ parameters.customTestInitSteps }} + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + - template: /eng/docker-tools/templates/jobs/test-images-linux-client.yml@self + parameters: + name: Linux_arm32 + pool: ${{ parameters.linuxArm32Pool }} + matrix: dependencies.GenerateTestMatrix.outputs['matrix.LinuxArm32'] + testJobTimeout: ${{ parameters.linuxArmTestJobTimeout }} + internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} + customInitSteps: ${{ parameters.customTestInitSteps }} + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + - template: /eng/docker-tools/templates/jobs/test-images-windows-client.yml@self + parameters: + name: Windows1809_amd64 + pool: ${{ parameters.windows1809Pool }} + matrix: dependencies.GenerateTestMatrix.outputs['matrix.Windows1809Amd64'] + testJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }} + internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} + customInitSteps: ${{ parameters.customTestInitSteps }} + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + - template: /eng/docker-tools/templates/jobs/test-images-windows-client.yml@self + parameters: + name: Windows2022_amd64 + pool: ${{ parameters.windows2022Pool }} + matrix: dependencies.GenerateTestMatrix.outputs['matrix.WindowsLtsc2022Amd64'] + testJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }} + internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} + customInitSteps: ${{ parameters.customTestInitSteps }} + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + - template: /eng/docker-tools/templates/jobs/test-images-windows-client.yml@self + parameters: + name: Windows2025_amd64 + pool: ${{ parameters.windows2025Pool }} + matrix: dependencies.GenerateTestMatrix.outputs['matrix.WindowsLtsc2025Amd64'] + testJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }} + internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} + customInitSteps: ${{ parameters.customTestInitSteps }} + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + - template: /eng/docker-tools/templates/jobs/test-images-windows-client.yml@self + parameters: + name: WindowsLtsc2016_amd64 + pool: ${{ parameters.windows2016Pool }} + matrix: dependencies.GenerateTestMatrix.outputs['matrix.WindowsLtsc2016Amd64'] + testJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }} + internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} + customInitSteps: ${{ parameters.customTestInitSteps }} + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} diff --git a/eng/docker-tools/templates/stages/dotnet/build-and-test.yml b/eng/docker-tools/templates/stages/dotnet/build-and-test.yml new file mode 100644 index 000000000..311b16133 --- /dev/null +++ b/eng/docker-tools/templates/stages/dotnet/build-and-test.yml @@ -0,0 +1,133 @@ +# A wrapper template around the common build-test-publish-repo template with settings +# specific to the .NET team's infrastructure. + +parameters: + linuxAmd64Pool: "" + + # (Optional) This service connection should be an Azure Resource Manager + # service connection to a storage account that's needed during image builds. + # It can be used to build images with access to private/internal bits. + # If specified, this service connection will be used to pass a storage + # account access token as `--build-arg ACCESSTOKEN=***` to all image builds. + storageAccountServiceConnection: null + + # Parameters for pre-build jobs + customGenerateMatrixInitSteps: [] + customCopyBaseImagesInitSteps: [] + + # Build parameters + noCache: false + publishConfig: null + buildMatrixType: platformDependencyGraph + buildMatrixCustomBuildLegGroupArgs: "" + linuxAmdBuildJobTimeout: 60 + linuxArmBuildJobTimeout: 60 + windowsAmdBuildJobTimeout: 60 + customBuildInitSteps: [] + + # Test parameters + testMatrixType: platformVersionedOs + testMatrixCustomBuildLegGroupArgs: "" + linuxAmdTestJobTimeout: 60 + linuxArmTestJobTimeout: 60 + windowsAmdTestJobTimeout: 60 + customTestInitSteps: [] + sourceBuildPipelineRunId: "" + + internalProjectName: null + publicProjectName: null + + versionsRepoRef: null + +stages: +- template: /eng/docker-tools/templates/stages/build-and-test.yml@self + parameters: + noCache: ${{ parameters.noCache }} + publishConfig: ${{ parameters.publishConfig }} + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + storageAccountServiceConnection: ${{ parameters.storageAccountServiceConnection }} + customGenerateMatrixInitSteps: ${{ parameters.customGenerateMatrixInitSteps }} + buildMatrixCustomBuildLegGroupArgs: ${{ parameters.buildMatrixCustomBuildLegGroupArgs }} + testMatrixCustomBuildLegGroupArgs: ${{ parameters.testMatrixCustomBuildLegGroupArgs }} + customCopyBaseImagesInitSteps: ${{ parameters.customCopyBaseImagesInitSteps}} + customBuildInitSteps: ${{ parameters.customBuildInitSteps }} + customTestInitSteps: ${{ parameters.customTestInitSteps }} + windowsAmdBuildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }} + windowsAmdTestJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }} + linuxAmdBuildJobTimeout: ${{ parameters.linuxAmdBuildJobTimeout }} + linuxArmBuildJobTimeout: ${{ parameters.linuxArmBuildJobTimeout }} + buildMatrixType: ${{ parameters.buildMatrixType }} + testMatrixType: ${{ parameters.testMatrixType }} + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + + # Only clone versions repo if we need to reference it during the build in order to cache images. + ${{ if eq(parameters.noCache, false) }}: + versionsRepoRef: ${{ parameters.versionsRepoRef }} + + # Linux AMD64 + linuxAmd64Pool: + ${{ if ne(parameters.linuxAmd64Pool, '') }}: + ${{ parameters.linuxAmd64Pool }} + ${{ elseif eq(variables['System.TeamProject'], parameters.publicProjectName) }}: + vmImage: $(defaultLinuxAmd64PoolImage) + ${{ elseif eq(variables['System.TeamProject'], parameters.internalProjectName) }}: + name: $(linuxAmd64InternalPoolName) + image: $(linuxAmd64InternalPoolImage) + os: linux + + # Linux Arm64 + linuxArm64Pool: + os: linux + hostArchitecture: Arm64 + image: $(linuxArm64PoolImage) + ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}: + name: $(linuxArm64PublicPoolName) + ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}: + name: $(linuxArm64InternalPoolName) + + # Linux Arm32 + linuxArm32Pool: + os: linux + hostArchitecture: Arm64 + image: $(linuxArm32PoolImage) + ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}: + name: $(linuxArm32PublicPoolName) + ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}: + name: $(linuxArm32InternalPoolName) + + # Windows Server 2016 + windows2016Pool: + os: windows + name: $(windowsServer2016PoolName) + ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}: + image: $(windowsServer2016PublicPoolImage) + ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}: + image: $(windowsServer2016InternalPoolImage) + + # Windows Server 2019 (1809) + windows1809Pool: + os: windows + name: $(windowsServer2019PoolName) + ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}: + image: $(windowsServer2019PublicPoolImage) + ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}: + image: $(windowsServer2019InternalPoolImage) + + # Windows Server 2022 + windows2022Pool: + os: windows + name: $(windowsServer2022PoolName) + ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}: + image: $(windowsServer2022PublicPoolImage) + ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}: + image: $(windowsServer2022InternalPoolImage) + + # Windows Server 2025 + windows2025Pool: + os: windows + name: $(windowsServer2025PoolName) + ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}: + image: $(windowsServer2025PublicPoolImage) + ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}: + image: $(windowsServer2025InternalPoolImage) diff --git a/eng/docker-tools/templates/stages/dotnet/build-test-publish-repo.yml b/eng/docker-tools/templates/stages/dotnet/build-test-publish-repo.yml new file mode 100644 index 000000000..c412f00d2 --- /dev/null +++ b/eng/docker-tools/templates/stages/dotnet/build-test-publish-repo.yml @@ -0,0 +1,74 @@ +# This template wraps the .NET-specific build-and-test and publish templates + +parameters: + linuxAmd64Pool: "" + + # Parameters for pre-build jobs + customGenerateMatrixInitSteps: [] + customCopyBaseImagesInitSteps: [] + + # Build parameters + noCache: false + publishConfig: null + buildMatrixType: platformDependencyGraph + buildMatrixCustomBuildLegGroupArgs: "" + linuxAmdBuildJobTimeout: 60 + linuxArmBuildJobTimeout: 60 + windowsAmdBuildJobTimeout: 60 + customBuildInitSteps: [] + + # Test parameters + testMatrixType: platformVersionedOs + testMatrixCustomBuildLegGroupArgs: "" + linuxAmdTestJobTimeout: 60 + linuxArmTestJobTimeout: 60 + windowsAmdTestJobTimeout: 60 + customTestInitSteps: [] + sourceBuildPipelineRunId: "" + + # Publish parameters + customPublishInitSteps: [] + + # Other common parameters + internalProjectName: null + publicProjectName: null + versionsRepoRef: "" + +stages: +- template: /eng/docker-tools/templates/stages/dotnet/build-and-test.yml@self + parameters: + linuxAmd64Pool: ${{ parameters.linuxAmd64Pool }} + # Pre-build + customGenerateMatrixInitSteps: ${{ parameters.customGenerateMatrixInitSteps }} + customCopyBaseImagesInitSteps: ${{ parameters.customCopyBaseImagesInitSteps }} + # Build + noCache: ${{ parameters.noCache }} + publishConfig: ${{ parameters.publishConfig }} + buildMatrixType: ${{ parameters.buildMatrixType }} + buildMatrixCustomBuildLegGroupArgs: ${{ parameters.buildMatrixCustomBuildLegGroupArgs }} + linuxAmdBuildJobTimeout: ${{ parameters.linuxAmdBuildJobTimeout }} + linuxArmBuildJobTimeout: ${{ parameters.linuxArmBuildJobTimeout }} + windowsAmdBuildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }} + customBuildInitSteps: ${{ parameters.customBuildInitSteps }} + # Test + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + testMatrixType: ${{ parameters.testMatrixType }} + testMatrixCustomBuildLegGroupArgs: ${{ parameters.testMatrixCustomBuildLegGroupArgs }} + linuxAmdTestJobTimeout: ${{ parameters.linuxAmdTestJobTimeout }} + linuxArmTestJobTimeout: ${{ parameters.linuxArmTestJobTimeout }} + windowsAmdTestJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }} + customTestInitSteps: ${{ parameters.customTestInitSteps }} + # Other + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} + +- template: /eng/docker-tools/templates/stages/dotnet/publish.yml@self + parameters: + pool: ${{ parameters.linuxAmd64Pool }} + customPublishInitSteps: ${{ parameters.customPublishInitSteps }} + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + publishConfig: ${{ parameters.publishConfig }} + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} diff --git a/eng/docker-tools/templates/stages/dotnet/publish-config-nonprod.yml b/eng/docker-tools/templates/stages/dotnet/publish-config-nonprod.yml new file mode 100644 index 000000000..eddd78593 --- /dev/null +++ b/eng/docker-tools/templates/stages/dotnet/publish-config-nonprod.yml @@ -0,0 +1,104 @@ +# This pipeline template injects the publish config for the dotnet-docker +# non-production (unofficial) environment. +# The overall structure of this file should stay mostly in-sync with the +# publish-config-prod.yml template. + +parameters: +# By default, images are staged in repos that are prefixed with this pipeline +# build ID. This is makes it easy to look up which pipeline images were built +# from and vice versa. +- name: sourceBuildPipelineRunId + type: string + default: $(Build.BuildId) + +# This prefix is added to the staging repo when pushing images. If the trailing +# slash is omitted, it will not be added automatically. +- name: stagingRepoPrefix + type: string + default: "build-staging/" + +# Images will have this prefix added to their repo name when pushed to the +# publishing ACR. If the trailing slash is omitted, it will not be added +# automatically. +- name: publishRepoPrefix + type: string + default: "public/" + +# This template will have the publishConfig, internalProjectName, and +# publicProjectName parameters passed to it automatically. +- name: stagesTemplate + type: string + +# These parameters will be passed to the template referred to by the +# stagesTemplate parameter. +# Note: publishConfig, internalProjectName, and publicProjectName are passed +# automatically by this template. Don't define them in this parameter - they +# will get overwritten. +- name: stagesTemplateParameters + type: object + default: {} + + +stages: +- template: ${{ parameters.stagesTemplate }} + parameters: + ${{ insert }}: ${{ parameters.stagesTemplateParameters }} + + internalProjectName: "internal" + publicProjectName: "public" + + publishConfig: + internalMirrorAcr: + server: $(acr-staging-test.server) + repoPrefix: $(mirrorRepoPrefix) + resourceGroup: $(testResourceGroup) + subscription: $(testSubscription) + serviceConnection: + name: $(internal-mirror-test.serviceConnectionName) + id: $(internal-mirror-test.serviceConnection.id) + clientId: $(internal-mirror-test.serviceConnection.clientId) + tenantId: $(testTenant) + + publicMirrorAcr: + server: $(public-mirror.server) + resourceGroup: $(public-mirror.resourceGroup) + subscription: $(public-mirror.subscription) + serviceConnection: + name: $(public-mirror.serviceConnectionName) + id: $(public-mirror.serviceConnection.id) + tenantId: $(public-mirror.serviceConnection.tenantId) + clientId: $(public-mirror.serviceConnection.clientId) + + buildAcr: + server: $(acr-staging-test.server) + resourceGroup: $(testResourceGroup) + subscription: $(testSubscription) + repoPrefix: "${{ parameters.stagingRepoPrefix }}${{ parameters.sourceBuildPipelineRunId }}/" + serviceConnection: + name: $(build-test.serviceConnectionName) + id: $(build-test.serviceConnection.id) + clientId: $(build-test.serviceConnection.clientId) + tenantId: $(testTenant) + + cleanServiceConnection: + name: $(clean-test.serviceConnectionName) + id: $(clean-test.serviceConnection.id) + clientId: $(clean-test.serviceConnection.clientId) + tenantId: $(testTenant) + + testServiceConnection: + name: $(test-nonprod.serviceConnectionName) + id: $(test-nonprod.serviceConnection.id) + clientId: $(test-nonprod.serviceConnection.clientId) + tenantId: $(testTenant) + + publishAcr: + server: $(acr-test.server) + resourceGroup: $(testResourceGroup) + subscription: $(testSubscription) + repoPrefix: "${{ parameters.publishRepoPrefix }}" + serviceConnection: + name: $(publish-test.serviceConnectionName) + id: $(publish-test.serviceConnection.id) + clientId: $(publish-test.serviceConnection.clientId) + tenantId: $(testTenant) diff --git a/eng/docker-tools/templates/stages/dotnet/publish-config-prod.yml b/eng/docker-tools/templates/stages/dotnet/publish-config-prod.yml new file mode 100644 index 000000000..d45807ddb --- /dev/null +++ b/eng/docker-tools/templates/stages/dotnet/publish-config-prod.yml @@ -0,0 +1,104 @@ +# This pipeline template injects the publish config for the dotnet-docker +# production (official) environment. +# The overall structure of this file should stay mostly in-sync with the +# publish-config-nonprod.yml template. + +parameters: +# By default, images are staged in repos that are prefixed with this pipeline +# build ID. This is makes it easy to look up which pipeline images were built +# from and vice versa. +- name: sourceBuildPipelineRunId + type: string + default: $(Build.BuildId) + +# This prefix is added to the staging repo when pushing images. If the trailing +# slash is omitted, it will not be added automatically. +- name: stagingRepoPrefix + type: string + default: "build-staging/" + +# Images will have this prefix added to their repo name when pushed to the +# publishing ACR. If the trailing slash is omitted, it will not be added +# automatically. +- name: publishRepoPrefix + type: string + default: "public/" + +# This template will have the publishConfig, internalProjectName, and +# publicProjectName parameters passed to it automatically. +- name: stagesTemplate + type: string + +# These parameters will be passed to the template referred to by the +# stagesTemplate parameter. +# Note: publishConfig, internalProjectName, and publicProjectName are passed +# automatically by this template. Don't define them in this parameter - they +# will get overwritten. +- name: stagesTemplateParameters + type: object + default: {} + + +stages: +- template: ${{ parameters.stagesTemplate }} + parameters: + ${{ insert }}: ${{ parameters.stagesTemplateParameters }} + + internalProjectName: "internal" + publicProjectName: "public" + + publishConfig: + internalMirrorAcr: + server: $(acr-staging.server) + repoPrefix: $(mirrorRepoPrefix) + resourceGroup: $(acr-staging.resourceGroup) + subscription: $(acr-staging.subscription) + serviceConnection: + name: $(internal-mirror.serviceConnectionName) + id: $(internal-mirror.serviceConnection.id) + clientId: $(internal-mirror.serviceConnection.clientId) + tenantId: $(internal-mirror.serviceConnection.tenantId) + + publicMirrorAcr: + server: $(public-mirror.server) + resourceGroup: $(public-mirror.resourceGroup) + subscription: $(public-mirror.subscription) + serviceConnection: + name: $(public-mirror.serviceConnectionName) + id: $(public-mirror.serviceConnection.id) + tenantId: $(public-mirror.serviceConnection.tenantId) + clientId: $(public-mirror.serviceConnection.clientId) + + buildAcr: + server: $(acr-staging.server) + resourceGroup: $(acr-staging.resourceGroup) + subscription: $(acr-staging.subscription) + repoPrefix: "${{ parameters.stagingRepoPrefix }}${{ parameters.sourceBuildPipelineRunId }}/" + serviceConnection: + name: $(build.serviceConnectionName) + id: $(build.serviceConnection.id) + clientId: $(build.serviceConnection.clientId) + tenantId: $(build.serviceConnection.tenantId) + + cleanServiceConnection: + name: $(clean.serviceConnectionName) + id: $(clean.serviceConnection.id) + clientId: $(clean.serviceConnection.clientId) + tenantId: $(clean.serviceConnection.tenantId) + + testServiceConnection: + name: $(test.serviceConnectionName) + id: $(test.serviceConnection.id) + clientId: $(test.serviceConnection.clientId) + tenantId: $(test.serviceConnection.tenantId) + + publishAcr: + server: $(acr.server) + resourceGroup: $(acr.resourceGroup) + subscription: $(acr.subscription) + repoPrefix: "${{ parameters.publishRepoPrefix }}" + serviceConnection: + name: $(publish.serviceConnectionName) + id: $(publish.serviceConnection.id) + clientId: $(publish.serviceConnection.clientId) + tenantId: $(publish.serviceConnection.tenantId) diff --git a/eng/docker-tools/templates/stages/dotnet/publish.yml b/eng/docker-tools/templates/stages/dotnet/publish.yml new file mode 100644 index 000000000..66cb2c8bd --- /dev/null +++ b/eng/docker-tools/templates/stages/dotnet/publish.yml @@ -0,0 +1,56 @@ +# This template wraps the common publish stage template with settings specific +# to the .NET team's infrastructure. + +parameters: + internalProjectName: null + publicProjectName: null + publishConfig: null + pool: "" + isStandalonePublish: false + customPublishInitSteps: [] + sourceBuildPipelineDefinitionId: '' + sourceBuildPipelineRunId: '' + versionsRepoRef: null + overrideImageInfoCommit: false + +stages: +- template: /eng/docker-tools/templates/stages/publish.yml@self + parameters: + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + publishConfig: ${{ parameters.publishConfig }} + isStandalonePublish: ${{ parameters.isStandalonePublish }} + sourceBuildPipelineDefinitionId: ${{ parameters.sourceBuildPipelineDefinitionId }} + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} + overrideImageInfoCommit: ${{ parameters.overrideImageInfoCommit }} + + customPublishInitSteps: + - pwsh: | + # When reporting the repo name in the publish notification, we don't want to include + # the org part of the repo name (e.g. we want "dotnet-docker", not "dotnet-dotnet-docker"). + # This also accounts for the different separators between AzDO and GitHub repo names. + + $repoName = "$(Build.Repository.Name)" + + $orgSeparatorIndex = $repoName.IndexOf("/") + if ($orgSeparatorIndex -eq -1) { + $orgSeparatorIndex = $repoName.IndexOf("-") + } + + if ($orgSeparatorIndex -ge 0) { + $repoName = $repoName.Substring($orgSeparatorIndex + 1) + } + echo "##vso[task.setvariable variable=publishNotificationRepoName]$repoName" + displayName: "Set Custom Repo Name Var" + - ${{ parameters.customPublishInitSteps }} + + pool: + ${{ if ne(parameters.pool, '') }}: + ${{ parameters.pool }} + ${{ elseif eq(variables['System.TeamProject'], parameters.publicProjectName) }}: + vmImage: $(defaultLinuxAmd64PoolImage) + ${{ elseif eq(variables['System.TeamProject'], parameters.internalProjectName) }}: + name: $(linuxAmd64InternalPoolName) + image: $(linuxAmd64InternalPoolImage) + os: linux diff --git a/eng/docker-tools/templates/stages/publish.yml b/eng/docker-tools/templates/stages/publish.yml new file mode 100644 index 000000000..ffdd594c2 --- /dev/null +++ b/eng/docker-tools/templates/stages/publish.yml @@ -0,0 +1,80 @@ +parameters: + customPublishInitSteps: [] + customPublishVariables: [] + + internalProjectName: null + publicProjectName: null + + publishConfig: null + + isStandalonePublish: false + + pool: + vmImage: $(defaultLinuxAmd64PoolImage) + + sourceBuildPipelineDefinitionId: '' + sourceBuildPipelineRunId: '' + + versionsRepoRef: null + versionsRepoPath: "versions" + + # When true, any updated images will have the SHA in their commit URL updated + # to the commit that this pipeline is running on, instead of the commit they + # were built from. Use in combination with isStandalonePublish to ensure that + # internally built images still reference public Dockerfiles. + overrideImageInfoCommit: false + +################################################################################ +# Publish Images +################################################################################ +stages: +- stage: Publish + ${{ if eq(parameters.isStandalonePublish, true) }}: + dependsOn: [] + ${{ else }}: + ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: + dependsOn: Test + ${{ else }}: + dependsOn: Post_Build + condition: " + and( + not(canceled()), + and( + contains(variables['stages'], 'publish'), + or( + or( + and( + and( + contains(variables['stages'], 'build'), + succeeded('Post_Build')), + and( + contains(variables['stages'], 'test'), + in(dependencies.Test.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'))), + or( + and( + not(contains(variables['stages'], 'build')), + and( + contains(variables['stages'], 'test'), + in(dependencies.Test.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'))), + and( + not(contains(variables['stages'], 'test')), + and( + contains(variables['stages'], 'build'), + succeeded('Post_Build'))))), + not( + or( + contains(variables['stages'], 'build'), + contains(variables['stages'], 'test'))))))" + jobs: + - template: /eng/docker-tools/templates/jobs/publish.yml@self + parameters: + pool: ${{ parameters.pool }} + internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} + customPublishVariables: ${{ parameters.customPublishVariables }} + customInitSteps: ${{ parameters.customPublishInitSteps }} + sourceBuildPipelineDefinitionId: ${{ parameters.sourceBuildPipelineDefinitionId }} + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} + versionsRepoPath: ${{ parameters.versionsRepoPath }} + overrideImageInfoCommit: ${{ parameters.overrideImageInfoCommit }} diff --git a/eng/docker-tools/templates/stages/setup-service-connections.yml b/eng/docker-tools/templates/stages/setup-service-connections.yml new file mode 100644 index 000000000..2ef74e90c --- /dev/null +++ b/eng/docker-tools/templates/stages/setup-service-connections.yml @@ -0,0 +1,38 @@ +# This stage exists to tell Azure DevOps about all of the service connections +# that will be used in the pipeline. A service connection will not work unless +# it is declared in this stage's parameters, even if your pipeline has already +# been granted access to the service connection. This stage also does not need +# to complete before the service connection is used. +parameters: +- name: pool + type: object + default: + name: $(default1ESInternalPoolName) + image: $(default1ESInternalPoolImage) + os: linux +# serviceConnections object shape: +# - name: string +- name: serviceConnections + type: object + default: [] + +stages: + +- stage: SetupServiceConnectionsStage + displayName: Setup service connections + jobs: + + - job: SetupServiceConnectionsJob + displayName: Setup service connections + pool: ${{ parameters.pool }} + steps: + - checkout: none + - ${{ each serviceConnection in parameters.serviceConnections }}: + - task: AzureCLI@2 + displayName: Setup ${{ serviceConnection.name }} + inputs: + azureSubscription: ${{ serviceConnection.name }} + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + az account show diff --git a/eng/docker-tools/templates/steps/annotate-eol-digests.yml b/eng/docker-tools/templates/steps/annotate-eol-digests.yml new file mode 100644 index 000000000..0cb090c1c --- /dev/null +++ b/eng/docker-tools/templates/steps/annotate-eol-digests.yml @@ -0,0 +1,48 @@ +parameters: +- name: acr + type: object +# Path to EOL annotation data JSON file generated by 'generateEolAnnotationData*' command +- name: dataFile + type: string + +steps: + - script: mkdir -p $(Build.ArtifactStagingDirectory)/annotation-digests + displayName: Create Annotation Digests Directory + - template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self + parameters: + displayName: Annotate EOL Images (${{ parameters.acr.server }}) + serviceConnections: + - name: acr + id: ${{ parameters.acr.serviceConnection.id }} + tenantId: ${{ parameters.acr.serviceConnection.tenantId }} + clientId: ${{ parameters.acr.serviceConnection.clientId }} + internalProjectName: internal + condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true')) + args: >- + annotateEolDigests + "${{ parameters.dataFile }}" + "${{ parameters.acr.server }}" + "${{ parameters.acr.repoPrefix }}" + $(artifactsPath)/annotation-digests/annotation-digests.txt + $(dryRunArg) + - template: /eng/docker-tools/templates/steps/publish-artifact.yml@self + parameters: + path: $(Build.ArtifactStagingDirectory)/annotation-digests + artifactName: annotation-digests-${{ parameters.acr.server }}-$(System.JobAttempt) + displayName: Publish Annotation Digests List (${{ parameters.acr.server }}) + internalProjectName: internal + publicProjectName: public + condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true')) + - template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self + parameters: + displayName: Wait for Annotation Ingestion (${{ parameters.acr.server }}) + serviceConnections: + - name: mar + id: $(marStatus.serviceConnection.id) + tenantId: $(marStatus.serviceConnection.tenantId) + clientId: $(marStatus.serviceConnection.clientId) + internalProjectName: internal + condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true'), eq(variables['waitForIngestionEnabled'], 'true')) + args: >- + waitForMarAnnotationIngestion + $(artifactsPath)/annotation-digests/annotation-digests.txt diff --git a/eng/docker-tools/templates/steps/clean-acr-images.yml b/eng/docker-tools/templates/steps/clean-acr-images.yml new file mode 100644 index 000000000..b791de305 --- /dev/null +++ b/eng/docker-tools/templates/steps/clean-acr-images.yml @@ -0,0 +1,33 @@ +parameters: + repo: null + acr: null + action: null + age: null + customArgs: "--dry-run" + internalProjectName: null + publishConfig: null +steps: + - template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self + parameters: + # Options are documented in CleanAcrImagesOptions.cs + ${{ if eq(parameters.action, 'delete') }}: + displayName: "Delete ${{ parameters.repo }}" + ${{ elseif parameters.age }}: + displayName: "Clean ${{ parameters.repo }} (${{ parameters.action }} > ${{ parameters.age }}d)" + ${{ else }}: + displayName: "Clean ${{ parameters.repo }} (${{ parameters.action }})" + serviceConnections: + - name: acr + id: ${{ parameters.publishConfig.cleanServiceConnection.id }} + tenantId: ${{ parameters.publishConfig.cleanServiceConnection.tenantId }} + clientId: ${{ parameters.publishConfig.cleanServiceConnection.clientId }} + internalProjectName: ${{ parameters.internalProjectName }} + args: >- + cleanAcrImages + ${{ parameters.repo }} + ${{ parameters.acr.subscription }} + ${{ parameters.acr.resourceGroup }} + ${{ parameters.acr.server }} + --action ${{ parameters.action }} + --age ${{ parameters.age }} + ${{ parameters.customArgs }} diff --git a/eng/docker-tools/templates/steps/cleanup-docker-linux.yml b/eng/docker-tools/templates/steps/cleanup-docker-linux.yml new file mode 100644 index 000000000..ddf0f9b09 --- /dev/null +++ b/eng/docker-tools/templates/steps/cleanup-docker-linux.yml @@ -0,0 +1,15 @@ +parameters: + condition: true + +steps: + ################################################################################ + # Cleanup local Docker server + ################################################################################ +- script: docker stop $(docker ps -q) || true + displayName: Stop Running Containers + condition: and(always(), ${{ parameters.condition }}) + continueOnError: true +- script: docker system prune -a -f --volumes + displayName: Cleanup Docker + condition: and(always(), ${{ parameters.condition }}) + continueOnError: true diff --git a/eng/docker-tools/templates/steps/cleanup-docker-windows.yml b/eng/docker-tools/templates/steps/cleanup-docker-windows.yml new file mode 100644 index 000000000..a2c41c20b --- /dev/null +++ b/eng/docker-tools/templates/steps/cleanup-docker-windows.yml @@ -0,0 +1,18 @@ +parameters: + condition: true + +steps: + ################################################################################ + # Cleanup Docker Resources + ################################################################################ +- powershell: $(engDockerToolsPath)/Invoke-CleanupDocker.ps1 + displayName: Cleanup Docker Images + condition: and(always(), ${{ parameters.condition }}) + continueOnError: true +- powershell: | + if (Test-Path $(Build.BinariesDirectory)\.Microsoft.DotNet.ImageBuilder) { + Remove-Item $(Build.BinariesDirectory)\.Microsoft.DotNet.ImageBuilder -Force -Recurse; + } + displayName: Cleanup Image Builder + condition: and(always(), ${{ parameters.condition }}) + continueOnError: true diff --git a/eng/docker-tools/templates/steps/copy-base-images.yml b/eng/docker-tools/templates/steps/copy-base-images.yml new file mode 100644 index 000000000..d54737083 --- /dev/null +++ b/eng/docker-tools/templates/steps/copy-base-images.yml @@ -0,0 +1,52 @@ +parameters: +- name: acr + type: object + default: + server: "" + serviceConnection: + tenantId: "" + clientId: "" + id: "" + subscription: "" + resourceGroup: "" +- name: repoPrefix + type: string + default: null +- name: additionalOptions + type: string + default: "" +- name: continueOnError + type: string + default: false +- name: forceDryRun + type: boolean + default: false + +steps: +- ${{ if or(eq(parameters.forceDryRun, true), eq(variables['System.TeamProject'], 'public'), eq(variables['Build.Reason'], 'PullRequest')) }}: + - script: echo "##vso[task.setvariable variable=dryRunArg]--dry-run" +- template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self + parameters: + displayName: Copy Base Images + serviceConnections: + - name: "acr" + tenantId: ${{ parameters.acr.serviceConnection.tenantId }} + clientId: ${{ parameters.acr.serviceConnection.clientId }} + id: ${{ parameters.acr.serviceConnection.id }} + continueOnError: ${{ parameters.continueOnError }} + internalProjectName: 'internal' + # Use environment variable to reference $(dryRunArg). Since $(dryRunArg) might be undefined, + # PowerShell will treat the Azure Pipelines variable macro syntax as a command and throw an + # error + args: >- + copyBaseImages + '${{ parameters.acr.subscription }}' + '${{ parameters.acr.resourceGroup }}' + $(dockerHubRegistryCreds) + $(customCopyBaseImagesArgs) + --repo-prefix '${{ parameters.repoPrefix }}' + --registry-override '${{ parameters.acr.server }}' + --os-type 'linux' + --architecture '*' + $env:DRYRUNARG + ${{ parameters.additionalOptions }} diff --git a/eng/docker-tools/templates/steps/download-build-artifact.yml b/eng/docker-tools/templates/steps/download-build-artifact.yml new file mode 100644 index 000000000..965f49320 --- /dev/null +++ b/eng/docker-tools/templates/steps/download-build-artifact.yml @@ -0,0 +1,40 @@ +parameters: + # Destination directory on the pipeline agent's filesystem, relative or absolute. + targetPath: "" + # The build/pipeline artifact to download. If the value is left empty, + # the task downloads all artifacts associated with the pipeline run. + artifactName: "" + # AKA pipeline/definition - optional. + # If this is left empty, use the current pipeline's definition ID. + # You can get this from the URL of the pipeline's overview page on Azure DevOps. + # Example: https://dev.azure.com/$org/$project/_build?definitionId=373 + pipelineDefinitionId: "" + # AKA runId/buildId/pipelineId - optional. + # The identifier of the pipeline run from which to download the artifacts. + # If this is left empty, then always download from the current pipeline + # You can get this from the URL of the specific pipeline run, for example: + # https://dev.azure.com/$org/$project/_build/results?buildId=2709155&view=results + pipelineRunId: "" + condition: true + continueOnError: false + +steps: +# https://learn.microsoft.com/azure/devops/pipelines/tasks/reference/download-pipeline-artifact-v2 +- task: DownloadPipelineArtifact@2 + inputs: + ${{ if ne(parameters.pipelineRunId, '') }}: + buildType: specific + project: $(System.TeamProject) + ${{ if ne(parameters.pipelineDefinitionId, '') }}: + definition: ${{ parameters.pipelineDefinitionId }} + ${{ else }}: + definition: $(System.DefinitionId) + buildId: ${{ parameters.pipelineRunId }} + buildVersionToDownload: specific + ${{ else }}: + buildType: current + targetPath: ${{ parameters.targetPath }} + artifactName: ${{ parameters.artifactName }} + displayName: Download Build Artifact(s) + condition: and(succeeded(), ${{ parameters.condition }}) + continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/docker-tools/templates/steps/init-common.yml b/eng/docker-tools/templates/steps/init-common.yml new file mode 100644 index 000000000..c22ad449c --- /dev/null +++ b/eng/docker-tools/templates/steps/init-common.yml @@ -0,0 +1,9 @@ +parameters: + condition: true + +steps: +- powershell: | + $sourceBranch=$Env:BUILD_SOURCEBRANCH -replace "refs/heads/","" -replace "refs/tags/","" -replace "refs/pull/","" + echo "##vso[task.setvariable variable=sourceBranch]$sourceBranch" + displayName: Define Source Branch Variable + condition: and(succeeded(), ${{ parameters.condition }}) diff --git a/eng/docker-tools/templates/steps/init-docker-linux.yml b/eng/docker-tools/templates/steps/init-docker-linux.yml new file mode 100644 index 000000000..c8669e19b --- /dev/null +++ b/eng/docker-tools/templates/steps/init-docker-linux.yml @@ -0,0 +1,90 @@ +parameters: + setupImageBuilder: true + setupTestRunner: false + cleanupDocker: false + condition: true + +steps: +- template: /eng/docker-tools/templates/steps/init-common.yml@self + parameters: + condition: ${{ parameters.condition }} +- script: echo "##vso[task.setvariable variable=artifactsPath]/artifacts" + displayName: Define Artifacts Path Variable + condition: and(succeeded(), ${{ parameters.condition }}) + + ################################################################################ + # Cleanup Docker Resources + ################################################################################ +- ${{ if eq(parameters.cleanupDocker, 'true') }}: + - template: /eng/docker-tools/templates/steps/cleanup-docker-linux.yml@self + parameters: + condition: ${{ parameters.condition }} + + ################################################################################ + # Setup Image Builder (Optional) + ################################################################################ +- ${{ if eq(parameters.setupImageBuilder, 'true') }}: + + - powershell: $(engDockerToolsPath)/Pull-Image.ps1 $(imageNames.imageBuilder) + displayName: Pull Image Builder + condition: and(succeeded(), ${{ parameters.condition }}) + + - script: >- + docker build + -t $(imageNames.imageBuilder.withrepo) + --build-arg IMAGE=$(imageNames.imageBuilder) + -f $(engDockerToolsPath)/Dockerfile.WithRepo . + displayName: Build Image for Image Builder + condition: and(succeeded(), ${{ parameters.condition }}) + + - task: PowerShell@2 + displayName: Define ImageBuilder Command Variables + condition: and(succeeded(), ${{ parameters.condition }}) + inputs: + targetType: 'inline' + script: | + $imageBuilderImageName = "$(imageNames.imageBuilder.withrepo)" + Write-Host "##vso[task.setvariable variable=imageBuilderImageName]$imageBuilderImageName" + + $dockerRunBaseCmd = @( + "docker run --rm" + ) + + $dockerRunArgs = @( + "-v /var/run/docker.sock:/var/run/docker.sock" + "-v $(Build.ArtifactStagingDirectory):$(artifactsPath)" + "-w /repo" + "$(imageBuilderDockerRunExtraOptions)" + "$(imageNames.imageBuilder.withrepo)" + ) + + $authedDockerRunArgs = @( + '-e' + 'SYSTEM_ACCESSTOKEN=$env:SYSTEM_ACCESSTOKEN' + '-e' + 'SYSTEM_OIDCREQUESTURI=$env:SYSTEM_OIDCREQUESTURI' + ) + + $dockerRunCmd = $dockerRunBaseCmd + $dockerRunArgs + $authedDockerRunCmd = $dockerRunBaseCmd + $authedDockerRunArgs + $dockerRunArgs + + $runImageBuilderCmd = $($dockerRunCmd -join ' ') + $runAuthedImageBuilderCmd = $($authedDockerRunCmd -join ' ') + + Write-Host "##vso[task.setvariable variable=runImageBuilderCmd]$runImageBuilderCmd" + Write-Host "##vso[task.setvariable variable=runAuthedImageBuilderCmd]$runAuthedImageBuilderCmd" + + ################################################################################ + # Setup Test Runner (Optional) + ################################################################################ +- ${{ if eq(parameters.setupTestRunner, 'true') }}: + - powershell: $(engDockerToolsPath)/Pull-Image.ps1 $(imageNames.testrunner) + displayName: Pull Test Runner + condition: and(succeeded(), ${{ parameters.condition }}) + - script: > + docker build + -t $(imageNames.testRunner.withrepo) + --build-arg IMAGE=$(imageNames.testrunner) + -f $(engDockerToolsPath)/Dockerfile.WithRepo . + displayName: Build Test Runner Image + condition: and(succeeded(), ${{ parameters.condition }}) diff --git a/eng/docker-tools/templates/steps/init-docker-windows.yml b/eng/docker-tools/templates/steps/init-docker-windows.yml new file mode 100644 index 000000000..e8593b5eb --- /dev/null +++ b/eng/docker-tools/templates/steps/init-docker-windows.yml @@ -0,0 +1,48 @@ +parameters: + setupImageBuilder: true + condition: true + +steps: +- template: /eng/docker-tools/templates/steps/init-common.yml@self + parameters: + condition: ${{ parameters.condition }} +- powershell: echo "##vso[task.setvariable variable=artifactsPath]$(Build.ArtifactStagingDirectory)" + displayName: Define Artifacts Path Variable + condition: and(succeeded(), ${{ parameters.condition }}) + + ################################################################################ + # Cleanup Docker Resources + ################################################################################ +- template: /eng/docker-tools/templates/steps/cleanup-docker-windows.yml@self + parameters: + condition: ${{ parameters.condition }} + + ################################################################################ + # Setup Image Builder (Optional) + ################################################################################ +- ${{ if eq(parameters.setupImageBuilder, 'true') }}: + - powershell: $(engDockerToolsPath)/Invoke-WithRetry.ps1 "docker pull $(imageNames.imageBuilder)" + displayName: Pull Image Builder + condition: and(succeeded(), ${{ parameters.condition }}) + - script: docker create --name setupImageBuilder-$(Build.BuildId)-$(System.JobId) $(imageNames.imageBuilder) + displayName: Create Setup Container + condition: and(succeeded(), ${{ parameters.condition }}) + - script: > + docker cp + setupImageBuilder-$(Build.BuildId)-$(System.JobId):/image-builder + $(Build.BinariesDirectory)/.Microsoft.DotNet.ImageBuilder + displayName: Copy Image Builder + condition: and(succeeded(), ${{ parameters.condition }}) + - script: docker rm -f setupImageBuilder-$(Build.BuildId)-$(System.JobId) + displayName: Cleanup Setup Container + condition: and(always(), ${{ parameters.condition }}) + continueOnError: true + - task: PowerShell@2 + displayName: Define runImageBuilderCmd Variables + condition: and(succeeded(), ${{ parameters.condition }}) + inputs: + targetType: 'inline' + script: | + $runImageBuilderCmd = "$(Build.BinariesDirectory)\.Microsoft.DotNet.ImageBuilder\Microsoft.DotNet.ImageBuilder.exe" + Write-Host "##vso[task.setvariable variable=runImageBuilderCmd]$runImageBuilderCmd" + Write-Host "##vso[task.setvariable variable=runAuthedImageBuilderCmd]$runImageBuilderCmd" diff --git a/eng/docker-tools/templates/steps/init-matrix-build-publish.yml b/eng/docker-tools/templates/steps/init-matrix-build-publish.yml new file mode 100644 index 000000000..8a555eccd --- /dev/null +++ b/eng/docker-tools/templates/steps/init-matrix-build-publish.yml @@ -0,0 +1,78 @@ +# Initialize common variables used in +# - Generating build matrix +# - Building images +# - Running tests +# - Publishing images + +parameters: + publishConfig: null + versionsRepoRef: "" + versionsRepoPath: "versions" + +steps: +- checkout: self +- ${{ if ne(parameters.versionsRepoRef, '') }}: + - checkout: ${{ parameters.versionsRepoRef }} + path: s/${{ parameters.versionsRepoPath }} + persistCredentials: true + fetchDepth: 1 + condition: succeeded() +- powershell: | + $commonMatrixAndBuildOptions = "--source-repo $(publicGitRepoUri)" + if ("$(System.TeamProject)" -eq "internal" -and "$(Build.Reason)" -ne "PullRequest") { + $commonMatrixAndBuildOptions = "$commonMatrixAndBuildOptions --source-repo-prefix ${{ parameters.publishConfig.internalMirrorAcr.repoPrefix }} --registry-override ${{ parameters.publishConfig.buildAcr.server }}" + } + + if ("$(System.TeamProject)" -eq "public" -and "$(public-mirror.server)" -ne "") { + $commonMatrixAndBuildOptions = "$commonMatrixAndBuildOptions --base-override-regex '^(?!mcr\.microsoft\.com)' --base-override-sub '$(public-mirror.server)/'" + } + + if ("${{ parameters.versionsRepoRef }}" -ne "") { + $versionsBasePath = "${{ parameters.versionsRepoPath }}/" + $pipelineDisabledCache = "false" + + $pathSeparatorIndex = "$(Build.Repository.Name)".IndexOf("/") + if ($pathSeparatorIndex -ge 0) { + $buildRepoName = "$(Build.Repository.Name)".Substring($pathSeparatorIndex + 1) + } + else { + $buildRepoName = "$(Build.Repository.Name)" + } + + $engDockerToolsPath = "$(Build.Repository.LocalPath)/$buildRepoName/$(engDockerToolsRelativePath)" + + $engPath = "$(Build.Repository.LocalPath)/$buildRepoName/eng" + $manifest = "$buildRepoName/$(manifest)" + $testResultsDirectory = "$buildRepoName/$testResultsDirectory" + + if ("$(testScriptPath)") { + $testScriptPath = "$buildRepoName/$(testScriptPath)" + } + + echo "##vso[task.setvariable variable=buildRepoName]$buildRepoName" + echo "##vso[task.setvariable variable=engDockerToolsPath]$engDockerToolsPath" + echo "##vso[task.setvariable variable=manifest]$manifest" + echo "##vso[task.setvariable variable=engPath]$engPath" + echo "##vso[task.setvariable variable=testScriptPath]$testScriptPath" + echo "##vso[task.setvariable variable=testResultsDirectory]$testResultsDirectory" + } + else { + $versionsBasePath = "" + $pipelineDisabledCache = "true" + } + + echo "##vso[task.setvariable variable=commonMatrixAndBuildOptions]$commonMatrixAndBuildOptions" + echo "##vso[task.setvariable variable=versionsBasePath]$versionsBasePath" + echo "##vso[task.setvariable variable=pipelineDisabledCache]$pipelineDisabledCache" + displayName: Set Common Variables for Matrix, Build, and Publish + +- ${{ if ne(parameters.versionsRepoRef, '') }}: + # Special logic is needed to copy the tsaoptions.json file to a well known location for the 1ES PT. + # This template has multiple checkouts and AzDO doesn't have support for dynamically determining the + # default repo path therefore the 1es-official logic can't calculate the repo's tsa config file path. + - task: CopyFiles@2 + displayName: Copy TSA Config + inputs: + SourceFolder: '$(Build.Repository.LocalPath)/$(buildRepoName)' + Contents: '.config/tsaoptions.json' + TargetFolder: '$(Build.SourcesDirectory)' diff --git a/eng/docker-tools/templates/steps/parse-test-arg-arrays.yml b/eng/docker-tools/templates/steps/parse-test-arg-arrays.yml new file mode 100644 index 000000000..5ab18214d --- /dev/null +++ b/eng/docker-tools/templates/steps/parse-test-arg-arrays.yml @@ -0,0 +1,15 @@ +steps: +- powershell: | + # Formats the OS versions in a compact human-readable form (e.g. "os1/os2") + $osVersionsDisplayName = '$(osVersions)' -Replace '--os-version ', '' -Replace ' ', '/' + + # Defines a PowerShell snippet in string-form that can be used to initialize an array of the OS versions + $osVersionsArrayInitStr = "@('" + $($osVersionsDisplayName -Replace "/", "', '") + "')" + + echo "##vso[task.setvariable variable=osVersionsDisplayName]$osVersionsDisplayName" + echo "##vso[task.setvariable variable=osVersionsArrayInitStr]$osVersionsArrayInitStr" + + # Defines a PowerShell snippet in string-form that can be used to initialize an array of the image builder paths + $pathInitStr = "@('" + $('$(imageBuilderPaths)' -Replace '--path', '' -Replace " ", "', '") + "')" + echo "##vso[task.setvariable variable=imageBuilderPathsArrayInitStr]$pathInitStr" + displayName: Parse Test Arg Arrays diff --git a/eng/docker-tools/templates/steps/publish-artifact.yml b/eng/docker-tools/templates/steps/publish-artifact.yml new file mode 100644 index 000000000..72ce47005 --- /dev/null +++ b/eng/docker-tools/templates/steps/publish-artifact.yml @@ -0,0 +1,28 @@ +parameters: +- name: path + type: string +- name: artifactName + type: string +- name: displayName + type: string +- name: internalProjectName + type: string +- name: publicProjectName + type: string +- name: condition + type: string + default: 'true' + +steps: +- ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}: + - task: 1ES.PublishPipelineArtifact@1 + inputs: + path: ${{ parameters.path }} + artifact: ${{ parameters.artifactName }} + displayName: ${{ parameters.displayName }} + condition: and(succeeded(), ${{ parameters.condition }}) +- ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}: + - publish: ${{ parameters.path }} + artifact: ${{ parameters.artifactName }} + displayName: ${{ parameters.displayName }} + condition: and(succeeded(), ${{ parameters.condition }}) diff --git a/eng/docker-tools/templates/steps/publish-readmes.yml b/eng/docker-tools/templates/steps/publish-readmes.yml new file mode 100644 index 000000000..270898050 --- /dev/null +++ b/eng/docker-tools/templates/steps/publish-readmes.yml @@ -0,0 +1,29 @@ +parameters: + dryRunArg: "" + condition: true + +steps: +- script: > + $(runImageBuilderCmd) publishMcrDocs + --manifest '$(manifest)' + --registry-override '${{ parameters.publishConfig.publishAcr.server }}' + '$(mcrDocsRepoInfo.userName)' + '$(mcrDocsRepoInfo.email)' + $(mcrDocsRepoInfo.authArgs) + '$(publicGitRepoUri)' + ${{ parameters.dryRunArg }} + $(manifestVariables) + $(imageBuilder.queueArgs) + --git-owner 'Microsoft' + --git-repo 'mcrdocs' + --git-branch 'main' + --git-path 'teams' + $(additionalPublishMcrDocsArgs) + name: PublishReadmes + displayName: Publish Readmes + condition: ${{ parameters.condition }} +- template: /eng/docker-tools/templates/steps/wait-for-mcr-doc-ingestion.yml@self + parameters: + commitDigest: $(PublishReadmes.readmeCommitDigest) + condition: and(${{ parameters.condition }}, ne(variables['PublishReadmes.readmeCommitDigest'], '')) + dryRunArg: ${{ parameters.dryRunArg }} diff --git a/eng/docker-tools/templates/steps/retain-build.yml b/eng/docker-tools/templates/steps/retain-build.yml new file mode 100644 index 000000000..bcf11d57e --- /dev/null +++ b/eng/docker-tools/templates/steps/retain-build.yml @@ -0,0 +1,9 @@ +steps: +- powershell: > + $(engDockerToolsPath)/Retain-Build.ps1 + -BuildId $(Build.BuildId) + -AzdoOrgUri '$(System.CollectionUri)' + -AzdoProject '$(System.TeamProject)' + -Token '$(System.AccessToken)' + displayName: Enable permanent build retention + condition: and(succeeded(), eq(variables.retainBuild, 'true')) diff --git a/eng/docker-tools/templates/steps/run-imagebuilder.yml b/eng/docker-tools/templates/steps/run-imagebuilder.yml new file mode 100644 index 000000000..3613d809b --- /dev/null +++ b/eng/docker-tools/templates/steps/run-imagebuilder.yml @@ -0,0 +1,76 @@ +parameters: +- name: name + type: string + default: "" +- name: displayName + type: string + default: "Run ImageBuilder" +- name: serviceConnections + type: object + default: + # name: the name of the service connection argument that will be passed to the ImageBuilder command. + # For example, if the argument is --acr-service-connection, the name would be "acr". + - name: "" + # The service connection's ID (GUID). + id: "" + # The client ID of the Managed Idendity backing the service connection (GUID). + clientId: "" + # The ID of the tenant that the Managed Identity is in (GUID). + tenantId: "" +- name: internalProjectName + type: string + default: null +- name: args + type: string + default: null +- name: condition + type: string + default: succeeded() +- name: continueOnError + type: boolean + default: false +- name: dockerClientOS + type: string + default: "linux" + +steps: +- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: + + - task: PowerShell@2 + ${{ if ne(parameters.name, '') }}: + name: ${{ parameters.name }} + displayName: ${{ parameters.displayName }} + continueOnError: ${{ parameters.continueOnError }} + condition: ${{ parameters.condition }} + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + SYSTEM_OIDCREQUESTURI: $(System.OidcRequestUri) + inputs: + targetType: 'inline' + script: | + $serviceConnections = '${{ convertToJson(parameters.serviceConnections) }}' + + Write-Host "Service connections:" + Write-Host "${serviceConnections}" + + $serviceConnectionsJson = $serviceConnections | ConvertFrom-Json + $serviceConnectionsArgs = @() + foreach ($connection in $serviceConnectionsJson) { + $serviceConnectionsArgs += "--$($connection.name)-service-connection" + $serviceConnectionsArgs += "$($connection.tenantId):$($connection.clientId):$($connection.id)" + } + + $(runAuthedImageBuilderCmd) ${{ parameters.args }} @serviceConnectionsArgs + +- ${{ else }}: + + - task: PowerShell@2 + ${{ if ne(parameters.name, '') }}: + name: ${{ parameters.name }} + displayName: ${{ parameters.displayName }} + continueOnError: ${{ parameters.continueOnError }} + condition: ${{ parameters.condition }} + inputs: + targetType: 'inline' + script: >- + $(runImageBuilderCmd) ${{ parameters.args }} diff --git a/eng/docker-tools/templates/steps/run-pwsh-with-auth.yml b/eng/docker-tools/templates/steps/run-pwsh-with-auth.yml new file mode 100644 index 000000000..84de1e1f5 --- /dev/null +++ b/eng/docker-tools/templates/steps/run-pwsh-with-auth.yml @@ -0,0 +1,39 @@ +parameters: +- name: name + type: string + default: "" +- name: displayName + type: string + default: "Run PowerShell" +- name: serviceConnection + type: string + default: "" +- name: command + type: string + default: null +- name: continueOnError + type: boolean + default: false +- name: dockerClientOS + type: string + default: "linux" +- name: condition + type: string + default: true + +steps: +- task: AzureCLI@2 + ${{ if ne(parameters.name, '') }}: + name: ${{ parameters.name }} + displayName: ${{ parameters.displayName }} (Authenticated) + continueOnError: ${{ parameters.continueOnError }} + condition: and(succeeded(), ${{ parameters.condition }}) + inputs: + azureSubscription: ${{ parameters.serviceConnection }} + addSpnToEnvironment: true + ${{ if eq(parameters.dockerClientOS, 'windows') }}: + scriptType: 'ps' + ${{ else }}: + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: ${{ parameters.command }}; diff --git a/eng/docker-tools/templates/steps/set-dry-run.yml b/eng/docker-tools/templates/steps/set-dry-run.yml new file mode 100644 index 000000000..44e3062b0 --- /dev/null +++ b/eng/docker-tools/templates/steps/set-dry-run.yml @@ -0,0 +1,30 @@ +parameters: + name: publishConfig + type: object + +steps: +- powershell: | + if ("$env:ONEESPT_BUILDTYPE" -eq "Unofficial") + { + # Don't use dry-run mode for unofficial builds, since they publish to a + # non-production environment + $dryRunArg="" + } + elseif ("$(System.TeamProject)" -eq "$(publicProjectName)") + { + # Public builds need to use dry-run mode since they don't publish anywhere. + $dryRunArg="--dry-run" + } + elseif (-not "$(officialRepoPrefixes)".Split(',').Contains("${{ parameters.publishConfig.publishAcr.repoPrefix }}")) + { + # If we're running an internal build on an official pipeline but not + # publishing to an official repo prefix, then use dry run mode. + $dryRunArg="--dry-run" + } + else + { + $dryRunArg="" + } + + echo "##vso[task.setvariable variable=dryRunArg]$dryRunArg" + displayName: Set dry-run arg for non-prod diff --git a/eng/docker-tools/templates/steps/set-image-info-path-var.yml b/eng/docker-tools/templates/steps/set-image-info-path-var.yml new file mode 100644 index 000000000..71d317fa2 --- /dev/null +++ b/eng/docker-tools/templates/steps/set-image-info-path-var.yml @@ -0,0 +1,19 @@ +parameters: + publicSourceBranch: null + +steps: +- powershell: | + $basePath = "$(gitHubVersionsRepoInfo.path)" + + $publicSourceBranch = "${{ parameters.publicSourceBranch }}" + + if ($publicSourceBranch -eq "") { + throw "publicSourceBranch variable is not set" + } + + $buildRepoName = "$(Build.Repository.Name)".Replace("/", "-") + $imageInfoName = "image-info.$buildRepoName-$publicSourceBranch$(imageInfoVariant).json" + + echo "##vso[task.setvariable variable=imageInfoVersionsPath]$basePath/$imageInfoName" + echo "##vso[task.setvariable variable=gitHubImageInfoVersionsPath]$(gitHubVersionsRepoInfo.path)/$imageInfoName" + displayName: Set Image Info Path Vars diff --git a/eng/docker-tools/templates/steps/test-images-linux-client.yml b/eng/docker-tools/templates/steps/test-images-linux-client.yml new file mode 100644 index 000000000..38426ab2a --- /dev/null +++ b/eng/docker-tools/templates/steps/test-images-linux-client.yml @@ -0,0 +1,107 @@ +parameters: + preBuildValidation: false + internalProjectName: null + publishConfig: null + condition: true + customInitSteps: [] + sourceBuildPipelineRunId: "" + +steps: +- template: /eng/docker-tools/templates/steps/init-docker-linux.yml@self + parameters: + setupImageBuilder: false + setupTestRunner: true + # Clean only up when we're running an internal build, not a PR, and not doing pre-build validation. + # i.e. when we're building something important. + cleanupDocker: ${{ and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.preBuildValidation, 'false')) }} + condition: ${{ parameters.condition }} +- ${{ parameters.customInitSteps }} +- script: | + echo "##vso[task.setvariable variable=testRunner.container]testrunner-$(Build.BuildId)-$(System.JobId)" + + additionalTestArgs="$ADDITIONALTESTARGS" + if [ "${{ parameters.preBuildValidation }}" == "true" ]; then + additionalTestArgs="$additionalTestArgs -TestCategories pre-build" + else + if [ "${{ variables['System.TeamProject'] }}" == "${{ parameters.internalProjectName }}" ] && [ "${{ variables['Build.Reason'] }}" != "PullRequest" ]; then + additionalTestArgs="$additionalTestArgs -PullImages -Registry ${{ parameters.publishConfig.buildAcr.server }} -RepoPrefix ${{ parameters.publishConfig.buildAcr.repoPrefix }} -ImageInfoPath $(artifactsPath)/image-info.json" + if [ "$TESTCATEGORIESOVERRIDE" != "" ]; then + additionalTestArgs="$additionalTestArgs -TestCategories $TESTCATEGORIESOVERRIDE" + fi + fi + fi + echo "##vso[task.setvariable variable=additionalTestArgs]$additionalTestArgs" + displayName: Set Test Variables + condition: and(succeeded(), ${{ parameters.condition }}) +- script: > + docker run -t -d + -v /var/run/docker.sock:/var/run/docker.sock + -v $(Build.ArtifactStagingDirectory):$(artifactsPath) + -e DOCKER_BUILDKIT=1 + -e RUNNING_TESTS_IN_CONTAINER=true + --name $(testRunner.container) + $(imageNames.testRunner.withrepo) + displayName: Start Test Runner Container + condition: and(succeeded(), ${{ parameters.condition }}) +- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: + - template: /eng/docker-tools/templates/steps/run-pwsh-with-auth.yml@self + parameters: + displayName: Docker login + serviceConnection: ${{ parameters.publishConfig.testServiceConnection.name }} + condition: and(succeeded(), ${{ parameters.condition }}) + command: >- + $azLoginArgs = '--service-principal --tenant $env:AZURE_TENANT_ID -u $env:AZURE_CLIENT_ID --federated-token $env:AZURE_FEDERATED_TOKEN'; + docker exec -e AZURE_TENANT_ID=$env:tenantId -e AZURE_CLIENT_ID=$env:servicePrincipalId -e AZURE_FEDERATED_TOKEN=$env:idToken $(testRunner.container) pwsh + -File $(engDockerToolsRelativePath)/Invoke-WithRetry.ps1 + "az login $azLoginArgs; az acr login -n ${{ parameters.publishConfig.buildAcr.server }}" + - ${{ if eq(parameters.preBuildValidation, 'false') }}: + - template: /eng/docker-tools/templates/steps/download-build-artifact.yml@self + parameters: + targetPath: $(Build.ArtifactStagingDirectory) + artifactName: image-info + condition: ${{ parameters.condition }} + pipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} +- template: /eng/docker-tools/templates/steps/parse-test-arg-arrays.yml@self +- powershell: > + $(test.init); + docker exec + $(testRunner.options) + $(testRunner.container) + pwsh + -Command "$(testScriptPath) + -Paths $(imageBuilderPathsArrayInitStr) + -OSVersions $(osVersionsArrayInitStr) + -Architecture '$(architecture)' + $(additionalTestArgs)" + displayName: Test Images + condition: and(succeeded(), ${{ parameters.condition }}) +- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: + - script: docker exec $(testRunner.container) docker logout ${{ parameters.publishConfig.buildAcr.server }} + displayName: Docker logout + condition: and(always(), ${{ parameters.condition }}) + continueOnError: true +- powershell: > + docker cp + $(testRunner.container):/repo/$(testResultsDirectory) + $(Common.TestResultsDirectory)/. + displayName: Copy Test Results + condition: and(always(), ${{ parameters.condition }}) + continueOnError: true +- task: PublishTestResults@2 + displayName: Publish Test Results + condition: and(always(), ${{ parameters.condition }}) + continueOnError: true + inputs: + testRunner: vSTest + testResultsFiles: '**/*.trx' + searchFolder: $(Common.TestResultsDirectory) + mergeTestResults: true + publishRunAttachments: true + ${{ if eq(parameters.preBuildValidation, 'false') }}: + testRunTitle: $(productVersion) $(osVersionsDisplayName) $(architecture) + ${{ if eq(parameters.preBuildValidation, 'true') }}: + testRunTitle: Pre-Build Validation +- script: docker rm -f $(testRunner.container) + displayName: Cleanup TestRunner Container + condition: and(always(), ${{ parameters.condition }}) + continueOnError: true diff --git a/eng/docker-tools/templates/steps/test-images-windows-client.yml b/eng/docker-tools/templates/steps/test-images-windows-client.yml new file mode 100644 index 000000000..ad84cff0e --- /dev/null +++ b/eng/docker-tools/templates/steps/test-images-windows-client.yml @@ -0,0 +1,66 @@ +parameters: + internalProjectName: null + publishConfig: null + condition: true + customInitSteps: [] + sourceBuildPipelineRunId: "" + +steps: +- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: + - template: /eng/docker-tools/templates/steps/init-docker-windows.yml@self + parameters: + cleanupDocker: true + setupImageBuilder: false + condition: ${{ parameters.condition }} + - template: /eng/docker-tools/templates/steps/run-pwsh-with-auth.yml@self + parameters: + displayName: Docker login + serviceConnection: ${{ parameters.publishConfig.testServiceConnection.name }} + dockerClientOS: windows + condition: and(succeeded(), ${{ parameters.condition }}) + command: >- + az login --service-principal --tenant $env:tenantId -u $env:servicePrincipalId --federated-token $env:idToken; + $accessToken = $(az acr login -n ${{ parameters.publishConfig.buildAcr.server }} --expose-token --query accessToken --output tsv); + docker login ${{ parameters.publishConfig.buildAcr.server }} -u 00000000-0000-0000-0000-000000000000 -p $accessToken +- ${{ parameters.customInitSteps }} +- powershell: | + if ("${{ variables['System.TeamProject'] }}" -eq "${{ parameters.internalProjectName }}" -and "${{ variables['Build.Reason'] }}" -ne "PullRequest") { + $additionalTestArgs="$env:ADDITIONALTESTARGS -PullImages -Registry ${{ parameters.publishConfig.buildAcr.server }} -RepoPrefix ${{ parameters.publishConfig.buildAcr.repoPrefix }} -ImageInfoPath $(artifactsPath)/image-info.json" + } + echo "##vso[task.setvariable variable=additionalTestArgs]$additionalTestArgs" + displayName: Set Test Variables + condition: and(succeeded(), ${{ parameters.condition }}) +- powershell: Get-ChildItem -Path tests -r | Where {$_.Extension -match "trx"} | Remove-Item + displayName: Cleanup Old Test Results + condition: and(succeeded(), ${{ parameters.condition }}) +- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: + - template: /eng/docker-tools/templates/steps/download-build-artifact.yml@self + parameters: + targetPath: $(Build.ArtifactStagingDirectory) + artifactName: image-info + condition: ${{ parameters.condition }} + pipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} +- template: /eng/docker-tools/templates/steps/parse-test-arg-arrays.yml@self +- powershell: > + $(test.init); + $(testScriptPath) + -Paths $(imageBuilderPathsArrayInitStr) + -OSVersions $(osVersionsArrayInitStr) + $(additionalTestArgs) + displayName: Test Images + condition: and(succeeded(), ${{ parameters.condition }}) +- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: + - script: docker logout ${{ parameters.publishConfig.buildAcr.server }} + displayName: Docker logout + condition: and(always(), ${{ parameters.condition }}) + continueOnError: true +- task: PublishTestResults@2 + displayName: Publish Test Results + condition: and(always(), ${{ parameters.condition }}) + continueOnError: true + inputs: + testRunner: vSTest + testResultsFiles: '$(testResultsDirectory)/**/*.trx' + mergeTestResults: true + publishRunAttachments: true + testRunTitle: $(productVersion) $(osVersionsDisplayName) amd64 diff --git a/eng/docker-tools/templates/steps/validate-branch.yml b/eng/docker-tools/templates/steps/validate-branch.yml new file mode 100644 index 000000000..0fb1a841d --- /dev/null +++ b/eng/docker-tools/templates/steps/validate-branch.yml @@ -0,0 +1,52 @@ +parameters: + publishConfig: null + internalProjectName: null + +steps: +- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: + - powershell: | + if ("$env:ONEESPT_BUILDTYPE" -eq "Unofficial") + { + echo "Build is from an unofficial pipeline, continuing." + exit 0 + } + + $isOfficialRepoPrefix = "$(officialRepoPrefixes)".Split(',').Contains("${{ parameters.publishConfig.publishAcr.repoPrefix }}") + if (-not $isOfficialRepoPrefix) + { + echo "This build will not publish to an official repo prefix, continuing." + echo "Publish repo prefix: ${{ parameters.publishConfig.publishAcr.repoPrefix }}" + echo "Official repo prefixes: $(officialRepoPrefixes)" + exit 0 + } + + $isOfficialBranch = "$(officialBranches)".Split(',').Contains("$(sourceBranch)") + if ($isOfficialBranch) + { + echo "$(sourceBranch) is an official branch, continuing." + echo "Official branches: $(officialBranches)" + exit 0 + } + + $hasOfficialBranchPrefix = $false + foreach ($prefix in "$(officialBranchPrefixes)".Split(',')) { + if ("$(sourceBranch)".StartsWith($prefix)) { + $hasOfficialBranchPrefix = $true + break + } + } + + if ($hasOfficialBranchPrefix) + { + echo "$(sourceBranch) has an official branch prefix, continuing." + echo "Official branch prefixes: $(officialBranchPrefixes)" + exit 0 + } + + echo "##vso[task.logissue type=error]Official builds must be done from an official branch ($(officialBranches)) and repo prefix ($(officialRepoPrefixes))." + echo "Build definition: $(Build.DefinitionName)" + echo "1ESPT build type: $(OneESPT.BuildType)" + echo "Current branch: $(sourceBranch)" + echo "Publish repo prefix: ${{ parameters.publishConfig.publishAcr.repoPrefix }}" + exit 1 + displayName: Validate Branch diff --git a/eng/docker-tools/templates/steps/wait-for-mcr-doc-ingestion.yml b/eng/docker-tools/templates/steps/wait-for-mcr-doc-ingestion.yml new file mode 100644 index 000000000..5f64438bb --- /dev/null +++ b/eng/docker-tools/templates/steps/wait-for-mcr-doc-ingestion.yml @@ -0,0 +1,21 @@ +parameters: + commitDigest: null + condition: true + dryRunArg: "" + +steps: +- template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self + parameters: + displayName: Wait for MCR Doc Ingestion + condition: and(${{ parameters.condition }}, eq(variables['waitForIngestionEnabled'], 'true')) + serviceConnections: + - name: mar + id: $(marStatus.serviceConnection.id) + tenantId: $(marStatus.serviceConnection.tenantId) + clientId: $(marStatus.serviceConnection.clientId) + internalProjectName: 'internal' + args: >- + waitForMcrDocIngestion + '${{ parameters.commitDigest }}' + --timeout '$(mcrDocIngestionTimeout)' + ${{ parameters.dryRunArg }} diff --git a/eng/docker-tools/templates/steps/wait-for-mcr-image-ingestion.yml b/eng/docker-tools/templates/steps/wait-for-mcr-image-ingestion.yml new file mode 100644 index 000000000..6c7236439 --- /dev/null +++ b/eng/docker-tools/templates/steps/wait-for-mcr-image-ingestion.yml @@ -0,0 +1,37 @@ +parameters: +- name: publishConfig + type: object + +- name: imageInfoPath + type: string + +- name: minQueueTime + type: string + +- name: dryRunArg + type: string + +- name: condition + type: string + default: "true" + +steps: +- template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self + parameters: + displayName: Wait for Image Ingestion + condition: and(${{ parameters.condition }}, eq(variables['waitForIngestionEnabled'], 'true')) + serviceConnections: + - name: mar + id: $(marStatus.serviceConnection.id) + tenantId: $(marStatus.serviceConnection.tenantId) + clientId: $(marStatus.serviceConnection.clientId) + internalProjectName: 'internal' + args: >- + waitForMcrImageIngestion + '${{ parameters.imageInfoPath }}' + --manifest '$(manifest)' + --repo-prefix '${{ parameters.publishConfig.publishAcr.repoPrefix }}' + --min-queue-time '${{ parameters.minQueueTime }}' + --timeout '$(mcrImageIngestionTimeout)' + $(manifestVariables) + ${{ parameters.dryRunArg }} diff --git a/eng/docker-tools/templates/task-prefix-decorator.yml b/eng/docker-tools/templates/task-prefix-decorator.yml new file mode 100644 index 000000000..598bfbe24 --- /dev/null +++ b/eng/docker-tools/templates/task-prefix-decorator.yml @@ -0,0 +1,63 @@ +# This Azure Pipelines template adds a prefix to the display name of each +# task passed through the `stages` parameter. When used in conjunction with +# an "extends" template which injects a lot of tasks into the pipeline, the +# added prefix helps to identify which tasks were passed through this template +# and which tasks were injected by the `baseTemplate`. +# +# This template assumes that `baseTemplate` uses the `stages` parameter. If it +# doesn't, this template likely won't work as expected. + +parameters: +# The pipeline will behave as if it were originally extended from this template, +# except with updated task display names. +- name: baseTemplate + type: string + default: "" + +# These parameters are passed directly to `baseTemplate` +- name: templateParameters + type: object + default: null + +# These stages will be modified and passed to the `baseTemplate` as the +# `stages` parameter. +- name: stages + type: stageList + default: [] + +# This prefix will be added to the display name of each task. +- name: taskPrefix + type: string + default: "🟪" + + +extends: + template: ${{ parameters.baseTemplate }} + parameters: + ${{ insert }}: ${{ parameters.templateParameters }} + stages: + - ${{ each stage in parameters.stages }}: + - stage: ${{ stage.stage }} + ${{ each property in stage }}: + ${{ if notIn(property.key, 'stage', 'jobs') }}: + ${{ property.key }} : ${{ property.value }} + jobs: + - ${{ each job in stage.jobs }}: + - job: ${{ job.job }} + ${{ each property in job }}: + ${{ if notIn(property.key, 'job', 'steps') }}: + ${{ property.key }} : ${{ property.value }} + steps: + - ${{ each step in job.steps }}: + # Special case for Azure Pipelines checkout task: + # https://learn.microsoft.com/azure/devops/extend/develop/pipeline-decorator-context?view=azure-devops#task-names-and-guids + # The checkout task does not have a name - it is special and built directly into the agent. + # Avoid modifying the checkout task, or else it will show up in the UI as a task with no name. + - ${{ if contains(step.task, '6d15af64-176c-496d-b583-fd2ae21d4df4') }}: + - ${{ step }} + - ${{ else }}: + - task: ${{ step.task }} + ${{ each property in step }}: + ${{ if notIn(property.key, 'task', 'displayName') }}: + ${{ property.key }} : ${{ property.value }} + displayName: ${{ parameters.taskPrefix }} ${{ step.displayName }} diff --git a/eng/docker-tools/templates/variables/common-paths.yml b/eng/docker-tools/templates/variables/common-paths.yml new file mode 100644 index 000000000..d8a520025 --- /dev/null +++ b/eng/docker-tools/templates/variables/common-paths.yml @@ -0,0 +1,5 @@ +variables: + engDockerToolsRelativePath: eng/docker-tools + engDockerToolsPath: $(Build.Repository.LocalPath)/$(engDockerToolsRelativePath) + engPath: $(Build.Repository.LocalPath)/eng + testScriptPath: "" diff --git a/eng/docker-tools/templates/variables/common.yml b/eng/docker-tools/templates/variables/common.yml new file mode 100644 index 000000000..1a994f684 --- /dev/null +++ b/eng/docker-tools/templates/variables/common.yml @@ -0,0 +1,79 @@ +variables: +- template: /eng/docker-tools/templates/variables/docker-images.yml@self +- template: /eng/docker-tools/templates/variables/common-paths.yml@self + +- name: publishReadme + value: true +- name: publishImageInfo + value: true +- name: ingestKustoImageInfo + value: true + # CG is disabled by default because projects are built within Dockerfiles and CG step do not scan artifacts + # that are built within Dockerfiles. A separate CG pipeline exists for this reason. +- name: skipComponentGovernanceDetection + value: false +- name: build.imageBuilderDockerRunExtraOptions + value: "" +- name: imageBuilderDockerRunExtraOptions + value: "" +- name: generateEolAnnotationDataExtraOptions + value: "" +- name: productVersionComponents + value: 2 +- name: imageInfoVariant + value: "" +- name: publishNotificationsEnabled + value: false +- name: manifestVariables + value: "" +- name: mcrImageIngestionTimeout + value: "00:20:00" +- name: mcrDocIngestionTimeout + value: "00:05:00" +- name: officialBranches + # comma-delimited list of branch names + value: main +- name: mirrorRepoPrefix + value: 'mirror/' +- name: cgBuildGrepArgs + value: "''" +- name: test.init + value: "" +- name: testRunner.options + value: "" +- name: customCopyBaseImagesArgs + value: "" +- name: additionalGenerateBuildMatrixOptions + value: "" +- name: trimCachedImagesForMatrix + value: false + +- name: defaultLinuxAmd64PoolImage + value: ubuntu-latest +- name: defaultLinuxArm32PoolImage + value: null +- name: defaultLinuxArm64PoolImage + value: null +- name: defaultWindows2016PoolImage + value: vs2017-win2016 +- name: defaultWindows1809PoolImage + value: windows-2019 +- name: defaultWindows2022PoolImage + value: windows-2022 +- name: defaultWindows2025PoolImage + value: windows-2025 + +- name: default1ESInternalPoolName + value: NetCore1ESPool-Internal +- name: default1ESInternalPoolImage + value: 1es-ubuntu-2204 + +- template: /eng/docker-tools/templates/variables/sdl-pool.yml@self + +# Define these as placeholder values to allow string validation to succeed since we don't have the +# variable group with the actual values in public builds. For internal builds, the variable group +# will cause these values to be overridden with the real values. +- name: acr.subscription + value: 00000000-0000-0000-0000-000000000000 +- name: acr-staging.subscription + value: 00000000-0000-0000-0000-000000000000 diff --git a/eng/docker-tools/templates/variables/docker-images.yml b/eng/docker-tools/templates/variables/docker-images.yml new file mode 100644 index 000000000..fff171d69 --- /dev/null +++ b/eng/docker-tools/templates/variables/docker-images.yml @@ -0,0 +1,7 @@ +variables: + imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder:2852433 + imageNames.imageBuilder: $(imageNames.imageBuilderName) + imageNames.imageBuilder.withrepo: imagebuilder-withrepo:$(Build.BuildId)-$(System.JobId) + imageNames.testRunner: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux3.0-docker-testrunner + imageNames.testRunner.withrepo: testrunner-withrepo:$(Build.BuildId)-$(System.JobId) + imageNames.syft: anchore/syft:v1.31.0-debug diff --git a/eng/docker-tools/templates/variables/dotnet/build-test-publish.yml b/eng/docker-tools/templates/variables/dotnet/build-test-publish.yml new file mode 100644 index 000000000..96a0cf500 --- /dev/null +++ b/eng/docker-tools/templates/variables/dotnet/build-test-publish.yml @@ -0,0 +1,48 @@ +# Common variables for building/testing/publishing in the .NET team's pipelines + +variables: +- template: /eng/docker-tools/templates/variables/dotnet/common.yml@self + +- name: commonVersionsImageInfoPath + value: build-info/docker +- name: publicGitRepoUri + value: https://github.com/dotnet/dotnet-docker +- name: testScriptPath + value: ./tests/run-tests.ps1 +- name: testResultsDirectory + value: tests/Microsoft.DotNet.Docker.Tests/TestResults/ + +- name: officialRepoPrefixes + value: public/,internal/private/,unlisted/ + readonly: true + +- name: mcrDocsRepoInfo.userName + value: $(gitHubApp.marDocsUpdater.userName) +- name: mcrDocsRepoInfo.email + value: $(gitHubApp.marDocsUpdater.email) + +- name: publishNotificationsEnabled + value: true +- name: gitHubNotificationsRepoInfo.org + value: dotnet +- name: gitHubNotificationsRepoInfo.repo + value: dotnet-docker-internal +# $(gitHubNotificationsRepoInfo.authArgs) is needed by the "Post Publish +# Notification" step in eng/docker-tools/templates/jobs/publish.yml#L271, even during +# a dry-run. This value is a placeholder that gets replaced when referencing +# the secrets.yml variable template. +- name: gitHubNotificationsRepoInfo.authArgs + value: --gh-token 'placeholder' + +- name: gitHubVersionsRepoInfo.org + value: dotnet +- name: gitHubVersionsRepoInfo.repo + value: versions +- name: gitHubVersionsRepoInfo.branch + value: main +- name: gitHubVersionsRepoInfo.path + value: ${{ variables.commonVersionsImageInfoPath }} +- name: gitHubVersionsRepoInfo.userName + value: $(dotnetDockerBot.userName) +- name: gitHubVersionsRepoInfo.email + value: $(dotnetDockerBot.email) diff --git a/eng/docker-tools/templates/variables/dotnet/common.yml b/eng/docker-tools/templates/variables/dotnet/common.yml new file mode 100644 index 000000000..8f98c0921 --- /dev/null +++ b/eng/docker-tools/templates/variables/dotnet/common.yml @@ -0,0 +1,63 @@ +variables: +- template: /eng/docker-tools/templates/variables/common.yml@self + +- name: publicProjectName + value: public +- name: internalProjectName + value: internal + +# $(dockerHubRegistryCreds) is needed by the copy-base-images step in +# eng/docker-tools/templates/stages/build-and-test.yml#L73-L78, even during a dry-run. +# This is a placeholder that gets replaced when referencing the secrets.yml +# variable template. +- name: dockerHubRegistryCreds + value: --registry-creds 'docker.io=placeholder;placeholder' + +- name: linuxAmd64InternalPoolImage + value: 1es-ubuntu-2204 +- name: linuxAmd64InternalPoolName + value: NetCore1ESPool-Internal + +- name: linuxArm64PoolImage + value: Mariner-2-Docker-ARM64 +- name: linuxArm64PublicPoolName + value: Docker-Linux-Arm-Public +- name: linuxArm64InternalPoolName + value: Docker-Linux-Arm-Internal + +- name: linuxArm32PoolImage + value: Mariner-2-Docker-ARM64 +- name: linuxArm32PublicPoolName + value: Docker-Linux-Arm-Public +- name: linuxArm32InternalPoolName + value: Docker-Linux-Arm-Internal + +- name: windowsServer2016PublicPoolImage + value: Server2016-NESDockerBuilds +- name: windowsServer2016InternalPoolImage + value: Server2016-NESDockerBuilds-1ESPT +- name: windowsServer2016PoolName + value: Docker-2016-${{ variables['System.TeamProject'] }} + +- name: windowsServer2019PublicPoolImage + value: Server2019-1809-NESDockerBuilds +- name: windowsServer2019InternalPoolImage + value: Server2019-1809-NESDockerBuilds-1ESPT +- name: windowsServer2019PoolName + value: Docker-1809-${{ variables['System.TeamProject'] }} + +- name: windowsServer2022PublicPoolImage + value: Server2022-NESDockerBuilds +- name: windowsServer2022InternalPoolImage + value: Server2022-NESDockerBuilds-1ESPT +- name: windowsServer2022PoolName + value: Docker-2022-${{ variables['System.TeamProject'] }} + +- name: windowsServer2025PublicPoolImage + value: Server2025-NESDockerBuilds +- name: windowsServer2025InternalPoolImage + value: Server2025-NESDockerBuilds-1ESPT +- name: windowsServer2025PoolName + value: Docker-2025-${{ variables['System.TeamProject'] }} + +- group: DotNet-Docker-Common-2 diff --git a/eng/docker-tools/templates/variables/dotnet/secrets-unofficial.yml b/eng/docker-tools/templates/variables/dotnet/secrets-unofficial.yml new file mode 100644 index 000000000..1744ad288 --- /dev/null +++ b/eng/docker-tools/templates/variables/dotnet/secrets-unofficial.yml @@ -0,0 +1,5 @@ +variables: +- group: DotNet-Docker-Secrets-Low + +- name: dockerHubRegistryCreds + value: --registry-creds 'docker.io=$(dotnet-dockerhub-bot-username);$(dotnet-dockerhub-bot-pat-low)' diff --git a/eng/docker-tools/templates/variables/dotnet/secrets.yml b/eng/docker-tools/templates/variables/dotnet/secrets.yml new file mode 100644 index 000000000..0224c441e --- /dev/null +++ b/eng/docker-tools/templates/variables/dotnet/secrets.yml @@ -0,0 +1,17 @@ +variables: +- group: DotNet-Docker-Secrets + +- name: dockerHubRegistryCreds + value: --registry-creds 'docker.io=$(dotnetDockerHubBot.userName);$(BotAccount-dotnet-dockerhub-bot-PAT)' + +- name: gitHubNotificationsRepoInfo.authArgs + value: --gh-token '$(BotAccount-dotnet-docker-bot-PAT)' + +- name: gitHubVersionsRepoInfo.authArgs + value: --gh-token '$(BotAccount-dotnet-docker-bot-PAT)' + +- name: mcrDocsRepoInfo.authArgs + value: >- + --gh-private-key '$(GitHubApp-NET-Docker-MAR-Docs-Updater-PrivateKey)' + --gh-app-client-id '$(gitHubApp.marDocsUpdater.clientId)' + --gh-app-installation-id '$(gitHubApp.marDocsUpdater.microsoft.installationId)' diff --git a/eng/docker-tools/templates/variables/sdl-pool.yml b/eng/docker-tools/templates/variables/sdl-pool.yml new file mode 100644 index 000000000..18ae9b9fd --- /dev/null +++ b/eng/docker-tools/templates/variables/sdl-pool.yml @@ -0,0 +1,9 @@ +# This template provides default variables for the source analysis pool and +# image used by 1ES pipeline templates for the "SDL Sources Analysis" stage. +variables: +- name: defaultSourceAnalysisPoolName + value: NetCore1ESPool-Internal + readonly: true +- name: defaultSourceAnalysisPoolImage + value: 1es-windows-2022 + readonly: true From 20d739e2a2187cbc07a835175b3b8da216685fc2 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Mon, 8 Dec 2025 15:07:24 -0800 Subject: [PATCH 2/3] Delete eng/common --- eng/common/Dockerfile.WithRepo | 6 - eng/common/Dockerfile.syft | 16 - eng/common/Get-BaseImageStatus.ps1 | 33 -- eng/common/Get-ImageBuilder.ps1 | 13 - eng/common/Get-ImageNameVars.ps1 | 12 - eng/common/Install-DotNetSdk.ps1 | 64 --- eng/common/Invoke-CleanupDocker.ps1 | 20 - eng/common/Invoke-ImageBuilder.ps1 | 105 ----- eng/common/Invoke-WithRetry.ps1 | 41 -- eng/common/Pull-Image.ps1 | 18 - eng/common/Retain-Build.ps1 | 43 -- eng/common/build.ps1 | 76 ---- eng/common/pull-image.sh | 37 -- eng/common/readme.md | 28 -- eng/common/templates/1es-official.yml | 70 ---- eng/common/templates/1es-unofficial.yml | 79 ---- eng/common/templates/1es.yml | 75 ---- eng/common/templates/jobs/build-images.yml | 131 ------ .../templates/jobs/cg-build-projects.yml | 58 --- .../jobs/copy-base-images-staging.yml | 29 -- .../templates/jobs/copy-base-images.yml | 39 -- eng/common/templates/jobs/generate-matrix.yml | 72 ---- eng/common/templates/jobs/post-build.yml | 87 ---- eng/common/templates/jobs/publish.yml | 312 --------------- .../jobs/test-images-linux-client.yml | 30 -- .../jobs/test-images-windows-client.yml | 25 -- .../templates/jobs/validate-image-sizes.yml | 50 --- .../templates/jobs/wait-for-ingestion.yml | 26 -- .../templates/stages/build-and-test.yml | 343 ---------------- .../stages/build-test-publish-repo.yml | 378 ------------------ .../stages/dotnet/build-and-test.yml | 133 ------ .../stages/dotnet/build-test-publish-repo.yml | 74 ---- .../stages/dotnet/publish-config-nonprod.yml | 104 ----- .../stages/dotnet/publish-config-prod.yml | 104 ----- .../templates/stages/dotnet/publish.yml | 56 --- eng/common/templates/stages/publish.yml | 80 ---- .../stages/setup-service-connections.yml | 38 -- .../templates/steps/annotate-eol-digests.yml | 48 --- .../templates/steps/clean-acr-images.yml | 33 -- .../templates/steps/cleanup-docker-linux.yml | 15 - .../steps/cleanup-docker-windows.yml | 18 - .../common-init-for-matrix-and-build.yml | 74 ---- .../steps/copy-base-images-staging.yml | 19 - .../templates/steps/copy-base-images.yml | 52 --- .../steps/download-build-artifact.yml | 40 -- eng/common/templates/steps/init-common.yml | 9 - .../templates/steps/init-docker-linux.yml | 90 ----- .../templates/steps/init-docker-windows.yml | 48 --- .../steps/init-matrix-build-publish.yml | 78 ---- .../templates/steps/parse-test-arg-arrays.yml | 15 - .../templates/steps/publish-artifact.yml | 28 -- .../templates/steps/publish-readmes.yml | 29 -- eng/common/templates/steps/retain-build.yml | 9 - .../templates/steps/run-imagebuilder.yml | 76 ---- .../templates/steps/run-pwsh-with-auth.yml | 39 -- eng/common/templates/steps/set-dry-run.yml | 30 -- .../steps/set-image-info-path-var.yml | 19 - .../steps/test-images-linux-client.yml | 107 ----- .../steps/test-images-windows-client.yml | 66 --- .../templates/steps/validate-branch.yml | 52 --- .../templates/steps/validate-image-sizes.yml | 16 - .../steps/wait-for-mcr-doc-ingestion.yml | 21 - .../steps/wait-for-mcr-image-ingestion.yml | 37 -- .../templates/task-prefix-decorator.yml | 63 --- eng/common/templates/variables/codeql.yml | 16 - .../templates/variables/common-paths.yml | 5 - eng/common/templates/variables/common.yml | 82 ---- .../templates/variables/docker-images.yml | 7 - .../variables/dotnet/build-test-publish.yml | 48 --- .../templates/variables/dotnet/common.yml | 63 --- .../variables/dotnet/secrets-unofficial.yml | 5 - .../templates/variables/dotnet/secrets.yml | 17 - 72 files changed, 4279 deletions(-) delete mode 100644 eng/common/Dockerfile.WithRepo delete mode 100644 eng/common/Dockerfile.syft delete mode 100644 eng/common/Get-BaseImageStatus.ps1 delete mode 100644 eng/common/Get-ImageBuilder.ps1 delete mode 100644 eng/common/Get-ImageNameVars.ps1 delete mode 100644 eng/common/Install-DotNetSdk.ps1 delete mode 100644 eng/common/Invoke-CleanupDocker.ps1 delete mode 100644 eng/common/Invoke-ImageBuilder.ps1 delete mode 100644 eng/common/Invoke-WithRetry.ps1 delete mode 100644 eng/common/Pull-Image.ps1 delete mode 100644 eng/common/Retain-Build.ps1 delete mode 100644 eng/common/build.ps1 delete mode 100755 eng/common/pull-image.sh delete mode 100644 eng/common/readme.md delete mode 100644 eng/common/templates/1es-official.yml delete mode 100644 eng/common/templates/1es-unofficial.yml delete mode 100644 eng/common/templates/1es.yml delete mode 100644 eng/common/templates/jobs/build-images.yml delete mode 100644 eng/common/templates/jobs/cg-build-projects.yml delete mode 100644 eng/common/templates/jobs/copy-base-images-staging.yml delete mode 100644 eng/common/templates/jobs/copy-base-images.yml delete mode 100644 eng/common/templates/jobs/generate-matrix.yml delete mode 100644 eng/common/templates/jobs/post-build.yml delete mode 100644 eng/common/templates/jobs/publish.yml delete mode 100644 eng/common/templates/jobs/test-images-linux-client.yml delete mode 100644 eng/common/templates/jobs/test-images-windows-client.yml delete mode 100644 eng/common/templates/jobs/validate-image-sizes.yml delete mode 100644 eng/common/templates/jobs/wait-for-ingestion.yml delete mode 100644 eng/common/templates/stages/build-and-test.yml delete mode 100644 eng/common/templates/stages/build-test-publish-repo.yml delete mode 100644 eng/common/templates/stages/dotnet/build-and-test.yml delete mode 100644 eng/common/templates/stages/dotnet/build-test-publish-repo.yml delete mode 100644 eng/common/templates/stages/dotnet/publish-config-nonprod.yml delete mode 100644 eng/common/templates/stages/dotnet/publish-config-prod.yml delete mode 100644 eng/common/templates/stages/dotnet/publish.yml delete mode 100644 eng/common/templates/stages/publish.yml delete mode 100644 eng/common/templates/stages/setup-service-connections.yml delete mode 100644 eng/common/templates/steps/annotate-eol-digests.yml delete mode 100644 eng/common/templates/steps/clean-acr-images.yml delete mode 100644 eng/common/templates/steps/cleanup-docker-linux.yml delete mode 100644 eng/common/templates/steps/cleanup-docker-windows.yml delete mode 100644 eng/common/templates/steps/common-init-for-matrix-and-build.yml delete mode 100644 eng/common/templates/steps/copy-base-images-staging.yml delete mode 100644 eng/common/templates/steps/copy-base-images.yml delete mode 100644 eng/common/templates/steps/download-build-artifact.yml delete mode 100644 eng/common/templates/steps/init-common.yml delete mode 100644 eng/common/templates/steps/init-docker-linux.yml delete mode 100644 eng/common/templates/steps/init-docker-windows.yml delete mode 100644 eng/common/templates/steps/init-matrix-build-publish.yml delete mode 100644 eng/common/templates/steps/parse-test-arg-arrays.yml delete mode 100644 eng/common/templates/steps/publish-artifact.yml delete mode 100644 eng/common/templates/steps/publish-readmes.yml delete mode 100644 eng/common/templates/steps/retain-build.yml delete mode 100644 eng/common/templates/steps/run-imagebuilder.yml delete mode 100644 eng/common/templates/steps/run-pwsh-with-auth.yml delete mode 100644 eng/common/templates/steps/set-dry-run.yml delete mode 100644 eng/common/templates/steps/set-image-info-path-var.yml delete mode 100644 eng/common/templates/steps/test-images-linux-client.yml delete mode 100644 eng/common/templates/steps/test-images-windows-client.yml delete mode 100644 eng/common/templates/steps/validate-branch.yml delete mode 100644 eng/common/templates/steps/validate-image-sizes.yml delete mode 100644 eng/common/templates/steps/wait-for-mcr-doc-ingestion.yml delete mode 100644 eng/common/templates/steps/wait-for-mcr-image-ingestion.yml delete mode 100644 eng/common/templates/task-prefix-decorator.yml delete mode 100644 eng/common/templates/variables/codeql.yml delete mode 100644 eng/common/templates/variables/common-paths.yml delete mode 100644 eng/common/templates/variables/common.yml delete mode 100644 eng/common/templates/variables/docker-images.yml delete mode 100644 eng/common/templates/variables/dotnet/build-test-publish.yml delete mode 100644 eng/common/templates/variables/dotnet/common.yml delete mode 100644 eng/common/templates/variables/dotnet/secrets-unofficial.yml delete mode 100644 eng/common/templates/variables/dotnet/secrets.yml diff --git a/eng/common/Dockerfile.WithRepo b/eng/common/Dockerfile.WithRepo deleted file mode 100644 index d1126b3d1..000000000 --- a/eng/common/Dockerfile.WithRepo +++ /dev/null @@ -1,6 +0,0 @@ -# Use this Dockerfile to create an ImageBuilder image -ARG IMAGE -FROM $IMAGE - -WORKDIR /repo -COPY . . diff --git a/eng/common/Dockerfile.syft b/eng/common/Dockerfile.syft deleted file mode 100644 index 2e564e2ae..000000000 --- a/eng/common/Dockerfile.syft +++ /dev/null @@ -1,16 +0,0 @@ -ARG SYFT_IMAGE_NAME -ARG TARGET_IMAGE_NAME - -FROM ${SYFT_IMAGE_NAME} AS syft -FROM ${TARGET_IMAGE_NAME} AS scan-image - -FROM syft AS run-scan -ARG TARGET_IMAGE_NAME -ENV SYFT_CHECK_FOR_APP_UPDATE=0 \ - SYFT_SOURCE_NAME=${TARGET_IMAGE_NAME} -USER root -RUN --mount=from=scan-image,source=/,target=/rootfs \ - ["/syft", "scan", "/rootfs/", "--select-catalogers", "image", "--output", "spdx-json=/manifest.spdx.json"] - -FROM scratch AS output -COPY --from=run-scan /manifest.spdx.json /manifest.spdx.json diff --git a/eng/common/Get-BaseImageStatus.ps1 b/eng/common/Get-BaseImageStatus.ps1 deleted file mode 100644 index 6b2546265..000000000 --- a/eng/common/Get-BaseImageStatus.ps1 +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env pwsh - -<# -.SYNOPSIS -Outputs the status of external base images referenced in the Dockerfiles. -#> -[cmdletbinding()] -param( - # Path to the manifest file to use - [string] - $Manifest = "manifest.json", - - # Architecture to filter Dockerfiles to - [string] - $Architecture = "*", - - # A value indicating whether to run the script continously - [switch] - $Continuous, - - # Number of seconds to wait between each iteration - [int] - $ContinuousDelay = 10 -) - -Set-StrictMode -Version Latest - -$imageBuilderArgs = "getBaseImageStatus --manifest $Manifest --architecture $Architecture" -if ($Continuous) { - $imageBuilderArgs += " --continuous --continuous-delay $ContinuousDelay" -} - -& "$PSScriptRoot/Invoke-ImageBuilder.ps1" -ImageBuilderArgs $imageBuilderArgs diff --git a/eng/common/Get-ImageBuilder.ps1 b/eng/common/Get-ImageBuilder.ps1 deleted file mode 100644 index 3eb35240b..000000000 --- a/eng/common/Get-ImageBuilder.ps1 +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env pwsh - -# Load common image names -$imageNameVars = & $PSScriptRoot/Get-ImageNameVars.ps1 -foreach ($varName in $imageNameVars.Keys) { - Set-Variable -Name $varName -Value $imageNameVars[$varName] -Scope Global -} - -& docker inspect ${imageNames.imagebuilderName} | Out-Null -if (-not $?) { - Write-Output "Pulling" - & $PSScriptRoot/Invoke-WithRetry.ps1 "docker pull ${imageNames.imagebuilderName}" -} diff --git a/eng/common/Get-ImageNameVars.ps1 b/eng/common/Get-ImageNameVars.ps1 deleted file mode 100644 index 0b255c925..000000000 --- a/eng/common/Get-ImageNameVars.ps1 +++ /dev/null @@ -1,12 +0,0 @@ -# Returns a hashtable of variable name-to-value mapping representing the image name variables -# used by the common build infrastructure. - -$vars = @{} -Get-Content $PSScriptRoot/templates/variables/docker-images.yml | - Where-Object { $_.Trim().Length -gt 0 -and $_.Trim() -notlike 'variables:' -and $_.Trim() -notlike '# *' } | - ForEach-Object { - $parts = $_.Split(':', 2) - $vars[$parts[0].Trim()] = $parts[1].Trim() - } - -return $vars diff --git a/eng/common/Install-DotNetSdk.ps1 b/eng/common/Install-DotNetSdk.ps1 deleted file mode 100644 index 114faa260..000000000 --- a/eng/common/Install-DotNetSdk.ps1 +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env pwsh -# -# Copyright (c) .NET Foundation and contributors. All rights reserved. -# Licensed under the MIT license. See LICENSE file in the project root for full license information. -# - -<# -.SYNOPSIS -Install the .NET Core SDK at the specified path. - -.PARAMETER InstallPath -The path where the .NET Core SDK is to be installed. - -.PARAMETER Channel -The version of the .NET Core SDK to be installed. - -#> -[cmdletbinding()] -param( - [string] - $InstallPath, - [string] - $Channel = "9.0" -) - -Set-StrictMode -Version Latest -$ErrorActionPreference = 'Stop' - -if (!(Test-Path "$InstallPath")) { - mkdir "$InstallPath" | Out-Null -} - -$IsRunningOnUnix = $PSVersionTable.contains("Platform") -and $PSVersionTable.Platform -eq "Unix" -if ($IsRunningOnUnix) { - $DotnetInstallScript = "dotnet-install.sh" -} -else { - $DotnetInstallScript = "dotnet-install.ps1" -} - -$DotnetInstallScriptPath = Join-Path -Path $InstallPath -ChildPath $DotnetInstallScript - -if (!(Test-Path $DotnetInstallScriptPath)) { - [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; - & "$PSScriptRoot/Invoke-WithRetry.ps1" "Invoke-WebRequest 'https://builds.dotnet.microsoft.com/dotnet/scripts/v1/$DotnetInstallScript' -OutFile $DotnetInstallScriptPath" -} - -$DotnetChannel = $Channel - -$InstallFailed = $false -if ($IsRunningOnUnix) { - & chmod +x $DotnetInstallScriptPath - & "$PSScriptRoot/Invoke-WithRetry.ps1" "$DotnetInstallScriptPath --channel $DotnetChannel --install-dir $InstallPath" -Retries 5 - $InstallFailed = ($LASTEXITCODE -ne 0) -} -else { - & "$PSScriptRoot/Invoke-WithRetry.ps1" "$DotnetInstallScriptPath -Channel $DotnetChannel -InstallDir $InstallPath" -Retries 5 - $InstallFailed = (-not $?) -} - -# See https://github.com/NuGet/NuGet.Client/pull/4259 -$Env:NUGET_EXPERIMENTAL_CHAIN_BUILD_RETRY_POLICY = "6,1500" - -if ($InstallFailed) { throw "Failed to install the .NET Core SDK" } diff --git a/eng/common/Invoke-CleanupDocker.ps1 b/eng/common/Invoke-CleanupDocker.ps1 deleted file mode 100644 index ad637872c..000000000 --- a/eng/common/Invoke-CleanupDocker.ps1 +++ /dev/null @@ -1,20 +0,0 @@ -Set-StrictMode -Version Latest -$ErrorActionPreference = 'Stop' - -docker ps -a -q | ForEach-Object { docker rm -f $_ } - -docker volume prune -f - -# Preserve the tagged Windows base images and the common eng infra images (e.g. ImageBuilder) -# to avoid the expense of having to repull continuously. -$imageNameVars = & $PSScriptRoot/Get-ImageNameVars.ps1 - -docker images --format "{{.Repository}}:{{.Tag}} {{.ID}}" | - Where-Object { - $localImage = $_ - $localImage.Contains(": ")` - -Or -Not ($localImage.StartsWith("mcr.microsoft.com/windows")` - -Or ($imageNameVars.Values.Where({ $localImage.StartsWith($_) }, 'First').Count -gt 0)) } | - ForEach-Object { $_.Split(' ', [System.StringSplitOptions]::RemoveEmptyEntries)[1] } | - Select-Object -Unique | - ForEach-Object { docker rmi -f $_ } diff --git a/eng/common/Invoke-ImageBuilder.ps1 b/eng/common/Invoke-ImageBuilder.ps1 deleted file mode 100644 index fb349557b..000000000 --- a/eng/common/Invoke-ImageBuilder.ps1 +++ /dev/null @@ -1,105 +0,0 @@ -#!/usr/bin/env pwsh - -<# -.SYNOPSIS -Executes ImageBuilder with the specified args. - -.PARAMETER ImageBuilderArgs -The args to pass to ImageBuilder. - -.PARAMETER ReuseImageBuilderImage -Indicates that a previously built ImageBuilder image is presumed to exist locally and that -it should be used for this execution of the script. This allows some optimization when -multiple calls are being made to this script that don't require a fresh image (i.e. the -repo contents in the image don't need to be or should not be updated with each call to -this script). - -.PARAMETER OnCommandExecuted -A ScriptBlock that will be invoked after the ImageBuilder command has been executed. -This allows the caller to execute extra logic in the context of the ImageBuilder while -its container is still running. -The ScriptBlock is passed the following argument values: - 1. Container name -#> -[cmdletbinding()] -param( - [string] - $ImageBuilderArgs, - - [switch] - $ReuseImageBuilderImage, - - [scriptblock] - $OnCommandExecuted -) - -Set-StrictMode -Version Latest -$ErrorActionPreference = 'Stop' - -function Log { - param ([string] $Message) - - Write-Output $Message -} - -function Exec { - param ([string] $Cmd) - - Log "Executing: '$Cmd'" - Invoke-Expression $Cmd - if ($LASTEXITCODE -ne 0) { - $host.SetShouldExit($LASTEXITCODE) - exit $LASTEXITCODE - throw "Failed: '$Cmd'" - } -} - -$imageBuilderContainerName = "ImageBuilder-$(Get-Date -Format yyyyMMddhhmmss)" -$containerCreated = $false - -pushd $PSScriptRoot/../../ -try { - $activeOS = docker version -f "{{ .Server.Os }}" - if ($activeOS -eq "linux") { - # On Linux, ImageBuilder is run within a container. - $imageBuilderImageName = "microsoft-dotnet-imagebuilder-withrepo" - if ($ReuseImageBuilderImage -ne $True) { - & ./eng/common/Get-ImageBuilder.ps1 - Exec ("docker build -t $imageBuilderImageName --build-arg " ` - + "IMAGE=${imageNames.imageBuilderName} -f eng/common/Dockerfile.WithRepo .") - } - - $imageBuilderCmd = "docker run --name $imageBuilderContainerName -v /var/run/docker.sock:/var/run/docker.sock $imageBuilderImageName" - $containerCreated = $true - } - else { - # On Windows, ImageBuilder is run locally due to limitations with running Docker client within a container. - # Remove when https://github.com/dotnet/docker-tools/issues/159 is resolved - $imageBuilderFolder = ".Microsoft.DotNet.ImageBuilder" - $imageBuilderCmd = [System.IO.Path]::Combine($imageBuilderFolder, "Microsoft.DotNet.ImageBuilder.exe") - if (-not (Test-Path -Path "$imageBuilderCmd" -PathType Leaf)) { - & ./eng/common/Get-ImageBuilder.ps1 - Exec "docker create --name $imageBuilderContainerName ${imageNames.imageBuilderName}" - $containerCreated = $true - if (Test-Path -Path $imageBuilderFolder) - { - Remove-Item -Recurse -Force -Path $imageBuilderFolder - } - - Exec "docker cp ${imageBuilderContainerName}:/image-builder $imageBuilderFolder" - } - } - - Exec "$imageBuilderCmd $ImageBuilderArgs" - - if ($OnCommandExecuted) { - Invoke-Command $OnCommandExecuted -ArgumentList $imageBuilderContainerName - } -} -finally { - if ($containerCreated) { - Exec "docker container rm -f $imageBuilderContainerName" - } - - popd -} diff --git a/eng/common/Invoke-WithRetry.ps1 b/eng/common/Invoke-WithRetry.ps1 deleted file mode 100644 index 0aa2c2737..000000000 --- a/eng/common/Invoke-WithRetry.ps1 +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env pwsh - -# Executes a command and retries if it fails. -[cmdletbinding()] -param ( - [Parameter(Mandatory = $true)][string]$Cmd, - [int]$Retries = 2, - [int]$WaitFactor = 6 - ) - -Set-StrictMode -Version Latest -$ErrorActionPreference = 'Stop' - -$count = 0 -$completed = $false - -Write-Output "Executing '$Cmd'" - -while (-not $completed) { - try { - Invoke-Expression $Cmd - if (-not $(Test-Path variable:LASTEXITCODE) -or $LASTEXITCODE -eq 0) { - $completed = $true - continue - } - } - catch { - } - - $count++ - - if ($count -lt $Retries) { - $wait = [Math]::Pow($WaitFactor, $count - 1) - Write-Output "Retry $count/$Retries, retrying in $wait seconds..." - Start-Sleep $wait - } - else { - Write-Output "Retry $count/$Retries, no more retries left." - throw "Failed to execute '$Cmd'" - } -} diff --git a/eng/common/Pull-Image.ps1 b/eng/common/Pull-Image.ps1 deleted file mode 100644 index 2d0a82cf5..000000000 --- a/eng/common/Pull-Image.ps1 +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env pwsh - -[cmdletbinding()] -param( - [Parameter(Mandatory = $true, Position = 0)] - [string]$Image, - - [Parameter(Mandatory = $false)] - [int]$Retries = 2, - - [Parameter(Mandatory = $false)] - [int]$WaitFactor = 6 -) - -Set-StrictMode -Version Latest -$ErrorActionPreference = 'Stop' - -& "$PSScriptRoot/Invoke-WithRetry.ps1" "docker pull $Image" -Retries $Retries -WaitFactor $WaitFactor diff --git a/eng/common/Retain-Build.ps1 b/eng/common/Retain-Build.ps1 deleted file mode 100644 index ee601c950..000000000 --- a/eng/common/Retain-Build.ps1 +++ /dev/null @@ -1,43 +0,0 @@ -# Adapted from https://github.com/dotnet/arcade/blob/main/eng/common/retain-build.ps1 -Param( - [Parameter(Mandatory = $true)][int] $BuildId, - [Parameter(Mandatory = $true)][string] $AzdoOrgUri, - [Parameter(Mandatory = $true)][string] $AzdoProject, - [Parameter(Mandatory = $true)][string] $Token -) - -$ErrorActionPreference = 'Stop' -Set-StrictMode -Version 2.0 - -function Get-AzDOHeaders( - [string] $Token) { - $base64AuthInfo = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(":${Token}")) - $headers = @{"Authorization" = "Basic $base64AuthInfo" } - return $headers -} - -function Update-BuildRetention( - [string] $AzdoOrgUri, - [string] $AzdoProject, - [int] $BuildId, - [string] $Token) { - $headers = Get-AzDOHeaders -Token $Token - $requestBody = "{ - `"keepForever`": `"true`" - }" - - $requestUri = "${AzdoOrgUri}/${AzdoProject}/_apis/build/builds/${BuildId}?api-version=6.0" - Write-Host "Attempting to retain build using the following URI: ${requestUri} ..." - - try { - Invoke-RestMethod -Uri $requestUri -Method Patch -Body $requestBody -Header $headers -contentType "application/json" - Write-Host "Updated retention settings for build ${BuildId}." - } - catch { - Write-Host "##[error] Failed to update retention settings for build: $($_.Exception.Response.StatusDescription)" - exit 1 - } -} - -Update-BuildRetention -AzdoOrgUri $AzdoOrgUri -AzdoProject $AzdoProject -BuildId $BuildId -Token $Token -exit 0 diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 deleted file mode 100644 index 845c4e081..000000000 --- a/eng/common/build.ps1 +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env pwsh - -<# - .SYNOPSIS - Builds the Dockerfiles -#> - -[cmdletbinding()] -param( - # Product versions to filter by - [string[]]$Version = "*", - - # Names of OS to filter by - [string[]]$OS, - - # Type of architecture to filter by - [string]$Architecture, - - # Additional custom path filters - [string[]]$Paths, - - # Path to manifest file - [string]$Manifest = "manifest.json", - - # Additional args to pass to ImageBuilder - [string]$OptionalImageBuilderArgs -) - -Set-StrictMode -Version Latest -$ErrorActionPreference = 'Stop' - -function Log { - param ([string] $Message) - - Write-Output $Message -} - -function Exec { - param ([string] $Cmd) - - Log "Executing: '$Cmd'" - Invoke-Expression $Cmd - if ($LASTEXITCODE -ne 0) { - throw "Failed: '$Cmd'" - } -} - -pushd $PSScriptRoot/../.. -try { - $args = $OptionalImageBuilderArgs - - if ($Version) { - $args += ($Version | foreach { ' --version "{0}"' -f $_ }) - } - - if ($OS) { - $args += ($OS | foreach { ' --os-version "{0}"' -f $_ }) - } - - if ($Architecture) { - $args += ' --architecture "{0}"' -f $Architecture - } - - if ($Paths) { - $args += ($Paths | foreach { ' --path "{0}"' -f $_ }) - } - - if ($Manifest) { - $args += ' --manifest "{0}"' -f $Manifest - } - - ./eng/common/Invoke-ImageBuilder.ps1 "build $args" -} -finally { - popd -} diff --git a/eng/common/pull-image.sh b/eng/common/pull-image.sh deleted file mode 100755 index d6c6ec85b..000000000 --- a/eng/common/pull-image.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash - -# Stop script on NZEC -set -e -# Stop script if unbound variable found (use ${var:-} if intentional) -set -u - -say_err() { - printf "%b\n" "Error: $1" >&2 -} - -# Executes a command and retries if it fails. -execute() { - local count=0 - until "$@"; do - local exit=$? - count=$(( $count + 1 )) - if [ $count -lt $retries ]; then - local wait=$(( waitFactor ** (( count - 1 )) )) - echo "Retry $count/$retries exited $exit, retrying in $wait seconds..." - sleep $wait - else - say_err "Retry $count/$retries exited $exit, no more retries left." - return $exit - fi - done - - return 0 -} - -scriptName=$0 -retries=5 -waitFactor=6 -image=$1 - -echo "Pulling Docker image $image" -execute docker pull $image diff --git a/eng/common/readme.md b/eng/common/readme.md deleted file mode 100644 index c0fe1f2eb..000000000 --- a/eng/common/readme.md +++ /dev/null @@ -1,28 +0,0 @@ -# Don't touch this folder - - uuuuuuuuuuuuuuuuuuuu - u" uuuuuuuuuuuuuuuuuu "u - u" u$$$$$$$$$$$$$$$$$$$$u "u - u" u$$$$$$$$$$$$$$$$$$$$$$$$u "u - u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u - u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u - u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u - $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $ - $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $ - $ $$$" ... "$... ...$" ... "$$$ ... "$$$ $ - $ $$$u `"$$$$$$$ $$$ $$$$$ $$ $$$ $$$ $ - $ $$$$$$uu "$$$$ $$$ $$$$$ $$ """ u$$$ $ - $ $$$""$$$ $$$$ $$$u "$$$" u$$ $$$$$$$$ $ - $ $$$$....,$$$$$..$$$$$....,$$$$..$$$$$$$$ $ - $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $ - "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u" - "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u" - "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u" - "u "$$$$$$$$$$$$$$$$$$$$$$$$" u" - "u "$$$$$$$$$$$$$$$$$$$$" u" - "u """""""""""""""""" u" - """""""""""""""""""" - -!!! Changes made in this directory are subject to being overwritten by automation !!! - -The files in this directory are shared by all .NET Docker repos. If you need to make changes to these files, open an issue or submit a pull request in https://github.com/dotnet/docker-tools. \ No newline at end of file diff --git a/eng/common/templates/1es-official.yml b/eng/common/templates/1es-official.yml deleted file mode 100644 index 49721aa1c..000000000 --- a/eng/common/templates/1es-official.yml +++ /dev/null @@ -1,70 +0,0 @@ -# When extending this template, pipelines using a repository resource containing versions files for image caching must -# do the following: -# -# - Do not rely on any source code from the versions repo so as to not circumvent SDL and CG guidelines -# - The versions repo resource must be named `VersionsRepo` to avoid SDL scans -# - The versions repo must be checked out to `$(Build.SourcesDirectory)/versions` to avoid CG scans -# -# If the pipeline is not using a separate repository resource, ensure that there is no source code checked out in -# `$(Build.SourcesDirectory)/versions`, as it will not be scanned. -# -# The `cgDryRun` parameter will run CG but not submit the results, for testing purposes. - -parameters: -- name: cgDryRun - type: boolean - default: false -- name: stages - type: stageList - default: [] -- name: serviceConnections - type: object - default: [] -- name: pool - type: object - default: - name: $(default1ESInternalPoolName) - image: $(default1ESInternalPoolImage) - os: linux -- name: sourceAnalysisPool - type: object - default: - name: $(defaultSourceAnalysisPoolName) - image: $(defaultSourceAnalysisPoolImage) - os: windows - -resources: - repositories: - - repository: 1ESPipelineTemplates - type: git - name: 1ESPipelineTemplates/1ESPipelineTemplates - ref: refs/tags/release - -extends: - template: /eng/common/templates/task-prefix-decorator.yml@self - parameters: - baseTemplate: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates - templateParameters: - pool: ${{ parameters.pool }} - sdl: - binskim: - enabled: true - componentgovernance: - ignoreDirectories: $(Build.SourcesDirectory)/versions - whatIf: ${{ parameters.cgDryRun }} - showAlertLink: true - policheck: - enabled: true - sourceRepositoriesToScan: - exclude: - - repository: VersionsRepo - sourceAnalysisPool: ${{ parameters.sourceAnalysisPool }} - tsa: - enabled: true - stages: - - ${{ if gt(length(parameters.serviceConnections), 0) }}: - - template: /eng/common/templates/stages/setup-service-connections.yml@self - parameters: - pool: ${{ parameters.pool }} - serviceConnections: ${{ parameters.serviceConnections }} - - ${{ parameters.stages }} diff --git a/eng/common/templates/1es-unofficial.yml b/eng/common/templates/1es-unofficial.yml deleted file mode 100644 index 541f92f29..000000000 --- a/eng/common/templates/1es-unofficial.yml +++ /dev/null @@ -1,79 +0,0 @@ -# This unofficial template will always run CG in "what if" mode, which will not submit results to the CG. SDL tools may -# also be disabled for testing purposes. -# -# When extending this template, pipelines using a repository resource containing versions files for image caching must -# do the following: -# -# - Do not rely on any source code from the versions repo so as to not circumvent SDL and CG guidelines -# - The versions repo resource must be named `InternalVersionsRepo` or `PublicVersionsRepo` to avoid SDL scans -# - The versions repo must be checked out to `$(Build.SourcesDirectory)/versions` to avoid CG scans -# -# If the pipeline is not using a separate repository resource, ensure that there is no source code checked out in -# `$(Build.SourcesDirectory)/versions`, as it will not be scanned. - -parameters: -- name: disableSDL - type: boolean - default: false - displayName: Disable SDL -- name: stages - type: stageList - default: [] - # 1ES Pipeline Template parameters -- name: serviceConnections - type: object - default: [] -- name: pool - type: object - default: - name: $(default1ESInternalPoolName) - image: $(default1ESInternalPoolImage) - os: linux -- name: sourceAnalysisPool - type: object - default: - name: $(defaultSourceAnalysisPoolName) - image: $(defaultSourceAnalysisPoolImage) - os: windows - -resources: - repositories: - - repository: 1ESPipelineTemplates - type: git - name: 1ESPipelineTemplates/1ESPipelineTemplates - ref: refs/tags/release - -extends: - template: /eng/common/templates/task-prefix-decorator.yml@self - parameters: - # Use a unique task prefix for unofficial pipelines - taskPrefix: "🟦" - baseTemplate: v1/1ES.Unofficial.PipelineTemplate.yml@1ESPipelineTemplates - templateParameters: - pool: ${{ parameters.pool }} - sdl: - binskim: - enabled: true - componentgovernance: - ignoreDirectories: $(Build.SourcesDirectory)/versions - whatIf: true - showAlertLink: true - enableAllTools: ${{ not(parameters.disableSDL) }} - policheck: - enabled: true - sbom: - enabled: true - sourceRepositoriesToScan: - exclude: - - repository: InternalVersionsRepo - - repository: PublicVersionsRepo - sourceAnalysisPool: ${{ parameters.sourceAnalysisPool }} - tsa: - enabled: true - stages: - - ${{ if gt(length(parameters.serviceConnections), 0) }}: - - template: /eng/common/templates/stages/setup-service-connections.yml@self - parameters: - pool: ${{ parameters.pool }} - serviceConnections: ${{ parameters.serviceConnections }} - - ${{ parameters.stages }} diff --git a/eng/common/templates/1es.yml b/eng/common/templates/1es.yml deleted file mode 100644 index 95a44b2a1..000000000 --- a/eng/common/templates/1es.yml +++ /dev/null @@ -1,75 +0,0 @@ -# When extending this template, pipelines using a repository resource containing versions files for image caching must -# do the following: -# -# - Do not rely on any source code from the versions repo so as to not circumvent SDL and CG guidelines -# - The versions repo resource must be named `VersionsRepo` to avoid SDL scans -# - The versions repo must be checked out to `$(Build.SourcesDirectory)/versions` to avoid CG scans -# -# If the pipeline is not using a separate repository resource, ensure that there is no source code checked out in -# `$(Build.SourcesDirectory)/versions`, as it will not be scanned. - -parameters: -- name: stages - type: stageList - default: [] -# List of repositories that will be excluded from SDL scanning. This should -# only be used when including other repos without building their source code. -# E.g. for the dotnet/versions repo. -- name: reposToExcludeFromScanning - type: object - default: [] -# The pool that will be used for initializing service connections. -- name: pool - type: object - default: - name: $(default1ESInternalPoolName) - image: $(default1ESInternalPoolImage) - os: linux -# The pool that will be used for SDL jobs. -- name: sourceAnalysisPool - type: object - default: - name: $(defaultSourceAnalysisPoolName) - image: $(defaultSourceAnalysisPoolImage) - os: windows -# Container image SBOMs are generated manually during the build job. 1ESPT's -# automatic SBOM generation only adds unnecessary steps and artifacts to -# builds. SBOM is not needed for JSON outputs. If a pipeline outputs binary -# artifacts that ship to customers, then set this parameter to true. -- name: enableSbom - type: boolean - default: false - -resources: - repositories: - - repository: 1ESPipelineTemplates - type: git - name: 1ESPipelineTemplates/1ESPipelineTemplates - ref: refs/tags/release - -extends: - template: /eng/common/templates/task-prefix-decorator.yml@self - parameters: - baseTemplate: v1/1ES.${{ iif(contains(variables['Build.DefinitionName'], '-official'), 'Official', 'Unofficial') }}.PipelineTemplate.yml@1ESPipelineTemplates - templateParameters: - pool: ${{ parameters.pool }} - sdl: - sbom: - enabled: ${{ parameters.enableSbom }} - binskim: - enabled: true - componentgovernance: - ignoreDirectories: $(Build.SourcesDirectory)/versions - showAlertLink: true - policheck: - enabled: true - ${{ if ne(length(parameters.reposToExcludeFromScanning), 0) }}: - sourceRepositoriesToScan: - exclude: - - ${{ each repo in parameters.reposToExcludeFromScanning }}: - - repository: ${{ repo }} - sourceAnalysisPool: ${{ parameters.sourceAnalysisPool }} - tsa: - enabled: true - stages: - - ${{ parameters.stages }} diff --git a/eng/common/templates/jobs/build-images.yml b/eng/common/templates/jobs/build-images.yml deleted file mode 100644 index 24fa3e795..000000000 --- a/eng/common/templates/jobs/build-images.yml +++ /dev/null @@ -1,131 +0,0 @@ -parameters: - name: null - pool: {} - matrix: {} - dockerClientOS: null - buildJobTimeout: 60 - commonInitStepsForMatrixAndBuild: [] - customInitSteps: [] - publishConfig: null - noCache: false - internalProjectName: null - publicProjectName: null - storageAccountServiceConnection: null - -jobs: -- job: ${{ parameters.name }} - condition: and(${{ parameters.matrix }}, not(canceled()), in(dependencies.PreBuildValidation.result, 'Succeeded', 'SucceededWithIssues', 'Skipped')) - dependsOn: - - PreBuildValidation - - CopyBaseImages - - GenerateBuildMatrix - pool: ${{ parameters.pool }} - strategy: - matrix: $[ ${{ parameters.matrix }} ] - timeoutInMinutes: ${{ parameters.buildJobTimeout }} - variables: - imageBuilderDockerRunExtraOptions: $(build.imageBuilderDockerRunExtraOptions) - sbomDirectory: $(Build.ArtifactStagingDirectory)/sbom - imageInfoHostDir: $(Build.ArtifactStagingDirectory)/imageInfo - imageInfoContainerDir: $(artifactsPath)/imageInfo - steps: - - ${{ parameters.commonInitStepsForMatrixAndBuild }} - - template: /eng/common/templates/jobs/${{ format('../steps/init-docker-{0}.yml', parameters.dockerClientOS) }}@self - parameters: - cleanupDocker: true - - ${{ parameters.customInitSteps }} - - template: /eng/common/templates/steps/set-image-info-path-var.yml@self - parameters: - publicSourceBranch: $(publicSourceBranch) - - powershell: echo "##vso[task.setvariable variable=imageBuilderBuildArgs]" - condition: eq(variables.imageBuilderBuildArgs, '') - displayName: Initialize Image Builder Build Args - - powershell: | - New-Item -Path $(imageInfoHostDir) -ItemType Directory -Force - - # Reference the existing imageBuilderBuildArgs variable as an environment variable rather than injecting it directly - # with the $(imageBuilderBuildArgs) syntax. This is to avoid issues where the string may contain single quotes $ chars - # which really mess up assigning to a variable. It would require assigning the string with single quotes but also needing - # to escape the single quotes that are in the string which would need to be done outside the context of PowerShell. Since - # all we need is for that value to be in a PowerShell variable, we can get that by the fact that AzDO automatically creates - # the environment variable for us. - $imageBuilderBuildArgs = "$env:IMAGEBUILDERBUILDARGS $env:IMAGEBUILDER_QUEUEARGS --image-info-output-path $(imageInfoContainerDir)/$(legName)-image-info.json $(commonMatrixAndBuildOptions)" - if ($env:SYSTEM_TEAMPROJECT -eq "${{ parameters.internalProjectName }}" -and $env:BUILD_REASON -ne "PullRequest") { - $imageBuilderBuildArgs = "$imageBuilderBuildArgs --repo-prefix ${{ parameters.publishConfig.buildAcr.repoPrefix }} --push" - } - - # If the pipeline isn't configured to disable the cache and a build variable hasn't been set to disable the cache - if ("$(pipelineDisabledCache)" -ne "true" -and "${{ parameters.noCache }}" -ne "true") { - $imageBuilderBuildArgs = "$imageBuilderBuildArgs --image-info-source-path $(versionsBasePath)$(imageInfoVersionsPath)" - } - - echo "imageBuilderBuildArgs: $imageBuilderBuildArgs" - echo "##vso[task.setvariable variable=imageBuilderBuildArgs]$imageBuilderBuildArgs" - displayName: Set Image Builder Build Args - - template: /eng/common/templates/steps/run-imagebuilder.yml@self - parameters: - name: BuildImages - displayName: Build Images - serviceConnections: - # "name" here refers to the argument name, not the service connection name. - # It should probably be changed to "argName". - - name: acr - id: ${{ parameters.publishConfig.buildAcr.serviceConnection.id }} - tenantId: ${{ parameters.publishConfig.buildAcr.serviceConnection.tenantId }} - clientId: ${{ parameters.publishConfig.buildAcr.serviceConnection.clientId }} - - ${{ if parameters.storageAccountServiceConnection }}: - - name: storage - id: ${{ parameters.storageAccountServiceConnection.id }} - tenantId: ${{ parameters.storageAccountServiceConnection.tenantId }} - clientId: ${{ parameters.storageAccountServiceConnection.clientId }} - internalProjectName: ${{ parameters.internalProjectName }} - dockerClientOS: ${{ parameters.dockerClientOS }} - args: >- - build - --manifest $(manifest) - $(imageBuilderPaths) - $(osVersions) - --os-type $(osType) - --architecture $(architecture) - --retry - --digests-out-var 'builtImages' - --acr-subscription '${{ parameters.publishConfig.buildAcr.subscription }}' - --acr-resource-group '${{ parameters.publishConfig.buildAcr.resourceGroup }}' - $(manifestVariables) - $(imageBuilderBuildArgs) - - template: /eng/common/templates/steps/publish-artifact.yml@self - parameters: - path: $(imageInfoHostDir) - artifactName: $(legName)-image-info-$(System.JobAttempt) - displayName: Publish Image Info File Artifact - internalProjectName: ${{ parameters.internalProjectName }} - publicProjectName: ${{ parameters.publicProjectName }} - - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.dockerClientOS, 'linux')) }}: - - powershell: | - $images = "$(BuildImages.builtImages)" - if (-not $images) { return 0 } - $syftImageName = "${{ parameters.publishConfig.publicMirrorAcr.server }}/$(imageNames.syft)" - & $(engCommonPath)/Pull-Image.ps1 $syftImageName - $images -Split ',' | ForEach-Object { - echo "Generating SBOM for $_"; - $targetImageName = "$_"; - $formattedImageName = $targetImageName.Replace('${{ parameters.publishConfig.buildAcr.server }}/${{ parameters.publishConfig.buildAcr.repoPrefix }}', "").Replace('/', '_').Replace(':', '_'); - $sbomChildDir = "$(sbomDirectory)/$formattedImageName"; - New-Item -Type Directory -Path $sbomChildDir > $null; - docker build --output=$sbomChildDir -f $(engCommonPath)/Dockerfile.syft --build-arg SYFT_IMAGE_NAME=$syftImageName --build-arg TARGET_IMAGE_NAME=$targetImageName -t syft-sbom $(engCommonPath); - } - displayName: Generate SBOMs - condition: and(succeeded(), ne(variables['BuildImages.builtImages'], '')) - - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: - - template: /eng/common/templates/jobs/${{ format('../steps/test-images-{0}-client.yml', parameters.dockerClientOS) }}@self - parameters: - condition: ne(variables.testScriptPath, '') - - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.dockerClientOS, 'linux')) }}: - - template: /eng/common/templates/steps/publish-artifact.yml@self - parameters: - path: $(sbomDirectory) - artifactName: $(legName)-sboms - displayName: Publish SBOM - internalProjectName: ${{ parameters.internalProjectName }} - publicProjectName: ${{ parameters.publicProjectName }} - condition: ne(variables['BuildImages.builtImages'], '') diff --git a/eng/common/templates/jobs/cg-build-projects.yml b/eng/common/templates/jobs/cg-build-projects.yml deleted file mode 100644 index de372c4c4..000000000 --- a/eng/common/templates/jobs/cg-build-projects.yml +++ /dev/null @@ -1,58 +0,0 @@ -# This job builds all projects in the repository. It is intended to be used for CG purposes. -# The 1ES CG step does not scan artifacts that are built within Dockerfiles therefore they -# need to be built outside of Dockerfiles. -parameters: -# Setting cgDryRun will run CG but not submit the results -- name: cgDryRun - type: boolean - default: false - displayName: CG Dry Run -# See https://learn.microsoft.com/en-us/dotnet/core/tools/dotnet-install-script#options for possible Channel values -- name: dotnetVersionChannel - type: string - default: '9.0' - displayName: .NET Version - -jobs: -- job: BuildProjects - displayName: Build Projects - pool: - name: $(default1ESInternalPoolName) - image: $(default1ESInternalPoolImage) - os: linux - steps: - - powershell: > - ./eng/common/Install-DotNetSdk.ps1 -Channel ${{ parameters.dotnetVersionChannel }} -InstallPath "/usr/share/.dotnet" - displayName: Run Dotnet Install Script - - script: > - find . -name '*.csproj' | grep $(cgBuildGrepArgs) | xargs -n 1 /usr/share/.dotnet/dotnet build - displayName: Build Projects - - # Component Detection is only automatically run on production branches. - # To run Component Detection on non-production branches, the task must be manually injected. - - ${{ if eq(parameters.cgDryRun, true) }}: - - powershell: | - Write-Host "##vso[build.updatebuildnumber]$env:BUILD_BUILDNUMBER (Dry run)" - Write-Host "##vso[build.addbuildtag]dry-run" - - if ("$(officialBranches)".Split(',').Contains("$(Build.SourceBranch)")) - { - Write-Host "##vso[task.logissue type=error]Cannot run a CG dry-run build from an official branch ($(officialBranches))." - Write-Host "##vso[task.logissue type=error]Run the pipeline again from a different branch to avoid registering scan results." - exit 1 - } - displayName: Update Build Number - - task: ComponentGovernanceComponentDetection@0 - displayName: Component Detection (manually injected) - inputs: - # Running CG with `whatIf: true` or `scanType: LogOnly` outputs a list of detected components, but doesn't show - # which components would trigger an alert on a production build. - # As long as the build isn't ran on a tracked branch (typically main or nightly), it's OK to submit components - # to CG for analysis. Only the results for tracked branches matter. - scanType: Register - whatIf: false - alertWarningLevel: Low - failOnAlert: false - ignoreDirectories: $(Build.SourcesDirectory)/versions - showAlertLink: true - timeoutInMinutes: 10 diff --git a/eng/common/templates/jobs/copy-base-images-staging.yml b/eng/common/templates/jobs/copy-base-images-staging.yml deleted file mode 100644 index 17f4d359c..000000000 --- a/eng/common/templates/jobs/copy-base-images-staging.yml +++ /dev/null @@ -1,29 +0,0 @@ -parameters: -- name: name - type: string - default: null -- name: pool - type: object - default: {} -- name: publishConfig - type: object - default: null -- name: customInitSteps - type: stepList - default: [] -- name: additionalOptions - type: string - default: '' -- name: continueOnError - type: string - default: false - -jobs: -- template: /eng/common/templates/jobs/copy-base-images.yml@self - parameters: - name: ${{ parameters.name }} - pool: ${{ parameters.pool }} - customInitSteps: ${{ parameters.customInitSteps }} - additionalOptions: ${{ parameters.additionalOptions }} - acr: ${{ parameters.publishConfig.internalMirrorAcr }} - repoPrefix: ${{ parameters.publishConfig.internalMirrorAcr.repoPrefix }} diff --git a/eng/common/templates/jobs/copy-base-images.yml b/eng/common/templates/jobs/copy-base-images.yml deleted file mode 100644 index 3068a43cb..000000000 --- a/eng/common/templates/jobs/copy-base-images.yml +++ /dev/null @@ -1,39 +0,0 @@ -parameters: -- name: name - type: string - default: null -- name: pool - type: object - default: {} -- name: acr - type: object - default: null -- name: repoPrefix - type: string - default: null -- name: customInitSteps - type: stepList - default: [] -- name: additionalOptions - type: string - default: '' -- name: continueOnError - type: string - default: false -- name: forceDryRun - type: boolean - default: false - -jobs: -- job: ${{ parameters.name }} - pool: ${{ parameters.pool }} - steps: - - template: /eng/common/templates/steps/init-docker-linux.yml@self - - ${{ parameters.customInitSteps }} - - template: /eng/common/templates/steps/copy-base-images.yml@self - parameters: - acr: ${{ parameters.acr }} - repoPrefix: ${{ parameters.repoPrefix }} - additionalOptions: ${{ parameters.additionalOptions }} - continueOnError: ${{ parameters.continueOnError }} - forceDryRun: ${{ parameters.forceDryRun }} diff --git a/eng/common/templates/jobs/generate-matrix.yml b/eng/common/templates/jobs/generate-matrix.yml deleted file mode 100644 index 663aeb01f..000000000 --- a/eng/common/templates/jobs/generate-matrix.yml +++ /dev/null @@ -1,72 +0,0 @@ -parameters: - matrixType: null - name: null - pool: {} - customBuildLegGroupArgs: "" - isTestStage: false - internalProjectName: null - noCache: false - publishConfig: null - customInitSteps: [] - commonInitStepsForMatrixAndBuild: [] - sourceBuildPipelineRunId: "" - -jobs: -- job: ${{ parameters.name }} - pool: ${{ parameters.pool }} - steps: - - ${{ parameters.commonInitStepsForMatrixAndBuild }} - - template: /eng/common/templates/steps/retain-build.yml@self - - template: /eng/common/templates/steps/init-docker-linux.yml@self - - ${{ parameters.customInitSteps }} - - template: /eng/common/templates/steps/validate-branch.yml@self - parameters: - publishConfig: ${{ parameters.publishConfig }} - internalProjectName: ${{ parameters.internalProjectName }} - - template: /eng/common/templates/steps/set-image-info-path-var.yml - parameters: - publicSourceBranch: $(publicSourceBranch) - - ${{ if eq(parameters.isTestStage, true) }}: - - template: /eng/common/templates/steps/download-build-artifact.yml@self - parameters: - targetPath: $(Build.ArtifactStagingDirectory) - artifactName: image-info - pipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} - - powershell: | - $additionalGenerateBuildMatrixOptions = "$(additionalGenerateBuildMatrixOptions)" - - if ("${{ parameters.isTestStage}}" -eq "true") { - $additionalGenerateBuildMatrixOptions = "$additionalGenerateBuildMatrixOptions --image-info $(artifactsPath)/image-info.json" - } - elseif ("$(pipelineDisabledCache)" -ne "true" -and "${{ parameters.noCache }}" -ne "true" -and "$(trimCachedImagesForMatrix)" -eq "true") { - # If the pipeline isn't configured to disable the cache and a build variable hasn't been set to disable the cache - $additionalGenerateBuildMatrixOptions = "$additionalGenerateBuildMatrixOptions --image-info $(versionsBasePath)$(imageInfoVersionsPath) --trim-cached-images" - } - - echo "##vso[task.setvariable variable=additionalGenerateBuildMatrixOptions]$additionalGenerateBuildMatrixOptions" - displayName: Set GenerateBuildMatrix Variables - - script: > - echo "##vso[task.setvariable variable=generateBuildMatrixCommand] - generateBuildMatrix - --manifest $(manifest) - --type ${{ parameters.matrixType }} - --os-type '*' - --architecture '*' - --product-version-components $(productVersionComponents) - ${{ parameters.customBuildLegGroupArgs }} - $(imageBuilder.pathArgs) - $(manifestVariables) - $(commonMatrixAndBuildOptions) - $(additionalGenerateBuildMatrixOptions)" - displayName: Set GenerateBuildMatrix Command - - template: /eng/common/templates/steps/run-imagebuilder.yml@self - parameters: - name: matrix - displayName: Generate ${{ parameters.matrixType }} Matrix - serviceConnections: - - name: acr - tenantId: $(build.serviceConnection.tenantId) - clientId: $(build.serviceConnection.clientId) - id: $(build.serviceConnection.id) - internalProjectName: internal - args: $(generateBuildMatrixCommand) diff --git a/eng/common/templates/jobs/post-build.yml b/eng/common/templates/jobs/post-build.yml deleted file mode 100644 index 31ad2eff0..000000000 --- a/eng/common/templates/jobs/post-build.yml +++ /dev/null @@ -1,87 +0,0 @@ -parameters: - pool: {} - internalProjectName: null - publicProjectName: null - -jobs: -- job: Build - pool: ${{ parameters.pool }} - variables: - imageInfosSubDir: "/image-infos" - imageInfosHostDir: "$(Build.ArtifactStagingDirectory)$(imageInfosSubDir)" - imageInfosContainerDir: "$(artifactsPath)$(imageInfosSubDir)" - imageInfosOutputSubDir: "/output" - sbomOutputDir: "$(Build.ArtifactStagingDirectory)/sbom" - steps: - - template: /eng/common/templates/steps/init-docker-linux.yml@self - - template: /eng/common/templates/steps/download-build-artifact.yml@self - parameters: - targetPath: $(Build.ArtifactStagingDirectory) - # This can fail if no build jobs ran to produce any artifacts - continueOnError: true - - powershell: | - # Move all image-info artifacts to their own directory - New-Item -ItemType Directory -Path $(imageInfosHostDir) - Get-ChildItem -Directory -Filter "*-image-info-*" $(Build.ArtifactStagingDirectory) | - Move-Item -Verbose -Destination $(imageInfosHostDir) - displayName: Collect Image Info Files - - powershell: | - # Move the contents of all the SBOM artifact directories to a single location - New-Item -ItemType Directory -Path $(sbomOutputDir) - Get-ChildItem -Directory -Filter "*-sboms" $(Build.ArtifactStagingDirectory) | - ForEach-Object { - Get-ChildItem $_ -Directory | Move-Item -Force -Verbose -Destination $(sbomOutputDir) - } - displayName: Consolidate SBOMs to Single Directory - - powershell: | - # Deletes the artifacts from all the unsuccessful jobs - Get-ChildItem $(imageInfosHostDir) -Directory | - ForEach-Object { - [pscustomobject]@{ - # Parse the artifact name to separate the base of the name from the job attempt number - BaseName = $_.Name.Substring(0, $_.Name.LastIndexOf('-')); - JobAttempt = $_.Name.Substring($_.Name.LastIndexOf('-') + 1) - FullName = $_.FullName - } - } | - Group-Object BaseName | - # Delete all but the last artifact from each base name - ForEach-Object { - $_.Group | - Sort-Object JobAttempt | - Select-Object -ExpandProperty FullName -SkipLast 1 | - Remove-Item -Recurse -Force - } - displayName: Prune Publish Artifacts - - powershell: | - $imageInfoFiles = Get-ChildItem "$(imageInfosHostDir)" - if ($imageInfoFiles.Count -eq 0) { - echo "No image info files found." - echo "##vso[task.setvariable variable=noImageInfos;isOutput=true]true" - exit 0 - } - - New-Item -ItemType Directory -Path $(imageInfosHostDir)$(imageInfosOutputSubDir) -Force - $(runImageBuilderCmd) mergeImageInfo ` - --manifest $(manifest) ` - $(imageInfosContainerDir) ` - $(imageInfosContainerDir)$(imageInfosOutputSubDir)/image-info.json ` - $(manifestVariables) - name: MergeImageInfoFiles - displayName: Merge Image Info Files - - template: /eng/common/templates/steps/publish-artifact.yml@self - parameters: - condition: and(succeeded(), ne(variables['MergeImageInfoFiles.noImageInfos'], 'true')) - path: $(sbomOutputDir) - artifactName: sboms - displayName: Publish SBOM Artifact - internalProjectName: ${{ parameters.internalProjectName }} - publicProjectName: ${{ parameters.publicProjectName }} - - template: /eng/common/templates/steps/publish-artifact.yml@self - parameters: - condition: and(succeeded(), ne(variables['MergeImageInfoFiles.noImageInfos'], 'true')) - path: $(imageInfosHostDir)$(imageInfosOutputSubDir) - artifactName: image-info - displayName: Publish Image Info File Artifact - internalProjectName: ${{ parameters.internalProjectName }} - publicProjectName: ${{ parameters.publicProjectName }} diff --git a/eng/common/templates/jobs/publish.yml b/eng/common/templates/jobs/publish.yml deleted file mode 100644 index 463d9af45..000000000 --- a/eng/common/templates/jobs/publish.yml +++ /dev/null @@ -1,312 +0,0 @@ -parameters: - pool: {} - internalProjectName: null - publishConfig: null - customInitSteps: [] - customPublishVariables: [] - sourceBuildPipelineDefinitionId: "" - sourceBuildPipelineRunId: "" - versionsRepoRef: null - versionsRepoPath: "" - # When true, overrides the commit SHA in merged image info files to use the current repository commit. - # This ensures that updated images reference the correct commit in their commitUrl properties. - overrideImageInfoCommit: false - -jobs: -- job: Publish - pool: ${{ parameters.pool }} - timeoutInMinutes: 90 - - variables: - - name: imageBuilder.commonCmdArgs - value: >- - --manifest '$(manifest)' - --registry-override '${{ parameters.publishConfig.publishAcr.server }}' - $(manifestVariables) - $(imageBuilder.queueArgs) - - name: publishNotificationRepoName - value: $(Build.Repository.Name) - - name: branchName - ${{ if startsWith(variables['Build.SourceBranch'], 'refs/heads/') }}: - value: $[ replace(variables['Build.SourceBranch'], 'refs/heads/', '') ] - ${{ if startsWith(variables['Build.SourceBranch'], 'refs/pull/') }}: - value: $[ replace(variables['System.PullRequest.SourceBranch'], 'refs/heads/', '') ] - - name: imageInfoHostDir - value: $(Build.ArtifactStagingDirectory)/imageInfo - - name: imageInfoContainerDir - value: $(artifactsPath)/imageInfo - - name: sourceBuildIdOutputDir - value: $(Build.ArtifactStagingDirectory)/sourceBuildId - - name: commitOverrideArg - ${{ if eq(parameters.overrideImageInfoCommit, true) }}: - value: --commit-override $(Build.SourceVersion) - ${{ else }}: - value: '' - - ${{ parameters.customPublishVariables }} - - steps: - - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self - parameters: - publishConfig: ${{ parameters.publishConfig }} - versionsRepoRef: ${{ parameters.versionsRepoRef }} - - - template: /eng/common/templates/steps/retain-build.yml@self - - - template: /eng/common/templates/steps/init-docker-linux.yml@self - - - pwsh: | - $azdoOrgName = Split-Path -Leaf $Env:SYSTEM_COLLECTIONURI - echo "##vso[task.setvariable variable=azdoOrgName]$azdoOrgName" - $versionsRepoRoot = "$(Pipeline.Workspace)/s/${{ parameters.versionsRepoPath }}" - echo "##vso[task.setvariable variable=versionsRepoRoot]$versionsRepoRoot" - displayName: Set Publish Variables - - - ${{ parameters.customInitSteps }} - - - template: /eng/common/templates/steps/validate-branch.yml@self - parameters: - internalProjectName: ${{ parameters.internalProjectName }} - - - template: /eng/common/templates/steps/download-build-artifact.yml@self - parameters: - targetPath: $(imageInfoHostDir) - artifactName: image-info - piplineDefinitionId: ${{ parameters.sourceBuildPipelineDefinitionId }} - pipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} - # This can fail in scenarios where no build jobs have run to produce any artifacts - continueOnError: true - - - template: /eng/common/templates/steps/set-image-info-path-var.yml@self - parameters: - publicSourceBranch: $(publicSourceBranch) - - - template: /eng/common/templates/steps/set-dry-run.yml@self - parameters: - publishConfig: ${{ parameters.publishConfig }} - - - script: echo "##vso[task.setvariable variable=imageQueueTime]$(date --rfc-2822)" - displayName: Set Publish Variables - - - script: > - $(runImageBuilderCmd) trimUnchangedPlatforms - '$(imageInfoContainerDir)/image-info.json' - displayName: Trim Unchanged Images - - - template: /eng/common/templates/steps/run-imagebuilder.yml@self - parameters: - displayName: Copy Images - serviceConnections: - - name: acr - id: ${{ parameters.publishConfig.publishAcr.serviceConnection.id }} - tenantId: ${{ parameters.publishConfig.publishAcr.serviceConnection.tenantId }} - clientId: ${{ parameters.publishConfig.publishAcr.serviceConnection.clientId }} - internalProjectName: ${{ parameters.internalProjectName }} - args: >- - copyAcrImages - '${{ parameters.publishConfig.buildAcr.subscription }}' - '${{ parameters.publishConfig.buildAcr.resourceGroup }}' - '${{ parameters.publishConfig.buildAcr.repoPrefix }}' - '${{ parameters.publishConfig.buildAcr.server }}' - --os-type '*' - --architecture '*' - --repo-prefix '${{ parameters.publishConfig.publishAcr.repoPrefix }}' - --image-info '$(imageInfoContainerDir)/image-info.json' - $(dryRunArg) - $(imageBuilder.pathArgs) - $(imageBuilder.commonCmdArgs) - - - template: /eng/common/templates/steps/run-imagebuilder.yml@self - parameters: - displayName: Publish Manifest - serviceConnections: - - name: acr - id: ${{ parameters.publishConfig.publishAcr.serviceConnection.id }} - tenantId: ${{ parameters.publishConfig.publishAcr.serviceConnection.tenantId }} - clientId: ${{ parameters.publishConfig.publishAcr.serviceConnection.clientId }} - internalProjectName: ${{ parameters.internalProjectName }} - dockerClientOS: ${{ parameters.dockerClientOS }} - args: >- - publishManifest - '$(imageInfoContainerDir)/image-info.json' - --repo-prefix '${{ parameters.publishConfig.publishAcr.repoPrefix }}' - --os-type '*' - --architecture '*' - $(dryRunArg) - $(imageBuilder.pathArgs) - $(imageBuilder.commonCmdArgs) - - - template: /eng/common/templates/steps/publish-artifact.yml@self - parameters: - path: $(imageInfoHostDir) - artifactName: image-info-final-$(System.JobAttempt) - displayName: Publish Image Info File Artifact - internalProjectName: ${{ parameters.internalProjectName }} - publicProjectName: ${{ parameters.publicProjectName }} - - - template: /eng/common/templates/steps/wait-for-mcr-image-ingestion.yml@self - parameters: - publishConfig: ${{ parameters.publishConfig }} - imageInfoPath: '$(imageinfoContainerDir)/image-info.json' - minQueueTime: $(imageQueueTime) - dryRunArg: $(dryRunArg) - condition: succeeded() - - - template: /eng/common/templates/steps/publish-readmes.yml@self - parameters: - dryRunArg: $(dryRunArg) - condition: and(succeeded(), eq(variables['publishReadme'], 'true')) - - - script: mkdir -p $(Build.ArtifactStagingDirectory)/eol-annotation-data - displayName: Create EOL Annotation Data Directory - - - script: |- - cd $(versionsRepoRoot) - git pull origin $(gitHubVersionsRepoInfo.branch) - condition: and(succeeded(), eq(variables['publishImageInfo'], 'true')) - displayName: Pull Latest Changes from Versions Repo - - - script: >- - cp $(versionsRepoRoot)/$(gitHubImageInfoVersionsPath) $(imageInfoHostDir)/full-image-info-orig.json - condition: and(succeeded(), eq(variables['publishImageInfo'], 'true')) - displayName: Copy Latest Image Info from Versions Repo - - - script: > - $(runImageBuilderCmd) mergeImageInfo - $(imageInfoContainerDir) - $(imageInfoContainerDir)/full-image-info-new.json - $(manifestVariables) - $(dryRunArg) - --manifest $(manifest) - --publish - --initial-image-info-path $(imageInfoContainerDir)/full-image-info-orig.json - $(commitOverrideArg) - condition: and(succeeded(), eq(variables['publishImageInfo'], 'true')) - displayName: Merge Image Info - - - template: /eng/common/templates/steps/run-imagebuilder.yml@self - parameters: - displayName: Ingest Kusto Image Info - serviceConnections: - - name: kusto - id: $(kusto.serviceConnection.id) - tenantId: $(kusto.serviceConnection.tenantId) - clientId: $(kusto.serviceConnection.clientId) - internalProjectName: ${{ parameters.internalProjectName }} - condition: and(succeeded(), eq(variables['ingestKustoImageInfo'], 'true')) - args: >- - ingestKustoImageInfo - '$(imageInfoContainerDir)/image-info.json' - '$(kusto.cluster)' - '$(kusto.database)' - '$(kusto.imageTable)' - '$(kusto.layerTable)' - --os-type '*' - --architecture '*' - $(dryRunArg) - $(imageBuilder.commonCmdArgs) - - - template: /eng/common/templates/steps/run-imagebuilder.yml@self - parameters: - displayName: Generate EOL Annotation Data - serviceConnections: - - name: acr - id: ${{ parameters.publishConfig.publishAcr.serviceConnection.id }} - tenantId: ${{ parameters.publishConfig.publishAcr.serviceConnection.tenantId }} - clientId: ${{ parameters.publishConfig.publishAcr.serviceConnection.clientId }} - internalProjectName: internal - condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true')) - args: >- - generateEolAnnotationDataForPublish - '${{ parameters.publishConfig.publishAcr.server }}' - '${{ parameters.publishConfig.publishAcr.repoPrefix }}' - '$(artifactsPath)/eol-annotation-data/eol-annotation-data.json' - '$(imageInfoContainerDir)/full-image-info-orig.json' - '$(imageInfoContainerDir)/full-image-info-new.json' - $(generateEolAnnotationDataExtraOptions) - $(dryRunArg) - - - template: /eng/common/templates/steps/publish-artifact.yml@self - parameters: - path: $(Build.ArtifactStagingDirectory)/eol-annotation-data - artifactName: eol-annotation-data-$(System.JobAttempt) - displayName: Publish EOL Annotation Data Artifact - internalProjectName: internal - publicProjectName: public - condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true')) - - - template: /eng/common/templates/steps/annotate-eol-digests.yml@self - parameters: - acr: ${{ parameters.publishConfig.publishAcr }} - dataFile: $(artifactsPath)/eol-annotation-data/eol-annotation-data.json - - - script: > - $(runImageBuilderCmd) publishImageInfo - '$(imageInfoContainerDir)/full-image-info-new.json' - '$(gitHubVersionsRepoInfo.userName)' - '$(gitHubVersionsRepoInfo.email)' - $(gitHubVersionsRepoInfo.authArgs) - --git-owner '$(gitHubVersionsRepoInfo.org)' - --git-repo '$(gitHubVersionsRepoInfo.repo)' - --git-branch '$(gitHubVersionsRepoInfo.branch)' - --git-path '$(gitHubImageInfoVersionsPath)' - $(dryRunArg) - $(imageBuilder.commonCmdArgs) - condition: and(succeeded(), eq(variables['publishImageInfo'], 'true')) - displayName: Publish Image Info - - # Task displayNames names are hardcoded to reference the task prefix used by 1ES official - # pipelines in eng/common/templates/1es-official.yml. - # - # These will fail if they are dependend on by an unofficial pipeline since they use a unique task - # prefix compared to official pipelines (see eng/common/templates/1es-unofficial.yml). This is - # acceptable because unofficial pipelines should not publish images. - # - # https://github.com/dotnet/docker-tools/issues/1698 tracks making this command no longer depend - # on individual step displayNames. - - script: > - $(runImageBuilderCmd) postPublishNotification - '$(publishNotificationRepoName)' - '$(branchName)' - '$(imageInfoContainerDir)/image-info.json' - $(Build.BuildId) - '$(System.AccessToken)' - '$(azdoOrgName)' - '$(System.TeamProject)' - $(gitHubNotificationsRepoInfo.authArgs) - '$(gitHubNotificationsRepoInfo.org)' - '$(gitHubNotificationsRepoInfo.repo)' - --repo-prefix '${{ parameters.publishConfig.publishAcr.repoPrefix }}' - --task "🟪 Copy Images" - --task "🟪 Publish Manifest" - --task "🟪 Wait for Image Ingestion" - --task "🟪 Publish Readmes" - --task "🟪 Wait for MCR Doc Ingestion" - --task "🟪 Publish Image Info" - --task "🟪 Ingest Kusto Image Info" - --task "🟪 Generate EOL Annotation Data" - --task "🟪 Annotate EOL Images (${{ parameters.publishConfig.publishAcr.server }})" - --task "🟪 Wait for Annotation Ingestion (${{ parameters.publishConfig.publishAcr.server }})" - $(dryRunArg) - $(imageBuilder.commonCmdArgs) - displayName: Post Publish Notification - condition: and(always(), eq(variables['publishNotificationsEnabled'], 'true')) - - - powershell: | - # Default to current build number if parameter was not overridden - $buildId = "${{ parameters.sourceBuildPipelineRunId }}" - if ($buildId -eq "") { - $buildId = "$(Build.BuildNumber)" - } - - New-Item -ItemType Directory -Path $(sourceBuildIdOutputDir) - Set-Content -Path $(sourceBuildIdOutputDir)/source-build-id.txt -Value "$buildId" - condition: succeeded() - displayName: Write Source Build ID to File - - - template: /eng/common/templates/steps/publish-artifact.yml@self - parameters: - path: $(sourceBuildIdOutputDir) - artifactName: source-build-id - displayName: Publish Source Build ID Artifact - internalProjectName: ${{ parameters.internalProjectName }} - publicProjectName: ${{ parameters.publicProjectName }} diff --git a/eng/common/templates/jobs/test-images-linux-client.yml b/eng/common/templates/jobs/test-images-linux-client.yml deleted file mode 100644 index 9aa30e659..000000000 --- a/eng/common/templates/jobs/test-images-linux-client.yml +++ /dev/null @@ -1,30 +0,0 @@ -parameters: - name: null - pool: {} - matrix: {} - testJobTimeout: 60 - preBuildValidation: false - internalProjectName: null - publishConfig: null - customInitSteps: [] - sourceBuildPipelineRunId: "" - -jobs: -- job: ${{ parameters.name }} - ${{ if eq(parameters.preBuildValidation, 'false') }}: - condition: and(succeeded(), ${{ parameters.matrix }}) - dependsOn: GenerateTestMatrix - strategy: - matrix: $[ ${{ parameters.matrix }} ] - ${{ if eq(parameters.preBuildValidation, 'true') }}: - condition: and(succeeded(), ne(variables.testScriptPath, '')) - pool: ${{ parameters.pool }} - timeoutInMinutes: ${{ parameters.testJobTimeout }} - steps: - - template: /eng/common/templates/steps/test-images-linux-client.yml@self - parameters: - preBuildValidation: ${{ parameters.preBuildValidation }} - internalProjectName: ${{ parameters.internalProjectName }} - publishConfig: ${{ parameters.publishConfig }} - customInitSteps: ${{ parameters.customInitSteps }} - sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} diff --git a/eng/common/templates/jobs/test-images-windows-client.yml b/eng/common/templates/jobs/test-images-windows-client.yml deleted file mode 100644 index 94965f4cf..000000000 --- a/eng/common/templates/jobs/test-images-windows-client.yml +++ /dev/null @@ -1,25 +0,0 @@ -parameters: - name: null - pool: {} - matrix: {} - testJobTimeout: 60 - internalProjectName: null - publishConfig: null - customInitSteps: [] - sourceBuildPipelineRunId: "" - -jobs: -- job: ${{ parameters.name }} - condition: and(succeeded(), ${{ parameters.matrix }}) - dependsOn: GenerateTestMatrix - pool: ${{ parameters.pool }} - strategy: - matrix: $[ ${{ parameters.matrix }} ] - timeoutInMinutes: ${{ parameters.testJobTimeout }} - steps: - - template: /eng/common/templates/steps/test-images-windows-client.yml@self - parameters: - internalProjectName: ${{ parameters.internalProjectName }} - publishConfig: ${{ parameters.publishConfig }} - customInitSteps: ${{ parameters.customInitSteps }} - sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} diff --git a/eng/common/templates/jobs/validate-image-sizes.yml b/eng/common/templates/jobs/validate-image-sizes.yml deleted file mode 100644 index 18639a00b..000000000 --- a/eng/common/templates/jobs/validate-image-sizes.yml +++ /dev/null @@ -1,50 +0,0 @@ -parameters: - internalProjectName: null - publicProjectName: null - -jobs: -- ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}: - # Split the Linux images into separate jobs in order to compensate for less - # disk space on the Linux agents. See https://github.com/dotnet/dotnet-docker/issues/1588 - - job: LinuxAmd64PerfTests - pool: - vmImage: $(defaultLinuxAmd64PoolImage) - workspace: - clean: all - steps: - - template: ../steps/validate-image-sizes.yml - parameters: - dockerClientOS: linux - architecture: amd64 - validationMode: size - - job: LinuxArmPerfTests - pool: - vmImage: $(defaultLinuxAmd64PoolImage) - workspace: - clean: all - steps: - - template: ../steps/validate-image-sizes.yml - parameters: - dockerClientOS: linux - architecture: arm* - validationMode: size -- job: LinuxBaselineIntegrityPerfTests - pool: - vmImage: $(defaultLinuxAmd64PoolImage) - workspace: - clean: all - steps: - - template: ../steps/validate-image-sizes.yml - parameters: - dockerClientOS: linux - validationMode: integrity -- job: WindowsPerfTests - pool: Docker-2022-${{ variables['System.TeamProject'] }} - workspace: - clean: all - steps: - - template: ../steps/validate-image-sizes.yml - parameters: - dockerClientOS: windows - ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}: - validationMode: integrity diff --git a/eng/common/templates/jobs/wait-for-ingestion.yml b/eng/common/templates/jobs/wait-for-ingestion.yml deleted file mode 100644 index 38bc25f6d..000000000 --- a/eng/common/templates/jobs/wait-for-ingestion.yml +++ /dev/null @@ -1,26 +0,0 @@ -jobs: -- job: WaitForIngestion - pool: - vmImage: $(defaultLinuxAmd64PoolImage) - steps: - - template: ../steps/init-docker-linux.yml - - template: ../steps/download-build-artifact.yml - parameters: - targetPath: $(Build.ArtifactStagingDirectory) - condition: ne(variables['sourceBuildId'], '') - - powershell: | - # Get the last image info artifact that was published - $imageInfoDirName = @($(dir $(Build.ArtifactStagingDirectory)/image-info-final-* | select -ExpandProperty Name))[-1] - $imageInfoPath = "$(artifactsPath)/$imageInfoDirName/image-info.json" - echo "##vso[task.setvariable variable=imageInfoPath]$imageInfoPath" - displayName: Get Image Info Path - condition: and(succeeded(), ne(variables['sourceBuildId'], '')) - - template: ../steps/wait-for-mcr-image-ingestion.yml - parameters: - imageInfoPath: $(imageInfoPath) - minQueueTime: $(minQueueTime) - condition: and(succeeded(), ne(variables['sourceBuildId'], '')) - - template: ../steps/wait-for-mcr-doc-ingestion.yml - parameters: - commitDigest: $(readmeCommitDigest) - condition: and(succeeded(), ne(variables['readmeCommitDigest'], '')) diff --git a/eng/common/templates/stages/build-and-test.yml b/eng/common/templates/stages/build-and-test.yml deleted file mode 100644 index 2d5ed7026..000000000 --- a/eng/common/templates/stages/build-and-test.yml +++ /dev/null @@ -1,343 +0,0 @@ -parameters: - buildMatrixType: platformDependencyGraph - testMatrixType: platformVersionedOs - buildMatrixCustomBuildLegGroupArgs: "" - testMatrixCustomBuildLegGroupArgs: "" - customCopyBaseImagesInitSteps: [] - customGenerateMatrixInitSteps: [] - customBuildInitSteps: [] - customTestInitSteps: [] - sourceBuildPipelineRunId: "" - - linuxAmdBuildJobTimeout: 60 - linuxArmBuildJobTimeout: 60 - windowsAmdBuildJobTimeout: 60 - - linuxAmdTestJobTimeout: 60 - linuxArmTestJobTimeout: 60 - windowsAmdTestJobTimeout: 60 - - noCache: false - publishConfig: null - - internalProjectName: null - publicProjectName: null - - versionsRepoRef: "" - - storageAccountServiceConnection: null - - linuxAmd64Pool: - vmImage: $(defaultLinuxAmd64PoolImage) - linuxArm32Pool: - vmImage: $(defaultLinuxArm32PoolImage) - linuxArm64Pool: - vmImage: $(defaultLinuxArm64PoolImage) - windows2016Pool: - vmImage: $(defaultWindows2016PoolImage) - windows1809Pool: - vmImage: $(defaultWindows1809PoolImage) - windows2022Pool: - vmImage: $(defaultWindows2022PoolImage) - windows2025Pool: - vmImage: $(defaultWindows2025PoolImage) - - -################################################################################ -# Build Images -################################################################################ -stages: -- stage: Build - condition: and(succeeded(), contains(variables['stages'], 'build')) - dependsOn: [] - jobs: - - - template: /eng/common/templates/jobs/test-images-linux-client.yml@self - parameters: - name: PreBuildValidation - pool: ${{ parameters.linuxAmd64Pool }} - testJobTimeout: ${{ parameters.linuxAmdTestJobTimeout }} - preBuildValidation: true - internalProjectName: ${{ parameters.internalProjectName }} - publishConfig: ${{ parameters.publishConfig }} - customInitSteps: - - ${{ parameters.customTestInitSteps }} - # These variables are normally set by the matrix. Since this test job is not generated - # by a matrix, we need to set them manually. They can be set to empty values since their - # values aren't actually used for the pre-build tests. - - powershell: | - echo "##vso[task.setvariable variable=productVersion]" - echo "##vso[task.setvariable variable=imageBuilderPaths]" - echo "##vso[task.setvariable variable=osVersions]" - echo "##vso[task.setvariable variable=architecture]" - displayName: Initialize Test Variables - - - template: /eng/common/templates/jobs/copy-base-images-staging.yml@self - parameters: - name: CopyBaseImages - publishConfig: ${{ parameters.publishConfig }} - pool: ${{ parameters.linuxAmd64Pool }} - additionalOptions: "--manifest '$(manifest)' $(imageBuilder.pathArgs) $(manifestVariables)" - customInitSteps: ${{ parameters.customCopyBaseImagesInitSteps }} - - - template: /eng/common/templates/jobs/generate-matrix.yml@self - parameters: - matrixType: ${{ parameters.buildMatrixType }} - name: GenerateBuildMatrix - pool: ${{ parameters.linuxAmd64Pool }} - customBuildLegGroupArgs: ${{ parameters.buildMatrixCustomBuildLegGroupArgs }} - internalProjectName: ${{ parameters.internalProjectName }} - noCache: ${{ parameters.noCache }} - publishConfig: ${{ parameters.publishConfig }} - customInitSteps: ${{ parameters.customGenerateMatrixInitSteps }} - commonInitStepsForMatrixAndBuild: - - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self - parameters: - publishConfig: ${{ parameters.publishConfig }} - versionsRepoRef: ${{ parameters.versionsRepoRef }} - - - template: /eng/common/templates/jobs/build-images.yml@self - parameters: - name: Linux_amd64 - pool: ${{ parameters.linuxAmd64Pool }} - matrix: dependencies.GenerateBuildMatrix.outputs['matrix.LinuxAmd64'] - dockerClientOS: linux - buildJobTimeout: ${{ parameters.linuxAmdBuildJobTimeout }} - commonInitStepsForMatrixAndBuild: - - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self - parameters: - publishConfig: ${{ parameters.publishConfig }} - versionsRepoRef: ${{ parameters.versionsRepoRef }} - customInitSteps: ${{ parameters.customBuildInitSteps }} - noCache: ${{ parameters.noCache }} - publishConfig: ${{ parameters.publishConfig }} - internalProjectName: ${{ parameters.internalProjectName }} - publicProjectName: ${{ parameters.publicProjectName }} - storageAccountServiceConnection: ${{ parameters.storageAccountServiceConnection }} - - template: /eng/common/templates/jobs/build-images.yml@self - parameters: - name: Linux_arm64 - pool: ${{ parameters.linuxArm64Pool }} - matrix: dependencies.GenerateBuildMatrix.outputs['matrix.LinuxArm64'] - dockerClientOS: linux - buildJobTimeout: ${{ parameters.linuxArmBuildJobTimeout }} - commonInitStepsForMatrixAndBuild: - - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self - parameters: - publishConfig: ${{ parameters.publishConfig }} - versionsRepoRef: ${{ parameters.versionsRepoRef }} - customInitSteps: ${{ parameters.customBuildInitSteps }} - noCache: ${{ parameters.noCache }} - publishConfig: ${{ parameters.publishConfig }} - internalProjectName: ${{ parameters.internalProjectName }} - publicProjectName: ${{ parameters.publicProjectName }} - storageAccountServiceConnection: ${{ parameters.storageAccountServiceConnection }} - - template: /eng/common/templates/jobs/build-images.yml@self - parameters: - name: Linux_arm32 - pool: ${{ parameters.linuxArm32Pool }} - matrix: dependencies.GenerateBuildMatrix.outputs['matrix.LinuxArm32'] - dockerClientOS: linux - buildJobTimeout: ${{ parameters.linuxArmBuildJobTimeout }} - commonInitStepsForMatrixAndBuild: - - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self - parameters: - publishConfig: ${{ parameters.publishConfig }} - versionsRepoRef: ${{ parameters.versionsRepoRef }} - customInitSteps: ${{ parameters.customBuildInitSteps }} - noCache: ${{ parameters.noCache }} - publishConfig: ${{ parameters.publishConfig }} - internalProjectName: ${{ parameters.internalProjectName }} - publicProjectName: ${{ parameters.publicProjectName }} - storageAccountServiceConnection: ${{ parameters.storageAccountServiceConnection }} - - template: /eng/common/templates/jobs/build-images.yml@self - parameters: - name: Windows1809_amd64 - pool: ${{ parameters.windows1809Pool }} - matrix: dependencies.GenerateBuildMatrix.outputs['matrix.Windows1809Amd64'] - dockerClientOS: windows - buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }} - commonInitStepsForMatrixAndBuild: - - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self - parameters: - publishConfig: ${{ parameters.publishConfig }} - versionsRepoRef: ${{ parameters.versionsRepoRef }} - customInitSteps: ${{ parameters.customBuildInitSteps }} - noCache: ${{ parameters.noCache }} - publishConfig: ${{ parameters.publishConfig }} - internalProjectName: ${{ parameters.internalProjectName }} - publicProjectName: ${{ parameters.publicProjectName }} - storageAccountServiceConnection: ${{ parameters.storageAccountServiceConnection }} - - template: /eng/common/templates/jobs/build-images.yml@self - parameters: - name: Windows2022_amd64 - pool: ${{ parameters.windows2022Pool }} - matrix: dependencies.GenerateBuildMatrix.outputs['matrix.WindowsLtsc2022Amd64'] - dockerClientOS: windows - buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }} - commonInitStepsForMatrixAndBuild: - - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self - parameters: - publishConfig: ${{ parameters.publishConfig }} - versionsRepoRef: ${{ parameters.versionsRepoRef }} - customInitSteps: ${{ parameters.customBuildInitSteps }} - noCache: ${{ parameters.noCache }} - publishConfig: ${{ parameters.publishConfig }} - internalProjectName: ${{ parameters.internalProjectName }} - publicProjectName: ${{ parameters.publicProjectName }} - storageAccountServiceConnection: ${{ parameters.storageAccountServiceConnection }} - - template: /eng/common/templates/jobs/build-images.yml@self - parameters: - name: Windows2025_amd64 - pool: ${{ parameters.windows2025Pool }} - matrix: dependencies.GenerateBuildMatrix.outputs['matrix.WindowsLtsc2025Amd64'] - dockerClientOS: windows - buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }} - commonInitStepsForMatrixAndBuild: - - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self - parameters: - publishConfig: ${{ parameters.publishConfig }} - versionsRepoRef: ${{ parameters.versionsRepoRef }} - customInitSteps: ${{ parameters.customBuildInitSteps }} - noCache: ${{ parameters.noCache }} - publishConfig: ${{ parameters.publishConfig }} - internalProjectName: ${{ parameters.internalProjectName }} - publicProjectName: ${{ parameters.publicProjectName }} - versionsRepoRef: ${{ parameters.versionsRepoRef }} - storageAccountServiceConnection: ${{ parameters.storageAccountServiceConnection }} - - template: /eng/common/templates/jobs/build-images.yml@self - parameters: - name: WindowsLtsc2016_amd64 - pool: ${{ parameters.windows2016Pool }} - matrix: dependencies.GenerateBuildMatrix.outputs['matrix.WindowsLtsc2016Amd64'] - dockerClientOS: windows - buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }} - commonInitStepsForMatrixAndBuild: - - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self - parameters: - publishConfig: ${{ parameters.publishConfig }} - versionsRepoRef: ${{ parameters.versionsRepoRef }} - customInitSteps: ${{ parameters.customBuildInitSteps }} - noCache: ${{ parameters.noCache }} - publishConfig: ${{ parameters.publishConfig }} - internalProjectName: ${{ parameters.internalProjectName }} - publicProjectName: ${{ parameters.publicProjectName }} - storageAccountServiceConnection: ${{ parameters.storageAccountServiceConnection }} - -################################################################################ -# Post-Build -################################################################################ -- stage: Post_Build - dependsOn: Build - condition: and(succeeded(), contains(variables['stages'], 'build')) - jobs: - - template: /eng/common/templates/jobs/post-build.yml@self - parameters: - pool: ${{ parameters.linuxAmd64Pool }} - internalProjectName: ${{ parameters.internalProjectName }} - publicProjectName: ${{ parameters.publicProjectName }} - -################################################################################ -# Test Images -################################################################################ -- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: - - stage: Test - dependsOn: Post_Build - condition: " - and( - ne(stageDependencies.Post_Build.outputs['Build.MergeImageInfoFiles.noImageInfos'], 'true'), - ne(variables['testScriptPath'], ''), - and( - contains(variables['stages'], 'test'), - or( - and( - succeeded(), - contains(variables['stages'], 'build')), - not(contains(variables['stages'], 'build')))))" - jobs: - - template: /eng/common/templates/jobs/generate-matrix.yml@self - parameters: - matrixType: ${{ parameters.testMatrixType }} - name: GenerateTestMatrix - pool: ${{ parameters.linuxAmd64Pool }} - customBuildLegGroupArgs: ${{ parameters.testMatrixCustomBuildLegGroupArgs }} - isTestStage: true - internalProjectName: ${{ parameters.internalProjectName }} - publicProjectName: ${{ parameters.publicProjectName }} - customInitSteps: ${{ parameters.customGenerateMatrixInitSteps }} - sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} - commonInitStepsForMatrixAndBuild: - - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self - parameters: - versionsRepoRef: ${{ parameters.versionsRepoRef }} - publishConfig: ${{ parameters.publishConfig }} - - template: /eng/common/templates/jobs/test-images-linux-client.yml@self - parameters: - name: Linux_amd64 - pool: ${{ parameters.linuxAmd64Pool }} - matrix: dependencies.GenerateTestMatrix.outputs['matrix.LinuxAmd64'] - testJobTimeout: ${{ parameters.linuxAmdTestJobTimeout }} - internalProjectName: ${{ parameters.internalProjectName }} - publishConfig: ${{ parameters.publishConfig }} - customInitSteps: ${{ parameters.customTestInitSteps }} - sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} - - template: /eng/common/templates/jobs/test-images-linux-client.yml@self - parameters: - name: Linux_arm64 - pool: ${{ parameters.linuxArm64Pool }} - matrix: dependencies.GenerateTestMatrix.outputs['matrix.LinuxArm64'] - testJobTimeout: ${{ parameters.linuxArmTestJobTimeout }} - internalProjectName: ${{ parameters.internalProjectName }} - publishConfig: ${{ parameters.publishConfig }} - customInitSteps: ${{ parameters.customTestInitSteps }} - sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} - - template: /eng/common/templates/jobs/test-images-linux-client.yml@self - parameters: - name: Linux_arm32 - pool: ${{ parameters.linuxArm32Pool }} - matrix: dependencies.GenerateTestMatrix.outputs['matrix.LinuxArm32'] - testJobTimeout: ${{ parameters.linuxArmTestJobTimeout }} - internalProjectName: ${{ parameters.internalProjectName }} - publishConfig: ${{ parameters.publishConfig }} - customInitSteps: ${{ parameters.customTestInitSteps }} - sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} - - template: /eng/common/templates/jobs/test-images-windows-client.yml@self - parameters: - name: Windows1809_amd64 - pool: ${{ parameters.windows1809Pool }} - matrix: dependencies.GenerateTestMatrix.outputs['matrix.Windows1809Amd64'] - testJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }} - internalProjectName: ${{ parameters.internalProjectName }} - publishConfig: ${{ parameters.publishConfig }} - customInitSteps: ${{ parameters.customTestInitSteps }} - sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} - - template: /eng/common/templates/jobs/test-images-windows-client.yml@self - parameters: - name: Windows2022_amd64 - pool: ${{ parameters.windows2022Pool }} - matrix: dependencies.GenerateTestMatrix.outputs['matrix.WindowsLtsc2022Amd64'] - testJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }} - internalProjectName: ${{ parameters.internalProjectName }} - publishConfig: ${{ parameters.publishConfig }} - customInitSteps: ${{ parameters.customTestInitSteps }} - sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} - - template: /eng/common/templates/jobs/test-images-windows-client.yml@self - parameters: - name: Windows2025_amd64 - pool: ${{ parameters.windows2025Pool }} - matrix: dependencies.GenerateTestMatrix.outputs['matrix.WindowsLtsc2025Amd64'] - testJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }} - internalProjectName: ${{ parameters.internalProjectName }} - publishConfig: ${{ parameters.publishConfig }} - customInitSteps: ${{ parameters.customTestInitSteps }} - sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} - - template: /eng/common/templates/jobs/test-images-windows-client.yml@self - parameters: - name: WindowsLtsc2016_amd64 - pool: ${{ parameters.windows2016Pool }} - matrix: dependencies.GenerateTestMatrix.outputs['matrix.WindowsLtsc2016Amd64'] - testJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }} - internalProjectName: ${{ parameters.internalProjectName }} - publishConfig: ${{ parameters.publishConfig }} - customInitSteps: ${{ parameters.customTestInitSteps }} - sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} diff --git a/eng/common/templates/stages/build-test-publish-repo.yml b/eng/common/templates/stages/build-test-publish-repo.yml deleted file mode 100644 index aa29b2ba1..000000000 --- a/eng/common/templates/stages/build-test-publish-repo.yml +++ /dev/null @@ -1,378 +0,0 @@ -parameters: - buildMatrixType: platformDependencyGraph - testMatrixType: platformVersionedOs - buildMatrixCustomBuildLegGroupArgs: "" - testMatrixCustomBuildLegGroupArgs: "" - customCopyBaseImagesInitSteps: [] - customGenerateMatrixInitSteps: [] - customBuildInitSteps: [] - customTestInitSteps: [] - customPublishInitSteps: [] - customPublishVariables: [] - - linuxAmdBuildJobTimeout: 60 - linuxArmBuildJobTimeout: 60 - windowsAmdBuildJobTimeout: 60 - - linuxAmdTestJobTimeout: 60 - linuxArmTestJobTimeout: 60 - windowsAmdTestJobTimeout: 60 - - noCache: false - - internalProjectName: null - publicProjectName: null - - internalVersionsRepoRef: null - publicVersionsRepoRef: null - - isInternalServicingValidation: false - - linuxAmd64Pool: - vmImage: $(defaultLinuxAmd64PoolImage) - linuxArm32Pool: - vmImage: $(defaultLinuxArm32PoolImage) - linuxArm64Pool: - vmImage: $(defaultLinuxArm64PoolImage) - windows2016Pool: - vmImage: $(defaultWindows2016PoolImage) - windows1809Pool: - vmImage: $(defaultWindows1809PoolImage) - windows2022Pool: - vmImage: $(defaultWindows2022PoolImage) - windows2025Pool: - vmImage: $(defaultWindows2025PoolImage) - -stages: - -################################################################################ -# Build Images -################################################################################ -- stage: Build - condition: and(succeeded(), contains(variables['stages'], 'build')) - jobs: - - template: /eng/common/templates/jobs/test-images-linux-client.yml@self - parameters: - name: PreBuildValidation - pool: ${{ parameters.linuxAmd64Pool }} - testJobTimeout: ${{ parameters.linuxAmdTestJobTimeout }} - preBuildValidation: true - internalProjectName: ${{ parameters.internalProjectName }} - customInitSteps: - - ${{ parameters.customTestInitSteps }} - # These variables are normally set by the matrix. Since this test job is not generated - # by a matrix, we need to set them manually. They can be set to empty values since their - # values aren't actually used for the pre-build tests. - - powershell: | - echo "##vso[task.setvariable variable=productVersion]" - echo "##vso[task.setvariable variable=imageBuilderPaths]" - echo "##vso[task.setvariable variable=osVersions]" - echo "##vso[task.setvariable variable=architecture]" - displayName: Initialize Test Variables - - template: /eng/common/templates/jobs/copy-base-images-staging.yml@self - parameters: - name: CopyBaseImages - pool: ${{ parameters.linuxAmd64Pool }} - additionalOptions: "--manifest '$(manifest)' $(imageBuilder.pathArgs) $(manifestVariables)" - customInitSteps: ${{ parameters.customCopyBaseImagesInitSteps }} - - template: /eng/common/templates/jobs/generate-matrix.yml@self - parameters: - matrixType: ${{ parameters.buildMatrixType }} - name: GenerateBuildMatrix - pool: ${{ parameters.linuxAmd64Pool }} - customBuildLegGroupArgs: ${{ parameters.buildMatrixCustomBuildLegGroupArgs }} - internalProjectName: ${{ parameters.internalProjectName }} - noCache: ${{ parameters.noCache }} - customInitSteps: ${{ parameters.customGenerateMatrixInitSteps }} - commonInitStepsForMatrixAndBuild: - - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self - parameters: - noCache: ${{ parameters.noCache }} - internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }} - publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - - template: /eng/common/templates/jobs/build-images.yml@self - parameters: - name: Linux_amd64 - pool: ${{ parameters.linuxAmd64Pool }} - matrix: dependencies.GenerateBuildMatrix.outputs['matrix.LinuxAmd64'] - dockerClientOS: linux - buildJobTimeout: ${{ parameters.linuxAmdBuildJobTimeout }} - commonInitStepsForMatrixAndBuild: - - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self - parameters: - noCache: ${{ parameters.noCache }} - internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }} - publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - customInitSteps: ${{ parameters.customBuildInitSteps }} - noCache: ${{ parameters.noCache }} - internalProjectName: ${{ parameters.internalProjectName }} - publicProjectName: ${{ parameters.publicProjectName }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - - template: /eng/common/templates/jobs/build-images.yml@self - parameters: - name: Linux_arm64 - pool: ${{ parameters.linuxArm64Pool }} - matrix: dependencies.GenerateBuildMatrix.outputs['matrix.LinuxArm64'] - dockerClientOS: linux - buildJobTimeout: ${{ parameters.linuxArmBuildJobTimeout }} - commonInitStepsForMatrixAndBuild: - - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self - parameters: - noCache: ${{ parameters.noCache }} - internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }} - publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - customInitSteps: ${{ parameters.customBuildInitSteps }} - noCache: ${{ parameters.noCache }} - internalProjectName: ${{ parameters.internalProjectName }} - publicProjectName: ${{ parameters.publicProjectName }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - - template: /eng/common/templates/jobs/build-images.yml@self - parameters: - name: Linux_arm32 - pool: ${{ parameters.linuxArm32Pool }} - matrix: dependencies.GenerateBuildMatrix.outputs['matrix.LinuxArm32'] - dockerClientOS: linux - buildJobTimeout: ${{ parameters.linuxArmBuildJobTimeout }} - commonInitStepsForMatrixAndBuild: - - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self - parameters: - noCache: ${{ parameters.noCache }} - internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }} - publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - customInitSteps: ${{ parameters.customBuildInitSteps }} - noCache: ${{ parameters.noCache }} - internalProjectName: ${{ parameters.internalProjectName }} - publicProjectName: ${{ parameters.publicProjectName }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - - template: /eng/common/templates/jobs/build-images.yml@self - parameters: - name: Windows1809_amd64 - pool: ${{ parameters.windows1809Pool }} - matrix: dependencies.GenerateBuildMatrix.outputs['matrix.Windows1809Amd64'] - dockerClientOS: windows - buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }} - commonInitStepsForMatrixAndBuild: - - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self - parameters: - noCache: ${{ parameters.noCache }} - internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }} - publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - customInitSteps: ${{ parameters.customBuildInitSteps }} - noCache: ${{ parameters.noCache }} - internalProjectName: ${{ parameters.internalProjectName }} - publicProjectName: ${{ parameters.publicProjectName }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - - template: /eng/common/templates/jobs/build-images.yml@self - parameters: - name: Windows2022_amd64 - pool: ${{ parameters.windows2022Pool }} - matrix: dependencies.GenerateBuildMatrix.outputs['matrix.WindowsLtsc2022Amd64'] - dockerClientOS: windows - buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }} - commonInitStepsForMatrixAndBuild: - - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self - parameters: - noCache: ${{ parameters.noCache }} - internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }} - publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - customInitSteps: ${{ parameters.customBuildInitSteps }} - noCache: ${{ parameters.noCache }} - internalProjectName: ${{ parameters.internalProjectName }} - publicProjectName: ${{ parameters.publicProjectName }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - - template: /eng/common/templates/jobs/build-images.yml@self - parameters: - name: Windows2025_amd64 - pool: ${{ parameters.windows2025Pool }} - matrix: dependencies.GenerateBuildMatrix.outputs['matrix.WindowsLtsc2025Amd64'] - dockerClientOS: windows - buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }} - commonInitStepsForMatrixAndBuild: - - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self - parameters: - noCache: ${{ parameters.noCache }} - internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }} - publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - customInitSteps: ${{ parameters.customBuildInitSteps }} - noCache: ${{ parameters.noCache }} - internalProjectName: ${{ parameters.internalProjectName }} - publicProjectName: ${{ parameters.publicProjectName }} - internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }} - publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - - template: /eng/common/templates/jobs/build-images.yml@self - parameters: - name: WindowsLtsc2016_amd64 - pool: ${{ parameters.windows2016Pool }} - matrix: dependencies.GenerateBuildMatrix.outputs['matrix.WindowsLtsc2016Amd64'] - dockerClientOS: windows - buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }} - commonInitStepsForMatrixAndBuild: - - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self - parameters: - noCache: ${{ parameters.noCache }} - internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }} - publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - customInitSteps: ${{ parameters.customBuildInitSteps }} - noCache: ${{ parameters.noCache }} - internalProjectName: ${{ parameters.internalProjectName }} - publicProjectName: ${{ parameters.publicProjectName }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - -################################################################################ -# Post-Build -################################################################################ -- stage: Post_Build - dependsOn: Build - condition: and(succeeded(), contains(variables['stages'], 'build')) - jobs: - - template: /eng/common/templates/jobs/post-build.yml@self - parameters: - pool: ${{ parameters.linuxAmd64Pool }} - internalProjectName: ${{ parameters.internalProjectName }} - publicProjectName: ${{ parameters.publicProjectName }} - -################################################################################ -# Test Images -################################################################################ -- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.isInternalServicingValidation, 'false')) }}: - - stage: Test - dependsOn: Post_Build - condition: " - and( - ne(stageDependencies.Post_Build.outputs['Build.MergeImageInfoFiles.noImageInfos'], 'true'), - ne(variables['testScriptPath'], ''), - and( - contains(variables['stages'], 'test'), - or( - and( - succeeded(), - contains(variables['stages'], 'build')), - not(contains(variables['stages'], 'build')))))" - jobs: - - template: /eng/common/templates/jobs/generate-matrix.yml@self - parameters: - matrixType: ${{ parameters.testMatrixType }} - name: GenerateTestMatrix - pool: ${{ parameters.linuxAmd64Pool }} - customBuildLegGroupArgs: ${{ parameters.testMatrixCustomBuildLegGroupArgs }} - isTestStage: true - internalProjectName: ${{ parameters.internalProjectName }} - publicProjectName: ${{ parameters.publicProjectName }} - customInitSteps: ${{ parameters.customGenerateMatrixInitSteps }} - commonInitStepsForMatrixAndBuild: - - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self - parameters: - noCache: ${{ parameters.noCache }} - internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }} - publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }} - - template: /eng/common/templates/jobs/test-images-linux-client.yml@self - parameters: - name: Linux_amd64 - pool: ${{ parameters.linuxAmd64Pool }} - matrix: dependencies.GenerateTestMatrix.outputs['matrix.LinuxAmd64'] - testJobTimeout: ${{ parameters.linuxAmdTestJobTimeout }} - internalProjectName: ${{ parameters.internalProjectName }} - customInitSteps: ${{ parameters.customTestInitSteps }} - - template: /eng/common/templates/jobs/test-images-linux-client.yml@self - parameters: - name: Linux_arm64 - pool: ${{ parameters.linuxArm64Pool }} - matrix: dependencies.GenerateTestMatrix.outputs['matrix.LinuxArm64'] - testJobTimeout: ${{ parameters.linuxArmTestJobTimeout }} - internalProjectName: ${{ parameters.internalProjectName }} - customInitSteps: ${{ parameters.customTestInitSteps }} - - template: /eng/common/templates/jobs/test-images-linux-client.yml@self - parameters: - name: Linux_arm32 - pool: ${{ parameters.linuxArm32Pool }} - matrix: dependencies.GenerateTestMatrix.outputs['matrix.LinuxArm32'] - testJobTimeout: ${{ parameters.linuxArmTestJobTimeout }} - internalProjectName: ${{ parameters.internalProjectName }} - customInitSteps: ${{ parameters.customTestInitSteps }} - - template: /eng/common/templates/jobs/test-images-windows-client.yml@self - parameters: - name: Windows1809_amd64 - pool: ${{ parameters.windows1809Pool }} - matrix: dependencies.GenerateTestMatrix.outputs['matrix.Windows1809Amd64'] - testJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }} - internalProjectName: ${{ parameters.internalProjectName }} - customInitSteps: ${{ parameters.customTestInitSteps }} - - template: /eng/common/templates/jobs/test-images-windows-client.yml@self - parameters: - name: Windows2022_amd64 - pool: ${{ parameters.windows2022Pool }} - matrix: dependencies.GenerateTestMatrix.outputs['matrix.WindowsLtsc2022Amd64'] - testJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }} - internalProjectName: ${{ parameters.internalProjectName }} - customInitSteps: ${{ parameters.customTestInitSteps }} - - template: /eng/common/templates/jobs/test-images-windows-client.yml@self - parameters: - name: Windows2025_amd64 - pool: ${{ parameters.windows2025Pool }} - matrix: dependencies.GenerateTestMatrix.outputs['matrix.WindowsLtsc2025Amd64'] - testJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }} - internalProjectName: ${{ parameters.internalProjectName }} - customInitSteps: ${{ parameters.customTestInitSteps }} - - template: /eng/common/templates/jobs/test-images-windows-client.yml@self - parameters: - name: WindowsLtsc2016_amd64 - pool: ${{ parameters.windows2016Pool }} - matrix: dependencies.GenerateTestMatrix.outputs['matrix.WindowsLtsc2016Amd64'] - testJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }} - internalProjectName: ${{ parameters.internalProjectName }} - customInitSteps: ${{ parameters.customTestInitSteps }} - -################################################################################ -# Publish Images -################################################################################ -- ${{ if eq(parameters.isInternalServicingValidation, 'false') }}: - - stage: Publish - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: - dependsOn: Test - ${{ else }}: - dependsOn: Post_Build - condition: " - and( - not(canceled()), - and( - contains(variables['stages'], 'publish'), - or( - or( - and( - and( - contains(variables['stages'], 'build'), - succeeded('Post_Build')), - and( - contains(variables['stages'], 'test'), - in(dependencies.Test.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'))), - or( - and( - not(contains(variables['stages'], 'build')), - and( - contains(variables['stages'], 'test'), - in(dependencies.Test.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'))), - and( - not(contains(variables['stages'], 'test')), - and( - contains(variables['stages'], 'build'), - succeeded('Post_Build'))))), - not( - or( - contains(variables['stages'], 'build'), - contains(variables['stages'], 'test'))))))" - jobs: - - template: /eng/common/templates/jobs/publish.yml@self - parameters: - pool: ${{ parameters.linuxAmd64Pool }} - internalProjectName: ${{ parameters.internalProjectName }} - customPublishVariables: ${{ parameters.customPublishVariables }} - customInitSteps: ${{ parameters.customPublishInitSteps }} diff --git a/eng/common/templates/stages/dotnet/build-and-test.yml b/eng/common/templates/stages/dotnet/build-and-test.yml deleted file mode 100644 index dd8afd9ec..000000000 --- a/eng/common/templates/stages/dotnet/build-and-test.yml +++ /dev/null @@ -1,133 +0,0 @@ -# A wrapper template around the common build-test-publish-repo template with settings -# specific to the .NET team's infrastructure. - -parameters: - linuxAmd64Pool: "" - - # (Optional) This service connection should be an Azure Resource Manager - # service connection to a storage account that's needed during image builds. - # It can be used to build images with access to private/internal bits. - # If specified, this service connection will be used to pass a storage - # account access token as `--build-arg ACCESSTOKEN=***` to all image builds. - storageAccountServiceConnection: null - - # Parameters for pre-build jobs - customGenerateMatrixInitSteps: [] - customCopyBaseImagesInitSteps: [] - - # Build parameters - noCache: false - publishConfig: null - buildMatrixType: platformDependencyGraph - buildMatrixCustomBuildLegGroupArgs: "" - linuxAmdBuildJobTimeout: 60 - linuxArmBuildJobTimeout: 60 - windowsAmdBuildJobTimeout: 60 - customBuildInitSteps: [] - - # Test parameters - testMatrixType: platformVersionedOs - testMatrixCustomBuildLegGroupArgs: "" - linuxAmdTestJobTimeout: 60 - linuxArmTestJobTimeout: 60 - windowsAmdTestJobTimeout: 60 - customTestInitSteps: [] - sourceBuildPipelineRunId: "" - - internalProjectName: null - publicProjectName: null - - versionsRepoRef: null - -stages: -- template: /eng/common/templates/stages/build-and-test.yml@self - parameters: - noCache: ${{ parameters.noCache }} - publishConfig: ${{ parameters.publishConfig }} - internalProjectName: ${{ parameters.internalProjectName }} - publicProjectName: ${{ parameters.publicProjectName }} - storageAccountServiceConnection: ${{ parameters.storageAccountServiceConnection }} - customGenerateMatrixInitSteps: ${{ parameters.customGenerateMatrixInitSteps }} - buildMatrixCustomBuildLegGroupArgs: ${{ parameters.buildMatrixCustomBuildLegGroupArgs }} - testMatrixCustomBuildLegGroupArgs: ${{ parameters.testMatrixCustomBuildLegGroupArgs }} - customCopyBaseImagesInitSteps: ${{ parameters.customCopyBaseImagesInitSteps}} - customBuildInitSteps: ${{ parameters.customBuildInitSteps }} - customTestInitSteps: ${{ parameters.customTestInitSteps }} - windowsAmdBuildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }} - windowsAmdTestJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }} - linuxAmdBuildJobTimeout: ${{ parameters.linuxAmdBuildJobTimeout }} - linuxArmBuildJobTimeout: ${{ parameters.linuxArmBuildJobTimeout }} - buildMatrixType: ${{ parameters.buildMatrixType }} - testMatrixType: ${{ parameters.testMatrixType }} - sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} - - # Only clone versions repo if we need to reference it during the build in order to cache images. - ${{ if eq(parameters.noCache, false) }}: - versionsRepoRef: ${{ parameters.versionsRepoRef }} - - # Linux AMD64 - linuxAmd64Pool: - ${{ if ne(parameters.linuxAmd64Pool, '') }}: - ${{ parameters.linuxAmd64Pool }} - ${{ elseif eq(variables['System.TeamProject'], parameters.publicProjectName) }}: - vmImage: $(defaultLinuxAmd64PoolImage) - ${{ elseif eq(variables['System.TeamProject'], parameters.internalProjectName) }}: - name: $(linuxAmd64InternalPoolName) - image: $(linuxAmd64InternalPoolImage) - os: linux - - # Linux Arm64 - linuxArm64Pool: - os: linux - hostArchitecture: Arm64 - image: $(linuxArm64PoolImage) - ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}: - name: $(linuxArm64PublicPoolName) - ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}: - name: $(linuxArm64InternalPoolName) - - # Linux Arm32 - linuxArm32Pool: - os: linux - hostArchitecture: Arm64 - image: $(linuxArm32PoolImage) - ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}: - name: $(linuxArm32PublicPoolName) - ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}: - name: $(linuxArm32InternalPoolName) - - # Windows Server 2016 - windows2016Pool: - os: windows - name: $(windowsServer2016PoolName) - ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}: - image: $(windowsServer2016PublicPoolImage) - ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}: - image: $(windowsServer2016InternalPoolImage) - - # Windows Server 2019 (1809) - windows1809Pool: - os: windows - name: $(windowsServer2019PoolName) - ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}: - image: $(windowsServer2019PublicPoolImage) - ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}: - image: $(windowsServer2019InternalPoolImage) - - # Windows Server 2022 - windows2022Pool: - os: windows - name: $(windowsServer2022PoolName) - ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}: - image: $(windowsServer2022PublicPoolImage) - ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}: - image: $(windowsServer2022InternalPoolImage) - - # Windows Server 2025 - windows2025Pool: - os: windows - name: $(windowsServer2025PoolName) - ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}: - image: $(windowsServer2025PublicPoolImage) - ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}: - image: $(windowsServer2025InternalPoolImage) diff --git a/eng/common/templates/stages/dotnet/build-test-publish-repo.yml b/eng/common/templates/stages/dotnet/build-test-publish-repo.yml deleted file mode 100644 index ee8f43af7..000000000 --- a/eng/common/templates/stages/dotnet/build-test-publish-repo.yml +++ /dev/null @@ -1,74 +0,0 @@ -# This template wraps the .NET-specific build-and-test and publish templates - -parameters: - linuxAmd64Pool: "" - - # Parameters for pre-build jobs - customGenerateMatrixInitSteps: [] - customCopyBaseImagesInitSteps: [] - - # Build parameters - noCache: false - publishConfig: null - buildMatrixType: platformDependencyGraph - buildMatrixCustomBuildLegGroupArgs: "" - linuxAmdBuildJobTimeout: 60 - linuxArmBuildJobTimeout: 60 - windowsAmdBuildJobTimeout: 60 - customBuildInitSteps: [] - - # Test parameters - testMatrixType: platformVersionedOs - testMatrixCustomBuildLegGroupArgs: "" - linuxAmdTestJobTimeout: 60 - linuxArmTestJobTimeout: 60 - windowsAmdTestJobTimeout: 60 - customTestInitSteps: [] - sourceBuildPipelineRunId: "" - - # Publish parameters - customPublishInitSteps: [] - - # Other common parameters - internalProjectName: null - publicProjectName: null - versionsRepoRef: "" - -stages: -- template: /eng/common/templates/stages/dotnet/build-and-test.yml@self - parameters: - linuxAmd64Pool: ${{ parameters.linuxAmd64Pool }} - # Pre-build - customGenerateMatrixInitSteps: ${{ parameters.customGenerateMatrixInitSteps }} - customCopyBaseImagesInitSteps: ${{ parameters.customCopyBaseImagesInitSteps }} - # Build - noCache: ${{ parameters.noCache }} - publishConfig: ${{ parameters.publishConfig }} - buildMatrixType: ${{ parameters.buildMatrixType }} - buildMatrixCustomBuildLegGroupArgs: ${{ parameters.buildMatrixCustomBuildLegGroupArgs }} - linuxAmdBuildJobTimeout: ${{ parameters.linuxAmdBuildJobTimeout }} - linuxArmBuildJobTimeout: ${{ parameters.linuxArmBuildJobTimeout }} - windowsAmdBuildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }} - customBuildInitSteps: ${{ parameters.customBuildInitSteps }} - # Test - sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} - testMatrixType: ${{ parameters.testMatrixType }} - testMatrixCustomBuildLegGroupArgs: ${{ parameters.testMatrixCustomBuildLegGroupArgs }} - linuxAmdTestJobTimeout: ${{ parameters.linuxAmdTestJobTimeout }} - linuxArmTestJobTimeout: ${{ parameters.linuxArmTestJobTimeout }} - windowsAmdTestJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }} - customTestInitSteps: ${{ parameters.customTestInitSteps }} - # Other - internalProjectName: ${{ parameters.internalProjectName }} - publicProjectName: ${{ parameters.publicProjectName }} - versionsRepoRef: ${{ parameters.versionsRepoRef }} - -- template: /eng/common/templates/stages/dotnet/publish.yml@self - parameters: - pool: ${{ parameters.linuxAmd64Pool }} - customPublishInitSteps: ${{ parameters.customPublishInitSteps }} - internalProjectName: ${{ parameters.internalProjectName }} - publicProjectName: ${{ parameters.publicProjectName }} - publishConfig: ${{ parameters.publishConfig }} - sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} - versionsRepoRef: ${{ parameters.versionsRepoRef }} diff --git a/eng/common/templates/stages/dotnet/publish-config-nonprod.yml b/eng/common/templates/stages/dotnet/publish-config-nonprod.yml deleted file mode 100644 index eddd78593..000000000 --- a/eng/common/templates/stages/dotnet/publish-config-nonprod.yml +++ /dev/null @@ -1,104 +0,0 @@ -# This pipeline template injects the publish config for the dotnet-docker -# non-production (unofficial) environment. -# The overall structure of this file should stay mostly in-sync with the -# publish-config-prod.yml template. - -parameters: -# By default, images are staged in repos that are prefixed with this pipeline -# build ID. This is makes it easy to look up which pipeline images were built -# from and vice versa. -- name: sourceBuildPipelineRunId - type: string - default: $(Build.BuildId) - -# This prefix is added to the staging repo when pushing images. If the trailing -# slash is omitted, it will not be added automatically. -- name: stagingRepoPrefix - type: string - default: "build-staging/" - -# Images will have this prefix added to their repo name when pushed to the -# publishing ACR. If the trailing slash is omitted, it will not be added -# automatically. -- name: publishRepoPrefix - type: string - default: "public/" - -# This template will have the publishConfig, internalProjectName, and -# publicProjectName parameters passed to it automatically. -- name: stagesTemplate - type: string - -# These parameters will be passed to the template referred to by the -# stagesTemplate parameter. -# Note: publishConfig, internalProjectName, and publicProjectName are passed -# automatically by this template. Don't define them in this parameter - they -# will get overwritten. -- name: stagesTemplateParameters - type: object - default: {} - - -stages: -- template: ${{ parameters.stagesTemplate }} - parameters: - ${{ insert }}: ${{ parameters.stagesTemplateParameters }} - - internalProjectName: "internal" - publicProjectName: "public" - - publishConfig: - internalMirrorAcr: - server: $(acr-staging-test.server) - repoPrefix: $(mirrorRepoPrefix) - resourceGroup: $(testResourceGroup) - subscription: $(testSubscription) - serviceConnection: - name: $(internal-mirror-test.serviceConnectionName) - id: $(internal-mirror-test.serviceConnection.id) - clientId: $(internal-mirror-test.serviceConnection.clientId) - tenantId: $(testTenant) - - publicMirrorAcr: - server: $(public-mirror.server) - resourceGroup: $(public-mirror.resourceGroup) - subscription: $(public-mirror.subscription) - serviceConnection: - name: $(public-mirror.serviceConnectionName) - id: $(public-mirror.serviceConnection.id) - tenantId: $(public-mirror.serviceConnection.tenantId) - clientId: $(public-mirror.serviceConnection.clientId) - - buildAcr: - server: $(acr-staging-test.server) - resourceGroup: $(testResourceGroup) - subscription: $(testSubscription) - repoPrefix: "${{ parameters.stagingRepoPrefix }}${{ parameters.sourceBuildPipelineRunId }}/" - serviceConnection: - name: $(build-test.serviceConnectionName) - id: $(build-test.serviceConnection.id) - clientId: $(build-test.serviceConnection.clientId) - tenantId: $(testTenant) - - cleanServiceConnection: - name: $(clean-test.serviceConnectionName) - id: $(clean-test.serviceConnection.id) - clientId: $(clean-test.serviceConnection.clientId) - tenantId: $(testTenant) - - testServiceConnection: - name: $(test-nonprod.serviceConnectionName) - id: $(test-nonprod.serviceConnection.id) - clientId: $(test-nonprod.serviceConnection.clientId) - tenantId: $(testTenant) - - publishAcr: - server: $(acr-test.server) - resourceGroup: $(testResourceGroup) - subscription: $(testSubscription) - repoPrefix: "${{ parameters.publishRepoPrefix }}" - serviceConnection: - name: $(publish-test.serviceConnectionName) - id: $(publish-test.serviceConnection.id) - clientId: $(publish-test.serviceConnection.clientId) - tenantId: $(testTenant) diff --git a/eng/common/templates/stages/dotnet/publish-config-prod.yml b/eng/common/templates/stages/dotnet/publish-config-prod.yml deleted file mode 100644 index d45807ddb..000000000 --- a/eng/common/templates/stages/dotnet/publish-config-prod.yml +++ /dev/null @@ -1,104 +0,0 @@ -# This pipeline template injects the publish config for the dotnet-docker -# production (official) environment. -# The overall structure of this file should stay mostly in-sync with the -# publish-config-nonprod.yml template. - -parameters: -# By default, images are staged in repos that are prefixed with this pipeline -# build ID. This is makes it easy to look up which pipeline images were built -# from and vice versa. -- name: sourceBuildPipelineRunId - type: string - default: $(Build.BuildId) - -# This prefix is added to the staging repo when pushing images. If the trailing -# slash is omitted, it will not be added automatically. -- name: stagingRepoPrefix - type: string - default: "build-staging/" - -# Images will have this prefix added to their repo name when pushed to the -# publishing ACR. If the trailing slash is omitted, it will not be added -# automatically. -- name: publishRepoPrefix - type: string - default: "public/" - -# This template will have the publishConfig, internalProjectName, and -# publicProjectName parameters passed to it automatically. -- name: stagesTemplate - type: string - -# These parameters will be passed to the template referred to by the -# stagesTemplate parameter. -# Note: publishConfig, internalProjectName, and publicProjectName are passed -# automatically by this template. Don't define them in this parameter - they -# will get overwritten. -- name: stagesTemplateParameters - type: object - default: {} - - -stages: -- template: ${{ parameters.stagesTemplate }} - parameters: - ${{ insert }}: ${{ parameters.stagesTemplateParameters }} - - internalProjectName: "internal" - publicProjectName: "public" - - publishConfig: - internalMirrorAcr: - server: $(acr-staging.server) - repoPrefix: $(mirrorRepoPrefix) - resourceGroup: $(acr-staging.resourceGroup) - subscription: $(acr-staging.subscription) - serviceConnection: - name: $(internal-mirror.serviceConnectionName) - id: $(internal-mirror.serviceConnection.id) - clientId: $(internal-mirror.serviceConnection.clientId) - tenantId: $(internal-mirror.serviceConnection.tenantId) - - publicMirrorAcr: - server: $(public-mirror.server) - resourceGroup: $(public-mirror.resourceGroup) - subscription: $(public-mirror.subscription) - serviceConnection: - name: $(public-mirror.serviceConnectionName) - id: $(public-mirror.serviceConnection.id) - tenantId: $(public-mirror.serviceConnection.tenantId) - clientId: $(public-mirror.serviceConnection.clientId) - - buildAcr: - server: $(acr-staging.server) - resourceGroup: $(acr-staging.resourceGroup) - subscription: $(acr-staging.subscription) - repoPrefix: "${{ parameters.stagingRepoPrefix }}${{ parameters.sourceBuildPipelineRunId }}/" - serviceConnection: - name: $(build.serviceConnectionName) - id: $(build.serviceConnection.id) - clientId: $(build.serviceConnection.clientId) - tenantId: $(build.serviceConnection.tenantId) - - cleanServiceConnection: - name: $(clean.serviceConnectionName) - id: $(clean.serviceConnection.id) - clientId: $(clean.serviceConnection.clientId) - tenantId: $(clean.serviceConnection.tenantId) - - testServiceConnection: - name: $(test.serviceConnectionName) - id: $(test.serviceConnection.id) - clientId: $(test.serviceConnection.clientId) - tenantId: $(test.serviceConnection.tenantId) - - publishAcr: - server: $(acr.server) - resourceGroup: $(acr.resourceGroup) - subscription: $(acr.subscription) - repoPrefix: "${{ parameters.publishRepoPrefix }}" - serviceConnection: - name: $(publish.serviceConnectionName) - id: $(publish.serviceConnection.id) - clientId: $(publish.serviceConnection.clientId) - tenantId: $(publish.serviceConnection.tenantId) diff --git a/eng/common/templates/stages/dotnet/publish.yml b/eng/common/templates/stages/dotnet/publish.yml deleted file mode 100644 index d041b3d54..000000000 --- a/eng/common/templates/stages/dotnet/publish.yml +++ /dev/null @@ -1,56 +0,0 @@ -# This template wraps the common publish stage template with settings specific -# to the .NET team's infrastructure. - -parameters: - internalProjectName: null - publicProjectName: null - publishConfig: null - pool: "" - isStandalonePublish: false - customPublishInitSteps: [] - sourceBuildPipelineDefinitionId: '' - sourceBuildPipelineRunId: '' - versionsRepoRef: null - overrideImageInfoCommit: false - -stages: -- template: /eng/common/templates/stages/publish.yml@self - parameters: - internalProjectName: ${{ parameters.internalProjectName }} - publicProjectName: ${{ parameters.publicProjectName }} - publishConfig: ${{ parameters.publishConfig }} - isStandalonePublish: ${{ parameters.isStandalonePublish }} - sourceBuildPipelineDefinitionId: ${{ parameters.sourceBuildPipelineDefinitionId }} - sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} - versionsRepoRef: ${{ parameters.versionsRepoRef }} - overrideImageInfoCommit: ${{ parameters.overrideImageInfoCommit }} - - customPublishInitSteps: - - pwsh: | - # When reporting the repo name in the publish notification, we don't want to include - # the org part of the repo name (e.g. we want "dotnet-docker", not "dotnet-dotnet-docker"). - # This also accounts for the different separators between AzDO and GitHub repo names. - - $repoName = "$(Build.Repository.Name)" - - $orgSeparatorIndex = $repoName.IndexOf("/") - if ($orgSeparatorIndex -eq -1) { - $orgSeparatorIndex = $repoName.IndexOf("-") - } - - if ($orgSeparatorIndex -ge 0) { - $repoName = $repoName.Substring($orgSeparatorIndex + 1) - } - echo "##vso[task.setvariable variable=publishNotificationRepoName]$repoName" - displayName: "Set Custom Repo Name Var" - - ${{ parameters.customPublishInitSteps }} - - pool: - ${{ if ne(parameters.pool, '') }}: - ${{ parameters.pool }} - ${{ elseif eq(variables['System.TeamProject'], parameters.publicProjectName) }}: - vmImage: $(defaultLinuxAmd64PoolImage) - ${{ elseif eq(variables['System.TeamProject'], parameters.internalProjectName) }}: - name: $(linuxAmd64InternalPoolName) - image: $(linuxAmd64InternalPoolImage) - os: linux diff --git a/eng/common/templates/stages/publish.yml b/eng/common/templates/stages/publish.yml deleted file mode 100644 index f1569fbac..000000000 --- a/eng/common/templates/stages/publish.yml +++ /dev/null @@ -1,80 +0,0 @@ -parameters: - customPublishInitSteps: [] - customPublishVariables: [] - - internalProjectName: null - publicProjectName: null - - publishConfig: null - - isStandalonePublish: false - - pool: - vmImage: $(defaultLinuxAmd64PoolImage) - - sourceBuildPipelineDefinitionId: '' - sourceBuildPipelineRunId: '' - - versionsRepoRef: null - versionsRepoPath: "versions" - - # When true, any updated images will have the SHA in their commit URL updated - # to the commit that this pipeline is running on, instead of the commit they - # were built from. Use in combination with isStandalonePublish to ensure that - # internally built images still reference public Dockerfiles. - overrideImageInfoCommit: false - -################################################################################ -# Publish Images -################################################################################ -stages: -- stage: Publish - ${{ if eq(parameters.isStandalonePublish, true) }}: - dependsOn: [] - ${{ else }}: - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: - dependsOn: Test - ${{ else }}: - dependsOn: Post_Build - condition: " - and( - not(canceled()), - and( - contains(variables['stages'], 'publish'), - or( - or( - and( - and( - contains(variables['stages'], 'build'), - succeeded('Post_Build')), - and( - contains(variables['stages'], 'test'), - in(dependencies.Test.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'))), - or( - and( - not(contains(variables['stages'], 'build')), - and( - contains(variables['stages'], 'test'), - in(dependencies.Test.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'))), - and( - not(contains(variables['stages'], 'test')), - and( - contains(variables['stages'], 'build'), - succeeded('Post_Build'))))), - not( - or( - contains(variables['stages'], 'build'), - contains(variables['stages'], 'test'))))))" - jobs: - - template: /eng/common/templates/jobs/publish.yml@self - parameters: - pool: ${{ parameters.pool }} - internalProjectName: ${{ parameters.internalProjectName }} - publishConfig: ${{ parameters.publishConfig }} - customPublishVariables: ${{ parameters.customPublishVariables }} - customInitSteps: ${{ parameters.customPublishInitSteps }} - sourceBuildPipelineDefinitionId: ${{ parameters.sourceBuildPipelineDefinitionId }} - sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} - versionsRepoRef: ${{ parameters.versionsRepoRef }} - versionsRepoPath: ${{ parameters.versionsRepoPath }} - overrideImageInfoCommit: ${{ parameters.overrideImageInfoCommit }} diff --git a/eng/common/templates/stages/setup-service-connections.yml b/eng/common/templates/stages/setup-service-connections.yml deleted file mode 100644 index 2ef74e90c..000000000 --- a/eng/common/templates/stages/setup-service-connections.yml +++ /dev/null @@ -1,38 +0,0 @@ -# This stage exists to tell Azure DevOps about all of the service connections -# that will be used in the pipeline. A service connection will not work unless -# it is declared in this stage's parameters, even if your pipeline has already -# been granted access to the service connection. This stage also does not need -# to complete before the service connection is used. -parameters: -- name: pool - type: object - default: - name: $(default1ESInternalPoolName) - image: $(default1ESInternalPoolImage) - os: linux -# serviceConnections object shape: -# - name: string -- name: serviceConnections - type: object - default: [] - -stages: - -- stage: SetupServiceConnectionsStage - displayName: Setup service connections - jobs: - - - job: SetupServiceConnectionsJob - displayName: Setup service connections - pool: ${{ parameters.pool }} - steps: - - checkout: none - - ${{ each serviceConnection in parameters.serviceConnections }}: - - task: AzureCLI@2 - displayName: Setup ${{ serviceConnection.name }} - inputs: - azureSubscription: ${{ serviceConnection.name }} - scriptType: pscore - scriptLocation: inlineScript - inlineScript: | - az account show diff --git a/eng/common/templates/steps/annotate-eol-digests.yml b/eng/common/templates/steps/annotate-eol-digests.yml deleted file mode 100644 index 57c5221fc..000000000 --- a/eng/common/templates/steps/annotate-eol-digests.yml +++ /dev/null @@ -1,48 +0,0 @@ -parameters: -- name: acr - type: object -# Path to EOL annotation data JSON file generated by 'generateEolAnnotationData*' command -- name: dataFile - type: string - -steps: - - script: mkdir -p $(Build.ArtifactStagingDirectory)/annotation-digests - displayName: Create Annotation Digests Directory - - template: /eng/common/templates/steps/run-imagebuilder.yml@self - parameters: - displayName: Annotate EOL Images (${{ parameters.acr.server }}) - serviceConnections: - - name: acr - id: ${{ parameters.acr.serviceConnection.id }} - tenantId: ${{ parameters.acr.serviceConnection.tenantId }} - clientId: ${{ parameters.acr.serviceConnection.clientId }} - internalProjectName: internal - condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true')) - args: >- - annotateEolDigests - "${{ parameters.dataFile }}" - "${{ parameters.acr.server }}" - "${{ parameters.acr.repoPrefix }}" - $(artifactsPath)/annotation-digests/annotation-digests.txt - $(dryRunArg) - - template: /eng/common/templates/steps/publish-artifact.yml@self - parameters: - path: $(Build.ArtifactStagingDirectory)/annotation-digests - artifactName: annotation-digests-${{ parameters.acr.server }}-$(System.JobAttempt) - displayName: Publish Annotation Digests List (${{ parameters.acr.server }}) - internalProjectName: internal - publicProjectName: public - condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true')) - - template: /eng/common/templates/steps/run-imagebuilder.yml@self - parameters: - displayName: Wait for Annotation Ingestion (${{ parameters.acr.server }}) - serviceConnections: - - name: mar - id: $(marStatus.serviceConnection.id) - tenantId: $(marStatus.serviceConnection.tenantId) - clientId: $(marStatus.serviceConnection.clientId) - internalProjectName: internal - condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true'), eq(variables['waitForIngestionEnabled'], 'true')) - args: >- - waitForMarAnnotationIngestion - $(artifactsPath)/annotation-digests/annotation-digests.txt diff --git a/eng/common/templates/steps/clean-acr-images.yml b/eng/common/templates/steps/clean-acr-images.yml deleted file mode 100644 index 0361df442..000000000 --- a/eng/common/templates/steps/clean-acr-images.yml +++ /dev/null @@ -1,33 +0,0 @@ -parameters: - repo: null - acr: null - action: null - age: null - customArgs: "--dry-run" - internalProjectName: null - publishConfig: null -steps: - - template: /eng/common/templates/steps/run-imagebuilder.yml@self - parameters: - # Options are documented in CleanAcrImagesOptions.cs - ${{ if eq(parameters.action, 'delete') }}: - displayName: "Delete ${{ parameters.repo }}" - ${{ elseif parameters.age }}: - displayName: "Clean ${{ parameters.repo }} (${{ parameters.action }} > ${{ parameters.age }}d)" - ${{ else }}: - displayName: "Clean ${{ parameters.repo }} (${{ parameters.action }})" - serviceConnections: - - name: acr - id: ${{ parameters.publishConfig.cleanServiceConnection.id }} - tenantId: ${{ parameters.publishConfig.cleanServiceConnection.tenantId }} - clientId: ${{ parameters.publishConfig.cleanServiceConnection.clientId }} - internalProjectName: ${{ parameters.internalProjectName }} - args: >- - cleanAcrImages - ${{ parameters.repo }} - ${{ parameters.acr.subscription }} - ${{ parameters.acr.resourceGroup }} - ${{ parameters.acr.server }} - --action ${{ parameters.action }} - --age ${{ parameters.age }} - ${{ parameters.customArgs }} diff --git a/eng/common/templates/steps/cleanup-docker-linux.yml b/eng/common/templates/steps/cleanup-docker-linux.yml deleted file mode 100644 index ddf0f9b09..000000000 --- a/eng/common/templates/steps/cleanup-docker-linux.yml +++ /dev/null @@ -1,15 +0,0 @@ -parameters: - condition: true - -steps: - ################################################################################ - # Cleanup local Docker server - ################################################################################ -- script: docker stop $(docker ps -q) || true - displayName: Stop Running Containers - condition: and(always(), ${{ parameters.condition }}) - continueOnError: true -- script: docker system prune -a -f --volumes - displayName: Cleanup Docker - condition: and(always(), ${{ parameters.condition }}) - continueOnError: true diff --git a/eng/common/templates/steps/cleanup-docker-windows.yml b/eng/common/templates/steps/cleanup-docker-windows.yml deleted file mode 100644 index 98fc52969..000000000 --- a/eng/common/templates/steps/cleanup-docker-windows.yml +++ /dev/null @@ -1,18 +0,0 @@ -parameters: - condition: true - -steps: - ################################################################################ - # Cleanup Docker Resources - ################################################################################ -- powershell: $(engCommonPath)/Invoke-CleanupDocker.ps1 - displayName: Cleanup Docker Images - condition: and(always(), ${{ parameters.condition }}) - continueOnError: true -- powershell: | - if (Test-Path $(Build.BinariesDirectory)\.Microsoft.DotNet.ImageBuilder) { - Remove-Item $(Build.BinariesDirectory)\.Microsoft.DotNet.ImageBuilder -Force -Recurse; - } - displayName: Cleanup Image Builder - condition: and(always(), ${{ parameters.condition }}) - continueOnError: true diff --git a/eng/common/templates/steps/common-init-for-matrix-and-build.yml b/eng/common/templates/steps/common-init-for-matrix-and-build.yml deleted file mode 100644 index ce34eaf26..000000000 --- a/eng/common/templates/steps/common-init-for-matrix-and-build.yml +++ /dev/null @@ -1,74 +0,0 @@ -parameters: - noCache: false - internalVersionsRepoRef: null - publicVersionsRepoRef: null - versionsRepoPath: versions - isInternalServicingValidation: false - -steps: -- checkout: self -- ${{ if and(eq(variables['System.TeamProject'], 'public'), eq(parameters.noCache, false)) }}: - - checkout: ${{ parameters.publicVersionsRepoRef }} - path: s/${{ parameters.versionsRepoPath }} -- ${{ if and(eq(variables['System.TeamProject'], 'internal'), eq(parameters.noCache, false)) }}: - - checkout: ${{ parameters.internalVersionsRepoRef }} - path: s/${{ parameters.versionsRepoPath }} -- powershell: | - $commonMatrixAndBuildOptions = "--source-repo $(publicGitRepoUri)" - if ("$(System.TeamProject)" -eq "internal" -and "$(Build.Reason)" -ne "PullRequest" -and "${{ parameters.isInternalServicingValidation }}" -ne "true") { - $commonMatrixAndBuildOptions = "$commonMatrixAndBuildOptions --source-repo-prefix $(mirrorRepoPrefix) --registry-override $(acr-staging.server)" - } - - if ("$(System.TeamProject)" -eq "public" -and "$(public-mirror.server)" -ne "") { - $commonMatrixAndBuildOptions = "$commonMatrixAndBuildOptions --base-override-regex '^(?!mcr\.microsoft\.com)' --base-override-sub '$(public-mirror.server)/'" - } - - if ("${{ parameters.noCache }}" -eq "false") { - $versionsBasePath = "${{ parameters.versionsRepoPath }}/" - $pipelineDisabledCache = "false" - - $pathSeparatorIndex = "$(Build.Repository.Name)".IndexOf("/") - if ($pathSeparatorIndex -ge 0) { - $buildRepoName = "$(Build.Repository.Name)".Substring($pathSeparatorIndex + 1) - } - else { - $buildRepoName = "$(Build.Repository.Name)" - } - - $engCommonPath = "$(Build.Repository.LocalPath)/$buildRepoName/$(engCommonRelativePath)" - - $engPath = "$(Build.Repository.LocalPath)/$buildRepoName/eng" - $manifest = "$buildRepoName/$(manifest)" - $testResultsDirectory = "$buildRepoName/$testResultsDirectory" - - if ("$(testScriptPath)") { - $testScriptPath = "$buildRepoName/$(testScriptPath)" - } - - echo "##vso[task.setvariable variable=buildRepoName]$buildRepoName" - echo "##vso[task.setvariable variable=engCommonPath]$engCommonPath" - echo "##vso[task.setvariable variable=manifest]$manifest" - echo "##vso[task.setvariable variable=engPath]$engPath" - echo "##vso[task.setvariable variable=testScriptPath]$testScriptPath" - echo "##vso[task.setvariable variable=testResultsDirectory]$testResultsDirectory" - } - else { - $versionsBasePath = "" - $pipelineDisabledCache = "true" - } - - echo "##vso[task.setvariable variable=commonMatrixAndBuildOptions]$commonMatrixAndBuildOptions" - echo "##vso[task.setvariable variable=versionsBasePath]$versionsBasePath" - echo "##vso[task.setvariable variable=pipelineDisabledCache]$pipelineDisabledCache" - displayName: Set Common Variables for Matrix and Build - -- ${{ if eq(parameters.noCache, false) }}: - # Special logic is needed to copy the tsaoptions.json file to a well known location for the 1ES PT. - # This template has multiple checkouts and AzDO doesn't have support for dynamically determining the - # default repo path therefore the 1es-official logic can't calculate the repo's tsa config file path. - - task: CopyFiles@2 - displayName: Copy TSA Config - inputs: - SourceFolder: '$(Build.Repository.LocalPath)/$(buildRepoName)' - Contents: '.config/tsaoptions.json' - TargetFolder: '$(Build.SourcesDirectory)' diff --git a/eng/common/templates/steps/copy-base-images-staging.yml b/eng/common/templates/steps/copy-base-images-staging.yml deleted file mode 100644 index 9735fc9b0..000000000 --- a/eng/common/templates/steps/copy-base-images-staging.yml +++ /dev/null @@ -1,19 +0,0 @@ -parameters: -- name: additionalOptions - type: string - default: null -- name: continueOnError - type: string - default: false - -steps: -- template: /eng/common/templates/steps/copy-base-images.yml@self - parameters: - additionalOptions: ${{ parameters.additionalOptions }} - continueOnError: ${{ parameters.continueOnError }} - repoPrefix: $(mirrorRepoPrefix) - acr: - resourceGroup: $(acr-staging.resourceGroup) - server: $(acr-staging.server) - serviceConnection: $(acr-staging.serviceConnectionName) - subscription: $(acr-staging.subscription) diff --git a/eng/common/templates/steps/copy-base-images.yml b/eng/common/templates/steps/copy-base-images.yml deleted file mode 100644 index f78b1c5dc..000000000 --- a/eng/common/templates/steps/copy-base-images.yml +++ /dev/null @@ -1,52 +0,0 @@ -parameters: -- name: acr - type: object - default: - server: "" - serviceConnection: - tenantId: "" - clientId: "" - id: "" - subscription: "" - resourceGroup: "" -- name: repoPrefix - type: string - default: null -- name: additionalOptions - type: string - default: "" -- name: continueOnError - type: string - default: false -- name: forceDryRun - type: boolean - default: false - -steps: -- ${{ if or(eq(parameters.forceDryRun, true), eq(variables['System.TeamProject'], 'public'), eq(variables['Build.Reason'], 'PullRequest')) }}: - - script: echo "##vso[task.setvariable variable=dryRunArg]--dry-run" -- template: /eng/common/templates/steps/run-imagebuilder.yml@self - parameters: - displayName: Copy Base Images - serviceConnections: - - name: "acr" - tenantId: ${{ parameters.acr.serviceConnection.tenantId }} - clientId: ${{ parameters.acr.serviceConnection.clientId }} - id: ${{ parameters.acr.serviceConnection.id }} - continueOnError: ${{ parameters.continueOnError }} - internalProjectName: 'internal' - # Use environment variable to reference $(dryRunArg). Since $(dryRunArg) might be undefined, - # PowerShell will treat the Azure Pipelines variable macro syntax as a command and throw an - # error - args: >- - copyBaseImages - '${{ parameters.acr.subscription }}' - '${{ parameters.acr.resourceGroup }}' - $(dockerHubRegistryCreds) - $(customCopyBaseImagesArgs) - --repo-prefix '${{ parameters.repoPrefix }}' - --registry-override '${{ parameters.acr.server }}' - --os-type 'linux' - --architecture '*' - $env:DRYRUNARG - ${{ parameters.additionalOptions }} diff --git a/eng/common/templates/steps/download-build-artifact.yml b/eng/common/templates/steps/download-build-artifact.yml deleted file mode 100644 index 965f49320..000000000 --- a/eng/common/templates/steps/download-build-artifact.yml +++ /dev/null @@ -1,40 +0,0 @@ -parameters: - # Destination directory on the pipeline agent's filesystem, relative or absolute. - targetPath: "" - # The build/pipeline artifact to download. If the value is left empty, - # the task downloads all artifacts associated with the pipeline run. - artifactName: "" - # AKA pipeline/definition - optional. - # If this is left empty, use the current pipeline's definition ID. - # You can get this from the URL of the pipeline's overview page on Azure DevOps. - # Example: https://dev.azure.com/$org/$project/_build?definitionId=373 - pipelineDefinitionId: "" - # AKA runId/buildId/pipelineId - optional. - # The identifier of the pipeline run from which to download the artifacts. - # If this is left empty, then always download from the current pipeline - # You can get this from the URL of the specific pipeline run, for example: - # https://dev.azure.com/$org/$project/_build/results?buildId=2709155&view=results - pipelineRunId: "" - condition: true - continueOnError: false - -steps: -# https://learn.microsoft.com/azure/devops/pipelines/tasks/reference/download-pipeline-artifact-v2 -- task: DownloadPipelineArtifact@2 - inputs: - ${{ if ne(parameters.pipelineRunId, '') }}: - buildType: specific - project: $(System.TeamProject) - ${{ if ne(parameters.pipelineDefinitionId, '') }}: - definition: ${{ parameters.pipelineDefinitionId }} - ${{ else }}: - definition: $(System.DefinitionId) - buildId: ${{ parameters.pipelineRunId }} - buildVersionToDownload: specific - ${{ else }}: - buildType: current - targetPath: ${{ parameters.targetPath }} - artifactName: ${{ parameters.artifactName }} - displayName: Download Build Artifact(s) - condition: and(succeeded(), ${{ parameters.condition }}) - continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/common/templates/steps/init-common.yml b/eng/common/templates/steps/init-common.yml deleted file mode 100644 index c22ad449c..000000000 --- a/eng/common/templates/steps/init-common.yml +++ /dev/null @@ -1,9 +0,0 @@ -parameters: - condition: true - -steps: -- powershell: | - $sourceBranch=$Env:BUILD_SOURCEBRANCH -replace "refs/heads/","" -replace "refs/tags/","" -replace "refs/pull/","" - echo "##vso[task.setvariable variable=sourceBranch]$sourceBranch" - displayName: Define Source Branch Variable - condition: and(succeeded(), ${{ parameters.condition }}) diff --git a/eng/common/templates/steps/init-docker-linux.yml b/eng/common/templates/steps/init-docker-linux.yml deleted file mode 100644 index 8c554ad9b..000000000 --- a/eng/common/templates/steps/init-docker-linux.yml +++ /dev/null @@ -1,90 +0,0 @@ -parameters: - setupImageBuilder: true - setupTestRunner: false - cleanupDocker: false - condition: true - -steps: -- template: /eng/common/templates/steps/init-common.yml@self - parameters: - condition: ${{ parameters.condition }} -- script: echo "##vso[task.setvariable variable=artifactsPath]/artifacts" - displayName: Define Artifacts Path Variable - condition: and(succeeded(), ${{ parameters.condition }}) - - ################################################################################ - # Cleanup Docker Resources - ################################################################################ -- ${{ if eq(parameters.cleanupDocker, 'true') }}: - - template: /eng/common/templates/steps/cleanup-docker-linux.yml@self - parameters: - condition: ${{ parameters.condition }} - - ################################################################################ - # Setup Image Builder (Optional) - ################################################################################ -- ${{ if eq(parameters.setupImageBuilder, 'true') }}: - - - powershell: $(engCommonPath)/Pull-Image.ps1 $(imageNames.imageBuilder) - displayName: Pull Image Builder - condition: and(succeeded(), ${{ parameters.condition }}) - - - script: >- - docker build - -t $(imageNames.imageBuilder.withrepo) - --build-arg IMAGE=$(imageNames.imageBuilder) - -f $(engCommonPath)/Dockerfile.WithRepo . - displayName: Build Image for Image Builder - condition: and(succeeded(), ${{ parameters.condition }}) - - - task: PowerShell@2 - displayName: Define ImageBuilder Command Variables - condition: and(succeeded(), ${{ parameters.condition }}) - inputs: - targetType: 'inline' - script: | - $imageBuilderImageName = "$(imageNames.imageBuilder.withrepo)" - Write-Host "##vso[task.setvariable variable=imageBuilderImageName]$imageBuilderImageName" - - $dockerRunBaseCmd = @( - "docker run --rm" - ) - - $dockerRunArgs = @( - "-v /var/run/docker.sock:/var/run/docker.sock" - "-v $(Build.ArtifactStagingDirectory):$(artifactsPath)" - "-w /repo" - "$(imageBuilderDockerRunExtraOptions)" - "$(imageNames.imageBuilder.withrepo)" - ) - - $authedDockerRunArgs = @( - '-e' - 'SYSTEM_ACCESSTOKEN=$env:SYSTEM_ACCESSTOKEN' - '-e' - 'SYSTEM_OIDCREQUESTURI=$env:SYSTEM_OIDCREQUESTURI' - ) - - $dockerRunCmd = $dockerRunBaseCmd + $dockerRunArgs - $authedDockerRunCmd = $dockerRunBaseCmd + $authedDockerRunArgs + $dockerRunArgs - - $runImageBuilderCmd = $($dockerRunCmd -join ' ') - $runAuthedImageBuilderCmd = $($authedDockerRunCmd -join ' ') - - Write-Host "##vso[task.setvariable variable=runImageBuilderCmd]$runImageBuilderCmd" - Write-Host "##vso[task.setvariable variable=runAuthedImageBuilderCmd]$runAuthedImageBuilderCmd" - - ################################################################################ - # Setup Test Runner (Optional) - ################################################################################ -- ${{ if eq(parameters.setupTestRunner, 'true') }}: - - powershell: $(engCommonPath)/Pull-Image.ps1 $(imageNames.testrunner) - displayName: Pull Test Runner - condition: and(succeeded(), ${{ parameters.condition }}) - - script: > - docker build - -t $(imageNames.testRunner.withrepo) - --build-arg IMAGE=$(imageNames.testrunner) - -f $(engCommonPath)/Dockerfile.WithRepo . - displayName: Build Test Runner Image - condition: and(succeeded(), ${{ parameters.condition }}) diff --git a/eng/common/templates/steps/init-docker-windows.yml b/eng/common/templates/steps/init-docker-windows.yml deleted file mode 100644 index ae867f9e8..000000000 --- a/eng/common/templates/steps/init-docker-windows.yml +++ /dev/null @@ -1,48 +0,0 @@ -parameters: - setupImageBuilder: true - condition: true - -steps: -- template: /eng/common/templates/steps/init-common.yml@self - parameters: - condition: ${{ parameters.condition }} -- powershell: echo "##vso[task.setvariable variable=artifactsPath]$(Build.ArtifactStagingDirectory)" - displayName: Define Artifacts Path Variable - condition: and(succeeded(), ${{ parameters.condition }}) - - ################################################################################ - # Cleanup Docker Resources - ################################################################################ -- template: /eng/common/templates/steps/cleanup-docker-windows.yml@self - parameters: - condition: ${{ parameters.condition }} - - ################################################################################ - # Setup Image Builder (Optional) - ################################################################################ -- ${{ if eq(parameters.setupImageBuilder, 'true') }}: - - powershell: $(engCommonPath)/Invoke-WithRetry.ps1 "docker pull $(imageNames.imageBuilder)" - displayName: Pull Image Builder - condition: and(succeeded(), ${{ parameters.condition }}) - - script: docker create --name setupImageBuilder-$(Build.BuildId)-$(System.JobId) $(imageNames.imageBuilder) - displayName: Create Setup Container - condition: and(succeeded(), ${{ parameters.condition }}) - - script: > - docker cp - setupImageBuilder-$(Build.BuildId)-$(System.JobId):/image-builder - $(Build.BinariesDirectory)/.Microsoft.DotNet.ImageBuilder - displayName: Copy Image Builder - condition: and(succeeded(), ${{ parameters.condition }}) - - script: docker rm -f setupImageBuilder-$(Build.BuildId)-$(System.JobId) - displayName: Cleanup Setup Container - condition: and(always(), ${{ parameters.condition }}) - continueOnError: true - - task: PowerShell@2 - displayName: Define runImageBuilderCmd Variables - condition: and(succeeded(), ${{ parameters.condition }}) - inputs: - targetType: 'inline' - script: | - $runImageBuilderCmd = "$(Build.BinariesDirectory)\.Microsoft.DotNet.ImageBuilder\Microsoft.DotNet.ImageBuilder.exe" - Write-Host "##vso[task.setvariable variable=runImageBuilderCmd]$runImageBuilderCmd" - Write-Host "##vso[task.setvariable variable=runAuthedImageBuilderCmd]$runImageBuilderCmd" diff --git a/eng/common/templates/steps/init-matrix-build-publish.yml b/eng/common/templates/steps/init-matrix-build-publish.yml deleted file mode 100644 index 5ba018fee..000000000 --- a/eng/common/templates/steps/init-matrix-build-publish.yml +++ /dev/null @@ -1,78 +0,0 @@ -# Initialize common variables used in -# - Generating build matrix -# - Building images -# - Running tests -# - Publishing images - -parameters: - publishConfig: null - versionsRepoRef: "" - versionsRepoPath: "versions" - -steps: -- checkout: self -- ${{ if ne(parameters.versionsRepoRef, '') }}: - - checkout: ${{ parameters.versionsRepoRef }} - path: s/${{ parameters.versionsRepoPath }} - persistCredentials: true - fetchDepth: 1 - condition: succeeded() -- powershell: | - $commonMatrixAndBuildOptions = "--source-repo $(publicGitRepoUri)" - if ("$(System.TeamProject)" -eq "internal" -and "$(Build.Reason)" -ne "PullRequest") { - $commonMatrixAndBuildOptions = "$commonMatrixAndBuildOptions --source-repo-prefix ${{ parameters.publishConfig.internalMirrorAcr.repoPrefix }} --registry-override ${{ parameters.publishConfig.buildAcr.server }}" - } - - if ("$(System.TeamProject)" -eq "public" -and "$(public-mirror.server)" -ne "") { - $commonMatrixAndBuildOptions = "$commonMatrixAndBuildOptions --base-override-regex '^(?!mcr\.microsoft\.com)' --base-override-sub '$(public-mirror.server)/'" - } - - if ("${{ parameters.versionsRepoRef }}" -ne "") { - $versionsBasePath = "${{ parameters.versionsRepoPath }}/" - $pipelineDisabledCache = "false" - - $pathSeparatorIndex = "$(Build.Repository.Name)".IndexOf("/") - if ($pathSeparatorIndex -ge 0) { - $buildRepoName = "$(Build.Repository.Name)".Substring($pathSeparatorIndex + 1) - } - else { - $buildRepoName = "$(Build.Repository.Name)" - } - - $engCommonPath = "$(Build.Repository.LocalPath)/$buildRepoName/$(engCommonRelativePath)" - - $engPath = "$(Build.Repository.LocalPath)/$buildRepoName/eng" - $manifest = "$buildRepoName/$(manifest)" - $testResultsDirectory = "$buildRepoName/$testResultsDirectory" - - if ("$(testScriptPath)") { - $testScriptPath = "$buildRepoName/$(testScriptPath)" - } - - echo "##vso[task.setvariable variable=buildRepoName]$buildRepoName" - echo "##vso[task.setvariable variable=engCommonPath]$engCommonPath" - echo "##vso[task.setvariable variable=manifest]$manifest" - echo "##vso[task.setvariable variable=engPath]$engPath" - echo "##vso[task.setvariable variable=testScriptPath]$testScriptPath" - echo "##vso[task.setvariable variable=testResultsDirectory]$testResultsDirectory" - } - else { - $versionsBasePath = "" - $pipelineDisabledCache = "true" - } - - echo "##vso[task.setvariable variable=commonMatrixAndBuildOptions]$commonMatrixAndBuildOptions" - echo "##vso[task.setvariable variable=versionsBasePath]$versionsBasePath" - echo "##vso[task.setvariable variable=pipelineDisabledCache]$pipelineDisabledCache" - displayName: Set Common Variables for Matrix, Build, and Publish - -- ${{ if ne(parameters.versionsRepoRef, '') }}: - # Special logic is needed to copy the tsaoptions.json file to a well known location for the 1ES PT. - # This template has multiple checkouts and AzDO doesn't have support for dynamically determining the - # default repo path therefore the 1es-official logic can't calculate the repo's tsa config file path. - - task: CopyFiles@2 - displayName: Copy TSA Config - inputs: - SourceFolder: '$(Build.Repository.LocalPath)/$(buildRepoName)' - Contents: '.config/tsaoptions.json' - TargetFolder: '$(Build.SourcesDirectory)' diff --git a/eng/common/templates/steps/parse-test-arg-arrays.yml b/eng/common/templates/steps/parse-test-arg-arrays.yml deleted file mode 100644 index 5ab18214d..000000000 --- a/eng/common/templates/steps/parse-test-arg-arrays.yml +++ /dev/null @@ -1,15 +0,0 @@ -steps: -- powershell: | - # Formats the OS versions in a compact human-readable form (e.g. "os1/os2") - $osVersionsDisplayName = '$(osVersions)' -Replace '--os-version ', '' -Replace ' ', '/' - - # Defines a PowerShell snippet in string-form that can be used to initialize an array of the OS versions - $osVersionsArrayInitStr = "@('" + $($osVersionsDisplayName -Replace "/", "', '") + "')" - - echo "##vso[task.setvariable variable=osVersionsDisplayName]$osVersionsDisplayName" - echo "##vso[task.setvariable variable=osVersionsArrayInitStr]$osVersionsArrayInitStr" - - # Defines a PowerShell snippet in string-form that can be used to initialize an array of the image builder paths - $pathInitStr = "@('" + $('$(imageBuilderPaths)' -Replace '--path', '' -Replace " ", "', '") + "')" - echo "##vso[task.setvariable variable=imageBuilderPathsArrayInitStr]$pathInitStr" - displayName: Parse Test Arg Arrays diff --git a/eng/common/templates/steps/publish-artifact.yml b/eng/common/templates/steps/publish-artifact.yml deleted file mode 100644 index 72ce47005..000000000 --- a/eng/common/templates/steps/publish-artifact.yml +++ /dev/null @@ -1,28 +0,0 @@ -parameters: -- name: path - type: string -- name: artifactName - type: string -- name: displayName - type: string -- name: internalProjectName - type: string -- name: publicProjectName - type: string -- name: condition - type: string - default: 'true' - -steps: -- ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}: - - task: 1ES.PublishPipelineArtifact@1 - inputs: - path: ${{ parameters.path }} - artifact: ${{ parameters.artifactName }} - displayName: ${{ parameters.displayName }} - condition: and(succeeded(), ${{ parameters.condition }}) -- ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}: - - publish: ${{ parameters.path }} - artifact: ${{ parameters.artifactName }} - displayName: ${{ parameters.displayName }} - condition: and(succeeded(), ${{ parameters.condition }}) diff --git a/eng/common/templates/steps/publish-readmes.yml b/eng/common/templates/steps/publish-readmes.yml deleted file mode 100644 index 4cf80ecf7..000000000 --- a/eng/common/templates/steps/publish-readmes.yml +++ /dev/null @@ -1,29 +0,0 @@ -parameters: - dryRunArg: "" - condition: true - -steps: -- script: > - $(runImageBuilderCmd) publishMcrDocs - --manifest '$(manifest)' - --registry-override '${{ parameters.publishConfig.publishAcr.server }}' - '$(mcrDocsRepoInfo.userName)' - '$(mcrDocsRepoInfo.email)' - $(mcrDocsRepoInfo.authArgs) - '$(publicGitRepoUri)' - ${{ parameters.dryRunArg }} - $(manifestVariables) - $(imageBuilder.queueArgs) - --git-owner 'Microsoft' - --git-repo 'mcrdocs' - --git-branch 'main' - --git-path 'teams' - $(additionalPublishMcrDocsArgs) - name: PublishReadmes - displayName: Publish Readmes - condition: ${{ parameters.condition }} -- template: /eng/common/templates/steps/wait-for-mcr-doc-ingestion.yml@self - parameters: - commitDigest: $(PublishReadmes.readmeCommitDigest) - condition: and(${{ parameters.condition }}, ne(variables['PublishReadmes.readmeCommitDigest'], '')) - dryRunArg: ${{ parameters.dryRunArg }} diff --git a/eng/common/templates/steps/retain-build.yml b/eng/common/templates/steps/retain-build.yml deleted file mode 100644 index 0cb192c68..000000000 --- a/eng/common/templates/steps/retain-build.yml +++ /dev/null @@ -1,9 +0,0 @@ -steps: -- powershell: > - $(engCommonPath)/Retain-Build.ps1 - -BuildId $(Build.BuildId) - -AzdoOrgUri '$(System.CollectionUri)' - -AzdoProject '$(System.TeamProject)' - -Token '$(System.AccessToken)' - displayName: Enable permanent build retention - condition: and(succeeded(), eq(variables.retainBuild, 'true')) diff --git a/eng/common/templates/steps/run-imagebuilder.yml b/eng/common/templates/steps/run-imagebuilder.yml deleted file mode 100644 index 3613d809b..000000000 --- a/eng/common/templates/steps/run-imagebuilder.yml +++ /dev/null @@ -1,76 +0,0 @@ -parameters: -- name: name - type: string - default: "" -- name: displayName - type: string - default: "Run ImageBuilder" -- name: serviceConnections - type: object - default: - # name: the name of the service connection argument that will be passed to the ImageBuilder command. - # For example, if the argument is --acr-service-connection, the name would be "acr". - - name: "" - # The service connection's ID (GUID). - id: "" - # The client ID of the Managed Idendity backing the service connection (GUID). - clientId: "" - # The ID of the tenant that the Managed Identity is in (GUID). - tenantId: "" -- name: internalProjectName - type: string - default: null -- name: args - type: string - default: null -- name: condition - type: string - default: succeeded() -- name: continueOnError - type: boolean - default: false -- name: dockerClientOS - type: string - default: "linux" - -steps: -- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: - - - task: PowerShell@2 - ${{ if ne(parameters.name, '') }}: - name: ${{ parameters.name }} - displayName: ${{ parameters.displayName }} - continueOnError: ${{ parameters.continueOnError }} - condition: ${{ parameters.condition }} - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - SYSTEM_OIDCREQUESTURI: $(System.OidcRequestUri) - inputs: - targetType: 'inline' - script: | - $serviceConnections = '${{ convertToJson(parameters.serviceConnections) }}' - - Write-Host "Service connections:" - Write-Host "${serviceConnections}" - - $serviceConnectionsJson = $serviceConnections | ConvertFrom-Json - $serviceConnectionsArgs = @() - foreach ($connection in $serviceConnectionsJson) { - $serviceConnectionsArgs += "--$($connection.name)-service-connection" - $serviceConnectionsArgs += "$($connection.tenantId):$($connection.clientId):$($connection.id)" - } - - $(runAuthedImageBuilderCmd) ${{ parameters.args }} @serviceConnectionsArgs - -- ${{ else }}: - - - task: PowerShell@2 - ${{ if ne(parameters.name, '') }}: - name: ${{ parameters.name }} - displayName: ${{ parameters.displayName }} - continueOnError: ${{ parameters.continueOnError }} - condition: ${{ parameters.condition }} - inputs: - targetType: 'inline' - script: >- - $(runImageBuilderCmd) ${{ parameters.args }} diff --git a/eng/common/templates/steps/run-pwsh-with-auth.yml b/eng/common/templates/steps/run-pwsh-with-auth.yml deleted file mode 100644 index 84de1e1f5..000000000 --- a/eng/common/templates/steps/run-pwsh-with-auth.yml +++ /dev/null @@ -1,39 +0,0 @@ -parameters: -- name: name - type: string - default: "" -- name: displayName - type: string - default: "Run PowerShell" -- name: serviceConnection - type: string - default: "" -- name: command - type: string - default: null -- name: continueOnError - type: boolean - default: false -- name: dockerClientOS - type: string - default: "linux" -- name: condition - type: string - default: true - -steps: -- task: AzureCLI@2 - ${{ if ne(parameters.name, '') }}: - name: ${{ parameters.name }} - displayName: ${{ parameters.displayName }} (Authenticated) - continueOnError: ${{ parameters.continueOnError }} - condition: and(succeeded(), ${{ parameters.condition }}) - inputs: - azureSubscription: ${{ parameters.serviceConnection }} - addSpnToEnvironment: true - ${{ if eq(parameters.dockerClientOS, 'windows') }}: - scriptType: 'ps' - ${{ else }}: - scriptType: 'pscore' - scriptLocation: 'inlineScript' - inlineScript: ${{ parameters.command }}; diff --git a/eng/common/templates/steps/set-dry-run.yml b/eng/common/templates/steps/set-dry-run.yml deleted file mode 100644 index 44e3062b0..000000000 --- a/eng/common/templates/steps/set-dry-run.yml +++ /dev/null @@ -1,30 +0,0 @@ -parameters: - name: publishConfig - type: object - -steps: -- powershell: | - if ("$env:ONEESPT_BUILDTYPE" -eq "Unofficial") - { - # Don't use dry-run mode for unofficial builds, since they publish to a - # non-production environment - $dryRunArg="" - } - elseif ("$(System.TeamProject)" -eq "$(publicProjectName)") - { - # Public builds need to use dry-run mode since they don't publish anywhere. - $dryRunArg="--dry-run" - } - elseif (-not "$(officialRepoPrefixes)".Split(',').Contains("${{ parameters.publishConfig.publishAcr.repoPrefix }}")) - { - # If we're running an internal build on an official pipeline but not - # publishing to an official repo prefix, then use dry run mode. - $dryRunArg="--dry-run" - } - else - { - $dryRunArg="" - } - - echo "##vso[task.setvariable variable=dryRunArg]$dryRunArg" - displayName: Set dry-run arg for non-prod diff --git a/eng/common/templates/steps/set-image-info-path-var.yml b/eng/common/templates/steps/set-image-info-path-var.yml deleted file mode 100644 index 71d317fa2..000000000 --- a/eng/common/templates/steps/set-image-info-path-var.yml +++ /dev/null @@ -1,19 +0,0 @@ -parameters: - publicSourceBranch: null - -steps: -- powershell: | - $basePath = "$(gitHubVersionsRepoInfo.path)" - - $publicSourceBranch = "${{ parameters.publicSourceBranch }}" - - if ($publicSourceBranch -eq "") { - throw "publicSourceBranch variable is not set" - } - - $buildRepoName = "$(Build.Repository.Name)".Replace("/", "-") - $imageInfoName = "image-info.$buildRepoName-$publicSourceBranch$(imageInfoVariant).json" - - echo "##vso[task.setvariable variable=imageInfoVersionsPath]$basePath/$imageInfoName" - echo "##vso[task.setvariable variable=gitHubImageInfoVersionsPath]$(gitHubVersionsRepoInfo.path)/$imageInfoName" - displayName: Set Image Info Path Vars diff --git a/eng/common/templates/steps/test-images-linux-client.yml b/eng/common/templates/steps/test-images-linux-client.yml deleted file mode 100644 index e97f015c7..000000000 --- a/eng/common/templates/steps/test-images-linux-client.yml +++ /dev/null @@ -1,107 +0,0 @@ -parameters: - preBuildValidation: false - internalProjectName: null - publishConfig: null - condition: true - customInitSteps: [] - sourceBuildPipelineRunId: "" - -steps: -- template: /eng/common/templates/steps/init-docker-linux.yml@self - parameters: - setupImageBuilder: false - setupTestRunner: true - # Clean only up when we're running an internal build, not a PR, and not doing pre-build validation. - # i.e. when we're building something important. - cleanupDocker: ${{ and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.preBuildValidation, 'false')) }} - condition: ${{ parameters.condition }} -- ${{ parameters.customInitSteps }} -- script: | - echo "##vso[task.setvariable variable=testRunner.container]testrunner-$(Build.BuildId)-$(System.JobId)" - - additionalTestArgs="$ADDITIONALTESTARGS" - if [ "${{ parameters.preBuildValidation }}" == "true" ]; then - additionalTestArgs="$additionalTestArgs -TestCategories pre-build" - else - if [ "${{ variables['System.TeamProject'] }}" == "${{ parameters.internalProjectName }}" ] && [ "${{ variables['Build.Reason'] }}" != "PullRequest" ]; then - additionalTestArgs="$additionalTestArgs -PullImages -Registry ${{ parameters.publishConfig.buildAcr.server }} -RepoPrefix ${{ parameters.publishConfig.buildAcr.repoPrefix }} -ImageInfoPath $(artifactsPath)/image-info.json" - if [ "$TESTCATEGORIESOVERRIDE" != "" ]; then - additionalTestArgs="$additionalTestArgs -TestCategories $TESTCATEGORIESOVERRIDE" - fi - fi - fi - echo "##vso[task.setvariable variable=additionalTestArgs]$additionalTestArgs" - displayName: Set Test Variables - condition: and(succeeded(), ${{ parameters.condition }}) -- script: > - docker run -t -d - -v /var/run/docker.sock:/var/run/docker.sock - -v $(Build.ArtifactStagingDirectory):$(artifactsPath) - -e DOCKER_BUILDKIT=1 - -e RUNNING_TESTS_IN_CONTAINER=true - --name $(testRunner.container) - $(imageNames.testRunner.withrepo) - displayName: Start Test Runner Container - condition: and(succeeded(), ${{ parameters.condition }}) -- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: - - template: /eng/common/templates/steps/run-pwsh-with-auth.yml@self - parameters: - displayName: Docker login - serviceConnection: ${{ parameters.publishConfig.testServiceConnection.name }} - condition: and(succeeded(), ${{ parameters.condition }}) - command: >- - $azLoginArgs = '--service-principal --tenant $env:AZURE_TENANT_ID -u $env:AZURE_CLIENT_ID --federated-token $env:AZURE_FEDERATED_TOKEN'; - docker exec -e AZURE_TENANT_ID=$env:tenantId -e AZURE_CLIENT_ID=$env:servicePrincipalId -e AZURE_FEDERATED_TOKEN=$env:idToken $(testRunner.container) pwsh - -File $(engCommonRelativePath)/Invoke-WithRetry.ps1 - "az login $azLoginArgs; az acr login -n ${{ parameters.publishConfig.buildAcr.server }}" - - ${{ if eq(parameters.preBuildValidation, 'false') }}: - - template: /eng/common/templates/steps/download-build-artifact.yml@self - parameters: - targetPath: $(Build.ArtifactStagingDirectory) - artifactName: image-info - condition: ${{ parameters.condition }} - pipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} -- template: /eng/common/templates/steps/parse-test-arg-arrays.yml@self -- powershell: > - $(test.init); - docker exec - $(testRunner.options) - $(testRunner.container) - pwsh - -Command "$(testScriptPath) - -Paths $(imageBuilderPathsArrayInitStr) - -OSVersions $(osVersionsArrayInitStr) - -Architecture '$(architecture)' - $(additionalTestArgs)" - displayName: Test Images - condition: and(succeeded(), ${{ parameters.condition }}) -- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: - - script: docker exec $(testRunner.container) docker logout ${{ parameters.publishConfig.buildAcr.server }} - displayName: Docker logout - condition: and(always(), ${{ parameters.condition }}) - continueOnError: true -- powershell: > - docker cp - $(testRunner.container):/repo/$(testResultsDirectory) - $(Common.TestResultsDirectory)/. - displayName: Copy Test Results - condition: and(always(), ${{ parameters.condition }}) - continueOnError: true -- task: PublishTestResults@2 - displayName: Publish Test Results - condition: and(always(), ${{ parameters.condition }}) - continueOnError: true - inputs: - testRunner: vSTest - testResultsFiles: '**/*.trx' - searchFolder: $(Common.TestResultsDirectory) - mergeTestResults: true - publishRunAttachments: true - ${{ if eq(parameters.preBuildValidation, 'false') }}: - testRunTitle: $(productVersion) $(osVersionsDisplayName) $(architecture) - ${{ if eq(parameters.preBuildValidation, 'true') }}: - testRunTitle: Pre-Build Validation -- script: docker rm -f $(testRunner.container) - displayName: Cleanup TestRunner Container - condition: and(always(), ${{ parameters.condition }}) - continueOnError: true diff --git a/eng/common/templates/steps/test-images-windows-client.yml b/eng/common/templates/steps/test-images-windows-client.yml deleted file mode 100644 index 00fa062fd..000000000 --- a/eng/common/templates/steps/test-images-windows-client.yml +++ /dev/null @@ -1,66 +0,0 @@ -parameters: - internalProjectName: null - publishConfig: null - condition: true - customInitSteps: [] - sourceBuildPipelineRunId: "" - -steps: -- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: - - template: /eng/common/templates/steps/init-docker-windows.yml@self - parameters: - cleanupDocker: true - setupImageBuilder: false - condition: ${{ parameters.condition }} - - template: /eng/common/templates/steps/run-pwsh-with-auth.yml@self - parameters: - displayName: Docker login - serviceConnection: ${{ parameters.publishConfig.testServiceConnection.name }} - dockerClientOS: windows - condition: and(succeeded(), ${{ parameters.condition }}) - command: >- - az login --service-principal --tenant $env:tenantId -u $env:servicePrincipalId --federated-token $env:idToken; - $accessToken = $(az acr login -n ${{ parameters.publishConfig.buildAcr.server }} --expose-token --query accessToken --output tsv); - docker login ${{ parameters.publishConfig.buildAcr.server }} -u 00000000-0000-0000-0000-000000000000 -p $accessToken -- ${{ parameters.customInitSteps }} -- powershell: | - if ("${{ variables['System.TeamProject'] }}" -eq "${{ parameters.internalProjectName }}" -and "${{ variables['Build.Reason'] }}" -ne "PullRequest") { - $additionalTestArgs="$env:ADDITIONALTESTARGS -PullImages -Registry ${{ parameters.publishConfig.buildAcr.server }} -RepoPrefix ${{ parameters.publishConfig.buildAcr.repoPrefix }} -ImageInfoPath $(artifactsPath)/image-info.json" - } - echo "##vso[task.setvariable variable=additionalTestArgs]$additionalTestArgs" - displayName: Set Test Variables - condition: and(succeeded(), ${{ parameters.condition }}) -- powershell: Get-ChildItem -Path tests -r | Where {$_.Extension -match "trx"} | Remove-Item - displayName: Cleanup Old Test Results - condition: and(succeeded(), ${{ parameters.condition }}) -- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: - - template: /eng/common/templates/steps/download-build-artifact.yml@self - parameters: - targetPath: $(Build.ArtifactStagingDirectory) - artifactName: image-info - condition: ${{ parameters.condition }} - pipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} -- template: /eng/common/templates/steps/parse-test-arg-arrays.yml@self -- powershell: > - $(test.init); - $(testScriptPath) - -Paths $(imageBuilderPathsArrayInitStr) - -OSVersions $(osVersionsArrayInitStr) - $(additionalTestArgs) - displayName: Test Images - condition: and(succeeded(), ${{ parameters.condition }}) -- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: - - script: docker logout ${{ parameters.publishConfig.buildAcr.server }} - displayName: Docker logout - condition: and(always(), ${{ parameters.condition }}) - continueOnError: true -- task: PublishTestResults@2 - displayName: Publish Test Results - condition: and(always(), ${{ parameters.condition }}) - continueOnError: true - inputs: - testRunner: vSTest - testResultsFiles: '$(testResultsDirectory)/**/*.trx' - mergeTestResults: true - publishRunAttachments: true - testRunTitle: $(productVersion) $(osVersionsDisplayName) amd64 diff --git a/eng/common/templates/steps/validate-branch.yml b/eng/common/templates/steps/validate-branch.yml deleted file mode 100644 index 0fb1a841d..000000000 --- a/eng/common/templates/steps/validate-branch.yml +++ /dev/null @@ -1,52 +0,0 @@ -parameters: - publishConfig: null - internalProjectName: null - -steps: -- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: - - powershell: | - if ("$env:ONEESPT_BUILDTYPE" -eq "Unofficial") - { - echo "Build is from an unofficial pipeline, continuing." - exit 0 - } - - $isOfficialRepoPrefix = "$(officialRepoPrefixes)".Split(',').Contains("${{ parameters.publishConfig.publishAcr.repoPrefix }}") - if (-not $isOfficialRepoPrefix) - { - echo "This build will not publish to an official repo prefix, continuing." - echo "Publish repo prefix: ${{ parameters.publishConfig.publishAcr.repoPrefix }}" - echo "Official repo prefixes: $(officialRepoPrefixes)" - exit 0 - } - - $isOfficialBranch = "$(officialBranches)".Split(',').Contains("$(sourceBranch)") - if ($isOfficialBranch) - { - echo "$(sourceBranch) is an official branch, continuing." - echo "Official branches: $(officialBranches)" - exit 0 - } - - $hasOfficialBranchPrefix = $false - foreach ($prefix in "$(officialBranchPrefixes)".Split(',')) { - if ("$(sourceBranch)".StartsWith($prefix)) { - $hasOfficialBranchPrefix = $true - break - } - } - - if ($hasOfficialBranchPrefix) - { - echo "$(sourceBranch) has an official branch prefix, continuing." - echo "Official branch prefixes: $(officialBranchPrefixes)" - exit 0 - } - - echo "##vso[task.logissue type=error]Official builds must be done from an official branch ($(officialBranches)) and repo prefix ($(officialRepoPrefixes))." - echo "Build definition: $(Build.DefinitionName)" - echo "1ESPT build type: $(OneESPT.BuildType)" - echo "Current branch: $(sourceBranch)" - echo "Publish repo prefix: ${{ parameters.publishConfig.publishAcr.repoPrefix }}" - exit 1 - displayName: Validate Branch diff --git a/eng/common/templates/steps/validate-image-sizes.yml b/eng/common/templates/steps/validate-image-sizes.yml deleted file mode 100644 index 9cd6e95a0..000000000 --- a/eng/common/templates/steps/validate-image-sizes.yml +++ /dev/null @@ -1,16 +0,0 @@ -parameters: - dockerClientOS: null - architecture: "*" - validationMode: "all" - -steps: - - template: ${{ format('../steps/init-docker-{0}.yml', parameters.dockerClientOS) }} - parameters: - # Get some disk space back, pipeline run time is not a concern here - cleanupDocker: true - - powershell: > - ./tests/performance/Validate-ImageSize.ps1 - -ImageBuilderCustomArgs "--architecture '${{ parameters.architecture }}'" - -ValidationMode:${{ parameters.validationMode }} - -PullImages - displayName: Run Image Size Tests diff --git a/eng/common/templates/steps/wait-for-mcr-doc-ingestion.yml b/eng/common/templates/steps/wait-for-mcr-doc-ingestion.yml deleted file mode 100644 index 05112cc5b..000000000 --- a/eng/common/templates/steps/wait-for-mcr-doc-ingestion.yml +++ /dev/null @@ -1,21 +0,0 @@ -parameters: - commitDigest: null - condition: true - dryRunArg: "" - -steps: -- template: /eng/common/templates/steps/run-imagebuilder.yml@self - parameters: - displayName: Wait for MCR Doc Ingestion - condition: and(${{ parameters.condition }}, eq(variables['waitForIngestionEnabled'], 'true')) - serviceConnections: - - name: mar - id: $(marStatus.serviceConnection.id) - tenantId: $(marStatus.serviceConnection.tenantId) - clientId: $(marStatus.serviceConnection.clientId) - internalProjectName: 'internal' - args: >- - waitForMcrDocIngestion - '${{ parameters.commitDigest }}' - --timeout '$(mcrDocIngestionTimeout)' - ${{ parameters.dryRunArg }} diff --git a/eng/common/templates/steps/wait-for-mcr-image-ingestion.yml b/eng/common/templates/steps/wait-for-mcr-image-ingestion.yml deleted file mode 100644 index f8ad85d57..000000000 --- a/eng/common/templates/steps/wait-for-mcr-image-ingestion.yml +++ /dev/null @@ -1,37 +0,0 @@ -parameters: -- name: publishConfig - type: object - -- name: imageInfoPath - type: string - -- name: minQueueTime - type: string - -- name: dryRunArg - type: string - -- name: condition - type: string - default: "true" - -steps: -- template: /eng/common/templates/steps/run-imagebuilder.yml@self - parameters: - displayName: Wait for Image Ingestion - condition: and(${{ parameters.condition }}, eq(variables['waitForIngestionEnabled'], 'true')) - serviceConnections: - - name: mar - id: $(marStatus.serviceConnection.id) - tenantId: $(marStatus.serviceConnection.tenantId) - clientId: $(marStatus.serviceConnection.clientId) - internalProjectName: 'internal' - args: >- - waitForMcrImageIngestion - '${{ parameters.imageInfoPath }}' - --manifest '$(manifest)' - --repo-prefix '${{ parameters.publishConfig.publishAcr.repoPrefix }}' - --min-queue-time '${{ parameters.minQueueTime }}' - --timeout '$(mcrImageIngestionTimeout)' - $(manifestVariables) - ${{ parameters.dryRunArg }} diff --git a/eng/common/templates/task-prefix-decorator.yml b/eng/common/templates/task-prefix-decorator.yml deleted file mode 100644 index 598bfbe24..000000000 --- a/eng/common/templates/task-prefix-decorator.yml +++ /dev/null @@ -1,63 +0,0 @@ -# This Azure Pipelines template adds a prefix to the display name of each -# task passed through the `stages` parameter. When used in conjunction with -# an "extends" template which injects a lot of tasks into the pipeline, the -# added prefix helps to identify which tasks were passed through this template -# and which tasks were injected by the `baseTemplate`. -# -# This template assumes that `baseTemplate` uses the `stages` parameter. If it -# doesn't, this template likely won't work as expected. - -parameters: -# The pipeline will behave as if it were originally extended from this template, -# except with updated task display names. -- name: baseTemplate - type: string - default: "" - -# These parameters are passed directly to `baseTemplate` -- name: templateParameters - type: object - default: null - -# These stages will be modified and passed to the `baseTemplate` as the -# `stages` parameter. -- name: stages - type: stageList - default: [] - -# This prefix will be added to the display name of each task. -- name: taskPrefix - type: string - default: "🟪" - - -extends: - template: ${{ parameters.baseTemplate }} - parameters: - ${{ insert }}: ${{ parameters.templateParameters }} - stages: - - ${{ each stage in parameters.stages }}: - - stage: ${{ stage.stage }} - ${{ each property in stage }}: - ${{ if notIn(property.key, 'stage', 'jobs') }}: - ${{ property.key }} : ${{ property.value }} - jobs: - - ${{ each job in stage.jobs }}: - - job: ${{ job.job }} - ${{ each property in job }}: - ${{ if notIn(property.key, 'job', 'steps') }}: - ${{ property.key }} : ${{ property.value }} - steps: - - ${{ each step in job.steps }}: - # Special case for Azure Pipelines checkout task: - # https://learn.microsoft.com/azure/devops/extend/develop/pipeline-decorator-context?view=azure-devops#task-names-and-guids - # The checkout task does not have a name - it is special and built directly into the agent. - # Avoid modifying the checkout task, or else it will show up in the UI as a task with no name. - - ${{ if contains(step.task, '6d15af64-176c-496d-b583-fd2ae21d4df4') }}: - - ${{ step }} - - ${{ else }}: - - task: ${{ step.task }} - ${{ each property in step }}: - ${{ if notIn(property.key, 'task', 'displayName') }}: - ${{ property.key }} : ${{ property.value }} - displayName: ${{ parameters.taskPrefix }} ${{ step.displayName }} diff --git a/eng/common/templates/variables/codeql.yml b/eng/common/templates/variables/codeql.yml deleted file mode 100644 index 0973342b9..000000000 --- a/eng/common/templates/variables/codeql.yml +++ /dev/null @@ -1,16 +0,0 @@ -parameters: -- name: TSAEnabled - displayName: Publish CodeQL results to TSA - type: boolean - default: true - -variables: - # Force CodeQL enabled so it may be run on any branch -- name: Codeql.Enabled - value: true - # Do not let CodeQL 3000 Extension gate scan frequency -- name: Codeql.Cadence - value: 0 - # CodeQL needs this plumbed along as a variable to enable TSA -- name: Codeql.TSAEnabled - value: ${{ parameters.TSAEnabled }} diff --git a/eng/common/templates/variables/common-paths.yml b/eng/common/templates/variables/common-paths.yml deleted file mode 100644 index 6e8864e4a..000000000 --- a/eng/common/templates/variables/common-paths.yml +++ /dev/null @@ -1,5 +0,0 @@ -variables: - engCommonRelativePath: eng/common - engCommonPath: $(Build.Repository.LocalPath)/$(engCommonRelativePath) - engPath: $(Build.Repository.LocalPath)/eng - testScriptPath: "" diff --git a/eng/common/templates/variables/common.yml b/eng/common/templates/variables/common.yml deleted file mode 100644 index 525709f45..000000000 --- a/eng/common/templates/variables/common.yml +++ /dev/null @@ -1,82 +0,0 @@ -variables: -- template: /eng/common/templates/variables/docker-images.yml@self -- template: /eng/common/templates/variables/common-paths.yml@self - -- name: publishReadme - value: true -- name: publishImageInfo - value: true -- name: ingestKustoImageInfo - value: true - # CG is disabled by default because projects are built within Dockerfiles and CG step do not scan artifacts - # that are built within Dockerfiles. A separate CG pipeline exists for this reason. -- name: skipComponentGovernanceDetection - value: false -- name: build.imageBuilderDockerRunExtraOptions - value: "" -- name: imageBuilderDockerRunExtraOptions - value: "" -- name: generateEolAnnotationDataExtraOptions - value: "" -- name: productVersionComponents - value: 2 -- name: imageInfoVariant - value: "" -- name: publishNotificationsEnabled - value: false -- name: manifestVariables - value: "" -- name: mcrImageIngestionTimeout - value: "00:20:00" -- name: mcrDocIngestionTimeout - value: "00:05:00" -- name: officialBranches - # comma-delimited list of branch names - value: main -- name: mirrorRepoPrefix - value: 'mirror/' -- name: cgBuildGrepArgs - value: "''" -- name: test.init - value: "" -- name: testRunner.options - value: "" -- name: customCopyBaseImagesArgs - value: "" -- name: additionalGenerateBuildMatrixOptions - value: "" -- name: trimCachedImagesForMatrix - value: false - -- name: defaultLinuxAmd64PoolImage - value: ubuntu-latest -- name: defaultLinuxArm32PoolImage - value: null -- name: defaultLinuxArm64PoolImage - value: null -- name: defaultWindows2016PoolImage - value: vs2017-win2016 -- name: defaultWindows1809PoolImage - value: windows-2019 -- name: defaultWindows2022PoolImage - value: windows-2022 -- name: defaultWindows2025PoolImage - value: windows-2025 - -- name: default1ESInternalPoolName - value: NetCore1ESPool-Internal -- name: default1ESInternalPoolImage - value: 1es-ubuntu-2204 - -- name: defaultSourceAnalysisPoolName - value: NetCore1ESPool-Internal -- name: defaultSourceAnalysisPoolImage - value: 1es-windows-2022 - -# Define these as placeholder values to allow string validation to succeed since we don't have the -# variable group with the actual values in public builds. For internal builds, the variable group -# will cause these values to be overridden with the real values. -- name: acr.subscription - value: 00000000-0000-0000-0000-000000000000 -- name: acr-staging.subscription - value: 00000000-0000-0000-0000-000000000000 diff --git a/eng/common/templates/variables/docker-images.yml b/eng/common/templates/variables/docker-images.yml deleted file mode 100644 index 4eb56e122..000000000 --- a/eng/common/templates/variables/docker-images.yml +++ /dev/null @@ -1,7 +0,0 @@ -variables: - imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder:2830238 - imageNames.imageBuilder: $(imageNames.imageBuilderName) - imageNames.imageBuilder.withrepo: imagebuilder-withrepo:$(Build.BuildId)-$(System.JobId) - imageNames.testRunner: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux3.0-docker-testrunner - imageNames.testRunner.withrepo: testrunner-withrepo:$(Build.BuildId)-$(System.JobId) - imageNames.syft: anchore/syft:v1.31.0-debug diff --git a/eng/common/templates/variables/dotnet/build-test-publish.yml b/eng/common/templates/variables/dotnet/build-test-publish.yml deleted file mode 100644 index 94732039b..000000000 --- a/eng/common/templates/variables/dotnet/build-test-publish.yml +++ /dev/null @@ -1,48 +0,0 @@ -# Common variables for building/testing/publishing in the .NET team's pipelines - -variables: -- template: /eng/common/templates/variables/dotnet/common.yml@self - -- name: commonVersionsImageInfoPath - value: build-info/docker -- name: publicGitRepoUri - value: https://github.com/dotnet/dotnet-docker -- name: testScriptPath - value: ./tests/run-tests.ps1 -- name: testResultsDirectory - value: tests/Microsoft.DotNet.Docker.Tests/TestResults/ - -- name: officialRepoPrefixes - value: public/,internal/private/,unlisted/ - readonly: true - -- name: mcrDocsRepoInfo.userName - value: $(gitHubApp.marDocsUpdater.userName) -- name: mcrDocsRepoInfo.email - value: $(gitHubApp.marDocsUpdater.email) - -- name: publishNotificationsEnabled - value: true -- name: gitHubNotificationsRepoInfo.org - value: dotnet -- name: gitHubNotificationsRepoInfo.repo - value: dotnet-docker-internal -# $(gitHubNotificationsRepoInfo.authArgs) is needed by the "Post Publish -# Notification" step in eng/common/templates/jobs/publish.yml#L271, even during -# a dry-run. This value is a placeholder that gets replaced when referencing -# the secrets.yml variable template. -- name: gitHubNotificationsRepoInfo.authArgs - value: --gh-token 'placeholder' - -- name: gitHubVersionsRepoInfo.org - value: dotnet -- name: gitHubVersionsRepoInfo.repo - value: versions -- name: gitHubVersionsRepoInfo.branch - value: main -- name: gitHubVersionsRepoInfo.path - value: ${{ variables.commonVersionsImageInfoPath }} -- name: gitHubVersionsRepoInfo.userName - value: $(dotnetDockerBot.userName) -- name: gitHubVersionsRepoInfo.email - value: $(dotnetDockerBot.email) diff --git a/eng/common/templates/variables/dotnet/common.yml b/eng/common/templates/variables/dotnet/common.yml deleted file mode 100644 index bb39d82c2..000000000 --- a/eng/common/templates/variables/dotnet/common.yml +++ /dev/null @@ -1,63 +0,0 @@ -variables: -- template: /eng/common/templates/variables/common.yml@self - -- name: publicProjectName - value: public -- name: internalProjectName - value: internal - -# $(dockerHubRegistryCreds) is needed by the copy-base-images step in -# eng/common/templates/stages/build-and-test.yml#L73-L78, even during a dry-run. -# This is a placeholder that gets replaced when referencing the secrets.yml -# variable template. -- name: dockerHubRegistryCreds - value: --registry-creds 'docker.io=placeholder;placeholder' - -- name: linuxAmd64InternalPoolImage - value: 1es-ubuntu-2204 -- name: linuxAmd64InternalPoolName - value: NetCore1ESPool-Internal - -- name: linuxArm64PoolImage - value: Mariner-2-Docker-ARM64 -- name: linuxArm64PublicPoolName - value: Docker-Linux-Arm-Public -- name: linuxArm64InternalPoolName - value: Docker-Linux-Arm-Internal - -- name: linuxArm32PoolImage - value: Mariner-2-Docker-ARM64 -- name: linuxArm32PublicPoolName - value: Docker-Linux-Arm-Public -- name: linuxArm32InternalPoolName - value: Docker-Linux-Arm-Internal - -- name: windowsServer2016PublicPoolImage - value: Server2016-NESDockerBuilds -- name: windowsServer2016InternalPoolImage - value: Server2016-NESDockerBuilds-1ESPT -- name: windowsServer2016PoolName - value: Docker-2016-${{ variables['System.TeamProject'] }} - -- name: windowsServer2019PublicPoolImage - value: Server2019-1809-NESDockerBuilds -- name: windowsServer2019InternalPoolImage - value: Server2019-1809-NESDockerBuilds-1ESPT -- name: windowsServer2019PoolName - value: Docker-1809-${{ variables['System.TeamProject'] }} - -- name: windowsServer2022PublicPoolImage - value: Server2022-NESDockerBuilds -- name: windowsServer2022InternalPoolImage - value: Server2022-NESDockerBuilds-1ESPT -- name: windowsServer2022PoolName - value: Docker-2022-${{ variables['System.TeamProject'] }} - -- name: windowsServer2025PublicPoolImage - value: Server2025-NESDockerBuilds -- name: windowsServer2025InternalPoolImage - value: Server2025-NESDockerBuilds-1ESPT -- name: windowsServer2025PoolName - value: Docker-2025-${{ variables['System.TeamProject'] }} - -- group: DotNet-Docker-Common-2 diff --git a/eng/common/templates/variables/dotnet/secrets-unofficial.yml b/eng/common/templates/variables/dotnet/secrets-unofficial.yml deleted file mode 100644 index 1744ad288..000000000 --- a/eng/common/templates/variables/dotnet/secrets-unofficial.yml +++ /dev/null @@ -1,5 +0,0 @@ -variables: -- group: DotNet-Docker-Secrets-Low - -- name: dockerHubRegistryCreds - value: --registry-creds 'docker.io=$(dotnet-dockerhub-bot-username);$(dotnet-dockerhub-bot-pat-low)' diff --git a/eng/common/templates/variables/dotnet/secrets.yml b/eng/common/templates/variables/dotnet/secrets.yml deleted file mode 100644 index 0224c441e..000000000 --- a/eng/common/templates/variables/dotnet/secrets.yml +++ /dev/null @@ -1,17 +0,0 @@ -variables: -- group: DotNet-Docker-Secrets - -- name: dockerHubRegistryCreds - value: --registry-creds 'docker.io=$(dotnetDockerHubBot.userName);$(BotAccount-dotnet-dockerhub-bot-PAT)' - -- name: gitHubNotificationsRepoInfo.authArgs - value: --gh-token '$(BotAccount-dotnet-docker-bot-PAT)' - -- name: gitHubVersionsRepoInfo.authArgs - value: --gh-token '$(BotAccount-dotnet-docker-bot-PAT)' - -- name: mcrDocsRepoInfo.authArgs - value: >- - --gh-private-key '$(GitHubApp-NET-Docker-MAR-Docs-Updater-PrivateKey)' - --gh-app-client-id '$(gitHubApp.marDocsUpdater.clientId)' - --gh-app-installation-id '$(gitHubApp.marDocsUpdater.microsoft.installationId)' From bfc630629718ebb97e3733dc556777f581d8b849 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Mon, 8 Dec 2025 15:10:26 -0800 Subject: [PATCH 3/3] Update eng/common references to eng/docker-tools --- .github/workflows/lint-code-base.yml | 2 +- build-and-test.ps1 | 4 ++-- .../Get-GeneratedDockerfiles.ps1 | 4 ++-- eng/pipelines/cg-detection.yml | 4 ++-- eng/pipelines/dotnet-framework-pr-no-cache.yml | 2 +- eng/pipelines/dotnet-framework-pr.yml | 2 +- eng/pipelines/dotnet-framework-samples.yml | 6 +++--- eng/pipelines/dotnet-framework.yml | 6 +++--- eng/pipelines/stages/dotnet-framework-base.yml | 4 ++-- eng/pipelines/update-readmes.yml | 8 ++++---- eng/pipelines/variables/common.yml | 2 +- eng/readme-templates/Get-GeneratedReadmes.ps1 | 4 ++-- tests/run-tests.ps1 | 12 ++++++------ 13 files changed, 30 insertions(+), 30 deletions(-) diff --git a/.github/workflows/lint-code-base.yml b/.github/workflows/lint-code-base.yml index 92e0275ad..205b8a9d0 100644 --- a/.github/workflows/lint-code-base.yml +++ b/.github/workflows/lint-code-base.yml @@ -25,6 +25,6 @@ jobs: uses: github/super-linter@v6 # https://github.com/github/super-linter env: DEFAULT_BRANCH: main - FILTER_REGEX_EXCLUDE: eng/common/.*|eng/readme-templates/.*|.portal-docs/.* + FILTER_REGEX_EXCLUDE: eng/docker-tools/.*|eng/readme-templates/.*|.portal-docs/.* GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} VALIDATE_MARKDOWN: true diff --git a/build-and-test.ps1 b/build-and-test.ps1 index 28a9f16c4..f6b5697ec 100644 --- a/build-and-test.ps1 +++ b/build-and-test.ps1 @@ -24,14 +24,14 @@ if ($Repos.Count -eq 0) { else { $Paths = @() $Repos | foreach { - + $Paths += "src/$_/$Version/$OS" } $testCategories = $Repos } if ($Mode -eq "BuildAndTest" -or $Mode -eq "Build") { - & ./eng/common/build.ps1 ` + & ./eng/docker-tools/build.ps1 ` -Version $Version ` -OS $OS ` -Paths $Paths ` diff --git a/eng/dockerfile-templates/Get-GeneratedDockerfiles.ps1 b/eng/dockerfile-templates/Get-GeneratedDockerfiles.ps1 index 275c7669b..21ad83024 100644 --- a/eng/dockerfile-templates/Get-GeneratedDockerfiles.ps1 +++ b/eng/dockerfile-templates/Get-GeneratedDockerfiles.ps1 @@ -22,10 +22,10 @@ $onDockerfilesGeneratedLinux = { # On Windows, ImageBuilder is run locally due to limitations with running Docker client within a container. # Remove when https://github.com/dotnet/docker-tools/issues/159 is resolved if ($dockerOs -eq "windows") { - & $PSScriptRoot/../common/Invoke-ImageBuilder.ps1 ` + & $PSScriptRoot/../docker-tools/Invoke-ImageBuilder.ps1 ` -ImageBuilderArgs $imageBuilderArgs } else { - & $PSScriptRoot/../common/Invoke-ImageBuilder.ps1 ` + & $PSScriptRoot/../docker-tools/Invoke-ImageBuilder.ps1 ` -ImageBuilderArgs $imageBuilderArgs ` -OnCommandExecuted $onDockerfilesGeneratedLinux } diff --git a/eng/pipelines/cg-detection.yml b/eng/pipelines/cg-detection.yml index 40c73395f..146cacc99 100644 --- a/eng/pipelines/cg-detection.yml +++ b/eng/pipelines/cg-detection.yml @@ -23,13 +23,13 @@ variables: value: false extends: - template: /eng/common/templates/1es-official.yml@self + template: /eng/docker-tools/templates/1es-official.yml@self parameters: cgDryRun: ${{ parameters.cgDryRun }} stages: - stage: CgDetection displayName: CG Detection jobs: - - template: /eng/common/templates/jobs/cg-build-projects.yml@self + - template: /eng/docker-tools/templates/jobs/cg-build-projects.yml@self parameters: cgDryRun: ${{ parameters.cgDryRun }} diff --git a/eng/pipelines/dotnet-framework-pr-no-cache.yml b/eng/pipelines/dotnet-framework-pr-no-cache.yml index d1634a9df..e58a5de4f 100644 --- a/eng/pipelines/dotnet-framework-pr-no-cache.yml +++ b/eng/pipelines/dotnet-framework-pr-no-cache.yml @@ -24,7 +24,7 @@ variables: readonly: true stages: -- template: /eng/common/templates/stages/dotnet/publish-config-nonprod.yml@self +- template: /eng/docker-tools/templates/stages/dotnet/publish-config-nonprod.yml@self parameters: stagesTemplate: /eng/pipelines/stages/dotnet-framework-base.yml@self stagesTemplateParameters: diff --git a/eng/pipelines/dotnet-framework-pr.yml b/eng/pipelines/dotnet-framework-pr.yml index 59bccfd5c..735e3ca14 100644 --- a/eng/pipelines/dotnet-framework-pr.yml +++ b/eng/pipelines/dotnet-framework-pr.yml @@ -23,7 +23,7 @@ variables: value: manifest.json stages: -- template: /eng/common/templates/stages/dotnet/publish-config-nonprod.yml@self +- template: /eng/docker-tools/templates/stages/dotnet/publish-config-nonprod.yml@self parameters: stagesTemplate: /eng/pipelines/stages/dotnet-framework-base.yml@self stagesTemplateParameters: diff --git a/eng/pipelines/dotnet-framework-samples.yml b/eng/pipelines/dotnet-framework-samples.yml index 2692b59df..d747611dd 100644 --- a/eng/pipelines/dotnet-framework-samples.yml +++ b/eng/pipelines/dotnet-framework-samples.yml @@ -28,19 +28,19 @@ parameters: variables: - template: /eng/pipelines/variables/common.yml@self -- template: /eng/common/templates/variables/dotnet/secrets${{ iif(contains(variables['Build.DefinitionName'], '-official'), '', '-unofficial') }}.yml@self +- template: /eng/docker-tools/templates/variables/dotnet/secrets${{ iif(contains(variables['Build.DefinitionName'], '-official'), '', '-unofficial') }}.yml@self - name: manifest value: manifest.samples.json - name: imageInfoVariant value: "-samples" extends: - template: /eng/common/templates/1es.yml@self + template: /eng/docker-tools/templates/1es.yml@self parameters: reposToExcludeFromScanning: - VersionsRepo stages: - - template: /eng/common/templates/stages/dotnet/publish-config-${{ iif(contains(variables['Build.DefinitionName'], '-official'), 'prod', 'nonprod') }}.yml@self + - template: /eng/docker-tools/templates/stages/dotnet/publish-config-${{ iif(contains(variables['Build.DefinitionName'], '-official'), 'prod', 'nonprod') }}.yml@self parameters: sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} stagesTemplate: /eng/pipelines/stages/dotnet-framework-base.yml@self diff --git a/eng/pipelines/dotnet-framework.yml b/eng/pipelines/dotnet-framework.yml index 5708171b6..372931ea2 100644 --- a/eng/pipelines/dotnet-framework.yml +++ b/eng/pipelines/dotnet-framework.yml @@ -28,19 +28,19 @@ parameters: variables: - template: /eng/pipelines/variables/common.yml@self -- template: /eng/common/templates/variables/dotnet/secrets${{ iif(contains(variables['Build.DefinitionName'], '-official'), '', '-unofficial') }}.yml@self +- template: /eng/docker-tools/templates/variables/dotnet/secrets${{ iif(contains(variables['Build.DefinitionName'], '-official'), '', '-unofficial') }}.yml@self - name: manifest value: manifest.json - name: mcrImageIngestionTimeout value: "00:30:00" extends: - template: /eng/common/templates/1es.yml@self + template: /eng/docker-tools/templates/1es.yml@self parameters: reposToExcludeFromScanning: - VersionsRepo stages: - - template: /eng/common/templates/stages/dotnet/publish-config-${{ iif(contains(variables['Build.DefinitionName'], '-official'), 'prod', 'nonprod') }}.yml@self + - template: /eng/docker-tools/templates/stages/dotnet/publish-config-${{ iif(contains(variables['Build.DefinitionName'], '-official'), 'prod', 'nonprod') }}.yml@self parameters: sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} stagesTemplate: /eng/pipelines/stages/dotnet-framework-base.yml@self diff --git a/eng/pipelines/stages/dotnet-framework-base.yml b/eng/pipelines/stages/dotnet-framework-base.yml index 641f02aea..81d1e4b37 100644 --- a/eng/pipelines/stages/dotnet-framework-base.yml +++ b/eng/pipelines/stages/dotnet-framework-base.yml @@ -24,7 +24,7 @@ parameters: stages: - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: - - template: /eng/common/templates/stages/setup-service-connections.yml@self + - template: /eng/docker-tools/templates/stages/setup-service-connections.yml@self parameters: serviceConnections: - name: ${{ parameters.publishConfig.internalMirrorAcr.serviceConnection.name }} @@ -33,7 +33,7 @@ stages: - ${{ each serviceConnection in parameters.additionalServiceConnections }}: - name: ${{ serviceConnection.name }} -- template: /eng/common/templates/stages/dotnet/build-test-publish-repo.yml@self +- template: /eng/docker-tools/templates/stages/dotnet/build-test-publish-repo.yml@self parameters: publishConfig: ${{ parameters.publishConfig }} internalProjectName: ${{ parameters.internalProjectName }} diff --git a/eng/pipelines/update-readmes.yml b/eng/pipelines/update-readmes.yml index c8e47480e..c629c767c 100644 --- a/eng/pipelines/update-readmes.yml +++ b/eng/pipelines/update-readmes.yml @@ -7,7 +7,7 @@ variables: value: manifest.json extends: - template: /eng/common/templates/1es-official.yml@self + template: /eng/docker-tools/templates/1es-official.yml@self parameters: stages: - stage: UpdateReadmes @@ -16,6 +16,6 @@ extends: - job: UpdateReadmes displayName: Update Readmes steps: - - template: /eng/common/templates/steps/init-docker-linux.yml@self - - template: /eng/common/templates/steps/publish-readmes.yml@self - - template: /eng/common/templates/steps/cleanup-docker-linux.yml@self \ No newline at end of file + - template: /eng/docker-tools/templates/steps/init-docker-linux.yml@self + - template: /eng/docker-tools/templates/steps/publish-readmes.yml@self + - template: /eng/docker-tools/templates/steps/cleanup-docker-linux.yml@self diff --git a/eng/pipelines/variables/common.yml b/eng/pipelines/variables/common.yml index 94d4af5a2..e61eed718 100644 --- a/eng/pipelines/variables/common.yml +++ b/eng/pipelines/variables/common.yml @@ -1,5 +1,5 @@ variables: -- template: /eng/common/templates/variables/dotnet/build-test-publish.yml@self +- template: /eng/docker-tools/templates/variables/dotnet/build-test-publish.yml@self - name: "publicGitRepoUri" value: "https://github.com/microsoft/dotnet-framework-docker" diff --git a/eng/readme-templates/Get-GeneratedReadmes.ps1 b/eng/readme-templates/Get-GeneratedReadmes.ps1 index 4f2d6b3e8..2819c8a6e 100644 --- a/eng/readme-templates/Get-GeneratedReadmes.ps1 +++ b/eng/readme-templates/Get-GeneratedReadmes.ps1 @@ -51,10 +51,10 @@ function Invoke-GenerateReadme { # On Windows, ImageBuilder is run locally due to limitations with running Docker client within a container. # Remove when https://github.com/dotnet/docker-tools/issues/159 is resolved if ($dockerOs -eq "windows") { - & $PSScriptRoot/../common/Invoke-ImageBuilder.ps1 ` + & $PSScriptRoot/../docker-tools/Invoke-ImageBuilder.ps1 ` -ImageBuilderArgs $imageBuilderArgs } else { - & $PSScriptRoot/../common/Invoke-ImageBuilder.ps1 ` + & $PSScriptRoot/../docker-tools/Invoke-ImageBuilder.ps1 ` -ImageBuilderArgs $imageBuilderArgs ` -OnCommandExecuted $onDockerfilesGeneratedLinux } diff --git a/tests/run-tests.ps1 b/tests/run-tests.ps1 index 177c1ccf5..4dd11eee3 100644 --- a/tests/run-tests.ps1 +++ b/tests/run-tests.ps1 @@ -12,15 +12,15 @@ param( [string[]]$Paths = @(), [string]$Architecture, - + [string[]]$OSVersions = @(), - + [string]$Registry, - + [string]$RepoPrefix, - + [switch]$PullImages, - + [string]$ImageInfoPath, [ValidateSet('runtime', 'sdk', 'aspnet', 'wcf', 'pre-build')] @@ -54,7 +54,7 @@ $ErrorActionPreference = 'Stop' # Install the .NET Core SDK $DotnetInstallDir = "$PSScriptRoot/../.dotnet" -& $PSScriptRoot/../eng/common/Install-DotNetSdk.ps1 $dotnetInstallDir +& $PSScriptRoot/../eng/docker-tools/Install-DotNetSdk.ps1 $dotnetInstallDir $activeOS = docker version -f "{{ .Server.Os }}"