Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion eng/Version.Details.props
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ This file should be imported by eng/Versions.props
<Project>
<PropertyGroup>
<!-- dotnet-arcade dependencies -->
<MicrosoftDotNetArcadeSdkPackageVersion>11.0.0-beta.26272.4</MicrosoftDotNetArcadeSdkPackageVersion>
<MicrosoftDotNetArcadeSdkPackageVersion>11.0.0-beta.26279.5</MicrosoftDotNetArcadeSdkPackageVersion>
<!-- dotnet-msbuild dependencies -->
<MicrosoftBuildPackageVersion>18.8.0-preview-26265-07</MicrosoftBuildPackageVersion>
<MicrosoftBuildFrameworkPackageVersion>18.8.0-preview-26265-07</MicrosoftBuildFrameworkPackageVersion>
Expand Down
4 changes: 2 additions & 2 deletions eng/Version.Details.xml
Original file line number Diff line number Diff line change
Expand Up @@ -76,9 +76,9 @@
</Dependency>
</ProductDependencies>
<ToolsetDependencies>
<Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="11.0.0-beta.26272.4">
<Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="11.0.0-beta.26279.5">
<Uri>https://github.com/dotnet/arcade</Uri>
<Sha>106da9762cd30768ee7cc944510daf476570a619</Sha>
<Sha>2bc908b1bdb34e2c42982eb511679bb458ddac2c</Sha>
</Dependency>
<Dependency Name="optimization.windows_nt-x64.MIBC.Runtime" Version="1.0.0-prerelease.26180.1">
<Uri>https://dev.azure.com/dnceng/internal/_git/dotnet-optimization</Uri>
Expand Down
20 changes: 19 additions & 1 deletion eng/common/build.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ Param(
[switch][Alias('pb')]$productBuild,
[switch]$fromVMR,
[switch][Alias('bl')]$binaryLog,
[string][Alias('bln')]$binaryLogName = '',
[switch][Alias('nobl')]$excludeCIBinarylog,
[switch] $ci,
[switch] $prepareMachine,
Expand All @@ -46,6 +47,7 @@ function Print-Usage() {
Write-Host " -platform <value> Platform configuration: 'x86', 'x64' or any valid Platform value to pass to msbuild"
Write-Host " -verbosity <value> Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic] (short: -v)"
Write-Host " -binaryLog Output binary log (short: -bl)"
Write-Host " -binaryLogName <value> Binary log file name or path; implies -binaryLog (short: -bln)"
Write-Host " -help Print help and exit"
Write-Host ""

Expand Down Expand Up @@ -102,7 +104,19 @@ function Build {
$toolsetBuildProj = InitializeToolset
InitializeCustomToolset

$bl = if ($binaryLog) { '/bl:' + (Join-Path $LogDir 'Build.binlog') } else { '' }
$bl = ''
if ($binaryLog) {
$binaryLogPath = if ([string]::IsNullOrEmpty($binaryLogName)) {
Join-Path $LogDir 'Build.binlog'
} elseif ([System.IO.Path]::IsPathRooted($binaryLogName)) {
$binaryLogName
} else {
Join-Path $LogDir $binaryLogName
}

Create-Directory (Split-Path -Parent $binaryLogPath)
$bl = '/bl:' + $binaryLogPath
}
$platformArg = if ($platform) { "/p:Platform=$platform" } else { '' }
$check = if ($buildCheck) { '/check' } else { '' }

Expand Down Expand Up @@ -162,6 +176,10 @@ try {
$nodeReuse = $false
}

if (-not [string]::IsNullOrEmpty($binaryLogName)) {
$binaryLog = $true
}

if ($nativeToolsOnMachine) {
$env:NativeToolsOnMachine = $true
}
Expand Down
19 changes: 18 additions & 1 deletion eng/common/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ usage()
echo " --configuration <value> Build configuration: 'Debug' or 'Release' (short: -c)"
echo " --verbosity <value> Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic] (short: -v)"
echo " --binaryLog Create MSBuild binary log (short: -bl)"
echo " --binaryLogName <value> Binary log file name or path; implies --binaryLog (short: -bln)"
echo " --help Print help and exit (short: -h)"
echo ""

Expand Down Expand Up @@ -83,6 +84,7 @@ warn_not_as_error=''
node_reuse=true
build_check=false
binary_log=false
binary_log_name=''
exclude_ci_binary_log=false

projects=''
Expand Down Expand Up @@ -114,6 +116,11 @@ while [[ $# -gt 0 ]]; do
-binarylog|-bl)
binary_log=true
;;
-binarylogname|-bln)
binary_log=true
binary_log_name=$2
shift
;;
-excludecibinarylog|-nobl)
exclude_ci_binary_log=true
;;
Expand Down Expand Up @@ -232,7 +239,17 @@ function Build {

local bl=""
if [[ "$binary_log" == true ]]; then
bl="/bl:\"$log_dir/Build.binlog\""
local binary_log_path=""
if [[ -z "$binary_log_name" ]]; then
binary_log_path="$log_dir/Build.binlog"
elif [[ "$binary_log_name" = /* ]]; then
binary_log_path="$binary_log_name"
else
binary_log_path="$log_dir/$binary_log_name"
fi

mkdir -p "$(dirname "$binary_log_path")"
bl="/bl:\"$binary_log_path\""
fi

local check=""
Expand Down
217 changes: 217 additions & 0 deletions eng/common/core-templates/job/helix-job-monitor.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,217 @@
parameters:
# Maximum run time of the monitor job in minutes. Also used for --max-wait-minutes.
- name: timeoutInMinutes
type: number
default: 360

# Owner segment of the source repository (e.g. 'dotnet' for 'dotnet/runtime') passed via --organization.
# Defaults to the owner segment of BUILD_REPOSITORY_NAME when empty.
- name: organization
type: string
default: ''

# Name of the source repository (e.g. 'runtime' for 'dotnet/runtime') passed via --repository.
# Defaults to the repo segment of BUILD_REPOSITORY_NAME when empty.
- name: repository
type: string
default: ''

# Optional dependency list for the generated job.
- name: dependsOn
type: object
default: []

# Optional condition for the generated job.
- name: condition
type: string
default: ''

# NuGet package id of the Helix job monitor tool.
- name: toolPackageId
type: string
default: Microsoft.DotNet.Helix.JobMonitor

# Console command exposed by the installed tool package.
- name: toolCommand
type: string
default: dotnet-helix-job-monitor

# Optional explicit tool version. Only honored when 'toolNupkgArtifactName' is set; in the
# default code path the version is taken from the consuming repo's .config/dotnet-tools.json.
- name: toolVersion
type: string
default: ''

# Base URI for the Helix service (--helix-base-uri).
- name: helixBaseUri
type: string
default: https://helix.dot.net/

# Helix API access token forwarded to the tool via the HELIX_ACCESSTOKEN environment variable.
- name: helixAccessToken
type: string
default: ''

# Polling interval in seconds (--polling-interval-seconds).
- name: pollingIntervalSeconds
type: number
default: 30

# Advanced: optional pipeline artifact (produced earlier in this run) that contains the tool
# nupkg. When set, the artifact is downloaded and the tool is installed from the nupkg into
# a local tool-path; this bypasses the repo's .config/dotnet-tools.json manifest and is
# primarily intended for the Arcade repository itself, where the Helix job monitor tool is
# built in the same pipeline that runs this template.
#
# When this parameter is empty (the default), the consuming repository must declare the tool
# in its .config/dotnet-tools.json manifest (alongside other local .NET tools); the template
# will check out the repo and run 'dotnet tool restore' to install the version pinned there.
- name: toolNupkgArtifactName
type: string
default: ''

# Advanced: sub-path within the downloaded artifact where the tool nupkg is located. Defaults
# to the standard Arcade non-shipping packages location for a Release build (relative to the
# pipeline artifact root, which is itself the build's 'artifacts' directory).
- name: toolNupkgArtifactSubPath
type: string
default: 'packages/Release/NonShipping'

jobs:
- job: HelixJobMonitor
displayName: Monitor Helix Jobs
timeoutInMinutes: ${{ parameters.timeoutInMinutes }}
${{ if ne(length(parameters.dependsOn), 0) }}:
dependsOn: ${{ parameters.dependsOn }}
${{ if ne(parameters.condition, '') }}:
condition: ${{ parameters.condition }}
pool:
${{ if eq(variables['System.TeamProject'], 'public') }}:
name: $(DncEngPublicBuildPool)
demands: ImageOverride -equals build.azurelinux.3.amd64.open
${{ else }}:
name: $(DncEngInternalBuildPool)
demands: ImageOverride -equals build.azurelinux.3.amd64
steps:
- checkout: self
fetchDepth: 1

- ${{ if ne(parameters.toolNupkgArtifactName, '') }}:
- task: DownloadPipelineArtifact@2
displayName: Download Helix Job Monitor artifact
inputs:
buildType: current
artifactName: ${{ parameters.toolNupkgArtifactName }}
itemPattern: '${{ parameters.toolNupkgArtifactSubPath }}/${{ parameters.toolPackageId }}.*.nupkg'
targetPath: $(Agent.TempDirectory)/helix-job-monitor-nupkg

- bash: |
set -euo pipefail

toolPath="$AGENT_TEMPDIRECTORY/helix-job-monitor-tool"
mkdir -p "$toolPath"

packageId='${{ parameters.toolPackageId }}'
toolVersion='${{ parameters.toolVersion }}'
nupkgArtifactSubPath='${{ parameters.toolNupkgArtifactSubPath }}'
nupkgDir="$AGENT_TEMPDIRECTORY/helix-job-monitor-nupkg/$nupkgArtifactSubPath"

if [ ! -d "$nupkgDir" ]; then
echo "Expected nupkg directory '$nupkgDir' was not produced by the artifact download." >&2
exit 1
fi

nupkg=$(find "$nupkgDir" -maxdepth 1 -type f -name "$packageId.*.nupkg" | head -n 1)
if [ -z "$nupkg" ]; then
echo "No '$packageId.*.nupkg' found in '$nupkgDir'." >&2
exit 1
fi

# Derive the version from the nupkg filename so the local package is selected
# deterministically instead of resolving against any other configured feed.
nupkgBase=$(basename "$nupkg" .nupkg)
derivedVersion="${nupkgBase#${packageId}.}"
if [ -z "$toolVersion" ]; then
toolVersion="$derivedVersion"
fi

echo "Using locally built '$packageId' version '$toolVersion' from '$nupkgDir'."

# Create a minimal NuGet.config that only references the local nupkg directory.
# This avoids conflicts with the repo's package source mapping which blocks --add-source.
toolNugetConfig="$AGENT_TEMPDIRECTORY/helix-job-monitor-nuget.config"
printf '<?xml version="1.0" encoding="utf-8"?>\n<configuration>\n <packageSources>\n <clear />\n <add key="local-tool" value="%s" />\n </packageSources>\n</configuration>\n' "$nupkgDir" > "$toolNugetConfig"

pushd "$(Build.SourcesDirectory)" > /dev/null
./eng/common/dotnet.sh tool install \
--tool-path "$toolPath" "$packageId" \
--version "$toolVersion" \
--configfile "$toolNugetConfig"

# Locate the tool DLL so the run step can invoke it via ./eng/common/dotnet.sh exec.
toolDll=$(find "$toolPath/.store" -path '*/tools/*/any/*.deps.json' -type f | head -n 1)
toolDll="${toolDll%.deps.json}.dll"
if [ ! -f "$toolDll" ]; then
echo "Could not find tool DLL in '$toolPath/.store'." >&2
exit 1
fi

echo "Tool DLL: $toolDll"
echo "##vso[task.setvariable variable=HelixJobMonitorDll]$toolDll"
displayName: Install Helix Job Monitor

- ${{ else }}:
- bash: ./eng/common/dotnet.sh tool restore
displayName: Restore Helix Job Monitor

- bash: |
set -euo pipefail

toolArgs=(
--helix-base-uri '${{ parameters.helixBaseUri }}'
--polling-interval-seconds '${{ parameters.pollingIntervalSeconds }}'
--max-wait-minutes "$((${{ parameters.timeoutInMinutes }} - 2))" # Set the tool's timeout slightly lower than the Azure DevOps job timeout to allow it to exit gracefully.
--stage-name '$(System.StageName)'
)

organization='${{ parameters.organization }}'
repository='${{ parameters.repository }}'

# Fall back to Azure DevOps-provided environment variables when the caller did not
# supply organization / repository explicitly. BUILD_REPOSITORY_NAME is typically
# 'owner/repo' for GitHub-backed builds.
if [ -z "$organization" ] || [ -z "$repository" ]; then
buildRepoName="${BUILD_REPOSITORY_NAME:-}"
if [ -n "$buildRepoName" ] && [[ "$buildRepoName" == */* ]]; then
repoOwner="${buildRepoName%%/*}"
repoName="${buildRepoName#*/}"
if [ -z "$organization" ]; then organization="$repoOwner"; fi
if [ -z "$repository" ]; then repository="$repoName"; fi
fi
fi

if [ -n "$organization" ]; then toolArgs+=( --organization "$organization" ); fi
if [ -n "$repository" ]; then toolArgs+=( --repository "$repository" ); fi

# Build.Reason and Build.SourceBranch are required to derive the Helix source filter
# the same way the Helix SDK submitter does (PR -> 'pr', internal -> 'official',
# otherwise -> 'ci'). Without these, manually-queued / scheduled / CI builds would
# be looked up under the wrong source prefix and find zero jobs.
toolArgs+=( --build-reason "$(Build.Reason)" )
toolArgs+=( --source-branch "$(Build.SourceBranch)" )

if [ -n '${{ parameters.toolNupkgArtifactName }}' ]; then
# Tool was installed from a local nupkg; run the DLL via the repo-local dotnet.
export DOTNET_ROOT="$(Build.SourcesDirectory)/.dotnet"
./eng/common/dotnet.sh exec "$(HelixJobMonitorDll)" "${toolArgs[@]}"
else
# Tool was restored from the local .config/dotnet-tools.json manifest; invoke it
# through the manifest from the repo root.
pushd "$BUILD_SOURCESDIRECTORY" > /dev/null
trap 'popd > /dev/null' EXIT
./eng/common/dotnet.sh tool run '${{ parameters.toolCommand }}' -- "${toolArgs[@]}"
fi
displayName: Monitor Helix Jobs
env:
SYSTEM_ACCESSTOKEN: $(System.AccessToken)
HELIX_ACCESSTOKEN: ${{ parameters.helixAccessToken }}
2 changes: 0 additions & 2 deletions eng/common/core-templates/steps/publish-logs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,6 @@ steps:
'$(publishing-dnceng-devdiv-code-r-build-re)'
'$(dn-bot-all-orgs-artifact-feeds-rw)'
'$(akams-client-id)'
'$(dnceng-symbol-server-pat)'
'$(dn-bot-all-orgs-build-rw-code-rw)'
'$(System.AccessToken)'
${{parameters.CustomSensitiveDataList}}
continueOnError: true
Expand Down
2 changes: 2 additions & 0 deletions eng/common/tools.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -748,6 +748,8 @@ function Stop-Processes() {
#
function MSBuild() {
if ($ci) {
InitializeToolset | Out-Null

$env:NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS = 20
$env:NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS = 20
Write-PipelineSetVariable -Name 'NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS' -Value '20'
Expand Down
2 changes: 2 additions & 0 deletions eng/common/tools.sh
Original file line number Diff line number Diff line change
Expand Up @@ -505,6 +505,8 @@ function MSBuild {
local args=( "$@" )

if [[ "$ci" == true ]]; then
InitializeToolset

export NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS=20
export NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS=20
Write-PipelineSetVariable -name "NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS" -value "20"
Expand Down
2 changes: 1 addition & 1 deletion global.json
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
"xcopy-msbuild": "18.0.0"
},
"msbuild-sdks": {
"Microsoft.DotNet.Arcade.Sdk": "11.0.0-beta.26272.4",
"Microsoft.DotNet.Arcade.Sdk": "11.0.0-beta.26279.5",
"Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.23255.2"
}
}
Loading