diff --git a/eng/pipelines/pr/jobs/test-buildproj-job.yml b/eng/pipelines/pr/jobs/test-buildproj-job.yml index ec2d292ec8..c232513e1f 100644 --- a/eng/pipelines/pr/jobs/test-buildproj-job.yml +++ b/eng/pipelines/pr/jobs/test-buildproj-job.yml @@ -18,6 +18,10 @@ parameters: - name: buildSuffix type: string + # Name of the artifact to upload the test results (including code coverage) to. + - name: testResultsArtifactBaseName + type: string + # Dotnet CLI verbosity level. - name: dotnetVerbosity type: string @@ -78,6 +82,10 @@ jobs: - job: "test_${{ parameters.platformDisplayName }}_${{ parameters.testDisplayName }}" displayName: "${{ parameters.testDisplayName }}_${{ parameters.platformDisplayName }}" + variables: + - name: testResultsPath + value: "$(REPO_ROOT)/test_results/${{ parameters.platformDisplayName }}_${{ parameters.testDisplayName }}" + pool: name: ${{ parameters.poolName }} demands: @@ -107,3 +115,13 @@ jobs: -p:BuildNumber='$(Build.BuildNumber)' -p:BuildSuffix='${{ parameters.buildSuffix }}' -p:TestFramework=${{ parameters.platformDotnet }} + -p:TestResultsFolderPath=${{ variables.testResultsPath }} + + # Publish test results + - template: /eng/pipelines/pr/steps/publish-test-results-step.yml@self + parameters: + buildConfiguration: ${{ parameters.buildConfiguration }} + platformDisplayName: ${{ parameters.platformDisplayName }} + testDisplayName: ${{ parameters.testDisplayName }} + testResultsArtifactBaseName: ${{ parameters.testResultsArtifactBaseName }} + testResultsPath: ${{ variables.testResultsPath }} diff --git a/eng/pipelines/pr/jobs/test-sqlclientmanual-job.yml b/eng/pipelines/pr/jobs/test-sqlclientmanual-job.yml index 9ca2c7c555..e1a5a4abd4 100644 --- a/eng/pipelines/pr/jobs/test-sqlclientmanual-job.yml +++ b/eng/pipelines/pr/jobs/test-sqlclientmanual-job.yml @@ -33,6 +33,10 @@ parameters: - name: stageNameSecrets type: string + # Name of the artifact to upload the test results (including code coverage) to. + - name: testResultsArtifactBaseName + type: string + # Platform Parameters ==================================================== # Display name of the platform. This will be formatted into the job's official name as well as @@ -128,6 +132,12 @@ jobs: - name: saPassword value: $[stageDependencies.${{ parameters.stageNameSecrets }}.secrets_job.outputs['SaPassword.Value']] + - name: testDisplayName + value: "sqlclient_manual_${{ parameters.configDisplayName}}_${{ parameters.testSet }}" + + - name: testResultsPath + value: "$(REPO_ROOT)/test_results/${{ parameters.platformDisplayName }}_$(testDisplayName)" + steps: # Install dotnet and the runtime that will run the tests (if it is not netframework) - template: /eng/pipelines/common/steps/install-dotnet.yml@self @@ -201,4 +211,13 @@ jobs: -p:BuildSuffix='${{ parameters.buildSuffix }}' -p:TestFramework=${{ parameters.platformDotnet }} -p:TestSet=${{ parameters.testSet }} + -p:TestResultsFolderPath=${{ variables.testResultsPath }} + # Publish test results + - template: /eng/pipelines/pr/steps/publish-test-results-step.yml@self + parameters: + buildConfiguration: ${{ parameters.buildConfiguration }} + platformDisplayName: ${{ parameters.platformDisplayName }} + testDisplayName: ${{ variables.testDisplayName }} + testResultsArtifactBaseName: ${{ parameters.testResultsArtifactBaseName }} + testResultsPath: ${{ variables.testResultsPath }} diff --git a/eng/pipelines/pr/pr-pipeline.yml b/eng/pipelines/pr/pr-pipeline.yml deleted file mode 100644 index daf393e502..0000000000 --- a/eng/pipelines/pr/pr-pipeline.yml +++ /dev/null @@ -1,94 +0,0 @@ -################################################################################# -# Licensed to the .NET Foundation under one or more agreements. # -# The .NET Foundation licenses this file to you under the MIT license. # -# See the LICENSE file in the project root for more information. # -################################################################################# - -name: $(DayOfYear)$(Rev:rr) - -parameters: - # General Parameters ===================================================== - - # Dotnet CLI verbosity level. - - name: dotnetVerbosity - displayName: dotnet CLI Verbosity - type: string - default: normal - values: - - quiet - - minimal - - normal - - detailed - - diagnostic - -variables: - - template: /eng/pipelines/common/variables/common-variables.yml@self - - template: /eng/pipelines/pr/variables/pr-variables.yml@self - -stages: - # Stage 1a: Build and pack all projects in the repository - - template: /eng/pipelines/pr/stages/pack-stage.yml - parameters: - buildConfiguration: Debug - buildSuffix: pr - stageName: ${{ variables.stageNamePack }} - - # Stage 1b: Generate secrets - - template: /eng/pipelines/pr/stages/generate-secrets-stage.yml - parameters: - stageName: ${{ variables.stageNameSecrets }} - - # Stage 2: Execute tests and collect code coverage - - template: /eng/pipelines/pr/stages/test-stages.yml - parameters: - buildConfiguration: Debug - buildSuffix: pr - dotnetVerbosity: ${{ parameters.dotnetVerbosity }} - poolName: $(PoolNameDefault) - stageNamePack: ${{ variables.stageNamePack }} - stageNameSecrets: ${{ variables.stageNameSecrets }} - - platforms: - - displayName: "windows_net462" - dotnet: "net462" - image: "ADO-MMS22-SQL22" - operatingSystem: "Windows" - - displayName: "windows_net8" - dotnet: "net8.0" - image: "ADO-MMS22-SQL22" - operatingSystem: "Windows" - - displayName: "windows_net9" - dotnet: "net9.0" - image: "ADO-MMS22-SQL22" - operatingSystem: "Windows" - - displayName: "windows_net10" - dotnet: "net10.0" - image: "ADO-MMS22-SQL22" - operatingSystem: "Windows" - - - displayName: "linux_net8" - dotnet: "net8.0" - image: "ADO-UB22-SQL22" - operatingSystem: "Linux" - - displayName: "linux_net9" - dotnet: "net9.0" - image: "ADO-UB22-SQL22" - operatingSystem: "Linux" - - displayName: "linux_net10" - dotnet: "net10.0" - image: "ADO-UB22-SQL22" - operatingSystem: "Linux" - - manualTestAzureKeyVaultUrl: $(AzureKeyVaultUrl) - manualTestAzureKeyVaultTenantId: $(AzureKeyVaultTenantId) - manualTestConnectionStringNpLocalhost: $(ConnectionStringNp_LocalhostDefault_UsernamePassword) - manualTestConnectionStringTcpLocalhost: $(ConnectionStringTcp_LocalhostDefault_UsernamePassword) - manualTestConnectionStringNpAzure: $(ConnectionStringNp_Azure_ManagedIdentity) - manualTestConnectionStringTcpAzure: $(ConnectionStringTcp_Azure_ManagedIdentity) - manualTestFileStreamDirectory: "$(Pipeline.Workspace)/filestream" - manualTestLocalDbAppName: $(LocalDbAppName) - manualTestLocalDbSharedInstanceName: $(LocalDbSharedInstanceName) - manualTestUserManagedIdentityClientId: $(UserManagedIdentityClientId) - - # Stage 3: Collect code coverage - # @TODO: diff --git a/eng/pipelines/pr/sqlclient-pr-pipeline.yml b/eng/pipelines/pr/sqlclient-pr-pipeline.yml new file mode 100644 index 0000000000..8df34ac600 --- /dev/null +++ b/eng/pipelines/pr/sqlclient-pr-pipeline.yml @@ -0,0 +1,147 @@ +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# + +# This pipeline builds and tests all products that we release. It is intended to run as validation +# for every PR. The set of tests it executes is not exhaustive, as many tests require special +# environments to provide complete coverage. Instead, this PR strikes a balance between complete +# confidence and speed of PR validation. +# +# It is triggered by pushes to PRs that target the main, dev/*, feat/*, and release/* branches in +# GitHub. The dev/* pattern includes dev/automation/* branches created by AI agents. +# +# It maps to the "sqlclient-pr" pipeline in the Public ADO project: +# https://sqlclientdrivers.visualstudio.com/public/_build?definitionId=2281 +# And the "sqlclient-pr" pipeline in the internal ADO project: +# https://dev.azure.com/SqlClientDrivers/ADO.Net/_build?definitionId=2271 + +# Set the pipeline run name to the day-of-year and the daily run counter. +name: $(DayOfYear)$(Rev:rr) + +# Trigger PR validation runs for all pushes to PRs that target the specified branches. +# https://learn.microsoft.com/en-us/azure/devops/pipelines/repos/github?view=azure-devops&tabs=yaml#pr-triggers +pr: + branches: + include: + # GitHub repo branch targets that will trigger PR validation builds. + - dev/* + - feat/* + - main + - release/* + + paths: + include: + - .azuredevops/* + - .config/* + - doc/* + - eng/pipelines/common/* + - eng/pipelines/pr/* + - src/* + - tools/* + - azurepipelines-coverage.yml + - build.proj + - Directory.Packages.props + - dotnet-tools.json + - global.json + - NuGet.config + +# Do not trigger commit or schedule runs for this pipeline. +trigger: none + +parameters: + # General Parameters ===================================================== + + # Dotnet CLI verbosity level. + - name: dotnetVerbosity + displayName: dotnet CLI Verbosity + type: string + default: normal + values: + - quiet + - minimal + - normal + - detailed + - diagnostic + + - name: platforms + type: object + default: + - displayName: "windows_net462" + dotnet: "net462" + image: "ADO-MMS22-SQL22" + operatingSystem: "Windows" + - displayName: "windows_net8" + dotnet: "net8.0" + image: "ADO-MMS22-SQL22" + operatingSystem: "Windows" + - displayName: "windows_net9" + dotnet: "net9.0" + image: "ADO-MMS22-SQL22" + operatingSystem: "Windows" + - displayName: "windows_net10" + dotnet: "net10.0" + image: "ADO-MMS22-SQL22" + operatingSystem: "Windows" + + - displayName: "linux_net8" + dotnet: "net8.0" + image: "ADO-UB22-SQL22" + operatingSystem: "Linux" + - displayName: "linux_net9" + dotnet: "net9.0" + image: "ADO-UB22-SQL22" + operatingSystem: "Linux" + - displayName: "linux_net10" + dotnet: "net10.0" + image: "ADO-UB22-SQL22" + operatingSystem: "Linux" + +variables: + - template: /eng/pipelines/common/variables/common-variables.yml@self + - template: /eng/pipelines/pr/variables/pr-variables.yml@self + +stages: + # Stage 1a: Build and pack all projects in the repository + - template: /eng/pipelines/pr/stages/pack-stage.yml@self + parameters: + buildConfiguration: Debug + buildSuffix: pr + stageName: ${{ variables.stageNamePack }} + + # Stage 1b: Generate secrets + - template: /eng/pipelines/pr/stages/generate-secrets-stage.yml@self + parameters: + stageName: ${{ variables.stageNameSecrets }} + + # Stage 2: Execute tests and collect code coverage + - template: /eng/pipelines/pr/stages/test-stages.yml@self + parameters: + buildConfiguration: Debug + buildSuffix: pr + dotnetVerbosity: ${{ parameters.dotnetVerbosity }} + poolName: $(PoolNameDefault) + stageNamePack: ${{ variables.stageNamePack }} + stageNameSecrets: ${{ variables.stageNameSecrets }} + testResultsArtifactBaseName: ${{ variables.testResultsArtifactBaseName }} + + platforms: ${{ parameters.platforms }} + + manualTestAzureKeyVaultUrl: $(AzureKeyVaultUrl) + manualTestAzureKeyVaultTenantId: $(AzureKeyVaultTenantId) + manualTestConnectionStringNpLocalhost: $(ConnectionStringNp_LocalhostDefault_UsernamePassword) + manualTestConnectionStringTcpLocalhost: $(ConnectionStringTcp_LocalhostDefault_UsernamePassword) + manualTestConnectionStringNpAzure: $(ConnectionStringNp_Azure_ManagedIdentity) + manualTestConnectionStringTcpAzure: $(ConnectionStringTcp_Azure_ManagedIdentity) + manualTestFileStreamDirectory: "$(Pipeline.Workspace)/filestream" + manualTestLocalDbAppName: $(LocalDbAppName) + manualTestLocalDbSharedInstanceName: $(LocalDbSharedInstanceName) + manualTestUserManagedIdentityClientId: $(UserManagedIdentityClientId) + + # Stage 3: Collect code coverage + - template: /eng/pipelines/pr/stages/collect-coverage-stage.yml@self + parameters: + dependsOn: + - ${{ each platform in parameters.platforms }}: + - "test_${{ platform.displayName }}" diff --git a/eng/pipelines/pr/stages/collect-coverage-stage.yml b/eng/pipelines/pr/stages/collect-coverage-stage.yml new file mode 100644 index 0000000000..2d03bb064f --- /dev/null +++ b/eng/pipelines/pr/stages/collect-coverage-stage.yml @@ -0,0 +1,75 @@ +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# + +parameters: + + # Names of the test stages that must complete before collecting coverage. + - name: dependsOn + type: object + default: [] + +stages: + - stage: collect_code_coverage + displayName: "Collect code coverage" + dependsOn: ${{ parameters.dependsOn }} + + jobs: + - job: collect_code_coverage + displayName: "Collect Code Coverage" + + pool: + vmImage: ubuntu-latest + + variables: + + # Set up a temporary directory that is cleaned up after each job run. This helps avoid + # disk space issues on pooled agents that may run many jobs before being retired. + - name: workingDir + value: "$(Agent.TempDirectory)/coverage" + + steps: + + # Part 1) Merge coverage reports ================================= + + # Install .NET SDK + - template: /eng/pipelines/common/steps/install-dotnet.yml@self + + # Restore additional dotnet tools + - template: /eng/pipelines/common/steps/restore-dotnet-tools.yml@self + + # Download coverage reports from the test jobs. + # NOTE: The artifact name is not specified, so *all* artifacts of the build will be + # checked for the coverage reports. + - task: DownloadPipelineArtifact@2 + displayName: Download coverage reports + inputs: + itemPattern: '**/*.coverage' + targetPath: "${{ variables.workingDir }}/originals" + + # Merge original files into a single Cobertura XML file + - script: >- + dotnet tool run dotnet-coverage -- merge "${{ variables.workingDir }}/originals/**/*.coverage" + --output "${{ variables.workingDir }}/merge/cobertura.xml" + --output-format cobertura + --log-file "${{ variables.workingDir }}/merge/merge.log" + --log-level Verbose + displayName: Merge coverage files + workingDirectory: $(REPO_ROOT) + + # Part 2) Publish merged results to the pipeline results/artifacts + + # Publish the merged coverage file as a pipeline artifact + - task: PublishPipelineArtifact@1 + displayName: Publish coverage artifact + inputs: + artifact: merged_coverage + targetPath: "${{ variables.workingDir }}/merge" + + # Publish the merged coverage file as coverage results so they can be viewed in ADO UI. + - task: PublishCodeCoverageResults@2 + displayName: Publish coverage results + inputs: + summaryFileLocation: "${{ variables.workingDir }}/merge/cobertura.xml" diff --git a/eng/pipelines/pr/stages/test-stages.yml b/eng/pipelines/pr/stages/test-stages.yml index 3a1e2c2011..f6023678b7 100644 --- a/eng/pipelines/pr/stages/test-stages.yml +++ b/eng/pipelines/pr/stages/test-stages.yml @@ -45,6 +45,12 @@ parameters: - name: stageNameSecrets type: string + # Name of the artifact to upload the test results (including code coverage) to. + # Note: This artifact will be used to store results from all jobs. Ensure that the folders used + # to store the results is unique to the test run. + - name: testResultsArtifactBaseName + type: string + # Manual Test Configuration Parameters ================================== # Azure Key Vault tenant ID that will be set in the config.jsonc file for sqlclient manual tests. @@ -129,6 +135,7 @@ stages: platformDotnet: ${{ platform.dotnet }} platformImage: ${{ platform.image }} poolName: ${{ parameters.poolName }} + testResultsArtifactBaseName: ${{ parameters.testResultsArtifactBaseName }} packageShortName: "Abstractions" testDisplayName: "abstractions" @@ -143,6 +150,7 @@ stages: platformDotnet: ${{ platform.dotnet }} platformImage: ${{ platform.image }} poolName: ${{ parameters.poolName }} + testResultsArtifactBaseName: ${{ parameters.testResultsArtifactBaseName }} packageShortName: "Azure" testDisplayName: "azure" @@ -157,6 +165,7 @@ stages: platformDotnet: ${{ platform.dotnet }} platformImage: ${{ platform.image }} poolName: ${{ parameters.poolName }} + testResultsArtifactBaseName: ${{ parameters.testResultsArtifactBaseName }} packageShortName: "SqlClient" testDisplayName: "sqlclient_functional" @@ -169,6 +178,7 @@ stages: buildSuffix: ${{ parameters.buildSuffix }} dotnetVerbosity: ${{ parameters.dotnetVerbosity }} stageNameSecrets: ${{ parameters.stageNameSecrets }} + testResultsArtifactBaseName: ${{ parameters.testResultsArtifactBaseName }} platformDisplayName: ${{ platform.displayName }} platformDotnet: ${{ platform.dotnet }} @@ -195,6 +205,7 @@ stages: buildSuffix: ${{ parameters.buildSuffix }} dotnetVerbosity: ${{ parameters.dotnetVerbosity }} stageNameSecrets: ${{ parameters.stageNameSecrets }} + testResultsArtifactBaseName: ${{ parameters.testResultsArtifactBaseName }} platformDisplayName: ${{ platform.displayName }} platformDotnet: ${{ platform.dotnet }} @@ -221,6 +232,7 @@ stages: buildSuffix: ${{ parameters.buildSuffix }} dotnetVerbosity: ${{ parameters.dotnetVerbosity }} stageNameSecrets: ${{ parameters.stageNameSecrets }} + testResultsArtifactBaseName: ${{ parameters.testResultsArtifactBaseName }} platformDisplayName: ${{ platform.displayName }} platformDotnet: ${{ platform.dotnet }} @@ -250,6 +262,7 @@ stages: platformDotnet: ${{ platform.dotnet }} platformImage: ${{ platform.image }} poolName: ${{ parameters.poolName }} + testResultsArtifactBaseName: ${{ parameters.testResultsArtifactBaseName }} packageShortName: "SqlClient" testDisplayName: "sqlclient_unit" diff --git a/eng/pipelines/pr/steps/publish-test-results-step.yml b/eng/pipelines/pr/steps/publish-test-results-step.yml new file mode 100644 index 0000000000..c64c3a974f --- /dev/null +++ b/eng/pipelines/pr/steps/publish-test-results-step.yml @@ -0,0 +1,63 @@ +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# + +# Publishes the test result files from the job to the pipeline and uploads the test results +# (including code coverage results) to a pipeline artifact. + +parameters: + + # Configuration to use to build the project. + - name: buildConfiguration + type: string + values: + - Debug + - Release + + # Display name of the platform. This will be formatted into the test result name. As such, only + # alphanumeric and _ characters are permitted. + - name: platformDisplayName + type: string + + # Display name of the test target that is being executed. This will be formatted into the test + # result name. As such, only alphanumeric and _ characters are permitted. + - name: testDisplayName + type: string + + # Name of the artifact to upload the test results (including code coverage) to. + - name: testResultsArtifactBaseName + type: string + + # Absolute path to the test results folder. This folder will be published to the artifact defined + # by ${{ testResultsArtifactBaseName }}_$(System.JobId) + - name: testResultsPath + type: string + +steps: + - pwsh: New-Item -ItemType Directory -Force -Path "${{ parameters.testResultsPath }}" + displayName: Ensure test results directory exists + condition: succeededOrFailed() + + # Publish the test results (trx files) to the pipeline + - task: PublishTestResults@2 + displayName: 'Publish Test Results' + inputs: + buildConfiguration: ${{ parameters.buildConfiguration }} + mergeTestResults: true + testResultsFiles: | + ${{ parameters.testResultsPath }}/*.trx + ${{ parameters.testResultsPath }}/**/*.coverage + testResultsFormat: VSTest + testRunTitle: "${{ parameters.platformDisplayName }}_${{ parameters.testDisplayName }}" + condition: succeededOrFailed() + + # Publish the test results as artifacts for the pipeline + - task: PublishPipelineArtifact@1 + displayName: 'Publish Test Artifacts' + inputs: + artifact: ${{ parameters.testResultsArtifactBaseName }}_$(System.JobId) + targetPath: ${{ parameters.testResultsPath }} + condition: succeededOrFailed() + diff --git a/eng/pipelines/pr/variables/pr-variables.yml b/eng/pipelines/pr/variables/pr-variables.yml index a9da73850e..cea536b0ba 100644 --- a/eng/pipelines/pr/variables/pr-variables.yml +++ b/eng/pipelines/pr/variables/pr-variables.yml @@ -24,8 +24,17 @@ variables: - group: sqlclient-testconfig-v1 # General Variables ====================================================== + + # Name to use for the packaging stage. This is to ensure that the packaging stage and the test + # stages (that will depend on the packaging stage passing) use the same name. - name: stageNamePack value: "pack_stage" + # Name to use for the secret generation stage. This is to ensure that the secret generation stage + # and the test stages (that will depend on the generated secret) use the same name. - name: stageNameSecrets value: "secrets_stage" + + # Base name of the test result artifacts. Job ID should be appended to this to prevent collision. + - name: testResultsArtifactBaseName + value: "test_results"