From ec19441b8f07b39ec068d18a1e128c1bfc79a76d Mon Sep 17 00:00:00 2001 From: Juan Sebastian Hoyos Ayala Date: Mon, 24 Jun 2024 18:04:07 -0700 Subject: [PATCH 1/3] Separate testing stage --- diagnostics-codeql.yml | 155 ---------------------------------------- diagnostics.yml | 64 ++++++++++------- eng/pipelines/build.yml | 34 +++------ 3 files changed, 49 insertions(+), 204 deletions(-) delete mode 100644 diagnostics-codeql.yml diff --git a/diagnostics-codeql.yml b/diagnostics-codeql.yml deleted file mode 100644 index 5cacd47b62..0000000000 --- a/diagnostics-codeql.yml +++ /dev/null @@ -1,155 +0,0 @@ -parameters: -- name: runtimeFeed - displayName: Feed for runtime installation - type: string - default: default - values: - - default -- name: runtimeFeedToken - displayName: Base 64 SAS Token for runtime installation - type: string - default: default - values: - - default - -trigger: - none - -schedules: - - cron: 0 3 * * SUN - displayName: Weekly Sunday CodeQL run - branches: - include: - - main - always: true - -variables: - # CG is handled in the primary CI pipeline - - name: skipComponentGovernanceDetection - value: true - -extends: - template: /eng/pipelines/pipeline-resources.yml - parameters: - stages: - - stage: build - displayName: Build and Test Diagnostics - jobs: - - template: /eng/pipelines/build.yml - parameters: - name: Windows - osGroup: Windows_NT - isCodeQLRun: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: x64 - Build_Release_x86: - _BuildConfig: Release - _BuildArch: x86 - Build_Release_arm: - _BuildConfig: Release - _BuildArch: arm - Build_Release_arm64: - _BuildConfig: Release - _BuildArch: arm64 - - - template: /eng/pipelines/build.yml - parameters: - name: Linux_x64 - osGroup: Linux - nativeBuildContainer: linux_x64 - isCodeQLRun: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: x64 - - - template: /eng/pipelines/build.yml - parameters: - name: Linux_musl - osGroup: Linux - nativeBuildContainer: linux_musl_x64 - isCodeQLRun: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: x64 - - - template: /eng/pipelines/build.yml - parameters: - name: MacOS - osGroup: MacOS - isCodeQLRun: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: x64 - - - template: /eng/pipelines/build.yml - parameters: - name: MacOS_arm64 - osGroup: MacOS_cross - crossBuild: true - isCodeQLRun: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: arm64 - - - template: /eng/pipelines/build.yml - parameters: - name: Linux_arm - osGroup: Linux - nativeBuildContainer: linux_arm - crossBuild: true - isCodeQLRun: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: arm - - - template: /eng/pipelines/build.yml - parameters: - name: Linux_arm64 - osGroup: Linux - nativeBuildContainer: linux_arm64 - crossBuild: true - isCodeQLRun: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: arm64 - - - template: /eng/pipelines/build.yml - parameters: - name: Linux_musl_arm - osGroup: Linux - nativeBuildContainer: linux_musl_arm - crossBuild: true - isCodeQLRun: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: arm - - - template: /eng/pipelines/build.yml - parameters: - name: Linux_musl_arm64 - osGroup: Linux - nativeBuildContainer: linux_musl_arm64 - crossBuild: true - isCodeQLRun: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: arm64 diff --git a/diagnostics.yml b/diagnostics.yml index 03678baa49..b975b6b748 100644 --- a/diagnostics.yml +++ b/diagnostics.yml @@ -41,32 +41,20 @@ extends: parameters: stages: - stage: build - displayName: Build and Test Diagnostics + displayName: Build Diagnostics Assets jobs: - ############################ - # # - # Source Build legs # - # # - ############################ - - template: ${{ variables.sourceBuildTemplate }} parameters: platform: name: Complete buildScript: ./eng/common/build.sh - ############################ - # # - # Build legs # - # # - ############################ - - template: /eng/pipelines/build.yml parameters: jobTemplate: ${{ variables.jobTemplate }} - name: Windows - osGroup: Windows_NT + osGroup: Windows + buildOnly: true buildConfigs: - configuration: Debug architecture: x64 @@ -123,6 +111,7 @@ extends: parameters: jobTemplate: ${{ variables.jobTemplate }} osGroup: MacOS + buildOnly: true buildConfigs: - configuration: Release architecture: x64 @@ -200,11 +189,39 @@ extends: artifactUploadPath: bin/linux.arm64.Release artifactTargetPath: bin/linux-musl.arm64.Release - ############################ - # # - # Test only legs # - # # - ############################ + - stage: test + displayName: Run Diagnostics Tests + jobs: + + - template: /eng/pipelines/build.yml + parameters: + jobTemplate: ${{ variables.jobTemplate }} + osGroup: Windows + testOnly: true + buildConfigs: + - configuration: Debug + architecture: x64 + - configuration: Release + architecture: x64 + - configuration: Release + architecture: x86 + - ${{ if ne(variables['System.TeamProject'], 'public') }}: + - configuration: Release + architecture: arm + - configuration: Release + architecture: arm64 + + - template: /eng/pipelines/build.yml + parameters: + jobTemplate: ${{ variables.jobTemplate }} + osGroup: MacOS + testOnly: true + buildConfigs: + - configuration: Release + architecture: x64 + - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: + - configuration: Debug + architecture: x64 - template: /eng/pipelines/build.yml parameters: @@ -212,7 +229,6 @@ extends: name: Ubuntu_20_04 osGroup: Linux container: test_ubuntu_20_04 - dependsOn: Linux testOnly: true buildConfigs: - configuration: Release @@ -227,7 +243,6 @@ extends: name: Ubuntu_22_04 osGroup: Linux container: test_ubuntu_22_04 - dependsOn: Linux testOnly: true buildConfigs: - configuration: Release @@ -243,7 +258,6 @@ extends: osGroup: Linux osSuffix: -musl container: test_linux_musl_x64 - dependsOn: Linux_musl testOnly: true disableComponentGovernance: true buildConfigs: @@ -261,7 +275,6 @@ extends: osGroup: Linux container: test_debian_11_amd64 dependsOn: Linux - testOnly: true buildConfigs: - configuration: Release architecture: x64 @@ -275,7 +288,6 @@ extends: name: Fedora_39 osGroup: Linux container: test_fedora - dependsOn: Linux testOnly: true buildConfigs: - configuration: Release @@ -287,6 +299,7 @@ extends: - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - stage: package displayName: Package, Sign, and Generate BAR Manifests + dependsOn: build jobs: - template: /eng/common/templates-official/job/job.yml parameters: @@ -366,6 +379,7 @@ extends: enableNugetValidation: true symbolPublishingAdditionalParameters: '/p:PublishSpecialClrFiles=false' publishInstallersAndChecksums: true + publishAssetsImmediately: true # This sets up the bits to do a Release. - template: /eng/pipelines/prepare-release.yml diff --git a/eng/pipelines/build.yml b/eng/pipelines/build.yml index 7cf00df2b5..ea6b97d20f 100644 --- a/eng/pipelines/build.yml +++ b/eng/pipelines/build.yml @@ -13,9 +13,9 @@ parameters: - name: osGroup type: string - default: Windows_NT + default: Windows values: - - Windows_NT + - Windows - Linux - MacOS - FreeBSD @@ -74,10 +74,6 @@ parameters: type: string default: '' -- name: isCodeQLRun - type: boolean - default: false - - name: disableComponentGovernance type: boolean default: false @@ -89,7 +85,7 @@ jobs: name: ${{ coalesce(parameters.name, parameters.osGroup) }}_${{ config.architecture }}_${{ config.configuration }} timeoutInMinutes: ${{ parameters.timeoutInMinutes }} enableMicrobuild: true - runAsPublic: ${{ parameters.isCodeQLRun }} + runAsPublic: ${{ eq(variables['System.TeamProject'], 'public') }} disableComponentGovernance: ${{ parameters.disableComponentGovernance }} templateContext: ${{ parameters.templateContext }} @@ -100,7 +96,7 @@ jobs: demands: ImageOverride -equals $(LinuxImage) os: linux - ${{ elseif eq(parameters.osGroup, 'Windows_NT') }}: + ${{ elseif eq(parameters.osGroup, 'Windows') }}: name: $(BuildPool) demands: ImageOverride -equals $(WindowsImage) os: windows @@ -139,9 +135,9 @@ jobs: - _TestArgs: '-test' - _Cross: '' - - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: + - ${{ if eq(parameters.osGroup, 'Windows') }}: - _buildScript: $(Build.SourcesDirectory)\build.cmd - - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: + - ${{ else }}: - _buildScript: $(Build.SourcesDirectory)/build.sh - ${{ if and(eq(parameters.testOnly, 'true'), eq(parameters.buildOnly, 'true')) }}: @@ -150,12 +146,12 @@ jobs: - ${{ if eq(parameters.testOnly, 'true') }}: - _TestArgs: '-test -skipnative' - - ${{ if or(eq(parameters.buildOnly, 'true'), eq(parameters.isCodeQLRun, 'true')) }}: + - ${{ if eq(parameters.buildOnly, 'true') }}: - _TestArgs: '' # For testing msrc's and service releases. The RuntimeSourceVersion is either "default" or the service release version to test - _InternalInstallArgs: '' - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.isCodeQLRun, 'false')) }}: + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - _InternalInstallArgs: -dotnetruntimeversion '$(DotnetRuntimeVersion)' -dotnetruntimedownloadversion '$(DotnetRuntimeDownloadVersion)' @@ -178,13 +174,11 @@ jobs: steps: - ${{ if eq(parameters.testOnly, true) }}: - - ${{ if ne(parameters.osGroup, 'Linux') }}: - - 'error, testOnly is only supported on Linux': error - task: DownloadPipelineArtifact@2 displayName: 'Download Build Artifacts' inputs: targetPath: '$(Build.ArtifactStagingDirectory)/__download__' - itemPattern: Build_${{ parameters.dependsOn }}_${{ config.architecture }}_${{ config.configuration }}/bin/** + itemPattern: Build_${{ parameters.osGroup }}${{ replace(parameters.osSuffix, '-', '_') }}_${{ config.architecture }}_${{ config.configuration }}/bin/** checkDownloadedFiles: true - task: CopyFiles@2 displayName: 'Binplace Downloaded Product' @@ -192,10 +186,6 @@ jobs: sourceFolder: $(Build.ArtifactStagingDirectory)/__download__/Build_${{ parameters.dependsOn }}_${{ config.architecture }}_${{ config.configuration }}/bin/linux${{ parameters.osSuffix }}.${{ config.architecture }}.${{ config.configuration }} targetFolder: '$(Build.SourcesDirectory)/artifacts/bin/linux.${{ config.architecture }}.${{ config.configuration }}' - - ${{ if eq(parameters.isCodeQLRun, 'true') }}: - - task: CodeQL3000Init@0 - displayName: CodeQL Initialize - - script: $(_buildScript) -ci -configuration ${{ config.configuration }} @@ -226,10 +216,6 @@ jobs: displayName: Build Native target: ${{ parameters.nativeBuildContainer }} - - ${{ if eq(parameters.isCodeQLRun, 'true') }}: - - task: CodeQL3000Finalize@0 - displayName: CodeQL Finalize - - ${{ if ne(config.artifactUploadPath, '') }}: - task: CopyFiles@2 displayName: Gather binaries for publish @@ -288,7 +274,7 @@ jobs: continueOnError: true condition: always() - - ${{ if and(ne(parameters.buildOnly, 'true'), ne(parameters.isCodeQLRun, 'true')) }}: + - ${{ if ne(parameters.buildOnly, 'true') }}: # Publish test results to Azure Pipelines - task: PublishTestResults@2 inputs: From c08e850f7637a0a9b168ad10d7d8f2d19d3f6eb3 Mon Sep 17 00:00:00 2001 From: Juan Sebastian Hoyos Ayala Date: Mon, 24 Jun 2024 21:40:29 -0700 Subject: [PATCH 2/3] Make prepare release always build --- diagnostics.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/diagnostics.yml b/diagnostics.yml index b975b6b748..4103cfe483 100644 --- a/diagnostics.yml +++ b/diagnostics.yml @@ -381,5 +381,5 @@ extends: publishInstallersAndChecksums: true publishAssetsImmediately: true - # This sets up the bits to do a Release. - - template: /eng/pipelines/prepare-release.yml + # This sets up the bits to do a Release. + - template: /eng/pipelines/prepare-release.yml From 9fc28172d9076c89bf7192aa46ec513f62dc2326 Mon Sep 17 00:00:00 2001 From: Juan Sebastian Hoyos Ayala Date: Mon, 24 Jun 2024 21:46:34 -0700 Subject: [PATCH 3/3] Remove debian dep --- diagnostics.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/diagnostics.yml b/diagnostics.yml index 4103cfe483..be04114332 100644 --- a/diagnostics.yml +++ b/diagnostics.yml @@ -274,7 +274,6 @@ extends: name: Debian_Bullseye osGroup: Linux container: test_debian_11_amd64 - dependsOn: Linux buildConfigs: - configuration: Release architecture: x64