From 92aa57ba85c15bcefa4bc970a15f7818ad844bb0 Mon Sep 17 00:00:00 2001 From: Matt Mitchell Date: Tue, 2 Jul 2024 07:20:51 -0700 Subject: [PATCH] Use federated connections for storage account and apiscan access (#6892) * Use federated connections for storage account and apiscan access * Remove other uses of dotnetbuilds storage account variables * Do not install pwsh in some jobs (already in container) * Don't set working directory for nodejs steps to .dotnet dir --- eng/pipelines/dotnet-monitor-compliance.yml | 47 ++++++++++++++++++--- eng/pipelines/jobs/build.yml | 4 +- eng/pipelines/jobs/pack-sign-publish.yml | 3 +- eng/pipelines/jobs/sign-binaries.yml | 4 +- eng/pipelines/jobs/test-binaries.yml | 17 +------- eng/pipelines/jobs/tpn.yml | 4 +- eng/pipelines/stages/preparerelease.yml | 23 +++++++++- 7 files changed, 77 insertions(+), 25 deletions(-) diff --git a/eng/pipelines/dotnet-monitor-compliance.yml b/eng/pipelines/dotnet-monitor-compliance.yml index 86caff41e50..4194e3e36a3 100644 --- a/eng/pipelines/dotnet-monitor-compliance.yml +++ b/eng/pipelines/dotnet-monitor-compliance.yml @@ -8,8 +8,6 @@ pr: none variables: - template: /eng/common/templates-official/variables/pool-providers.yml@self -- group: DotNet-ApiScan -- group: DotNetBuilds storage account read tokens - group: Release-Pipeline - name: _TeamName value: DotNetCore @@ -31,7 +29,7 @@ extends: workspace: clean: all - steps: + steps: - task: PowerShell@2 displayName: Get BAR ID inputs: @@ -41,7 +39,7 @@ extends: -TaskVariableName 'BuildBarId' env: SYSTEM_ACCESSTOKEN: $(System.AccessToken) - + - task: PowerShell@2 displayName: 'Get Build Version (Full)' inputs: @@ -61,6 +59,28 @@ extends: -TaskVariableName 'BuildMajorMinorVersion' -MajorMinorOnly + # Populate dotnetbuilds-internal-container-read-token + - template: /eng/common/templates-official/steps/get-delegation-sas.yml + parameters: + federatedServiceConnection: 'dotnetbuilds-internal-read' + outputVariableName: 'dotnetbuilds-internal-checksums-container-read-token' + expiryInHours: 1 + base64Encode: false + storageAccount: dotnetbuilds + container: internal-checksums + permissions: rl + + # Populate dotnetbuilds-internal-container-read-token + - template: /eng/common/templates-official/steps/get-delegation-sas.yml + parameters: + federatedServiceConnection: 'dotnetbuilds-internal-read' + outputVariableName: 'dotnetbuilds-internal-container-read-token' + expiryInHours: 1 + base64Encode: false + storageAccount: dotnetbuilds + container: internal + permissions: rl + # Only scan the files that are being shipped; use the same gathering procedure # that the asset staging process uses. - task: PowerShell@2 @@ -100,6 +120,23 @@ extends: -SourcePath '$(System.ArtifactsDirectory)\UnpackedBinaries' -TargetPath '$(System.ArtifactsDirectory)\ScannableBinaries' + - task: AzureCLI@2 + displayName: 'Login to APISCan Identity (Step 1)' + inputs: + azureSubscription: 'dotnet-apiscan' + scriptType: pscore + scriptLocation: inlineScript + addSpnToEnvironment: true + inlineScript: | + echo "##vso[task.setvariable variable=ARM_CLIENT_ID;issecret=true]$env:servicePrincipalId" + echo "##vso[task.setvariable variable=ARM_ID_TOKEN;issecret=true]$env:idToken" + echo "##vso[task.setvariable variable=ARM_TENANT_ID;issecret=true]$env:tenantId" + # Re-log in + - script: | + az login --service-principal -u $(ARM_CLIENT_ID) --tenant $(ARM_TENANT_ID) --allow-no-subscriptions --federated-token $(ARM_ID_TOKEN) + + displayName: 'Login to APISCan Identity (Step 2)' + - task: APIScan@2 displayName: Run APIScan inputs: @@ -109,7 +146,7 @@ extends: softwareBuildNum: '$(resources.pipeline.Build.runID)' symbolsFolder: 'SRV*http://symweb;$(System.ArtifactsDirectory)\UnpackedSymbols' env: - AzureServicesAuthConnectionString: runAs=App;AppId=$(apiscan-service-principal-app-id);TenantId=72f988bf-86f1-41af-91ab-2d7cd011db47;AppKey=$(apiscan-service-principal-app-secret) + AzureServicesAuthConnectionString: RunAs=Developer;DeveloperTool=AzureCli continueOnError: true - task: PublishSecurityAnalysisLogs@3 diff --git a/eng/pipelines/jobs/build.yml b/eng/pipelines/jobs/build.yml index 6c2c33613c4..61fc94a6538 100644 --- a/eng/pipelines/jobs/build.yml +++ b/eng/pipelines/jobs/build.yml @@ -108,7 +108,6 @@ jobs: - _CrossBuildArgs: '-cross' - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - - group: DotNetBuilds storage account read tokens - _InternalInstallArgs: >- -RuntimeSourceFeed https://dotnetbuilds.blob.core.windows.net/internal -RuntimeSourceFeedKey $(dotnetbuilds-internal-container-read-token-base64) @@ -140,6 +139,9 @@ jobs: - ${{ each step in parameters.preBuildSteps }}: - ${{ step }} + # Populate internal runtime access variables + - template: /eng/common/templates/steps/enable-internal-runtimes.yml + - script: >- $(Build.SourcesDirectory)/eng/cibuild$(scriptExt) -configuration ${{ parameters.configuration }} diff --git a/eng/pipelines/jobs/pack-sign-publish.yml b/eng/pipelines/jobs/pack-sign-publish.yml index c0aea9601e7..06cc2f61ec9 100644 --- a/eng/pipelines/jobs/pack-sign-publish.yml +++ b/eng/pipelines/jobs/pack-sign-publish.yml @@ -15,7 +15,6 @@ jobs: variables: - _BuildConfig: Release - _SignType: real - - group: DotNetBuilds storage account read tokens steps: - task: DownloadPipelineArtifact@2 displayName: Download Binaries @@ -35,6 +34,8 @@ jobs: - template: /eng/pipelines/steps/setup-nuget-sources.yml@self parameters: osGroup: Windows + # Populate internal runtime access variables + - template: /eng/common/templates/steps/enable-internal-runtimes.yml - script: >- $(Build.SourcesDirectory)/eng/cipacksignpublish.cmd /p:TeamName=$(_TeamName) diff --git a/eng/pipelines/jobs/sign-binaries.yml b/eng/pipelines/jobs/sign-binaries.yml index 79c8ba34241..00ac5543744 100644 --- a/eng/pipelines/jobs/sign-binaries.yml +++ b/eng/pipelines/jobs/sign-binaries.yml @@ -15,7 +15,6 @@ jobs: variables: - _BuildConfig: ${{ parameters.configuration }} - _SignType: real - - group: DotNetBuilds storage account read tokens steps: - task: DownloadPipelineArtifact@2 @@ -28,6 +27,9 @@ jobs: parameters: osGroup: Windows + # Populate internal runtime access variables + - template: /eng/common/templates/steps/enable-internal-runtimes.yml + - script: >- $(Build.SourcesDirectory)/restore.cmd -configuration ${{ parameters.configuration }} diff --git a/eng/pipelines/jobs/test-binaries.yml b/eng/pipelines/jobs/test-binaries.yml index 2e1a251315b..fbeb17f99be 100644 --- a/eng/pipelines/jobs/test-binaries.yml +++ b/eng/pipelines/jobs/test-binaries.yml @@ -79,21 +79,11 @@ jobs: - ${{ if eq(parameters.useHelix, 'true')}}: - ${{ if ne(parameters.osGroup, 'Linux_Musl')}}: - # Linux builds (of all variants) run in Mariner containers and do not include pwsh by default - ${{ if eq(parameters.osGroup, 'Linux') }}: - - script: | - $(Build.SourcesDirectory)/restore.sh - $(Build.SourcesDirectory)/.dotnet/dotnet tool install --global PowerShell - echo "##vso[task.prependpath]/home/cloudtest_azpcontainer/.dotnet/tools" - displayName: Install pwsh - # Calculate the fully qualified Nodejs version first so that any new releases will result in a new cache key - script: | - pwsh ../eng/helix/GetNodejsVersion.ps1 -MajorVersion $(NodeMajorVersion) -TaskVariableName 'FqNodejsVersion' + pwsh ./eng/helix/GetNodejsVersion.ps1 -MajorVersion $(NodeMajorVersion) -TaskVariableName 'FqNodejsVersion' displayName: Calculate Node.js version Linux - workingDirectory: "$(Build.SourcesDirectory)/.dotnet" - env: - DOTNET_ROOT: "$(Build.SourcesDirectory)/.dotnet" - task: Cache@2 displayName: Node.js Cache Linux @@ -103,11 +93,8 @@ jobs: path: $(HelixNodejsPayloadPath) - script: | - pwsh ../eng/helix/InstallNodejs.ps1 -Version $(FqNodejsVersion) -Architecture ${{ parameters.architecture }} -DestinationFolder "$(HelixNodejsPayloadPath)" + pwsh ./eng/helix/InstallNodejs.ps1 -Version $(FqNodejsVersion) -Architecture ${{ parameters.architecture }} -DestinationFolder "$(HelixNodejsPayloadPath)" displayName: Hydrate Node.js Installation Linux - workingDirectory: "$(Build.SourcesDirectory)/.dotnet" - env: - DOTNET_ROOT: "$(Build.SourcesDirectory)/.dotnet" - ${{ else }}: - pwsh: eng/helix/GetNodejsVersion.ps1 diff --git a/eng/pipelines/jobs/tpn.yml b/eng/pipelines/jobs/tpn.yml index 300f868479d..7ec25caa3b4 100644 --- a/eng/pipelines/jobs/tpn.yml +++ b/eng/pipelines/jobs/tpn.yml @@ -8,7 +8,6 @@ jobs: variables: - _InternalInstallArgs: '' - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - - group: DotNetBuilds storage account read tokens - _InternalInstallArgs: >- /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) @@ -17,6 +16,9 @@ jobs: parameters: osGroup: Windows + # Populate internal runtime access variables + - template: /eng/common/templates/steps/enable-internal-runtimes.yml + # Only restore the projects that are shipped so only packages we ship get included in the below CG scan - script: >- $(Build.SourcesDirectory)/restore.cmd -ci diff --git a/eng/pipelines/stages/preparerelease.yml b/eng/pipelines/stages/preparerelease.yml index 743f9aaf2d4..32f6f9782fc 100644 --- a/eng/pipelines/stages/preparerelease.yml +++ b/eng/pipelines/stages/preparerelease.yml @@ -20,7 +20,6 @@ stages: - group: DotNet-Diagnostics-Storage - group: DotNet-DotNetStage-Storage - group: Release-Pipeline - - group: DotNetBuilds storage account read tokens steps: - task: UseDotNet@2 displayName: 'Use .NET 6' @@ -49,6 +48,28 @@ stages: -MaestroToken $(MaestroAccessToken) -TaskVariableName 'BuildVersion' + # Populate dotnetbuilds-internal-container-read-token + - template: /eng/common/templates-official/steps/get-delegation-sas.yml + parameters: + federatedServiceConnection: 'dotnetbuilds-internal-read' + outputVariableName: 'dotnetbuilds-internal-checksums-container-read-token' + expiryInHours: 1 + base64Encode: false + storageAccount: dotnetbuilds + container: internal-checksums + permissions: rl + + # Populate dotnetbuilds-internal-container-read-token + - template: /eng/common/templates-official/steps/get-delegation-sas.yml + parameters: + federatedServiceConnection: 'dotnetbuilds-internal-read' + outputVariableName: 'dotnetbuilds-internal-container-read-token' + expiryInHours: 1 + base64Encode: false + storageAccount: dotnetbuilds + container: internal + permissions: rl + - task: AzureCLI@2 displayName: 'Download Build Assets' inputs: