Update build asset staging to use managed identity (#6899)

dotnet · Jun 28, 2024 · eea5e09 · eea5e09
1 parent ee7b2f5
commit eea5e09
Show file tree

Hide file tree

Showing 9 changed files with 71 additions and 99 deletions.
diff --git a/eng/pipelines/dotnet-monitor-compliance.yml b/eng/pipelines/dotnet-monitor-compliance.yml
@@ -71,8 +71,6 @@ extends:
             arguments: >-
               -BarBuildId "$(BuildBarId)"
               -AzdoToken "$(dn-bot-all-drop-rw-code-rw-release-all)"
-              -MaestroToken "$(MaestroAccessToken)"
-              -GitHubToken "$(BotAccount-dotnet-bot-repo-PAT)"
               -DownloadTargetPath "$(System.ArtifactsDirectory)\BuildAssets"
               -SasSuffixes "$(dotnetbuilds-internal-checksums-container-read-token),$(dotnetbuilds-internal-container-read-token)"
               -ReleaseVersion "$(BuildVersion)"

diff --git a/eng/pipelines/stages/preparerelease.yml b/eng/pipelines/stages/preparerelease.yml
@@ -28,14 +28,18 @@ stages:
         packageType: runtime
         version: 6.x
         installationPath: '$(Build.Repository.LocalPath)\.dotnet'
+
     - script: mkdir $(System.ArtifactsDirectory)\StagingToolLogs
       displayName: Create Staging Tool Logs Directory
+
     - script: '$(Build.SourcesDirectory)\dotnet.cmd build $(Build.Repository.LocalPath)\eng\release\DiagnosticsReleaseTool\DiagnosticsReleaseTool.csproj -c Release /bl'
       workingDirectory: '$(System.ArtifactsDirectory)\StagingToolLogs'
       displayName: 'Build Staging Tool'
+
     # Run tool for release and test release branches
     - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/internal/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/test/release/'))) }}:
       - template: /eng/common/templates-official/post-build/setup-maestro-vars.yml@self
+
       - task: PowerShell@2
         displayName: Get Build Version
         inputs:
@@ -44,55 +48,68 @@ stages:
             -BarId $(BARBuildId)
             -MaestroToken $(MaestroAccessToken)
             -TaskVariableName 'BuildVersion'
-      - task: PowerShell@2
+
+      - task: AzureCLI@2
         displayName: 'Download Build Assets'
         inputs:
-          targetType: filePath
-          filePath: '$(Build.Repository.LocalPath)/eng/release/Scripts/AcquireBuild.ps1'
+          azureSubscription: 'Darc: Maestro Production'
+          scriptType: ps
+          scriptPath: '$(Build.Repository.LocalPath)/eng/release/Scripts/AcquireBuild.ps1'
           arguments: >-
             -BarBuildId "$(BARBuildId)"
             -AzdoToken "$(dn-bot-all-drop-rw-code-rw-release-all)"
-            -MaestroToken "$(MaestroAccessToken)"
-            -GitHubToken "$(BotAccount-dotnet-bot-repo-PAT)"
             -DownloadTargetPath "$(System.ArtifactsDirectory)\BuildAssets"
             -SasSuffixes "$(dotnetbuilds-internal-checksums-container-read-token),$(dotnetbuilds-internal-container-read-token)"
             -ReleaseVersion "$(Build.BuildNumber)"
           workingDirectory: '$(Build.Repository.LocalPath)'
         continueOnError: true
-      - script: >-
-          $(Build.SourcesDirectory)\dotnet.cmd run --project $(Build.Repository.LocalPath)\eng\release\DiagnosticsReleaseTool\DiagnosticsReleaseTool.csproj -c Release
-          --
-          prepare-release
-          --input-drop-path "$(System.ArtifactsDirectory)\BuildAssets"
-          --tool-manifest "$(Build.Repository.LocalPath)\eng\release\tool-list.json"
-          --staging-directory "$(System.ArtifactsDirectory)\AssetsLayout"
-          --release-name "$(Build.BuildNumber)"
-          --build-version "$(BuildVersion)"
-          --account-name "$(dotnet-diagnostics-storage-accountname)"
-          --account-key "$(dotnetstage-storage-key)"
-          --sas-valid-days "$(dotnet-diagnostics-storage-retentiondays)"
-          -v True
-        workingDirectory: '$(System.ArtifactsDirectory)\StagingToolLogs'
-        displayName: 'Stage Build Assets and Manifest'
+
+      - task: AzureCLI@2
+        displayName: 'Manifest Generation and Asset Publishing'
+        inputs:
+          workingDirectory: '$(System.ArtifactsDirectory)\StagingToolLogs'
+          azureSubscription: 'dotnetstage-dotnet-monitor-rw'
+          scriptType: pscore
+          scriptLocation: inlineScript
+          addSpnToEnvironment: true
+          inlineScript: >-
+            $(Build.SourcesDirectory)\dotnet.cmd run
+            --project $(Build.Repository.LocalPath)\eng\release\DiagnosticsReleaseTool\DiagnosticsReleaseTool.csproj
+            -c Release
+            --
+            prepare-release
+            --input-drop-path "$(System.ArtifactsDirectory)\BuildAssets"
+            --tool-manifest "$(Build.Repository.LocalPath)\eng\release\tool-list.json"
+            --staging-directory "$(System.ArtifactsDirectory)\AssetsLayout"
+            --release-name "$(Build.BuildNumber)"
+            --build-version "$(BuildVersion)"
+            --account-name "$(dotnet-diagnostics-storage-accountname)"
+            --container-name $(dotnet-monitor-container-name)
+            --client-id "$env:servicePrincipalId"
+            -v True
+
       - template: /eng/pipelines/steps/publish-pipeline-artifact.yml@self
         parameters:
           displayName: 'Upload Assets Layout'
           targetPath: '$(System.ArtifactsDirectory)\AssetsLayout'
           artifact: 'StagingToolAssetsLayout'
           is1ESPipeline: ${{ parameters.is1ESPipeline }}
+
       # Only tag build from real release branches
       - ${{ if not(startsWith(variables['Build.SourceBranch'], 'refs/heads/test/release/')) }}:
         - task: Powershell@2
           displayName: 'Tag Build with MonitorRelease'
           inputs:
             targetType: inline
             script: Write-Host "##vso[build.addbuildtag]MonitorRelease"
+
       - template: /eng/pipelines/steps/publish-pipeline-artifact.yml@self
         parameters:
           displayName: 'Upload Staging Tool Logs'
           targetPath: '$(System.ArtifactsDirectory)\StagingToolLogs'
           artifact: 'StagingToolLogs'
           is1ESPipeline: ${{ parameters.is1ESPipeline }}
+
     - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
       - task: Powershell@2
         displayName: 'Tag Build with update-docker'

diff --git a/eng/release/DiagnosticsReleaseTool/Common/AzureBlobPublisher.cs b/eng/release/DiagnosticsReleaseTool/Common/AzureBlobPublisher.cs
@@ -1,10 +1,10 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
-using Azure.Storage;
+using Azure.Core;
+using Azure.Identity;
 using Azure.Storage.Blobs;
 using Azure.Storage.Blobs.Models;
-using Azure.Storage.Sas;
 using Microsoft.Extensions.Logging;
 using System;
 using System.Buffers;
@@ -17,17 +17,14 @@ namespace ReleaseTool.Core
 {
     public class AzureBlobBublisher : IPublisher
     {
-        private const int ClockSkewSec = 15 * 60;
         private const int MaxRetries = 15;
         private const int MaxFullLoopRetries = 5;
         private readonly TimeSpan FullLoopRetryDelay = TimeSpan.FromSeconds(1);
-        private const string AccessPolicyDownloadId = "DownloadDrop";
 
         private readonly string _accountName;
-        private readonly string _accountKey;
+        private readonly string _clientId;
         private readonly string _containerName;
         private readonly string _buildVersion;
-        private readonly int _sasValidDays;
         private readonly ILogger _logger;
 
         private BlobContainerClient _client;
@@ -40,12 +37,17 @@ private Uri AccountBlobUri
             }
         }
 
-        private StorageSharedKeyCredential AccountCredential
+        private TokenCredential Credentials
         {
             get
             {
-                StorageSharedKeyCredential credential = new StorageSharedKeyCredential(_accountName, _accountKey);
-                return credential;
+                if (_clientId == null)
+                {
+                    // Local development scenario. Use the default credential.
+                    return new DefaultAzureCredential();
+                }
+
+                return new DefaultAzureCredential(new DefaultAzureCredentialOptions { ManagedIdentityClientId = _clientId });
             }
         }
 
@@ -68,13 +70,12 @@ private BlobClientOptions BlobOptions
             }
         }
 
-        public AzureBlobBublisher(string accountName, string accountKey, string containerName, string buildVersion, int sasValidDays, ILogger logger)
+        public AzureBlobBublisher(string accountName, string clientId, string containerName, string buildVersion, ILogger logger)
         {
             _accountName = accountName;
-            _accountKey = accountKey;
+            _clientId = clientId;
             _containerName = containerName;
             _buildVersion = buildVersion;
-            _sasValidDays = sasValidDays;
             _logger = logger;
         }
 
@@ -107,20 +108,11 @@ public async Task<string> PublishFileAsync(FileMapping fileMap, CancellationToke
 
                     await blobClient.UploadAsync(srcStream, overwrite: true, ct);
 
-                    BlobSasBuilder sasBuilder = new BlobSasBuilder()
-                    {
-                        BlobContainerName = client.Name,
-                        BlobName = blobClient.Name,
-                        Identifier = AccessPolicyDownloadId,
-                        Protocol = SasProtocol.Https
-                    };
-                    Uri accessUri = blobClient.GenerateSasUri(sasBuilder);
-
                     using BlobDownloadStreamingResult blobStream = (await blobClient.DownloadStreamingAsync(cancellationToken: ct)).Value;
                     srcStream.Position = 0;
                     completed = await VerifyFileStreamsMatchAsync(srcStream, blobStream, ct);
 
-                    result = accessUri;
+                    result = blobClient.Uri;
                 }
                 catch (IOException ioEx) when (!(ioEx is PathTooLongException))
                 {
@@ -155,7 +147,7 @@ private async Task<BlobContainerClient> GetClient(CancellationToken ct)
         {
             if (_client == null)
             {
-                BlobServiceClient serviceClient = new BlobServiceClient(AccountBlobUri, AccountCredential, BlobOptions);
+                BlobServiceClient serviceClient = new BlobServiceClient(AccountBlobUri, Credentials, BlobOptions);
                 _logger.LogInformation($"Attempting to connect to {serviceClient.Uri} to store blobs.");
 
                 BlobContainerClient newClient;
@@ -176,31 +168,6 @@ private async Task<BlobContainerClient> GetClient(CancellationToken ct)
                         continue;
                     }
 
-                    try
-                    {
-                        DateTime baseTime = DateTime.UtcNow;
-                        // Add the new (or update existing) "download" policy to the container
-                        // This is used to mint the SAS tokens without an expiration policy
-                        // Expiration can be added later by modifying this policy
-                        BlobSignedIdentifier downloadPolicyIdentifier = new BlobSignedIdentifier()
-                        {
-                            Id = AccessPolicyDownloadId,
-                            AccessPolicy = new BlobAccessPolicy()
-                            {
-                                Permissions = "r",
-                                PolicyStartsOn = new DateTimeOffset(baseTime.AddSeconds(-ClockSkewSec)),
-                                PolicyExpiresOn = new DateTimeOffset(DateTime.UtcNow.AddDays(_sasValidDays).AddSeconds(ClockSkewSec)),
-                            }
-                        };
-                        _logger.LogInformation($"Writing download access policy: {AccessPolicyDownloadId} to {_containerName}.");
-                        await newClient.SetAccessPolicyAsync(PublicAccessType.None, new BlobSignedIdentifier[] { downloadPolicyIdentifier }, cancellationToken: ct);
-                    }
-                    catch (Exception ex)
-                    {
-                        _logger.LogWarning(ex, $"Failed to write access policy for {_containerName}, retrying.");
-                        continue;
-                    }
-
                     _logger.LogInformation($"Container {_containerName} is ready.");
                     _client = newClient;
                     break;

diff --git a/eng/release/DiagnosticsReleaseTool/Config.cs b/eng/release/DiagnosticsReleaseTool/Config.cs
@@ -14,9 +14,8 @@ internal class Config
         public string ReleaseName { get; }
         public string BuildVersion { get; }
         public string AccountName { get; }
-        public string AccountKey { get; }
+        public string ClientId { get; }
         public string ContainerName { get; }
-        public int SasValidDays { get; }
 
         public Config(
             FileInfo toolManifest,
@@ -26,9 +25,8 @@ public Config(
             string releaseName,
             string buildVersion,
             string accountName,
-            string accountKey,
-            string containerName,
-            int sasValidDays)
+            string clientId,
+            string containerName)
         {
             ToolManifest = toolManifest;
             ShouldVerifyManifest = verifyToolManifest;
@@ -37,9 +35,8 @@ public Config(
             ReleaseName = releaseName;
             BuildVersion = buildVersion;
             AccountName = accountName;
-            AccountKey = accountKey;
+            ClientId = clientId;
             ContainerName = containerName;
-            SasValidDays = sasValidDays;
         }
     }
 }
diff --git a/eng/release/DiagnosticsReleaseTool/DiagnosticsReleaseCommandLine.cs b/eng/release/DiagnosticsReleaseTool/DiagnosticsReleaseCommandLine.cs
@@ -107,8 +107,9 @@ private static Option<string> AzureStorageAccountNameOption() =>
 
         private static Option<string> AzureStorageAccountKeyOption() =>
             new Option<string>(
-                aliases: new[] { "-k", "--account-key" },
-                description: "Storage account key, in base 64 format.")
+                aliases: new[] { "-k", "--client-id" },
+                description: "Identity Client ID. If left blank, ambient identity will be used.",
+                getDefaultValue: () => null)
             {
                 IsRequired = true,
             };

diff --git a/eng/release/DiagnosticsReleaseTool/DiagnosticsReleaseRunner.cs b/eng/release/DiagnosticsReleaseTool/DiagnosticsReleaseRunner.cs
@@ -49,7 +49,7 @@ internal async static Task<int> PrepareRelease(Config releaseConfig, bool verbos
 
             IPublisher releasePublisher = dryRun ?
                 new SkipPublisher() :
-                new AzureBlobBublisher(releaseConfig.AccountName, releaseConfig.AccountKey, releaseConfig.ContainerName, releaseConfig.BuildVersion, releaseConfig.SasValidDays, logger);
+                new AzureBlobBublisher(releaseConfig.AccountName, releaseConfig.ClientId, releaseConfig.ContainerName, releaseConfig.BuildVersion, logger);
             IManifestGenerator manifestGenerator = new DiagnosticsManifestGenerator(releaseMetadata, releaseConfig.ToolManifest, logger);
 
             using var diagnosticsRelease = new Release(

diff --git a/eng/release/DiagnosticsReleaseTool/DiagnosticsReleaseTool.csproj b/eng/release/DiagnosticsReleaseTool/DiagnosticsReleaseTool.csproj
@@ -13,15 +13,16 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Extensions.Configuration" Version="[5.0.0]" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="[5.0.0]" />
-    <PackageReference Include="Microsoft.Extensions.Logging" Version="[5.0.0]" />
-    <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="[5.0.0]" />
-    <PackageReference Include="Microsoft.Extensions.Logging.Configuration" Version="[5.0.0]" />
-    <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="[5.0.0]" />
+    <PackageReference Include="Microsoft.Extensions.Configuration" Version="[6.0.0]" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="[6.0.0]" />
+    <PackageReference Include="Microsoft.Extensions.Logging" Version="[6.0.0]" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="[6.0.0]" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Configuration" Version="[6.0.0]" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="[6.0.0]" />
 
-    <PackageReference Include="Azure.Storage.Blobs" Version="[12.13.0]" />
-    <PackageReference Include="System.CommandLine" Version="2.0.0-beta1.20467.2" />
+    <PackageReference Include="Azure.Identity" Version="[1.12.0]" />
+    <PackageReference Include="Azure.Storage.Blobs" Version="[12.20.0]" />
+    <PackageReference Include="System.CommandLine" Version="2.0.0-beta1.20468.1" />
   </ItemGroup>
 
 </Project>
diff --git a/eng/release/Directory.Build.props b/eng/release/Directory.Build.props
@@ -1,4 +1,3 @@
 <Project>
-    <Import Project="$(MSBuildThisFileDirectory)..\..\Directory.Build.props"/>
-    <Import Project="Sdk.props" Sdk="Microsoft.DotNet.Arcade.Sdk" />
+    <Import Project="$(MSBuildThisFileDirectory)..\Directory.Build.props"/>
 </Project>
diff --git a/eng/release/Scripts/AcquireBuild.ps1 b/eng/release/Scripts/AcquireBuild.ps1
@@ -4,9 +4,6 @@ param(
   [Parameter(Mandatory=$true)][string] $DownloadTargetPath,
   [Parameter(Mandatory=$true)][string] $SasSuffixes,
   [Parameter(Mandatory=$true)][string] $AzdoToken,
-  [Parameter(Mandatory=$true)][string] $MaestroToken,
-  [Parameter(Mandatory=$true)][string] $GitHubToken,
-  [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com',
   [Parameter(Mandatory=$false)][string] $DarcVersion = $null,
   [Parameter(Mandatory=$false)][bool] $Separated = $true,
   [switch] $help,
@@ -19,9 +16,6 @@ function Write-Help() {
     Write-Host "  -DownloadTargetPath <value>       Path to download the build to."
     Write-Host "  -SasSuffixes <value>              Comma separated list of potential uri suffixes that can be used if anonymous access to a blob uri fails. Appended directly to the end of the URI. Use full SAS syntax with ?."
     Write-Host "  -AzdoToken <value>                Azure DevOps token to use for builds queries"
-    Write-Host "  -MaestroToken <value>             Maestro token to use for querying BAR"
-    Write-Host "  -GitHubToken <value>              GitHub token to use for querying repository information"
-    Write-Host "  -MaestroApiEndPoint <value>       BAR endpoint to use for build queries."
     Write-Host "  -Separated <`$true|`$false>       Download files to their repo separated locations."
     Write-Host ""
 }
@@ -62,12 +56,10 @@ try {
         --output-dir $DownloadTargetPath `
         --overwrite `
         --sas-suffixes $SasSuffixes `
-        --github-pat $GitHubToken `
         --azdev-pat $AzdoToken `
-        --bar-uri $MaestroApiEndPoint `
-        --password $MaestroToken `
         --verbose `
         --continue-on-error `
+        --ci `
         $separatedArgs
 
     if ($LastExitCode -ne 0) {