Merge main to feature/9.x (#6914)

.github/workflows/spellcheck.yml

@@ -17,23 +17,23 @@ jobs:
         with:
           persist-credentials: false
 
-      - uses: streetsidesoftware/cspell-action@104110db58e8c9a11c1c6be025e2082f4dded3bb
+      - uses: streetsidesoftware/cspell-action@542d05c6a8980c81277ec229f9beadf4ab3f5a34
         name: Documentation spellcheck
         if: ${{ !cancelled() }}
         with:
             files: '**/*.md'
             inline: error
             incremental_files_only: true
 
-      - uses: streetsidesoftware/cspell-action@104110db58e8c9a11c1c6be025e2082f4dded3bb
+      - uses: streetsidesoftware/cspell-action@542d05c6a8980c81277ec229f9beadf4ab3f5a34
         name: Resx spellcheck
         if: ${{ !cancelled() }}
         with:
             files: 'src/**/*.resx'
             inline: error
             incremental_files_only: true
 
-      - uses: streetsidesoftware/cspell-action@104110db58e8c9a11c1c6be025e2082f4dded3bb
+      - uses: streetsidesoftware/cspell-action@542d05c6a8980c81277ec229f9beadf4ab3f5a34
         name: Source code spellcheck
         if: ${{ !cancelled() }}
         with:

eng/dependabot/nuget.org/Versions.props

@@ -2,6 +2,6 @@
   <!-- Import references updated by Dependabot. -->
   <PropertyGroup>
     <!-- dotnet/diagnostics references -->
-    <MicrosoftDiagnosticsMonitoringShippedVersion>8.0.510501</MicrosoftDiagnosticsMonitoringShippedVersion>
+    <MicrosoftDiagnosticsMonitoringShippedVersion>8.0.532401</MicrosoftDiagnosticsMonitoringShippedVersion>
   </PropertyGroup>
 </Project>

eng/pipelines/dotnet-monitor-compliance.yml

@@ -71,8 +71,6 @@ extends:
             arguments: >-
               -BarBuildId "$(BuildBarId)"
               -AzdoToken "$(dn-bot-all-drop-rw-code-rw-release-all)"
-              -MaestroToken "$(MaestroAccessToken)"
-              -GitHubToken "$(BotAccount-dotnet-bot-repo-PAT)"
               -DownloadTargetPath "$(System.ArtifactsDirectory)\BuildAssets"
               -SasSuffixes "$(dotnetbuilds-internal-checksums-container-read-token),$(dotnetbuilds-internal-container-read-token)"
               -ReleaseVersion "$(BuildVersion)"

eng/pipelines/dotnet-monitor-release.yml

@@ -70,7 +70,7 @@ extends:
               -BarId $(BarId)
               -MaestroToken $(MaestroAccessToken)
               -TaskVariableName 'ReleaseVersion'
-        
+
         - task: PowerShell@2
           displayName: Get Build Version
           inputs:
@@ -107,10 +107,15 @@ extends:
 
         variables:
         - ${{ if eq(parameters.IsTestRun, 'true') }}:
-          - group: DotNet-Diagnostics-Storage-Test
+          - name: DestinationAccountName
+            value: monitortestcli
+          - name: ChecksumsAccountName
+            value: monitortestchecksums
         - ${{ else }}:
-          - group: DotNetCli storage account tokens
-        - group: DotNet-DotNetStage-Storage
+          - name: DestinationAccountName
+            value: dotnetcli
+          - name: ChecksumsAccountName
+            value: dotnetclichecksums
 
         workspace:
           clean: all
@@ -164,42 +169,28 @@ extends:
               - powershell: Install-Module Az.Storage -Force -Scope CurrentUser -AllowClobber -Verbose -RequiredVersion 5.10.1
                 displayName: Install Az.Storage Module 5.10.1
 
-              - powershell: |
-                  Write-Host "##vso[task.setvariable variable=DestinationAccountName]$env:DESTINATION_ACCOUNT_NAME"
-                  Write-Host "##vso[task.setvariable variable=DestinationSasTokenBase64;issecret=true]$env:DESTINATION_SAS_TOKEN_BASE64"
-                  Write-Host "##vso[task.setvariable variable=ChecksumsAccountName]$env:CHECKSUMS_ACCOUNT_NAME"
-                  Write-Host "##vso[task.setvariable variable=ChecksumsSasTokenBase64;issecret=true]$env:CHECKSUMS_SAS_TOKEN_BASE64"
-                displayName: Set Storage Accounts
-                ${{ if eq(parameters.IsTestRun, 'true') }}:
-                  env:
-                    # Variables provided by DotNet-Diagnostics-Storage-Test group
-                    DESTINATION_ACCOUNT_NAME: $(dotnet-monitor-test-storage-accountname)
-                    DESTINATION_SAS_TOKEN_BASE64: $(dotnet-monitor-test-blob-write-token-base64)
-                    CHECKSUMS_ACCOUNT_NAME: $(dotnet-monitor-checksums-test-storage-accountname)
-                    CHECKSUMS_SAS_TOKEN_BASE64: $(dotnet-monitor-checksums-test-blob-write-token-base64)
-                ${{ else }}:
-                  env:
-                    # Variables provided by "DotNetCli storage account tokens" group
-                    DESTINATION_ACCOUNT_NAME: dotnetcli
-                    DESTINATION_SAS_TOKEN_BASE64: $(dotnetcli-account-blob-write-token-base64)
-                    CHECKSUMS_ACCOUNT_NAME: dotnetclichecksums
-                    CHECKSUMS_SAS_TOKEN_BASE64: $(dotnetclichecksums-account-blob-write-token-base64)
-
-              - task: PowerShell@2
+              - task: AzureCLI@2
                 displayName: Publish Assets
                 inputs:
-                  filePath: $(Build.SourcesDirectory)/eng/release/Scripts/PublishToBlobAccounts.ps1
+                  # It seems that azureSubscription can't use runtime expressions, so we need to use a compile time expression
+                  # to set it.
+                  ${{ if eq(parameters.IsTestRun, 'true') }}:
+                    azureSubscription: dotnet-monitor-test-publish
+                  ${{ else }}:
+                    azureSubscription: dotnet-monitor-cli-storage-accounts-publish
+                  # Save the service principal details to the environment so that azcopy can use them
+                  addSpnToEnvironment: true
+                  scriptType: ps
+                  scriptLocation: scriptPath
+                  scriptPath: $(Build.SourcesDirectory)/eng/release/Scripts/PublishToBlobAccounts.ps1
                   arguments: >-
                     -AzCopyPath $(AzCopyPath)
                     -BuildVersion $(BuildVersion)
                     -ReleaseVersion $(ReleaseVersion)
-                    -DotnetStageAccountKey $(dotnetstage-storage-key)
                     -DestinationAccountName $(DestinationAccountName)
-                    -DestinationSasTokenBase64 $(DestinationSasTokenBase64)
                     -ChecksumsAccountName $(ChecksumsAccountName)
-                    -ChecksumsSasTokenBase64 $(ChecksumsSasTokenBase64)
                     -WhatIf:${{ format('${0}', parameters.IsDryRun) }}
-              
+
               - task: 1ES.PublishBuildArtifacts@1
                 displayName: Publish Logs
                 inputs:

eng/pipelines/stages/preparerelease.yml

@@ -28,14 +28,18 @@ stages:
         packageType: runtime
         version: 6.x
         installationPath: '$(Build.Repository.LocalPath)\.dotnet'
+
     - script: mkdir $(System.ArtifactsDirectory)\StagingToolLogs
       displayName: Create Staging Tool Logs Directory
+
     - script: '$(Build.SourcesDirectory)\dotnet.cmd build $(Build.Repository.LocalPath)\eng\release\DiagnosticsReleaseTool\DiagnosticsReleaseTool.csproj -c Release /bl'
       workingDirectory: '$(System.ArtifactsDirectory)\StagingToolLogs'
       displayName: 'Build Staging Tool'
+
     # Run tool for release and test release branches
     - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/internal/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/test/release/'))) }}:
       - template: /eng/common/templates-official/post-build/setup-maestro-vars.yml@self
+
       - task: PowerShell@2
         displayName: Get Build Version
         inputs:
@@ -44,55 +48,68 @@ stages:
             -BarId $(BARBuildId)
             -MaestroToken $(MaestroAccessToken)
             -TaskVariableName 'BuildVersion'
-      - task: PowerShell@2
+
+      - task: AzureCLI@2
         displayName: 'Download Build Assets'
         inputs:
-          targetType: filePath
-          filePath: '$(Build.Repository.LocalPath)/eng/release/Scripts/AcquireBuild.ps1'
+          azureSubscription: 'Darc: Maestro Production'
+          scriptType: ps
+          scriptPath: '$(Build.Repository.LocalPath)/eng/release/Scripts/AcquireBuild.ps1'
           arguments: >-
             -BarBuildId "$(BARBuildId)"
             -AzdoToken "$(dn-bot-all-drop-rw-code-rw-release-all)"
-            -MaestroToken "$(MaestroAccessToken)"
-            -GitHubToken "$(BotAccount-dotnet-bot-repo-PAT)"
             -DownloadTargetPath "$(System.ArtifactsDirectory)\BuildAssets"
             -SasSuffixes "$(dotnetbuilds-internal-checksums-container-read-token),$(dotnetbuilds-internal-container-read-token)"
             -ReleaseVersion "$(Build.BuildNumber)"
           workingDirectory: '$(Build.Repository.LocalPath)'
         continueOnError: true
-      - script: >-
-          $(Build.SourcesDirectory)\dotnet.cmd run --project $(Build.Repository.LocalPath)\eng\release\DiagnosticsReleaseTool\DiagnosticsReleaseTool.csproj -c Release
-          --
-          prepare-release
-          --input-drop-path "$(System.ArtifactsDirectory)\BuildAssets"
-          --tool-manifest "$(Build.Repository.LocalPath)\eng\release\tool-list.json"
-          --staging-directory "$(System.ArtifactsDirectory)\AssetsLayout"
-          --release-name "$(Build.BuildNumber)"
-          --build-version "$(BuildVersion)"
-          --account-name "$(dotnet-diagnostics-storage-accountname)"
-          --account-key "$(dotnetstage-storage-key)"
-          --sas-valid-days "$(dotnet-diagnostics-storage-retentiondays)"
-          -v True
-        workingDirectory: '$(System.ArtifactsDirectory)\StagingToolLogs'
-        displayName: 'Stage Build Assets and Manifest'
+
+      - task: AzureCLI@2
+        displayName: 'Manifest Generation and Asset Publishing'
+        inputs:
+          workingDirectory: '$(System.ArtifactsDirectory)\StagingToolLogs'
+          azureSubscription: 'dotnetstage-dotnet-monitor-rw'
+          scriptType: pscore
+          scriptLocation: inlineScript
+          addSpnToEnvironment: true
+          inlineScript: >-
+            $(Build.SourcesDirectory)\dotnet.cmd run
+            --project $(Build.Repository.LocalPath)\eng\release\DiagnosticsReleaseTool\DiagnosticsReleaseTool.csproj
+            -c Release
+            --
+            prepare-release
+            --input-drop-path "$(System.ArtifactsDirectory)\BuildAssets"
+            --tool-manifest "$(Build.Repository.LocalPath)\eng\release\tool-list.json"
+            --staging-directory "$(System.ArtifactsDirectory)\AssetsLayout"
+            --release-name "$(Build.BuildNumber)"
+            --build-version "$(BuildVersion)"
+            --account-name "$(dotnet-diagnostics-storage-accountname)"
+            --container-name $(dotnet-monitor-container-name)
+            --client-id "$env:servicePrincipalId"
+            -v True
+
       - template: /eng/pipelines/steps/publish-pipeline-artifact.yml@self
         parameters:
           displayName: 'Upload Assets Layout'
           targetPath: '$(System.ArtifactsDirectory)\AssetsLayout'
           artifact: 'StagingToolAssetsLayout'
           is1ESPipeline: ${{ parameters.is1ESPipeline }}
+
       # Only tag build from real release branches
       - ${{ if not(startsWith(variables['Build.SourceBranch'], 'refs/heads/test/release/')) }}:
         - task: Powershell@2
           displayName: 'Tag Build with MonitorRelease'
           inputs:
             targetType: inline
             script: Write-Host "##vso[build.addbuildtag]MonitorRelease"
+
       - template: /eng/pipelines/steps/publish-pipeline-artifact.yml@self
         parameters:
           displayName: 'Upload Staging Tool Logs'
           targetPath: '$(System.ArtifactsDirectory)\StagingToolLogs'
           artifact: 'StagingToolLogs'
           is1ESPipeline: ${{ parameters.is1ESPipeline }}
+
     - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
       - task: Powershell@2
         displayName: 'Tag Build with update-docker'

eng/release/DiagnosticsReleaseTool/Common/AzureBlobPublisher.cs

@@ -1,10 +1,10 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
-using Azure.Storage;
+using Azure.Core;
+using Azure.Identity;
 using Azure.Storage.Blobs;
 using Azure.Storage.Blobs.Models;
-using Azure.Storage.Sas;
 using Microsoft.Extensions.Logging;
 using System;
 using System.Buffers;
@@ -17,17 +17,14 @@ namespace ReleaseTool.Core
 {
     public class AzureBlobBublisher : IPublisher
     {
-        private const int ClockSkewSec = 15 * 60;
         private const int MaxRetries = 15;
         private const int MaxFullLoopRetries = 5;
         private readonly TimeSpan FullLoopRetryDelay = TimeSpan.FromSeconds(1);
-        private const string AccessPolicyDownloadId = "DownloadDrop";
 
         private readonly string _accountName;
-        private readonly string _accountKey;
+        private readonly string _clientId;
         private readonly string _containerName;
         private readonly string _buildVersion;
-        private readonly int _sasValidDays;
         private readonly ILogger _logger;
 
         private BlobContainerClient _client;
@@ -40,12 +37,17 @@ private Uri AccountBlobUri
             }
         }
 
-        private StorageSharedKeyCredential AccountCredential
+        private TokenCredential Credentials
         {
             get
             {
-                StorageSharedKeyCredential credential = new StorageSharedKeyCredential(_accountName, _accountKey);
-                return credential;
+                if (_clientId == null)
+                {
+                    // Local development scenario. Use the default credential.
+                    return new DefaultAzureCredential();
+                }
+
+                return new DefaultAzureCredential(new DefaultAzureCredentialOptions { ManagedIdentityClientId = _clientId });
             }
         }
 
@@ -68,13 +70,12 @@ private BlobClientOptions BlobOptions
             }
         }
 
-        public AzureBlobBublisher(string accountName, string accountKey, string containerName, string buildVersion, int sasValidDays, ILogger logger)
+        public AzureBlobBublisher(string accountName, string clientId, string containerName, string buildVersion, ILogger logger)
         {
             _accountName = accountName;
-            _accountKey = accountKey;
+            _clientId = clientId;
             _containerName = containerName;
             _buildVersion = buildVersion;
-            _sasValidDays = sasValidDays;
             _logger = logger;
         }
 
@@ -107,20 +108,11 @@ public async Task<string> PublishFileAsync(FileMapping fileMap, CancellationToke
 
                     await blobClient.UploadAsync(srcStream, overwrite: true, ct);
 
-                    BlobSasBuilder sasBuilder = new BlobSasBuilder()
-                    {
-                        BlobContainerName = client.Name,
-                        BlobName = blobClient.Name,
-                        Identifier = AccessPolicyDownloadId,
-                        Protocol = SasProtocol.Https
-                    };
-                    Uri accessUri = blobClient.GenerateSasUri(sasBuilder);
-
                     using BlobDownloadStreamingResult blobStream = (await blobClient.DownloadStreamingAsync(cancellationToken: ct)).Value;
                     srcStream.Position = 0;
                     completed = await VerifyFileStreamsMatchAsync(srcStream, blobStream, ct);
 
-                    result = accessUri;
+                    result = blobClient.Uri;
                 }
                 catch (IOException ioEx) when (!(ioEx is PathTooLongException))
                 {
@@ -155,7 +147,7 @@ private async Task<BlobContainerClient> GetClient(CancellationToken ct)
         {
             if (_client == null)
             {
-                BlobServiceClient serviceClient = new BlobServiceClient(AccountBlobUri, AccountCredential, BlobOptions);
+                BlobServiceClient serviceClient = new BlobServiceClient(AccountBlobUri, Credentials, BlobOptions);
                 _logger.LogInformation($"Attempting to connect to {serviceClient.Uri} to store blobs.");
 
                 BlobContainerClient newClient;
@@ -176,31 +168,6 @@ private async Task<BlobContainerClient> GetClient(CancellationToken ct)
                         continue;
                     }
 
-                    try
-                    {
-                        DateTime baseTime = DateTime.UtcNow;
-                        // Add the new (or update existing) "download" policy to the container
-                        // This is used to mint the SAS tokens without an expiration policy
-                        // Expiration can be added later by modifying this policy
-                        BlobSignedIdentifier downloadPolicyIdentifier = new BlobSignedIdentifier()
-                        {
-                            Id = AccessPolicyDownloadId,
-                            AccessPolicy = new BlobAccessPolicy()
-                            {
-                                Permissions = "r",
-                                PolicyStartsOn = new DateTimeOffset(baseTime.AddSeconds(-ClockSkewSec)),
-                                PolicyExpiresOn = new DateTimeOffset(DateTime.UtcNow.AddDays(_sasValidDays).AddSeconds(ClockSkewSec)),
-                            }
-                        };
-                        _logger.LogInformation($"Writing download access policy: {AccessPolicyDownloadId} to {_containerName}.");
-                        await newClient.SetAccessPolicyAsync(PublicAccessType.None, new BlobSignedIdentifier[] { downloadPolicyIdentifier }, cancellationToken: ct);
-                    }
-                    catch (Exception ex)
-                    {
-                        _logger.LogWarning(ex, $"Failed to write access policy for {_containerName}, retrying.");
-                        continue;
-                    }
-
                     _logger.LogInformation($"Container {_containerName} is ready.");
                     _client = newClient;
                     break;