diff --git a/eng/pipelines/dotnet-monitor-compliance.yml b/eng/pipelines/dotnet-monitor-compliance.yml index e443bd43067..86caff41e50 100644 --- a/eng/pipelines/dotnet-monitor-compliance.yml +++ b/eng/pipelines/dotnet-monitor-compliance.yml @@ -71,8 +71,6 @@ extends: arguments: >- -BarBuildId "$(BuildBarId)" -AzdoToken "$(dn-bot-all-drop-rw-code-rw-release-all)" - -MaestroToken "$(MaestroAccessToken)" - -GitHubToken "$(BotAccount-dotnet-bot-repo-PAT)" -DownloadTargetPath "$(System.ArtifactsDirectory)\BuildAssets" -SasSuffixes "$(dotnetbuilds-internal-checksums-container-read-token),$(dotnetbuilds-internal-container-read-token)" -ReleaseVersion "$(BuildVersion)" diff --git a/eng/pipelines/stages/preparerelease.yml b/eng/pipelines/stages/preparerelease.yml index 6a728d15ebf..743f9aaf2d4 100644 --- a/eng/pipelines/stages/preparerelease.yml +++ b/eng/pipelines/stages/preparerelease.yml @@ -28,14 +28,18 @@ stages: packageType: runtime version: 6.x installationPath: '$(Build.Repository.LocalPath)\.dotnet' + - script: mkdir $(System.ArtifactsDirectory)\StagingToolLogs displayName: Create Staging Tool Logs Directory + - script: '$(Build.SourcesDirectory)\dotnet.cmd build $(Build.Repository.LocalPath)\eng\release\DiagnosticsReleaseTool\DiagnosticsReleaseTool.csproj -c Release /bl' workingDirectory: '$(System.ArtifactsDirectory)\StagingToolLogs' displayName: 'Build Staging Tool' + # Run tool for release and test release branches - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/internal/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/test/release/'))) }}: - template: /eng/common/templates-official/post-build/setup-maestro-vars.yml@self + - task: PowerShell@2 displayName: Get Build Version inputs: @@ -44,42 +48,53 @@ stages: -BarId $(BARBuildId) -MaestroToken $(MaestroAccessToken) -TaskVariableName 'BuildVersion' - - task: PowerShell@2 + + - task: AzureCLI@2 displayName: 'Download Build Assets' inputs: - targetType: filePath - filePath: '$(Build.Repository.LocalPath)/eng/release/Scripts/AcquireBuild.ps1' + azureSubscription: 'Darc: Maestro Production' + scriptType: ps + scriptPath: '$(Build.Repository.LocalPath)/eng/release/Scripts/AcquireBuild.ps1' arguments: >- -BarBuildId "$(BARBuildId)" -AzdoToken "$(dn-bot-all-drop-rw-code-rw-release-all)" - -MaestroToken "$(MaestroAccessToken)" - -GitHubToken "$(BotAccount-dotnet-bot-repo-PAT)" -DownloadTargetPath "$(System.ArtifactsDirectory)\BuildAssets" -SasSuffixes "$(dotnetbuilds-internal-checksums-container-read-token),$(dotnetbuilds-internal-container-read-token)" -ReleaseVersion "$(Build.BuildNumber)" workingDirectory: '$(Build.Repository.LocalPath)' continueOnError: true - - script: >- - $(Build.SourcesDirectory)\dotnet.cmd run --project $(Build.Repository.LocalPath)\eng\release\DiagnosticsReleaseTool\DiagnosticsReleaseTool.csproj -c Release - -- - prepare-release - --input-drop-path "$(System.ArtifactsDirectory)\BuildAssets" - --tool-manifest "$(Build.Repository.LocalPath)\eng\release\tool-list.json" - --staging-directory "$(System.ArtifactsDirectory)\AssetsLayout" - --release-name "$(Build.BuildNumber)" - --build-version "$(BuildVersion)" - --account-name "$(dotnet-diagnostics-storage-accountname)" - --account-key "$(dotnetstage-storage-key)" - --sas-valid-days "$(dotnet-diagnostics-storage-retentiondays)" - -v True - workingDirectory: '$(System.ArtifactsDirectory)\StagingToolLogs' - displayName: 'Stage Build Assets and Manifest' + + - task: AzureCLI@2 + displayName: 'Manifest Generation and Asset Publishing' + inputs: + workingDirectory: '$(System.ArtifactsDirectory)\StagingToolLogs' + azureSubscription: 'dotnetstage-dotnet-monitor-rw' + scriptType: pscore + scriptLocation: inlineScript + addSpnToEnvironment: true + inlineScript: >- + $(Build.SourcesDirectory)\dotnet.cmd run + --project $(Build.Repository.LocalPath)\eng\release\DiagnosticsReleaseTool\DiagnosticsReleaseTool.csproj + -c Release + -- + prepare-release + --input-drop-path "$(System.ArtifactsDirectory)\BuildAssets" + --tool-manifest "$(Build.Repository.LocalPath)\eng\release\tool-list.json" + --staging-directory "$(System.ArtifactsDirectory)\AssetsLayout" + --release-name "$(Build.BuildNumber)" + --build-version "$(BuildVersion)" + --account-name "$(dotnet-diagnostics-storage-accountname)" + --container-name $(dotnet-monitor-container-name) + --client-id "$env:servicePrincipalId" + -v True + - template: /eng/pipelines/steps/publish-pipeline-artifact.yml@self parameters: displayName: 'Upload Assets Layout' targetPath: '$(System.ArtifactsDirectory)\AssetsLayout' artifact: 'StagingToolAssetsLayout' is1ESPipeline: ${{ parameters.is1ESPipeline }} + # Only tag build from real release branches - ${{ if not(startsWith(variables['Build.SourceBranch'], 'refs/heads/test/release/')) }}: - task: Powershell@2 @@ -87,12 +102,14 @@ stages: inputs: targetType: inline script: Write-Host "##vso[build.addbuildtag]MonitorRelease" + - template: /eng/pipelines/steps/publish-pipeline-artifact.yml@self parameters: displayName: 'Upload Staging Tool Logs' targetPath: '$(System.ArtifactsDirectory)\StagingToolLogs' artifact: 'StagingToolLogs' is1ESPipeline: ${{ parameters.is1ESPipeline }} + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - task: Powershell@2 displayName: 'Tag Build with update-docker' diff --git a/eng/release/DiagnosticsReleaseTool/Common/AzureBlobPublisher.cs b/eng/release/DiagnosticsReleaseTool/Common/AzureBlobPublisher.cs index 04becfb2293..945ac240ed8 100644 --- a/eng/release/DiagnosticsReleaseTool/Common/AzureBlobPublisher.cs +++ b/eng/release/DiagnosticsReleaseTool/Common/AzureBlobPublisher.cs @@ -1,10 +1,10 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using Azure.Storage; +using Azure.Core; +using Azure.Identity; using Azure.Storage.Blobs; using Azure.Storage.Blobs.Models; -using Azure.Storage.Sas; using Microsoft.Extensions.Logging; using System; using System.Buffers; @@ -17,17 +17,14 @@ namespace ReleaseTool.Core { public class AzureBlobBublisher : IPublisher { - private const int ClockSkewSec = 15 * 60; private const int MaxRetries = 15; private const int MaxFullLoopRetries = 5; private readonly TimeSpan FullLoopRetryDelay = TimeSpan.FromSeconds(1); - private const string AccessPolicyDownloadId = "DownloadDrop"; private readonly string _accountName; - private readonly string _accountKey; + private readonly string _clientId; private readonly string _containerName; private readonly string _buildVersion; - private readonly int _sasValidDays; private readonly ILogger _logger; private BlobContainerClient _client; @@ -40,12 +37,17 @@ private Uri AccountBlobUri } } - private StorageSharedKeyCredential AccountCredential + private TokenCredential Credentials { get { - StorageSharedKeyCredential credential = new StorageSharedKeyCredential(_accountName, _accountKey); - return credential; + if (_clientId == null) + { + // Local development scenario. Use the default credential. + return new DefaultAzureCredential(); + } + + return new DefaultAzureCredential(new DefaultAzureCredentialOptions { ManagedIdentityClientId = _clientId }); } } @@ -68,13 +70,12 @@ private BlobClientOptions BlobOptions } } - public AzureBlobBublisher(string accountName, string accountKey, string containerName, string buildVersion, int sasValidDays, ILogger logger) + public AzureBlobBublisher(string accountName, string clientId, string containerName, string buildVersion, ILogger logger) { _accountName = accountName; - _accountKey = accountKey; + _clientId = clientId; _containerName = containerName; _buildVersion = buildVersion; - _sasValidDays = sasValidDays; _logger = logger; } @@ -107,20 +108,11 @@ public async Task PublishFileAsync(FileMapping fileMap, CancellationToke await blobClient.UploadAsync(srcStream, overwrite: true, ct); - BlobSasBuilder sasBuilder = new BlobSasBuilder() - { - BlobContainerName = client.Name, - BlobName = blobClient.Name, - Identifier = AccessPolicyDownloadId, - Protocol = SasProtocol.Https - }; - Uri accessUri = blobClient.GenerateSasUri(sasBuilder); - using BlobDownloadStreamingResult blobStream = (await blobClient.DownloadStreamingAsync(cancellationToken: ct)).Value; srcStream.Position = 0; completed = await VerifyFileStreamsMatchAsync(srcStream, blobStream, ct); - result = accessUri; + result = blobClient.Uri; } catch (IOException ioEx) when (!(ioEx is PathTooLongException)) { @@ -155,7 +147,7 @@ private async Task GetClient(CancellationToken ct) { if (_client == null) { - BlobServiceClient serviceClient = new BlobServiceClient(AccountBlobUri, AccountCredential, BlobOptions); + BlobServiceClient serviceClient = new BlobServiceClient(AccountBlobUri, Credentials, BlobOptions); _logger.LogInformation($"Attempting to connect to {serviceClient.Uri} to store blobs."); BlobContainerClient newClient; @@ -176,31 +168,6 @@ private async Task GetClient(CancellationToken ct) continue; } - try - { - DateTime baseTime = DateTime.UtcNow; - // Add the new (or update existing) "download" policy to the container - // This is used to mint the SAS tokens without an expiration policy - // Expiration can be added later by modifying this policy - BlobSignedIdentifier downloadPolicyIdentifier = new BlobSignedIdentifier() - { - Id = AccessPolicyDownloadId, - AccessPolicy = new BlobAccessPolicy() - { - Permissions = "r", - PolicyStartsOn = new DateTimeOffset(baseTime.AddSeconds(-ClockSkewSec)), - PolicyExpiresOn = new DateTimeOffset(DateTime.UtcNow.AddDays(_sasValidDays).AddSeconds(ClockSkewSec)), - } - }; - _logger.LogInformation($"Writing download access policy: {AccessPolicyDownloadId} to {_containerName}."); - await newClient.SetAccessPolicyAsync(PublicAccessType.None, new BlobSignedIdentifier[] { downloadPolicyIdentifier }, cancellationToken: ct); - } - catch (Exception ex) - { - _logger.LogWarning(ex, $"Failed to write access policy for {_containerName}, retrying."); - continue; - } - _logger.LogInformation($"Container {_containerName} is ready."); _client = newClient; break; diff --git a/eng/release/DiagnosticsReleaseTool/Config.cs b/eng/release/DiagnosticsReleaseTool/Config.cs index 637fda3a393..475b84c8188 100644 --- a/eng/release/DiagnosticsReleaseTool/Config.cs +++ b/eng/release/DiagnosticsReleaseTool/Config.cs @@ -14,9 +14,8 @@ internal class Config public string ReleaseName { get; } public string BuildVersion { get; } public string AccountName { get; } - public string AccountKey { get; } + public string ClientId { get; } public string ContainerName { get; } - public int SasValidDays { get; } public Config( FileInfo toolManifest, @@ -26,9 +25,8 @@ public Config( string releaseName, string buildVersion, string accountName, - string accountKey, - string containerName, - int sasValidDays) + string clientId, + string containerName) { ToolManifest = toolManifest; ShouldVerifyManifest = verifyToolManifest; @@ -37,9 +35,8 @@ public Config( ReleaseName = releaseName; BuildVersion = buildVersion; AccountName = accountName; - AccountKey = accountKey; + ClientId = clientId; ContainerName = containerName; - SasValidDays = sasValidDays; } } } diff --git a/eng/release/DiagnosticsReleaseTool/DiagnosticsReleaseCommandLine.cs b/eng/release/DiagnosticsReleaseTool/DiagnosticsReleaseCommandLine.cs index 06072d77a3e..5884f3a3cb6 100644 --- a/eng/release/DiagnosticsReleaseTool/DiagnosticsReleaseCommandLine.cs +++ b/eng/release/DiagnosticsReleaseTool/DiagnosticsReleaseCommandLine.cs @@ -107,8 +107,9 @@ private static Option AzureStorageAccountNameOption() => private static Option AzureStorageAccountKeyOption() => new Option( - aliases: new[] { "-k", "--account-key" }, - description: "Storage account key, in base 64 format.") + aliases: new[] { "-k", "--client-id" }, + description: "Identity Client ID. If left blank, ambient identity will be used.", + getDefaultValue: () => null) { IsRequired = true, }; diff --git a/eng/release/DiagnosticsReleaseTool/DiagnosticsReleaseRunner.cs b/eng/release/DiagnosticsReleaseTool/DiagnosticsReleaseRunner.cs index 3ecfa62f936..e9024d55d48 100644 --- a/eng/release/DiagnosticsReleaseTool/DiagnosticsReleaseRunner.cs +++ b/eng/release/DiagnosticsReleaseTool/DiagnosticsReleaseRunner.cs @@ -49,7 +49,7 @@ internal async static Task PrepareRelease(Config releaseConfig, bool verbos IPublisher releasePublisher = dryRun ? new SkipPublisher() : - new AzureBlobBublisher(releaseConfig.AccountName, releaseConfig.AccountKey, releaseConfig.ContainerName, releaseConfig.BuildVersion, releaseConfig.SasValidDays, logger); + new AzureBlobBublisher(releaseConfig.AccountName, releaseConfig.ClientId, releaseConfig.ContainerName, releaseConfig.BuildVersion, logger); IManifestGenerator manifestGenerator = new DiagnosticsManifestGenerator(releaseMetadata, releaseConfig.ToolManifest, logger); using var diagnosticsRelease = new Release( diff --git a/eng/release/DiagnosticsReleaseTool/DiagnosticsReleaseTool.csproj b/eng/release/DiagnosticsReleaseTool/DiagnosticsReleaseTool.csproj index fc0720a3fb5..336e0567bce 100644 --- a/eng/release/DiagnosticsReleaseTool/DiagnosticsReleaseTool.csproj +++ b/eng/release/DiagnosticsReleaseTool/DiagnosticsReleaseTool.csproj @@ -13,15 +13,16 @@ - - - - - - + + + + + + - - + + + diff --git a/eng/release/Directory.Build.props b/eng/release/Directory.Build.props index a8133bfcf5e..100d8a4ebf8 100644 --- a/eng/release/Directory.Build.props +++ b/eng/release/Directory.Build.props @@ -1,4 +1,3 @@ - - + \ No newline at end of file diff --git a/eng/release/Scripts/AcquireBuild.ps1 b/eng/release/Scripts/AcquireBuild.ps1 index b76c6279a09..220c6ff13fd 100644 --- a/eng/release/Scripts/AcquireBuild.ps1 +++ b/eng/release/Scripts/AcquireBuild.ps1 @@ -4,9 +4,6 @@ param( [Parameter(Mandatory=$true)][string] $DownloadTargetPath, [Parameter(Mandatory=$true)][string] $SasSuffixes, [Parameter(Mandatory=$true)][string] $AzdoToken, - [Parameter(Mandatory=$true)][string] $MaestroToken, - [Parameter(Mandatory=$true)][string] $GitHubToken, - [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com', [Parameter(Mandatory=$false)][string] $DarcVersion = $null, [Parameter(Mandatory=$false)][bool] $Separated = $true, [switch] $help, @@ -19,9 +16,6 @@ function Write-Help() { Write-Host " -DownloadTargetPath Path to download the build to." Write-Host " -SasSuffixes Comma separated list of potential uri suffixes that can be used if anonymous access to a blob uri fails. Appended directly to the end of the URI. Use full SAS syntax with ?." Write-Host " -AzdoToken Azure DevOps token to use for builds queries" - Write-Host " -MaestroToken Maestro token to use for querying BAR" - Write-Host " -GitHubToken GitHub token to use for querying repository information" - Write-Host " -MaestroApiEndPoint BAR endpoint to use for build queries." Write-Host " -Separated <`$true|`$false> Download files to their repo separated locations." Write-Host "" } @@ -62,12 +56,10 @@ try { --output-dir $DownloadTargetPath ` --overwrite ` --sas-suffixes $SasSuffixes ` - --github-pat $GitHubToken ` --azdev-pat $AzdoToken ` - --bar-uri $MaestroApiEndPoint ` - --password $MaestroToken ` --verbose ` --continue-on-error ` + --ci ` $separatedArgs if ($LastExitCode -ne 0) {