[Ready for EngSys Review] ESRP Publishing

eng/pipelines/templates/jobs/pack.yml

@@ -55,15 +55,52 @@ jobs:
       ServiceDirectory: ${{ parameters.ServiceDirectory }}
       PackageInfoDirectory: $(Build.ArtifactStagingDirectory)/PackageInfo
 
-  - task: Powershell@2
-    displayName: "Pack Crates"
-    condition: and(succeeded(), ne(variables['NoPackagesChanged'],'true'))
-    inputs:
-      pwsh: true
-      filePath: $(Build.SourcesDirectory)/eng/scripts/Pack-Crates.ps1
-      arguments: >
-        -OutputPath '$(Build.ArtifactStagingDirectory)'
-        -PackageInfoDirectory '$(Build.ArtifactStagingDirectory)/PackageInfo'
+  - ${{ if eq('auto', parameters.ServiceDirectory) }}:
+    - task: Powershell@2
+      displayName: Pack Crates
+      condition: and(succeeded(), ne(variables['NoPackagesChanged'],'true'))
+      inputs:
+        pwsh: true
+        filePath: $(Build.SourcesDirectory)/eng/scripts/Pack-Crates.ps1
+        arguments: >
+          -OutputPath '$(Build.ArtifactStagingDirectory)'
+          -PackageInfoDirectory '$(Build.ArtifactStagingDirectory)/PackageInfo'
+
+  - ${{ else }}:
+    - pwsh: |
+        $artifacts = '${{ convertToJson(parameters.Artifacts) }}' | ConvertFrom-Json
+        $isReleaseBuild = $true
+        $artifactsToBuild = $artifacts | Where-Object { $_.releaseInBatch -eq 'True' }
+
+        if (!$artifactsToBuild) {
+          Write-Host "No packages to release. Building all packages in the service directory with no dependency validation."
+          $artifactsToBuild = $artifacts
+          $isReleaseBuild = $false
+        }
+
+        $packageNames = $artifactsToBuild.name
+
+        Write-Host "##vso[task.setvariable variable=PackageNames]$($packageNames -join ',')"
+        if ($isReleaseBuild) {
+          Write-Host "##vso[task.setvariable variable=AdditionalPackageParams]-Release"
+        } else {
+          Write-Host "##vso[task.setvariable variable=AdditionalPackageParams]"
+        }
+      displayName: Configure crate packing
+      condition: and(succeeded(), ne(variables['NoPackagesChanged'],'true'))
+
+    - task: Powershell@2
+      displayName: Pack Crates
+      condition: and(succeeded(), ne(variables['NoPackagesChanged'],'true'))
+      inputs:
+        pwsh: true
+        filePath: $(Build.SourcesDirectory)/eng/scripts/Pack-Crates.ps1
+        arguments: >
+          -OutputPath '$(Build.ArtifactStagingDirectory)'
+          -PackageNames $(PackageNames)
+          -OutBuildOrderFile '$(Build.ArtifactStagingDirectory)/release-order.json'
+          $(AdditionalPackageParams)
+
 
   - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml
     parameters:

eng/pipelines/templates/stages/archetype-rust-release.yml

@@ -14,16 +14,13 @@ parameters:
 - name: DevFeedName
   type: string
   default: 'public/azure-sdk-for-rust'
-- name: Environment
-  type: string
-  default: 'cratesio'
 
 stages:
 - ${{ if eq(variables['System.TeamProject'], 'internal') }}:
   - ${{ if in(variables['Build.Reason'], 'Manual', '') }}:
-    - ${{ each artifact in parameters.Artifacts }}:
-      - stage: Release_${{artifact.safeName}}
-        displayName: "Release: ${{artifact.name}}"
+    - ${{ if gt(length(parameters.Artifacts), 0) }}:
+      - stage: Release_Batch
+        displayName: "Releasing: ${{length(parameters.Artifacts)}} crates"
         dependsOn: ${{parameters.DependsOn}}
         condition: and(succeeded(), ne(variables['SetDevVersion'], 'true'), ne(variables['Skip.Release'], 'true'), ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-rust-pr'))
         variables:
@@ -50,16 +47,17 @@ stages:
 
           - template: /eng/common/pipelines/templates/steps/retain-run.yml
 
-          - script: |
-              echo "##vso[build.addbuildtag]${{artifact.name}}"
-            displayName: Add build tag '${{artifact.name}}'
+          - ${{ each artifact in parameters.Artifacts }}:
+            - script: |
+                echo "##vso[build.addbuildtag]${{artifact.name}}"
+              displayName: Add build tag '${{artifact.name}}'
 
-          - template: /eng/common/pipelines/templates/steps/create-tags-and-git-release.yml
-            parameters:
-              ArtifactLocation: $(Pipeline.Workspace)/${{parameters.PipelineArtifactName}}/${{artifact.name}}
-              PackageRepository: Crates.io
-              ReleaseSha: $(Build.SourceVersion)
-              WorkingDirectory: $(Pipeline.Workspace)/_work
+            - template: /eng/common/pipelines/templates/steps/create-tags-and-git-release.yml
+              parameters:
+                ArtifactLocation: $(Pipeline.Workspace)/${{parameters.PipelineArtifactName}}/${{artifact.name}}
+                PackageRepository: Crates.io
+                ReleaseSha: $(Build.SourceVersion)
+                WorkingDirectory: $(Pipeline.Workspace)/_work
 
         - deployment: PublishPackage
           displayName: "Publish to Crates.io"
@@ -71,7 +69,10 @@ stages:
             - input: pipelineArtifact  # Required, type of the input artifact
               artifactName: ${{parameters.PipelineArtifactName}}  # Required, name of the pipeline artifact
               targetPath: $(Pipeline.Workspace)/drop  # Optional, specifies where the artifact is downloaded to
-          environment: ${{parameters.Environment}}
+          ${{if parameters.TestPipeline}}:
+            environment: none
+          ${{else}}:
+            environment: package-publish
           # This timeout shouldn't be necessary once we're able to parallelize better. Right now,
           # this is here to ensure larger areas (30+) libraries don't time out.
           timeoutInMinutes: 120
@@ -84,33 +85,76 @@ stages:
             runOnce:
               deploy:
                 steps:
-                - template: /eng/pipelines/templates/steps/use-rust.yml@self
-                  parameters:
-                    Toolchain: stable
-
                 - pwsh: |
-                    $additionalOwners = @('heaths', 'hallipr')
-                    $token = $env:CARGO_REGISTRY_TOKEN
-                    $crateName = '${{artifact.name}}'
-
-                    $manifestPath = "$(Pipeline.Workspace)/drop/$crateName/contents/Cargo.toml"
-                    Write-Host "> cargo publish --manifest-path `"$manifestPath`""
-                    cargo publish --manifest-path $manifestPath
-                    if (!$?) {
-                      Write-Error "Failed to publish package: '$crateName'"
-                      exit 1
-                    }
-
-                    $existingOwners = (cargo owner --list $crateName) -replace " \(.*", ""
-                    $missingOwners = $additionalOwners | Where-Object { $existingOwners -notcontains $_ }
-
-                    foreach ($owner in $missingOwners) {
-                      Write-Host "> cargo owner --add $owner $crateName"
-                      cargo owner --add $owner $crateName
-                    }
-                  displayName: Publish Crate
-                  env:
-                    CARGO_REGISTRY_TOKEN: $(azure-sdk-cratesio-token)
+                    Write-Host "Setting ArtifactName to empty"
+                    Write-Host "##vso[task.setvariable variable=ArtifactName;]"
+                  displayName: Initialize ArtifactName variable
+
+                # This loop over artifacts is used to produce the correct number
+                # of ESRP release tasks. It has the side effect of also setting
+                # the artifact name by looking up the index of the current
+                # "artifact.name" in the parameters.Artifacts array, using that
+                # as an "index" and then using that same index to look up the
+                # actual artifact to release in the release-order.json file.
+                - ${{ each artifact in parameters.Artifacts }}:
+                  - pwsh: |
+                      # From the DevOps template artifact loop calculate the current index
+                      $indexItem = '${{ artifact.name }}'
+                      [array] $indexList = ConvertFrom-Json '${{ convertToJson(parameters.Artifacts.*.name) }}'
+                      $index = $indexList.IndexOf($indexItem)
+                      Write-Host "Index of template artifact: $index"
+
+                      [array] $artifacts = Get-Content '$(Pipeline.Workspace)/drop/release-order.json' | ConvertFrom-Json
+
+                      $artifactName = $artifacts[$index]
+
+                      Write-Host "Releasing artifact: $artifactName"
+
+                      $artifactRootPath = '$(Pipeline.Workspace)/drop'
+                      $outDir = '$(Pipeline.Workspace)/esrp-release'
+
+                      if (Test-Path $outDir) {
+                        Write-Host "Cleaning output directory: $outDir"
+                        Remove-Item -Path $outDir -Recurse -Force
+                      }
+                      New-Item -ItemType Directory -Path $outDir -Force | Out-Null
+
+                      $packageMetadataPath = "$artifactRootPath/PackageInfo/$artifactName.json"
+                      if (!(Test-Path $packageMetadataPath)) {
+                        Write-Error "Package metadata file not found: $packageMetadataPath"
+                        exit 1
+                      }
+
+                      $packageMetadata = Get-Content -Raw $packageMetadataPath | ConvertFrom-Json
+                      $packageVersion = $packageMetadata.version
+                      Write-Host "Package version: $packageVersion"
+
+                      $cratePath = "$artifactRootPath/$artifactName/$artifactName-$packageVersion.crate"
+                      Copy-Item `
+                        -Path $cratePath `
+                        -Destination $outDir
+                      Write-Host "Contents of $outDir"
+                      Get-ChildItem -Path $outDir | ForEach-Object { Write-Host $_.FullName }
+                    displayName: 'Copy crate for ESRP'
+
+                  - task: EsrpRelease@10
+                    displayName: 'ESRP Release'
+                    inputs:
+                      connectedservicename: 'Azure SDK PME Managed Identity'
+                      ClientId: '5f81938c-2544-4f1f-9251-dd9de5b8a81b'
+                      DomainTenantId: '975f013f-7f24-47e8-a7d3-abc4752bf346'
+                      Usemanagedidentity: true
+                      KeyVaultName: 'kv-azuresdk-codesign'
+                      SignCertName: 'azure-sdk-esrp-release-certificate'
+                      intent: 'packagedistribution'
+                      contenttype: 'Rust'
+                      contentsource: 'Folder'
+                      folderlocation: '$(Pipeline.Workspace)/esrp-release'
+                      waitforreleasecompletion: true
+                      owners: ${{ coalesce(variables['Build.RequestedForEmail'], '[email protected]') }}
+                      approvers: ${{ coalesce(variables['Build.RequestedForEmail'], '[email protected]') }}
+                      serviceendpointurl: 'https://api.esrp.microsoft.com/'
+                      mainpublisher: 'ESRPRELPACMANTEST'
 
         - job: UpdatePackageVersion
           displayName: "API Review and Package Version Update"
@@ -130,69 +174,32 @@ stages:
             displayName: Download ${{parameters.PipelineArtifactName}} artifact
             artifact: ${{parameters.PipelineArtifactName}}
 
-          - template: /eng/common/pipelines/templates/steps/create-apireview.yml
-            parameters:
-              ArtifactPath: $(Pipeline.Workspace)/${{parameters.PipelineArtifactName}}
-              Artifacts: ${{parameters.Artifacts}}
-              ConfigFileDir: $(Pipeline.Workspace)/${{parameters.PipelineArtifactName}}/PackageInfo
-              MarkPackageAsShipped: true
-              ArtifactName: ${{parameters.PipelineArtifactName}}
-              SourceRootPath: $(System.DefaultWorkingDirectory)
-              PackageName: ${{artifact.name}}
-
-          # Apply the version increment to each library, which updates the Cargo.toml and changelog files.
-          - task: PowerShell@2
-            displayName: Increment ${{artifact.name}} version
-            inputs:
-              targetType: filePath
-              filePath: $(Build.SourcesDirectory)/eng/scripts/Update-PackageVersion.ps1
-              arguments: >
-                -ServiceDirectory '${{parameters.ServiceDirectory}}'
-                -PackageName '${{artifact.name}}'
+          - ${{ each artifact in parameters.Artifacts }}:
+            - template: /eng/common/pipelines/templates/steps/create-apireview.yml
+              parameters:
+                ArtifactPath: $(Pipeline.Workspace)/${{parameters.PipelineArtifactName}}
+                Artifacts: ${{parameters.Artifacts}}
+                ConfigFileDir: $(Pipeline.Workspace)/${{parameters.PipelineArtifactName}}/PackageInfo
+                MarkPackageAsShipped: true
+                ArtifactName: ${{parameters.PipelineArtifactName}}
+                SourceRootPath: $(System.DefaultWorkingDirectory)
+                PackageName: ${{artifact.name}}
+
+            # Apply the version increment to each library, which updates the Cargo.toml and changelog files.
+            - task: PowerShell@2
+              displayName: Increment ${{artifact.name}} version
+              inputs:
+                targetType: filePath
+                filePath: $(Build.SourcesDirectory)/eng/scripts/Update-PackageVersion.ps1
+                arguments: >
+                  -ServiceDirectory '${{parameters.ServiceDirectory}}'
+                  -PackageName '${{artifact.name}}'
 
           - template: /eng/common/pipelines/templates/steps/create-pull-request.yml
             parameters:
               PRBranchName: increment-package-version-${{parameters.ServiceDirectory}}-$(Build.BuildId)
-              CommitMsg: "Increment package version after release of ${{ artifact.name }}"
+              CommitMsg: "Increment package version after release of ${{ join(', ', parameters.Artifacts.*.name) }}"
               PRTitle: "Increment versions for ${{parameters.ServiceDirectory}} releases"
               CloseAfterOpenForTesting: '${{parameters.TestPipeline}}'
               ${{ if startsWith(variables['Build.SourceBranch'], 'refs/pull/') }}:
                 BaseBranchName: main
-
-        - ${{ if eq(parameters.TestPipeline, true) }}:
-          - job: ManualApproval
-            displayName: "Manual approval"
-            dependsOn: PublishPackage
-            condition: ne(variables['Skip.PublishPackage'], 'true')
-            pool: server
-            timeoutInMinutes: 120 # 2 hours
-            steps:
-            - task: ManualValidation@1
-              timeoutInMinutes: 60 # 1 hour
-              inputs:
-                notifyUsers: '' # Required, but empty string allowed
-                allowApproversToApproveTheirOwnRuns: true
-                instructions: "Approve yank of ${{ artifact.name }}"
-                onTimeout: 'resume'
-
-          - job: YankCrates
-            displayName: "Yank Crates"
-            dependsOn: ManualApproval
-            condition: and(succeeded(), ne(variables['Skip.PublishPackage'], 'true'))
-            steps:
-            - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
-
-            - download: current
-              displayName: Download ${{parameters.PipelineArtifactName}} artifact
-              artifact: ${{parameters.PipelineArtifactName}}
-
-            - task: PowerShell@2
-              displayName: Yank Crates
-              env:
-                CARGO_REGISTRY_TOKEN: $(azure-sdk-cratesio-token)
-              inputs:
-                targetType: filePath
-                filePath: $(Build.SourcesDirectory)/eng/scripts/Yank-Crates.ps1
-                arguments:
-                  -CrateNames '${{artifact.name}}'
-                  -PackageInfoDirectory '$(Pipeline.Workspace)/${{parameters.PipelineArtifactName}}/PackageInfo'

eng/pipelines/templates/stages/archetype-sdk-client.yml

@@ -148,6 +148,9 @@ extends:
             parameters:
               DependsOn: "Build"
               ServiceDirectory: ${{ parameters.ServiceDirectory }}
-              Artifacts: ${{ parameters.Artifacts }}
+              Artifacts: 
+                - ${{ each artifact in parameters.Artifacts }}: 
+                  - ${{ if ne(artifact.releaseInBatch, 'false')}}: 
+                    - ${{ artifact }}
               TestPipeline: ${{ eq(parameters.ServiceDirectory, 'canary') }}
               PipelineArtifactName: packages

eng/scripts/Language-Settings.ps1

@@ -1,7 +1,7 @@
 $Language = "rust"
 $LanguageDisplayName = "Rust"
 $PackageRepository = "crates.io"
-$packagePattern = "Cargo.toml"
+$packagePattern = "*.crate"
 #$MetadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/main/_data/releases/latest/rust-packages.csv"
 $GithubUri = "https://github.com/Azure/azure-sdk-for-rust"
 $PackageRepositoryUri = "https://crates.io/crates"
@@ -139,15 +139,32 @@ function Get-rust-AdditionalValidationPackagesFromPackageSet ($packagesWithChang
   return $additionalPackages ?? @()
 }
 
+# $GetPackageInfoFromPackageFileFn = "Get-${Language}-PackageInfoFromPackageFile"
 function Get-rust-PackageInfoFromPackageFile([IO.FileInfo]$pkg, [string]$workingDirectory) {
-  #$pkg will be a FileInfo object for the Cargo.toml file in a package artifact directory
-  $package = cargo read-manifest --manifest-path $pkg.FullName | ConvertFrom-Json
+  # Create a temporary folder for extraction
+  $extractionPath = [System.IO.Path]::Combine([System.IO.Path]::GetTempPath(), [System.IO.Path]::GetRandomFileName())
+  New-Item -ItemType Directory -Path $extractionPath | Out-Null
+
+  # Extract the .crate file (which is a tarball) to the temporary folder
+  tar -xvf $pkg.FullName -C $extractionPath
+  $cargoTomlPath = [System.IO.Path]::Combine($extractionPath, $pkg.BaseName, 'Cargo.toml')
+
+  Write-Host "Reading package info from $cargoTomlPath"
+  if (!(Test-Path $cargoTomlPath)) {
+    $message = "The Cargo.toml file was not found in the package artifact at $cargoTomlPath"
+    LogError $message
+    throw $message
+  }
+
+  $package = cargo read-manifest --manifest-path $cargoTomlPath | ConvertFrom-Json
 
   $packageName = $package.name
   $packageVersion = $package.version
 
-  $changeLogLoc = Get-ChildItem -Path $pkg.DirectoryName -Filter "CHANGELOG.md" | Select-Object -First 1
-  $readmeContentLoc = Get-ChildItem -Path $pkg.DirectoryName -Filter "README.md" | Select-Object -First 1
+  $packageAssetPath = [System.IO.Path]::Combine($extractionPath, "$packageName-$packageVersion")
+
+  $changeLogLoc = Get-ChildItem -Path $packageAssetPath -Filter "CHANGELOG.md" | Select-Object -First 1
+  $readmeContentLoc = Get-ChildItem -Path $packageAssetPath -Filter "README.md" | Select-Object -First 1
 
   if ($changeLogLoc) {
     $releaseNotes = Get-ChangeLogEntryAsString -ChangeLogLocation $changeLogLoc -VersionString $packageVersion