diff --git a/eng/pipelines/templates/jobs/pack.yml b/eng/pipelines/templates/jobs/pack.yml index 1d0f51a814..cbd258b83a 100644 --- a/eng/pipelines/templates/jobs/pack.yml +++ b/eng/pipelines/templates/jobs/pack.yml @@ -55,15 +55,52 @@ jobs: ServiceDirectory: ${{ parameters.ServiceDirectory }} PackageInfoDirectory: $(Build.ArtifactStagingDirectory)/PackageInfo - - task: Powershell@2 - displayName: "Pack Crates" - condition: and(succeeded(), ne(variables['NoPackagesChanged'],'true')) - inputs: - pwsh: true - filePath: $(Build.SourcesDirectory)/eng/scripts/Pack-Crates.ps1 - arguments: > - -OutputPath '$(Build.ArtifactStagingDirectory)' - -PackageInfoDirectory '$(Build.ArtifactStagingDirectory)/PackageInfo' + - ${{ if eq('auto', parameters.ServiceDirectory) }}: + - task: Powershell@2 + displayName: Pack Crates + condition: and(succeeded(), ne(variables['NoPackagesChanged'],'true')) + inputs: + pwsh: true + filePath: $(Build.SourcesDirectory)/eng/scripts/Pack-Crates.ps1 + arguments: > + -OutputPath '$(Build.ArtifactStagingDirectory)' + -PackageInfoDirectory '$(Build.ArtifactStagingDirectory)/PackageInfo' + + - ${{ else }}: + - pwsh: | + $artifacts = '${{ convertToJson(parameters.Artifacts) }}' | ConvertFrom-Json + $isReleaseBuild = $true + $artifactsToBuild = $artifacts | Where-Object { $_.releaseInBatch -eq 'True' } + + if (!$artifactsToBuild) { + Write-Host "No packages to release. Building all packages in the service directory with no dependency validation." + $artifactsToBuild = $artifacts + $isReleaseBuild = $false + } + + $packageNames = $artifactsToBuild.name + + Write-Host "##vso[task.setvariable variable=PackageNames]$($packageNames -join ',')" + if ($isReleaseBuild) { + Write-Host "##vso[task.setvariable variable=AdditionalPackageParams]-Release" + } else { + Write-Host "##vso[task.setvariable variable=AdditionalPackageParams]" + } + displayName: Configure crate packing + condition: and(succeeded(), ne(variables['NoPackagesChanged'],'true')) + + - task: Powershell@2 + displayName: Pack Crates + condition: and(succeeded(), ne(variables['NoPackagesChanged'],'true')) + inputs: + pwsh: true + filePath: $(Build.SourcesDirectory)/eng/scripts/Pack-Crates.ps1 + arguments: > + -OutputPath '$(Build.ArtifactStagingDirectory)' + -PackageNames $(PackageNames) + -OutBuildOrderFile '$(Build.ArtifactStagingDirectory)/release-order.json' + $(AdditionalPackageParams) + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml parameters: diff --git a/eng/pipelines/templates/stages/archetype-rust-release.yml b/eng/pipelines/templates/stages/archetype-rust-release.yml index 5ac10abb01..71d485806c 100644 --- a/eng/pipelines/templates/stages/archetype-rust-release.yml +++ b/eng/pipelines/templates/stages/archetype-rust-release.yml @@ -14,16 +14,13 @@ parameters: - name: DevFeedName type: string default: 'public/azure-sdk-for-rust' -- name: Environment - type: string - default: 'cratesio' stages: - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - ${{ if in(variables['Build.Reason'], 'Manual', '') }}: - - ${{ each artifact in parameters.Artifacts }}: - - stage: Release_${{artifact.safeName}} - displayName: "Release: ${{artifact.name}}" + - ${{ if gt(length(parameters.Artifacts), 0) }}: + - stage: Release_Batch + displayName: "Releasing: ${{length(parameters.Artifacts)}} crates" dependsOn: ${{parameters.DependsOn}} condition: and(succeeded(), ne(variables['SetDevVersion'], 'true'), ne(variables['Skip.Release'], 'true'), ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-rust-pr')) variables: @@ -50,16 +47,17 @@ stages: - template: /eng/common/pipelines/templates/steps/retain-run.yml - - script: | - echo "##vso[build.addbuildtag]${{artifact.name}}" - displayName: Add build tag '${{artifact.name}}' + - ${{ each artifact in parameters.Artifacts }}: + - script: | + echo "##vso[build.addbuildtag]${{artifact.name}}" + displayName: Add build tag '${{artifact.name}}' - - template: /eng/common/pipelines/templates/steps/create-tags-and-git-release.yml - parameters: - ArtifactLocation: $(Pipeline.Workspace)/${{parameters.PipelineArtifactName}}/${{artifact.name}} - PackageRepository: Crates.io - ReleaseSha: $(Build.SourceVersion) - WorkingDirectory: $(Pipeline.Workspace)/_work + - template: /eng/common/pipelines/templates/steps/create-tags-and-git-release.yml + parameters: + ArtifactLocation: $(Pipeline.Workspace)/${{parameters.PipelineArtifactName}}/${{artifact.name}} + PackageRepository: Crates.io + ReleaseSha: $(Build.SourceVersion) + WorkingDirectory: $(Pipeline.Workspace)/_work - deployment: PublishPackage displayName: "Publish to Crates.io" @@ -71,7 +69,10 @@ stages: - input: pipelineArtifact # Required, type of the input artifact artifactName: ${{parameters.PipelineArtifactName}} # Required, name of the pipeline artifact targetPath: $(Pipeline.Workspace)/drop # Optional, specifies where the artifact is downloaded to - environment: ${{parameters.Environment}} + ${{if parameters.TestPipeline}}: + environment: none + ${{else}}: + environment: package-publish # This timeout shouldn't be necessary once we're able to parallelize better. Right now, # this is here to ensure larger areas (30+) libraries don't time out. timeoutInMinutes: 120 @@ -84,33 +85,76 @@ stages: runOnce: deploy: steps: - - template: /eng/pipelines/templates/steps/use-rust.yml@self - parameters: - Toolchain: stable - - pwsh: | - $additionalOwners = @('heaths', 'hallipr') - $token = $env:CARGO_REGISTRY_TOKEN - $crateName = '${{artifact.name}}' - - $manifestPath = "$(Pipeline.Workspace)/drop/$crateName/contents/Cargo.toml" - Write-Host "> cargo publish --manifest-path `"$manifestPath`"" - cargo publish --manifest-path $manifestPath - if (!$?) { - Write-Error "Failed to publish package: '$crateName'" - exit 1 - } - - $existingOwners = (cargo owner --list $crateName) -replace " \(.*", "" - $missingOwners = $additionalOwners | Where-Object { $existingOwners -notcontains $_ } - - foreach ($owner in $missingOwners) { - Write-Host "> cargo owner --add $owner $crateName" - cargo owner --add $owner $crateName - } - displayName: Publish Crate - env: - CARGO_REGISTRY_TOKEN: $(azure-sdk-cratesio-token) + Write-Host "Setting ArtifactName to empty" + Write-Host "##vso[task.setvariable variable=ArtifactName;]" + displayName: Initialize ArtifactName variable + + # This loop over artifacts is used to produce the correct number + # of ESRP release tasks. It has the side effect of also setting + # the artifact name by looking up the index of the current + # "artifact.name" in the parameters.Artifacts array, using that + # as an "index" and then using that same index to look up the + # actual artifact to release in the release-order.json file. + - ${{ each artifact in parameters.Artifacts }}: + - pwsh: | + # From the DevOps template artifact loop calculate the current index + $indexItem = '${{ artifact.name }}' + [array] $indexList = ConvertFrom-Json '${{ convertToJson(parameters.Artifacts.*.name) }}' + $index = $indexList.IndexOf($indexItem) + Write-Host "Index of template artifact: $index" + + [array] $artifacts = Get-Content '$(Pipeline.Workspace)/drop/release-order.json' | ConvertFrom-Json + + $artifactName = $artifacts[$index] + + Write-Host "Releasing artifact: $artifactName" + + $artifactRootPath = '$(Pipeline.Workspace)/drop' + $outDir = '$(Pipeline.Workspace)/esrp-release' + + if (Test-Path $outDir) { + Write-Host "Cleaning output directory: $outDir" + Remove-Item -Path $outDir -Recurse -Force + } + New-Item -ItemType Directory -Path $outDir -Force | Out-Null + + $packageMetadataPath = "$artifactRootPath/PackageInfo/$artifactName.json" + if (!(Test-Path $packageMetadataPath)) { + Write-Error "Package metadata file not found: $packageMetadataPath" + exit 1 + } + + $packageMetadata = Get-Content -Raw $packageMetadataPath | ConvertFrom-Json + $packageVersion = $packageMetadata.version + Write-Host "Package version: $packageVersion" + + $cratePath = "$artifactRootPath/$artifactName/$artifactName-$packageVersion.crate" + Copy-Item ` + -Path $cratePath ` + -Destination $outDir + Write-Host "Contents of $outDir" + Get-ChildItem -Path $outDir | ForEach-Object { Write-Host $_.FullName } + displayName: 'Copy crate for ESRP' + + - task: EsrpRelease@10 + displayName: 'ESRP Release' + inputs: + connectedservicename: 'Azure SDK PME Managed Identity' + ClientId: '5f81938c-2544-4f1f-9251-dd9de5b8a81b' + DomainTenantId: '975f013f-7f24-47e8-a7d3-abc4752bf346' + Usemanagedidentity: true + KeyVaultName: 'kv-azuresdk-codesign' + SignCertName: 'azure-sdk-esrp-release-certificate' + intent: 'packagedistribution' + contenttype: 'Rust' + contentsource: 'Folder' + folderlocation: '$(Pipeline.Workspace)/esrp-release' + waitforreleasecompletion: true + owners: ${{ coalesce(variables['Build.RequestedForEmail'], 'azuresdk@microsoft.com') }} + approvers: ${{ coalesce(variables['Build.RequestedForEmail'], 'azuresdk@microsoft.com') }} + serviceendpointurl: 'https://api.esrp.microsoft.com/' + mainpublisher: 'ESRPRELPACMANTEST' - job: UpdatePackageVersion displayName: "API Review and Package Version Update" @@ -130,69 +174,32 @@ stages: displayName: Download ${{parameters.PipelineArtifactName}} artifact artifact: ${{parameters.PipelineArtifactName}} - - template: /eng/common/pipelines/templates/steps/create-apireview.yml - parameters: - ArtifactPath: $(Pipeline.Workspace)/${{parameters.PipelineArtifactName}} - Artifacts: ${{parameters.Artifacts}} - ConfigFileDir: $(Pipeline.Workspace)/${{parameters.PipelineArtifactName}}/PackageInfo - MarkPackageAsShipped: true - ArtifactName: ${{parameters.PipelineArtifactName}} - SourceRootPath: $(System.DefaultWorkingDirectory) - PackageName: ${{artifact.name}} - - # Apply the version increment to each library, which updates the Cargo.toml and changelog files. - - task: PowerShell@2 - displayName: Increment ${{artifact.name}} version - inputs: - targetType: filePath - filePath: $(Build.SourcesDirectory)/eng/scripts/Update-PackageVersion.ps1 - arguments: > - -ServiceDirectory '${{parameters.ServiceDirectory}}' - -PackageName '${{artifact.name}}' + - ${{ each artifact in parameters.Artifacts }}: + - template: /eng/common/pipelines/templates/steps/create-apireview.yml + parameters: + ArtifactPath: $(Pipeline.Workspace)/${{parameters.PipelineArtifactName}} + Artifacts: ${{parameters.Artifacts}} + ConfigFileDir: $(Pipeline.Workspace)/${{parameters.PipelineArtifactName}}/PackageInfo + MarkPackageAsShipped: true + ArtifactName: ${{parameters.PipelineArtifactName}} + SourceRootPath: $(System.DefaultWorkingDirectory) + PackageName: ${{artifact.name}} + + # Apply the version increment to each library, which updates the Cargo.toml and changelog files. + - task: PowerShell@2 + displayName: Increment ${{artifact.name}} version + inputs: + targetType: filePath + filePath: $(Build.SourcesDirectory)/eng/scripts/Update-PackageVersion.ps1 + arguments: > + -ServiceDirectory '${{parameters.ServiceDirectory}}' + -PackageName '${{artifact.name}}' - template: /eng/common/pipelines/templates/steps/create-pull-request.yml parameters: PRBranchName: increment-package-version-${{parameters.ServiceDirectory}}-$(Build.BuildId) - CommitMsg: "Increment package version after release of ${{ artifact.name }}" + CommitMsg: "Increment package version after release of ${{ join(', ', parameters.Artifacts.*.name) }}" PRTitle: "Increment versions for ${{parameters.ServiceDirectory}} releases" CloseAfterOpenForTesting: '${{parameters.TestPipeline}}' ${{ if startsWith(variables['Build.SourceBranch'], 'refs/pull/') }}: BaseBranchName: main - - - ${{ if eq(parameters.TestPipeline, true) }}: - - job: ManualApproval - displayName: "Manual approval" - dependsOn: PublishPackage - condition: ne(variables['Skip.PublishPackage'], 'true') - pool: server - timeoutInMinutes: 120 # 2 hours - steps: - - task: ManualValidation@1 - timeoutInMinutes: 60 # 1 hour - inputs: - notifyUsers: '' # Required, but empty string allowed - allowApproversToApproveTheirOwnRuns: true - instructions: "Approve yank of ${{ artifact.name }}" - onTimeout: 'resume' - - - job: YankCrates - displayName: "Yank Crates" - dependsOn: ManualApproval - condition: and(succeeded(), ne(variables['Skip.PublishPackage'], 'true')) - steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - - - download: current - displayName: Download ${{parameters.PipelineArtifactName}} artifact - artifact: ${{parameters.PipelineArtifactName}} - - - task: PowerShell@2 - displayName: Yank Crates - env: - CARGO_REGISTRY_TOKEN: $(azure-sdk-cratesio-token) - inputs: - targetType: filePath - filePath: $(Build.SourcesDirectory)/eng/scripts/Yank-Crates.ps1 - arguments: - -CrateNames '${{artifact.name}}' - -PackageInfoDirectory '$(Pipeline.Workspace)/${{parameters.PipelineArtifactName}}/PackageInfo' diff --git a/eng/pipelines/templates/stages/archetype-sdk-client.yml b/eng/pipelines/templates/stages/archetype-sdk-client.yml index 141ec4de3a..9cd7dd8817 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-client.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-client.yml @@ -148,6 +148,9 @@ extends: parameters: DependsOn: "Build" ServiceDirectory: ${{ parameters.ServiceDirectory }} - Artifacts: ${{ parameters.Artifacts }} + Artifacts: + - ${{ each artifact in parameters.Artifacts }}: + - ${{ if ne(artifact.releaseInBatch, 'false')}}: + - ${{ artifact }} TestPipeline: ${{ eq(parameters.ServiceDirectory, 'canary') }} PipelineArtifactName: packages diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index a973cc38f3..d085a398fc 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -1,7 +1,7 @@ $Language = "rust" $LanguageDisplayName = "Rust" $PackageRepository = "crates.io" -$packagePattern = "Cargo.toml" +$packagePattern = "*.crate" #$MetadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/main/_data/releases/latest/rust-packages.csv" $GithubUri = "https://github.com/Azure/azure-sdk-for-rust" $PackageRepositoryUri = "https://crates.io/crates" @@ -139,15 +139,32 @@ function Get-rust-AdditionalValidationPackagesFromPackageSet ($packagesWithChang return $additionalPackages ?? @() } +# $GetPackageInfoFromPackageFileFn = "Get-${Language}-PackageInfoFromPackageFile" function Get-rust-PackageInfoFromPackageFile([IO.FileInfo]$pkg, [string]$workingDirectory) { - #$pkg will be a FileInfo object for the Cargo.toml file in a package artifact directory - $package = cargo read-manifest --manifest-path $pkg.FullName | ConvertFrom-Json + # Create a temporary folder for extraction + $extractionPath = [System.IO.Path]::Combine([System.IO.Path]::GetTempPath(), [System.IO.Path]::GetRandomFileName()) + New-Item -ItemType Directory -Path $extractionPath | Out-Null + + # Extract the .crate file (which is a tarball) to the temporary folder + tar -xvf $pkg.FullName -C $extractionPath + $cargoTomlPath = [System.IO.Path]::Combine($extractionPath, $pkg.BaseName, 'Cargo.toml') + + Write-Host "Reading package info from $cargoTomlPath" + if (!(Test-Path $cargoTomlPath)) { + $message = "The Cargo.toml file was not found in the package artifact at $cargoTomlPath" + LogError $message + throw $message + } + + $package = cargo read-manifest --manifest-path $cargoTomlPath | ConvertFrom-Json $packageName = $package.name $packageVersion = $package.version - $changeLogLoc = Get-ChildItem -Path $pkg.DirectoryName -Filter "CHANGELOG.md" | Select-Object -First 1 - $readmeContentLoc = Get-ChildItem -Path $pkg.DirectoryName -Filter "README.md" | Select-Object -First 1 + $packageAssetPath = [System.IO.Path]::Combine($extractionPath, "$packageName-$packageVersion") + + $changeLogLoc = Get-ChildItem -Path $packageAssetPath -Filter "CHANGELOG.md" | Select-Object -First 1 + $readmeContentLoc = Get-ChildItem -Path $packageAssetPath -Filter "README.md" | Select-Object -First 1 if ($changeLogLoc) { $releaseNotes = Get-ChangeLogEntryAsString -ChangeLogLocation $changeLogLoc -VersionString $packageVersion diff --git a/eng/scripts/Pack-Crates.ps1 b/eng/scripts/Pack-Crates.ps1 index b266667345..4b2c290cc1 100755 --- a/eng/scripts/Pack-Crates.ps1 +++ b/eng/scripts/Pack-Crates.ps1 @@ -8,22 +8,20 @@ param( [string[]]$PackageNames, [Parameter(ParameterSetName = 'PackageInfo')] [string]$PackageInfoDirectory, - [switch]$NoVerify + [switch]$Release, + [switch]$NoVerify, + [string]$OutBuildOrderFile ) $ErrorActionPreference = 'Stop' -. (Join-Path $PSScriptRoot '..' 'common' 'scripts' 'common.ps1') +. ([System.IO.Path]::Combine($PSScriptRoot, '..', 'common', 'scripts', 'common.ps1')) Write-Host @" Packing crates with RUSTFLAGS: '${env:RUSTFLAGS}' "@ -if ($OutputPath) { - $OutputPath = New-Item -ItemType Directory -Path $OutputPath -Force | Select-Object -ExpandProperty FullName -} - function Get-OutputPackageNames($workspacePackages) { $packablePackages = $workspacePackages | Where-Object -Property publish -NE -Value @() $packablePackageNames = $packablePackages.name @@ -57,15 +55,11 @@ function Get-OutputPackageNames($workspacePackages) { return $names } -function Get-CargoMetadata() { - cargo metadata --no-deps --format-version 1 --manifest-path "$RepoRoot/Cargo.toml" | ConvertFrom-Json -Depth 100 -AsHashtable -} - function Get-CargoPackages() { $metadata = Get-CargoMetadata - # path based depdenencies are assumed to be unreleased package versions - # they must be included in this build and build before packages that depend on them + # Path based dependencies are assumed to be unreleased package versions. In + # non-release builds these should be packed as well. foreach ($package in $metadata.packages) { $package.UnreleasedDependencies = @() foreach ($dependency in $package.dependencies) { @@ -83,9 +77,18 @@ function Get-PackagesToBuild() { $packages = Get-CargoPackages $outputPackageNames = Get-OutputPackageNames $packages - # We start with output packages, then recursively add unreleased dependencies to the list of packages that need to be built + # Force array in instances of a single package name + if ($outputPackageNames -isnot [array]) { + $outputPackageNames = @($outputPackageNames) + } + [array]$packagesToBuild = $packages | Where-Object { $outputPackageNames.Contains($_.name) } + if ($Release) { + return $packagesToBuild + } + + # If not releasing, expand dependencies into list of packages to build $toProcess = $packagesToBuild while ($toProcess.Length -gt 0) { $package = $toProcess[0] @@ -99,51 +102,11 @@ function Get-PackagesToBuild() { } } - $buildOrder = @() - - # Then we order the packages to that dependencies are built first - while ($packagesToBuild.Count -gt 0) { - # Pick any package with no unreleased dependencies, add it to the build order and remove it from the list of other packages' unreleased dependencies - $package = $packagesToBuild | Where-Object { $_.UnreleasedDependencies.Count -eq 0 } | Select-Object -First 1 - - if (-not $package) { - Write-Error "These packages cannot be built because they depend on unreleased dependencies that aren't being built." -ErrorAction Continue - foreach ($package in $packagesToBuild) { - Write-Error " $($package.name) -> $($package.UnreleasedDependencies -join ', ')" -ErrorAction Continue - } - exit 1 - } - - $package.OutputPackage = $outputPackageNames.Contains($package.name) - $buildOrder += $package - $packagesToBuild = @($packagesToBuild -ne $package) - - foreach ($otherPackage in $packagesToBuild) { - $otherPackage.UnreleasedDependencies = $otherPackage.UnreleasedDependencies -ne $package - } - } - - return $buildOrder -} - -function Initialize-VendorDirectory() { - $path = "$RepoRoot/target/vendor" - Invoke-LoggedCommand "cargo vendor $path" -GroupOutput | Out-Host - return $path + return $packagesToBuild } -function Add-CrateToLocalRegistry($LocalRegistryPath, $Package) { - $packageName = $Package.name - $packageVersion = $Package.version - - # create an index entry for the package - $packagePath = "$RepoRoot/target/package/$packageName-$packageVersion" - - Write-Host "Copying package '$packageName' to vendor directory '$LocalRegistryPath'" - Copy-Item -Path $packagePath -Destination $LocalRegistryPath -Recurse - - #write an empty checksum file - '{"files":{}}' | Out-File -FilePath "$LocalRegistryPath/$packageName-$packageVersion/.cargo-checksum.json" -Encoding utf8 +function Get-CargoMetadata() { + cargo metadata --no-deps --format-version 1 --manifest-path "$RepoRoot/Cargo.toml" | ConvertFrom-Json -Depth 100 -AsHashtable } function Create-ApiViewFile($package) { @@ -156,64 +119,73 @@ function Create-ApiViewFile($package) { "$packagePath/review/$packageName.rust.json" } -Push-Location $RepoRoot +$originalLocation = Get-Location try { - $localRegistryPath = Initialize-VendorDirectory + Set-Location $RepoRoot [array]$packages = Get-PackagesToBuild - - Write-Host "Building packages in the following order:" + $packageParams = @() foreach ($package in $packages) { - $packageName = $package.name - $type = if ($package.OutputPackage) { "output" } else { "dependency" } - Write-Host " $packageName ($type)" + $packageParams += "--package", $package.name } - foreach ($package in $packages) { - Write-Host "" - - $packageName = $package.name - $packageVersion = $package.version - - $command = "cargo publish --locked --dry-run --package $packageName --registry crates-io --config `"source.crates-io.replace-with='local'`" --config `"source.local.directory='$localRegistryPath'`" --allow-dirty" - - if ($NoVerify) { - $command += " --no-verify" - } - - Invoke-LoggedCommand -Command $command -GroupOutput + if ($NoVerify) { + $packageParams += "--no-verify" + } + LogGroupStart "cargo publish --locked --dry-run --allow-dirty $($packageParams -join ' ')" + Write-Host "cargo publish --locked --dry-run --allow-dirty $($packageParams -join ' ')" + & cargo publish --locked --dry-run --allow-dirty @packageParams 2>&1 ` + | Tee-Object -Variable packResult ` + | ForEach-Object { Write-Host $_ -ForegroundColor Gray } + LogGroupEnd + + Write-Host "Finished packing crates" + if ($LASTEXITCODE) { + Write-Host "cargo publish failed with exit code $LASTEXITCODE" + exit $LASTEXITCODE + } - # copy the package to the local registry - Add-CrateToLocalRegistry ` - -LocalRegistryPath $localRegistryPath ` - -Package $package + if ($OutputPath) { + $OutputPath = New-Item -ItemType Directory -Path $OutputPath -Force | Select-Object -ExpandProperty FullName - if ($OutputPath -and $package.OutputPackage) { - $sourcePath = "$RepoRoot/target/package/$packageName-$packageVersion" - $targetPath = "$OutputPath/$packageName" - $targetContentsPath = "$targetPath/contents" - $targetApiReviewFile = "$targetPath/$packageName.rust.json" + foreach ($package in $packages) { + $sourcePath = [System.IO.Path]::Combine($RepoRoot, "target", "package", "$($package.name)-$($package.version)") + $targetPath = [System.IO.Path]::Combine($OutputPath, $package.name) + $targetContentsPath = [System.IO.Path]::Combine($targetPath, "contents") + $targetApiReviewFile = [System.IO.Path]::Combine($targetPath, "$($package.name).rust.json") if (Test-Path -Path $targetContentsPath) { Remove-Item -Path $targetContentsPath -Recurse -Force } - Write-Host "Copying package '$packageName' to '$targetContentsPath'" + Write-Host "Copying package contents '$($package.name)' to '$targetContentsPath'" New-Item -ItemType Directory -Path $targetContentsPath -Force | Out-Null - Copy-Item -Path $sourcePath/* -Destination $targetContentsPath -Recurse -Exclude "Cargo.toml.orig" + Copy-Item -Path $sourcePath/* -Destination $targetContentsPath -Recurse + + Write-Host "Copying .crate file for '$($package.name)' to '$targetPath'" + Copy-Item -Path "$sourcePath.crate" -Destination $targetPath -Force Write-Host "Creating API review file" $apiReviewFile = Create-ApiViewFile $package - + Write-Host "Copying API review file to '$targetApiReviewFile'" Copy-Item -Path $apiReviewFile -Destination $targetApiReviewFile -Force } } - Write-Host "Removing local registry" - Remove-Item -Path $localRegistryPath -Recurse -Force | Out-Null + if ($OutBuildOrderFile) { + $buildOrder = @() + foreach ($line in $packResult) { + if ($line -match '^\s*Packaging (\w*) ([\w\d\.-]*)') { + $buildOrder += $matches[1] + } + } + + Write-Host "Build Order: $($buildOrder -join ', ')" + ConvertTo-Json $buildOrder -Depth 100 | Set-Content $OutBuildOrderFile + } } finally { - Pop-Location + Set-Location $originalLocation } diff --git a/sdk/canary/ci.yml b/sdk/canary/ci.yml index 0ffe09337e..55e0c848fd 100644 --- a/sdk/canary/ci.yml +++ b/sdk/canary/ci.yml @@ -10,12 +10,22 @@ trigger: include: - sdk/canary/ +parameters: +- name: release_azure_canary_core + displayName: azure_canary_core + type: boolean + default: false +- name: release_azure_canary + displayName: azure_canary + type: boolean + default: false + extends: template: /eng/pipelines/templates/stages/archetype-sdk-client.yml parameters: ServiceDirectory: canary Artifacts: - - name: azure_canary_core - safeName: AzureTemplateCore - name: azure_canary - safeName: AzureTemplate + releaseInBatch: ${{ parameters.release_azure_canary }} + - name: azure_canary_core + releaseInBatch: ${{ parameters.release_azure_canary_core }} diff --git a/sdk/core/ci.yml b/sdk/core/ci.yml index fcbcbd265a..c7dc313f70 100644 --- a/sdk/core/ci.yml +++ b/sdk/core/ci.yml @@ -10,22 +10,52 @@ trigger: include: - sdk/core/ +parameters: +- name: typespec + displayName: typespec + type: boolean + default: false +- name: typespec_macros + displayName: typespec_macros + type: boolean + default: false +- name: typespec_client_core + displayName: typespec_client_core + type: boolean + default: false +- name: release_azure_core + displayName: azure_core + type: boolean + default: false +- name: release_azure_core_macros + displayName: azure_core_macros + type: boolean + default: false +- name: release_azure_core_amqp + displayName: azure_core_amqp + type: boolean + default: false +- name: release_azure_core_opentelemetry + displayName: azure_core_opentelemetry + type: boolean + default: false + extends: template: /eng/pipelines/templates/stages/archetype-sdk-client.yml parameters: ServiceDirectory: core Artifacts: - name: typespec - safeName: Typespec + releaseInBatch: ${{ parameters.typespec }} - name: typespec_macros - safeName: TypespecMacros + releaseInBatch: ${{ parameters.typespec_macros }} - name: typespec_client_core - safeName: TypespecClientCore + releaseInBatch: ${{ parameters.typespec_client_core }} - name: azure_core - safeName: AzureCore + releaseInBatch: ${{ parameters.azure_core }} - name: azure_core_macros - safeName: AzureCoreMacros + releaseInBatch: ${{ parameters.azure_core_macros }} - name: azure_core_amqp - safeName: AzureCoreAmqp + releaseInBatch: ${{ parameters.azure_core_amqp }} - name: azure_core_opentelemetry - safeName: AzureCoreOpentelemetry + releaseInBatch: ${{ parameters.azure_core_opentelemetry }} diff --git a/sdk/cosmos/ci.yml b/sdk/cosmos/ci.yml index d5509e143a..73bccf9941 100644 --- a/sdk/cosmos/ci.yml +++ b/sdk/cosmos/ci.yml @@ -10,10 +10,16 @@ trigger: include: - sdk/cosmos/ +parameters: +- name: release_azure_data_cosmos + displayName: azure_data_cosmos + type: boolean + default: false + extends: template: /eng/pipelines/templates/stages/archetype-sdk-client.yml parameters: ServiceDirectory: cosmos Artifacts: - name: azure_data_cosmos - safeName: AzureDataCosmos + releaseInBatch: ${{ parameters.release_azure_data_cosmos }} diff --git a/sdk/eventhubs/ci.yml b/sdk/eventhubs/ci.yml index 917e2a89f1..7b4ed58c71 100644 --- a/sdk/eventhubs/ci.yml +++ b/sdk/eventhubs/ci.yml @@ -10,12 +10,22 @@ trigger: include: - sdk/eventhubs/ +parameters: +- name: release_azure_messaging_eventhubs + displayName: azure_messaging_eventhubs + type: boolean + default: false +- name: release_azure_messaging_eventhubs_checkpointstore_blob + displayName: azure_messaging_eventhubs_checkpointstore_blob + type: boolean + default: false + extends: template: /eng/pipelines/templates/stages/archetype-sdk-client.yml parameters: ServiceDirectory: eventhubs Artifacts: - name: azure_messaging_eventhubs - safeName: AzureMessagingEventHubs + releaseInBatch: ${{ parameters.release_azure_messaging_eventhubs }} - name: azure_messaging_eventhubs_checkpointstore_blob - safeName: AzureMessagingEventHubsBlobCheckpointStore + releaseInBatch: ${{ parameters.release_azure_messaging_eventhubs_checkpointstore_blob }} diff --git a/sdk/identity/ci.yml b/sdk/identity/ci.yml index 0e7a26102e..9908060d08 100644 --- a/sdk/identity/ci.yml +++ b/sdk/identity/ci.yml @@ -3,12 +3,18 @@ trigger: branches: include: - - main - - hotfix/* - - release/* + - main + - hotfix/* + - release/* paths: include: - - sdk/identity/ + - sdk/identity/ + +parameters: +- name: release_azure_identity + displayName: azure_identity + type: boolean + default: false extends: template: /eng/pipelines/templates/stages/archetype-sdk-client.yml @@ -16,7 +22,7 @@ extends: ServiceDirectory: identity Artifacts: - name: azure_identity - safeName: AzureIdentity + releaseInBatch: ${{ parameters.release_azure_identity }} ${{ if endsWith(variables['Build.DefinitionName'], 'weekly') }}: Location: uksouth diff --git a/sdk/keyvault/ci.yml b/sdk/keyvault/ci.yml index 44779bf03c..6ecd0157ac 100644 --- a/sdk/keyvault/ci.yml +++ b/sdk/keyvault/ci.yml @@ -1,10 +1,4 @@ # NOTE: Please refer to https://aka.ms/azsdk/engsys/ci-yaml before editing this file. -parameters: -- name: RunLiveTests - displayName: 'Run live tests' - type: boolean - default: false - trigger: branches: include: @@ -15,6 +9,24 @@ trigger: include: - sdk/keyvault/ +parameters: +- name: RunLiveTests + displayName: Run live tests + type: boolean + default: false +- name: release_azure_security_keyvault_secrets + displayName: azure_security_keyvault_secrets + type: boolean + default: false +- name: release_azure_security_keyvault_keys + displayName: azure_security_keyvault_keys + type: boolean + default: false +- name: release_azure_security_keyvault_certificates + displayName: azure_security_keyvault_certificates + type: boolean + default: false + extends: template: /eng/pipelines/templates/stages/archetype-sdk-client.yml parameters: @@ -22,8 +34,8 @@ extends: RunLiveTests: ${{ or(parameters.RunLiveTests, eq(variables['Build.Reason'], 'Schedule')) }} Artifacts: - name: azure_security_keyvault_secrets - safeName: AzureSecurityKeyvaultSecrets - - name: azure_security_keyvault_keys - safeName: AzureSecurityKeyvaultKeys + releaseInBatch: ${{ parameters.release_azure_security_keyvault_secrets }} - name: azure_security_keyvault_certificates - safeName: AzureSecurityKeyvaultCertificates + releaseInBatch: ${{ parameters.release_azure_security_keyvault_certificates }} + - name: azure_security_keyvault_keys + releaseInBatch: ${{ parameters.release_azure_security_keyvault_keys }} diff --git a/sdk/servicebus/ci.yml b/sdk/servicebus/ci.yml index 36c7d1c97a..c9d7068834 100644 --- a/sdk/servicebus/ci.yml +++ b/sdk/servicebus/ci.yml @@ -3,18 +3,24 @@ trigger: branches: include: - - main - - feature/* - - release/* - - hotfix/* + - main + - feature/* + - release/* + - hotfix/* paths: include: - - sdk/servicebus/ + - sdk/servicebus/ + +parameters: +- name: release_azure_messaging_servicebus + displayName: azure_messaging_servicebus + type: boolean + default: false extends: template: /eng/pipelines/templates/stages/archetype-sdk-client.yml parameters: ServiceDirectory: servicebus Artifacts: - - name: azure_messaging_servicebus - safeName: AzureMessagingServiceBus + - name: azure_messaging_servicebus + releaseInBatch: ${{ parameters.release_azure_messaging_servicebus }} diff --git a/sdk/storage/ci.yml b/sdk/storage/ci.yml index bd92c09b5e..361eb08c2a 100644 --- a/sdk/storage/ci.yml +++ b/sdk/storage/ci.yml @@ -1,10 +1,4 @@ # NOTE: Please refer to https://aka.ms/azsdk/engsys/ci-yaml before editing this file. -parameters: -- name: RunLiveTests - displayName: 'Run live tests' - type: boolean - default: false - trigger: branches: include: @@ -15,6 +9,24 @@ trigger: include: - sdk/storage/ +parameters: +- name: RunLiveTests + displayName: Run live tests + type: boolean + default: false +- name: release_azure_storage_common + displayName: azure_storage_common + type: boolean + default: false +- name: release_azure_storage_blob + displayName: azure_storage_blob + type: boolean + default: false +- name: release_azure_storage_queue + displayName: azure_storage_queue + type: boolean + default: false + extends: template: /eng/pipelines/templates/stages/archetype-sdk-client.yml parameters: @@ -22,9 +34,10 @@ extends: RunLiveTests: ${{ or(parameters.RunLiveTests, eq(variables['Build.Reason'], 'Schedule')) }} TestTimeoutInMinutes: 120 Artifacts: - - name: azure_storage_common - safeName: AzureStorageCommon + # Artifact order in this file maps to order evaluated by Pack-Crates.ps1 + # When the artifacts like blob and queue depend on common, common will need + # to be listed first. - name: azure_storage_blob - safeName: AzureStorageBlob + releaseInBatch: ${{ parameters.release_azure_storage_blob }} - name: azure_storage_queue - safeName: AzureStorageQueue + releaseInBatch: ${{ parameters.release_azure_storage_queue }}