From 83a7bae6cc6f27b7b9b6374e7e5dbdd64434ce07 Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Thu, 23 Jan 2025 08:35:32 -0800 Subject: [PATCH 1/3] Updates in preparation for PR pipeline --- eng/pipelines/templates/jobs/ci.tests.yml | 5 ++ eng/pipelines/templates/jobs/ci.yml | 69 +++++++++++++------ ...roject-list-and-cache-maven-repository.yml | 16 +++++ .../steps/initialize-test-environment.yml | 9 +++ eng/scripts/Stage-MavenPackageArtifacts.ps1 | 25 ++++++- .../Verify-Required-Maven-Artifacts.ps1 | 25 ++++++- 6 files changed, 125 insertions(+), 24 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index 181e7543f2e54..6195ff3979ac3 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -5,6 +5,9 @@ parameters: - name: ServiceDirectory type: string default: 'not-specified' # Set a default that breaks in obvious ways. + - name: ExcludePaths + type: object + default: [] - name: TestPipeline type: boolean default: false @@ -104,6 +107,8 @@ jobs: - template: /eng/pipelines/templates/steps/initialize-test-environment.yml parameters: CheckoutRecordings: true + ServiceDirectory: ${{parameters.ServiceDirectory}} + ExcludePaths: ${{parameters.ExcludePaths}} - template: /eng/common/testproxy/test-proxy-tool.yml parameters: diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index b3944bc6583f0..0a4222e130248 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -95,18 +95,11 @@ jobs: - '!sdk/**/test-recordings' - '!sdk/**/session-records' - # This is the case for the PullReqest pipeline which has a ServiceDirectory of 'auto'. - # For the PullRequest pipeline, the packageInfo files generated from the diff end up - # being the source of truth for the artifacts list. - - ${{ if and(eq(variables['Build.Reason'], 'PullRequest'), eq(parameters.ServiceDirectory, 'auto')) }}: - - template: /eng/common/pipelines/templates/steps/save-package-properties.yml - parameters: - ServiceDirectory: ${{parameters.ServiceDirectory}} - ExcludePaths: ${{parameters.ExcludePaths}} - - template: /eng/pipelines/templates/steps/generate-project-list-and-cache-maven-repository.yml parameters: JobType: 'Build' + ServiceDirectory: ${{parameters.ServiceDirectory}} + ExcludePaths: ${{parameters.ExcludePaths}} - task: UsePythonVersion@0 displayName: 'Use Python $(PythonVersion)' @@ -203,8 +196,11 @@ jobs: arguments: > -BuildOutputDirectory $(System.DefaultWorkingDirectory)/build -ArtifactsList ('$(ArtifactsJson)' | ConvertFrom-Json | Select-Object name, groupId) + -PackageInfoDir $(Build.ArtifactStagingDirectory)/PackageInfo -InformationAction Continue + # JRS - This only copies the ReleaseArtifacts but for the PR pipeline + # just copy all of the built artifacts? - task: PowerShell@2 displayName: 'Copy artifacts to staging' inputs: @@ -215,6 +211,7 @@ jobs: -SourceDirectory $(System.DefaultWorkingDirectory)/build -TargetDirectory $(Build.ArtifactStagingDirectory) -Artifacts ('${{ replace(convertToJson(parameters.ReleaseArtifacts), '''', '`''') }}' | ConvertFrom-Json | Where-Object -Not skipPublishPackage ) + -PackageInfoDir $(Build.ArtifactStagingDirectory)/PackageInfo -InformationAction Continue - task: Powershell@2 @@ -303,6 +300,8 @@ jobs: - template: /eng/pipelines/templates/steps/generate-project-list-and-cache-maven-repository.yml parameters: JobType: 'Analyze' + ServiceDirectory: ${{parameters.ServiceDirectory}} + ExcludePaths: ${{parameters.ExcludePaths}} - task: PowerShell@2 displayName: Generate directories variable for sparse checkout @@ -332,9 +331,14 @@ jobs: ServiceDirectory: "template" TestPipeline: ${{ parameters.TestPipeline }} - - template: /eng/common/pipelines/templates/steps/verify-readme.yml - parameters: - ScanPath: $(Build.SourcesDirectory)/sdk/${{ parameters.ServiceDirectory }} + - ${{ if and(eq(variables['Build.Reason'], 'PullRequest'), eq(parameters.ServiceDirectory, 'auto')) }}: + - template: /eng/common/pipelines/templates/steps/verify-readmes.yml + parameters: + PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo + - ${{ else }}: + - template: /eng/common/pipelines/templates/steps/verify-readme.yml + parameters: + ScanPath: $(Build.SourcesDirectory)/sdk/${{ parameters.ServiceDirectory }} - template: /eng/common/pipelines/templates/steps/verify-links.yml parameters: @@ -345,9 +349,24 @@ jobs: Directory: sdk/${{ parameters.ServiceDirectory }} CheckLinkGuidance: $true - - template: /eng/common/pipelines/templates/steps/verify-samples.yml - parameters: - ServiceDirectory: ${{ parameters.ServiceDirectory }} + - ${{ if and(eq(variables['Build.Reason'], 'PullRequest'), eq(parameters.ServiceDirectory, 'auto')) }}: + - pwsh: | + $diffJson = Get-Content '$(Build.ArtifactStagingDirectory)/diff/diff.json' -Raw + $diff = ConvertFrom-Json $diffJson + $changedServicesString = '' + if ($diff.ChangedServices) { + $changedServicesString = $diff.ChangedServices -join "," + } + Write-Host "changedServicesString='$changedServicesString'" + Write-Host "##vso[task.setvariable variable=SamplesServiceDirectories;]$changedServicesString" + displayName: Get samples service directories from PR diff + - template: /eng/common/pipelines/templates/steps/verify-samples.yml + parameters: + ServiceDirectories: $(SamplesServiceDirectories) + - ${{ else }}: + - template: /eng/common/pipelines/templates/steps/verify-samples.yml + parameters: + ServiceDirectory: ${{ parameters.ServiceDirectory }} # Use BasePathLength of 38 instead of the default 49 as some released files fail when the number is higher. - template: /eng/common/pipelines/templates/steps/verify-path-length.yml @@ -356,13 +375,18 @@ jobs: BasePathLength: 38 - ${{if ne(parameters.SDKType, 'data')}}: - - ${{ each artifact in parameters.Artifacts }}: - - ${{if ne(artifact.skipVerifyChangelog, 'true')}}: - - template: /eng/common/pipelines/templates/steps/verify-changelog.yml - parameters: - PackageName: ${{artifact.name}} - ServiceName: ${{parameters.ServiceDirectory}} - ForRelease: false + - ${{ if and(eq(variables['Build.Reason'], 'PullRequest'), eq(parameters.ServiceDirectory, 'auto')) }}: + - template: /eng/common/pipelines/templates/steps/verify-changelogs.yml + parameters: + PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo + - ${{ else }}: + - ${{ each artifact in parameters.Artifacts }}: + - ${{if ne(artifact.skipVerifyChangelog, 'true')}}: + - template: /eng/common/pipelines/templates/steps/verify-changelog.yml + parameters: + PackageName: ${{artifact.name}} + ServiceName: ${{parameters.ServiceDirectory}} + ForRelease: false # We `install` the code quality reports tooling into our local m2 cache separately from building the Maven project # reports. This means it is available as part of that, but also so that this is not documented in the project report. @@ -430,6 +454,7 @@ jobs: AdditionalParameters: SDKType: ${{ parameters.SDKType }} ServiceDirectory: ${{ parameters.ServiceDirectory }} + ExcludePaths: ${{ parameters.ExcludePaths }} TestPipeline: ${{ parameters.TestPipeline }} PreTestSteps: ${{ parameters.PreTestSteps }} PreBuildSteps: ${{ parameters.PreBuildSteps }} diff --git a/eng/pipelines/templates/steps/generate-project-list-and-cache-maven-repository.yml b/eng/pipelines/templates/steps/generate-project-list-and-cache-maven-repository.yml index 7eacf72434809..60acc7ea962b4 100644 --- a/eng/pipelines/templates/steps/generate-project-list-and-cache-maven-repository.yml +++ b/eng/pipelines/templates/steps/generate-project-list-and-cache-maven-repository.yml @@ -15,8 +15,24 @@ parameters: - name: UseCache type: boolean default: true + - name: ServiceDirectory + type: string + default: 'not-specified' # Set a default that breaks in obvious ways. + - name: ExcludePaths + type: object + default: [] steps: + +# This is the case for the PullReqest pipeline which has a ServiceDirectory of 'auto'. +# For the PullRequest pipeline, the packageInfo files generated from the diff end up +# being the source of truth for the artifacts list. +- ${{ if and(eq(variables['Build.Reason'], 'PullRequest'), eq(parameters.ServiceDirectory, 'auto')) }}: + - template: /eng/common/pipelines/templates/steps/save-package-properties.yml + parameters: + ServiceDirectory: ${{parameters.ServiceDirectory}} + ExcludePaths: ${{parameters.ExcludePaths}} + # Create the ProjectList and ProjectListSha256 variables. # The ProjectList is used by subsequent steps and tasks to run CI. # The ProjectListSha256 is used by the cache key, previously the cache key was using ProjectList but it was found diff --git a/eng/pipelines/templates/steps/initialize-test-environment.yml b/eng/pipelines/templates/steps/initialize-test-environment.yml index 62a437b7f87b0..4ee90d3fec21b 100644 --- a/eng/pipelines/templates/steps/initialize-test-environment.yml +++ b/eng/pipelines/templates/steps/initialize-test-environment.yml @@ -5,6 +5,12 @@ parameters: - name: OS type: string default: $(OSVmImage) + - name: ServiceDirectory + type: string + default: 'not-specified' # Set a default that breaks in obvious ways. + - name: ExcludePaths + type: object + default: [] steps: # Skip sparse checkout for the `azure-sdk-for--pr` private mirrored repositories @@ -14,6 +20,7 @@ steps: parameters: Paths: - '**/*.xml' + - '**/ci*.yml' # necessary for save-package-properties.yml - ${{ if not(parameters.CheckoutRecordings) }}: - '!sdk/**/test-recordings' - '!sdk/**/session-records' @@ -44,6 +51,8 @@ steps: - template: /eng/pipelines/templates/steps/generate-project-list-and-cache-maven-repository.yml parameters: JobType: 'test' + ServiceDirectory: ${{parameters.ServiceDirectory}} + ExcludePaths: ${{parameters.ExcludePaths}} # This call is used generate the sparse checkout variables for non-FromSource runs. - task: PowerShell@2 diff --git a/eng/scripts/Stage-MavenPackageArtifacts.ps1 b/eng/scripts/Stage-MavenPackageArtifacts.ps1 index c8c64084c1177..fbff85f63f5c4 100644 --- a/eng/scripts/Stage-MavenPackageArtifacts.ps1 +++ b/eng/scripts/Stage-MavenPackageArtifacts.ps1 @@ -1,7 +1,8 @@ param( [Parameter(Mandatory=$true)][string]$SourceDirectory, [Parameter(Mandatory=$true)][string]$TargetDirectory, - [Parameter(Mandatory=$true)][array]$Artifacts + [Parameter(Mandatory=$false)][array]$Artifacts, + [Parameter(Mandatory=$false)][string] $PackageInfoDir = $null ) $ErrorActionPreference = "Stop" @@ -11,6 +12,28 @@ Write-host "Target Directory is: $TargetDirectory" . $PSScriptRoot\MavenPackaging.ps1 +if ($Artifacts -eq $null) { + $Artifacts = @() +} + +if ($ArtifactsList.Count -eq 0) { + if (-not $PackageInfoDir -or (-not (Test-Path -Path $PackageInfoDir))) { + LogError "Artifacts list was empty and PackageInfoDir was null or incorrect." + exit(1) + } + Write-Host "Artifacts List was empty, getting Artifacts from PackageInfoDir=$PackageInfoDir" + [array]$packageInfoFiles = Get-ChildItem -Path $PackageInfoDir "*.json" + foreach($packageInfoFile in $packageInfoFiles) { + $packageInfoJson = Get-Content $packageInfoFile -Raw + $packageInfo = ConvertFrom-Json $packageInfoJson + $Artifacts += New-Object PSObject -Property @{ + groupId = $packageInfo.Group + name = $packageInfo.ArtifactName + } + + } +} + Write-Host "Searching for packages in: $SourceDirectory" $packageDetails = Get-MavenPackageDetails -ArtifactDirectory $SourceDirectory Write-Host "Found $($packageDetails.Count) packages in: $SourceDirectory" diff --git a/eng/scripts/Verify-Required-Maven-Artifacts.ps1 b/eng/scripts/Verify-Required-Maven-Artifacts.ps1 index c936aab2254c3..d3d1d4c85e8bd 100644 --- a/eng/scripts/Verify-Required-Maven-Artifacts.ps1 +++ b/eng/scripts/Verify-Required-Maven-Artifacts.ps1 @@ -9,7 +9,8 @@ param( [Parameter(Mandatory=$true)][string]$BuildOutputDirectory, # ArtifactsList will be using ('${{ convertToJson(parameters.Artifacts) }}' | ConvertFrom-Json | Select-Object name, groupId) - [Parameter(Mandatory=$true)][array] $ArtifactsList + [Parameter(Mandatory=$false)][array] $ArtifactsList, + [Parameter(Mandatory=$false)][string] $PackageInfoDir = $null ) . "${PSScriptRoot}/../common/scripts/common.ps1" @@ -17,6 +18,28 @@ Write-Host "BuildOutputDirectory=$($BuildOutputDirectory)" Write-Host "BuildOutputDirectory contents" Get-ChildItem -Path $BuildOutputDirectory -Recurse -Name +if (-not $ArtifactsList) { + $ArtifactsList = @() +} + +if ($ArtifactsList.Count -eq 0) { + if (-not $PackageInfoDir -or (-not (Test-Path -Path $PackageInfoDir))) { + LogError "ArtifactsList was empty and PackageInfoDir was null or incorrect." + exit(1) + } + Write-Host "Artifacts List was empty, getting Artifacts from PackageInfoDir=$PackageInfoDir" + [array]$packageInfoFiles = Get-ChildItem -Path $PackageInfoDir "*.json" + foreach($packageInfoFile in $packageInfoFiles) { + $packageInfoJson = Get-Content $packageInfoFile -Raw + $packageInfo = ConvertFrom-Json $packageInfoJson + $ArtifactsList += New-Object PSObject -Property @{ + groupId = $packageInfo.Group + name = $packageInfo.ArtifactName + } + + } +} + Write-Host "" Write-Host "ArtifactsList:" $ArtifactsList | Format-Table -Property GroupId, Name | Out-String | Write-Host From da9b1c056b3052c91bce8d9450c9b35e57c42e7f Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Thu, 23 Jan 2025 09:28:04 -0800 Subject: [PATCH 2/3] Fix variable name --- eng/scripts/Stage-MavenPackageArtifacts.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/scripts/Stage-MavenPackageArtifacts.ps1 b/eng/scripts/Stage-MavenPackageArtifacts.ps1 index fbff85f63f5c4..e5138fc45d220 100644 --- a/eng/scripts/Stage-MavenPackageArtifacts.ps1 +++ b/eng/scripts/Stage-MavenPackageArtifacts.ps1 @@ -16,7 +16,7 @@ if ($Artifacts -eq $null) { $Artifacts = @() } -if ($ArtifactsList.Count -eq 0) { +if ($Artifacts.Count -eq 0) { if (-not $PackageInfoDir -or (-not (Test-Path -Path $PackageInfoDir))) { LogError "Artifacts list was empty and PackageInfoDir was null or incorrect." exit(1) From 8c72388415091ca8bb7f809e401eb1a78b1ee605 Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Thu, 23 Jan 2025 09:37:25 -0800 Subject: [PATCH 3/3] Remove my initials from a comment and complete the comment --- eng/pipelines/templates/jobs/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 0a4222e130248..9e07e7a39dd90 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -199,8 +199,8 @@ jobs: -PackageInfoDir $(Build.ArtifactStagingDirectory)/PackageInfo -InformationAction Continue - # JRS - This only copies the ReleaseArtifacts but for the PR pipeline - # just copy all of the built artifacts? + # This only copies the ReleaseArtifacts but for the PR pipeline just copy + # all of the built artifacts. - task: PowerShell@2 displayName: 'Copy artifacts to staging' inputs: