Updates in preparation for PR pipeline

eng/pipelines/templates/jobs/ci.tests.yml

@@ -5,6 +5,9 @@ parameters:
   - name: ServiceDirectory
     type: string
     default: 'not-specified' # Set a default that breaks in obvious ways.
+  - name: ExcludePaths
+    type: object
+    default: []
   - name: TestPipeline
     type: boolean
     default: false
@@ -104,6 +107,8 @@ jobs:
       - template: /eng/pipelines/templates/steps/initialize-test-environment.yml
         parameters:
           CheckoutRecordings: true
+          ServiceDirectory: ${{parameters.ServiceDirectory}}
+          ExcludePaths: ${{parameters.ExcludePaths}}
 
       - template: /eng/common/testproxy/test-proxy-tool.yml
         parameters:

eng/pipelines/templates/jobs/ci.yml

@@ -95,18 +95,11 @@ jobs:
               - '!sdk/**/test-recordings'
               - '!sdk/**/session-records'
 
-      # This is the case for the PullReqest pipeline which has a ServiceDirectory of 'auto'.
-      # For the PullRequest pipeline, the packageInfo files generated from the diff end up
-      # being the source of truth for the artifacts list.
-      - ${{ if and(eq(variables['Build.Reason'], 'PullRequest'), eq(parameters.ServiceDirectory, 'auto')) }}:
-        - template: /eng/common/pipelines/templates/steps/save-package-properties.yml
-          parameters:
-            ServiceDirectory: ${{parameters.ServiceDirectory}}
-            ExcludePaths: ${{parameters.ExcludePaths}}
-
       - template: /eng/pipelines/templates/steps/generate-project-list-and-cache-maven-repository.yml
         parameters:
           JobType: 'Build'
+          ServiceDirectory: ${{parameters.ServiceDirectory}}
+          ExcludePaths: ${{parameters.ExcludePaths}}
 
       - task: UsePythonVersion@0
         displayName: 'Use Python $(PythonVersion)'
@@ -203,8 +196,11 @@ jobs:
           arguments: >
             -BuildOutputDirectory $(System.DefaultWorkingDirectory)/build
             -ArtifactsList ('$(ArtifactsJson)' | ConvertFrom-Json | Select-Object name, groupId)
+            -PackageInfoDir $(Build.ArtifactStagingDirectory)/PackageInfo
             -InformationAction Continue
 
+      # This only copies the ReleaseArtifacts but for the PR pipeline just copy
+      # all of the built artifacts.
       - task: PowerShell@2
         displayName: 'Copy artifacts to staging'
         inputs:
@@ -215,6 +211,7 @@ jobs:
             -SourceDirectory $(System.DefaultWorkingDirectory)/build
             -TargetDirectory $(Build.ArtifactStagingDirectory)
             -Artifacts ('${{ replace(convertToJson(parameters.ReleaseArtifacts), '''', '`''') }}' | ConvertFrom-Json | Where-Object -Not skipPublishPackage )
+            -PackageInfoDir $(Build.ArtifactStagingDirectory)/PackageInfo
             -InformationAction Continue
 
       - task: Powershell@2
@@ -303,6 +300,8 @@ jobs:
       - template: /eng/pipelines/templates/steps/generate-project-list-and-cache-maven-repository.yml
         parameters:
           JobType: 'Analyze'
+          ServiceDirectory: ${{parameters.ServiceDirectory}}
+          ExcludePaths: ${{parameters.ExcludePaths}}
 
       - task: PowerShell@2
         displayName: Generate directories variable for sparse checkout
@@ -332,9 +331,14 @@ jobs:
           ServiceDirectory: "template"
           TestPipeline: ${{ parameters.TestPipeline }}
 
-      - template: /eng/common/pipelines/templates/steps/verify-readme.yml
-        parameters:
-          ScanPath: $(Build.SourcesDirectory)/sdk/${{ parameters.ServiceDirectory }}
+      - ${{ if and(eq(variables['Build.Reason'], 'PullRequest'), eq(parameters.ServiceDirectory, 'auto')) }}:
+        - template: /eng/common/pipelines/templates/steps/verify-readmes.yml
+          parameters:
+            PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo
+      - ${{ else }}:
+        - template: /eng/common/pipelines/templates/steps/verify-readme.yml
+          parameters:
+            ScanPath: $(Build.SourcesDirectory)/sdk/${{ parameters.ServiceDirectory }}
 
       - template: /eng/common/pipelines/templates/steps/verify-links.yml
         parameters:
@@ -345,9 +349,24 @@ jobs:
             Directory: sdk/${{ parameters.ServiceDirectory }}
           CheckLinkGuidance: $true
 
-      - template: /eng/common/pipelines/templates/steps/verify-samples.yml
-        parameters:
-          ServiceDirectory: ${{ parameters.ServiceDirectory }}
+      - ${{ if and(eq(variables['Build.Reason'], 'PullRequest'), eq(parameters.ServiceDirectory, 'auto')) }}:
+        - pwsh: |
+            $diffJson = Get-Content '$(Build.ArtifactStagingDirectory)/diff/diff.json' -Raw
+            $diff = ConvertFrom-Json $diffJson
+            $changedServicesString = ''
+            if ($diff.ChangedServices) {
+                $changedServicesString = $diff.ChangedServices -join ","
+            }
+            Write-Host "changedServicesString='$changedServicesString'"
+            Write-Host "##vso[task.setvariable variable=SamplesServiceDirectories;]$changedServicesString"
+          displayName: Get samples service directories from PR diff
+        - template: /eng/common/pipelines/templates/steps/verify-samples.yml
+          parameters:
+            ServiceDirectories: $(SamplesServiceDirectories)
+      - ${{ else }}:
+        - template: /eng/common/pipelines/templates/steps/verify-samples.yml
+          parameters:
+            ServiceDirectory: ${{ parameters.ServiceDirectory }}
 
       # Use BasePathLength of 38 instead of the default 49 as some released files fail when the number is higher.
       - template: /eng/common/pipelines/templates/steps/verify-path-length.yml
@@ -356,13 +375,18 @@ jobs:
           BasePathLength: 38
 
       - ${{if ne(parameters.SDKType, 'data')}}:
-        - ${{ each artifact in parameters.Artifacts }}:
-          - ${{if ne(artifact.skipVerifyChangelog, 'true')}}:
-            - template: /eng/common/pipelines/templates/steps/verify-changelog.yml
-              parameters:
-                PackageName: ${{artifact.name}}
-                ServiceName: ${{parameters.ServiceDirectory}}
-                ForRelease: false
+        - ${{ if and(eq(variables['Build.Reason'], 'PullRequest'), eq(parameters.ServiceDirectory, 'auto')) }}:
+          - template: /eng/common/pipelines/templates/steps/verify-changelogs.yml
+            parameters:
+              PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo
+        - ${{ else }}:
+          - ${{ each artifact in parameters.Artifacts }}:
+            - ${{if ne(artifact.skipVerifyChangelog, 'true')}}:
+              - template: /eng/common/pipelines/templates/steps/verify-changelog.yml
+                parameters:
+                  PackageName: ${{artifact.name}}
+                  ServiceName: ${{parameters.ServiceDirectory}}
+                  ForRelease: false
 
       # We `install` the code quality reports tooling into our local m2 cache separately from building the Maven project
       # reports. This means it is available as part of that, but also so that this is not documented in the project report.
@@ -430,6 +454,7 @@ jobs:
       AdditionalParameters:
         SDKType: ${{ parameters.SDKType }}
         ServiceDirectory: ${{ parameters.ServiceDirectory }}
+        ExcludePaths: ${{ parameters.ExcludePaths }}
         TestPipeline: ${{ parameters.TestPipeline }}
         PreTestSteps: ${{ parameters.PreTestSteps }}
         PreBuildSteps: ${{ parameters.PreBuildSteps }}

eng/pipelines/templates/steps/generate-project-list-and-cache-maven-repository.yml

@@ -15,8 +15,24 @@ parameters:
   - name: UseCache
     type: boolean
     default: true
+  - name: ServiceDirectory
+    type: string
+    default: 'not-specified' # Set a default that breaks in obvious ways.
+  - name: ExcludePaths
+    type: object
+    default: []
 
 steps:
+
+# This is the case for the PullReqest pipeline which has a ServiceDirectory of 'auto'.
+# For the PullRequest pipeline, the packageInfo files generated from the diff end up
+# being the source of truth for the artifacts list.
+- ${{ if and(eq(variables['Build.Reason'], 'PullRequest'), eq(parameters.ServiceDirectory, 'auto')) }}:
+  - template: /eng/common/pipelines/templates/steps/save-package-properties.yml
+    parameters:
+      ServiceDirectory: ${{parameters.ServiceDirectory}}
+      ExcludePaths: ${{parameters.ExcludePaths}}
+
   # Create the ProjectList and ProjectListSha256 variables.
   # The ProjectList is used by subsequent steps and tasks to run CI.
   # The ProjectListSha256 is used by the cache key, previously the cache key was using ProjectList but it was found

eng/pipelines/templates/steps/initialize-test-environment.yml

@@ -5,6 +5,12 @@ parameters:
   - name: OS
     type: string
     default: $(OSVmImage)
+  - name: ServiceDirectory
+    type: string
+    default: 'not-specified' # Set a default that breaks in obvious ways.
+  - name: ExcludePaths
+    type: object
+    default: []
 
 steps:
   # Skip sparse checkout for the `azure-sdk-for-<lang>-pr` private mirrored repositories
@@ -14,6 +20,7 @@ steps:
       parameters:
         Paths:
           - '**/*.xml'
+          - '**/ci*.yml' # necessary for save-package-properties.yml
           - ${{ if not(parameters.CheckoutRecordings) }}:
             - '!sdk/**/test-recordings'
             - '!sdk/**/session-records'
@@ -44,6 +51,8 @@ steps:
   - template: /eng/pipelines/templates/steps/generate-project-list-and-cache-maven-repository.yml
     parameters:
       JobType: 'test'
+      ServiceDirectory: ${{parameters.ServiceDirectory}}
+      ExcludePaths: ${{parameters.ExcludePaths}}
 
   # This call is used generate the sparse checkout variables for non-FromSource runs.
   - task: PowerShell@2

eng/scripts/Stage-MavenPackageArtifacts.ps1

@@ -1,7 +1,8 @@
 param(
   [Parameter(Mandatory=$true)][string]$SourceDirectory,
   [Parameter(Mandatory=$true)][string]$TargetDirectory,
-  [Parameter(Mandatory=$true)][array]$Artifacts
+  [Parameter(Mandatory=$false)][array]$Artifacts,
+  [Parameter(Mandatory=$false)][string] $PackageInfoDir = $null
 )
 
 $ErrorActionPreference = "Stop"
@@ -11,6 +12,28 @@ Write-host "Target Directory is: $TargetDirectory"
 
 . $PSScriptRoot\MavenPackaging.ps1
 
+if ($Artifacts -eq $null) {
+  $Artifacts = @()
+}
+
+if ($Artifacts.Count -eq 0) {
+  if (-not $PackageInfoDir -or (-not (Test-Path -Path $PackageInfoDir))) {
+    LogError "Artifacts list was empty and PackageInfoDir was null or incorrect."
+    exit(1)
+  }
+  Write-Host "Artifacts List was empty, getting Artifacts from PackageInfoDir=$PackageInfoDir"
+  [array]$packageInfoFiles = Get-ChildItem -Path $PackageInfoDir "*.json"
+  foreach($packageInfoFile in $packageInfoFiles) {
+    $packageInfoJson = Get-Content $packageInfoFile -Raw
+    $packageInfo = ConvertFrom-Json $packageInfoJson
+    $Artifacts += New-Object PSObject -Property @{
+                          groupId = $packageInfo.Group
+                          name    = $packageInfo.ArtifactName
+                      }
+
+  }
+}
+
 Write-Host "Searching for packages in: $SourceDirectory"
 $packageDetails = Get-MavenPackageDetails -ArtifactDirectory $SourceDirectory
 Write-Host "Found $($packageDetails.Count) packages in: $SourceDirectory"

eng/scripts/Verify-Required-Maven-Artifacts.ps1

@@ -9,14 +9,37 @@
 param(
   [Parameter(Mandatory=$true)][string]$BuildOutputDirectory,
   # ArtifactsList will be using ('${{ convertToJson(parameters.Artifacts) }}' | ConvertFrom-Json | Select-Object name, groupId)
-  [Parameter(Mandatory=$true)][array] $ArtifactsList
+  [Parameter(Mandatory=$false)][array] $ArtifactsList,
+  [Parameter(Mandatory=$false)][string] $PackageInfoDir = $null
 )
 . "${PSScriptRoot}/../common/scripts/common.ps1"
 
 Write-Host "BuildOutputDirectory=$($BuildOutputDirectory)"
 Write-Host "BuildOutputDirectory contents"
 Get-ChildItem -Path $BuildOutputDirectory -Recurse -Name
 
+if (-not $ArtifactsList) {
+  $ArtifactsList = @()
+}
+
+if ($ArtifactsList.Count -eq 0) {
+  if (-not $PackageInfoDir -or (-not (Test-Path -Path $PackageInfoDir))) {
+    LogError "ArtifactsList was empty and PackageInfoDir was null or incorrect."
+    exit(1)
+  }
+  Write-Host "Artifacts List was empty, getting Artifacts from PackageInfoDir=$PackageInfoDir"
+  [array]$packageInfoFiles = Get-ChildItem -Path $PackageInfoDir "*.json"
+  foreach($packageInfoFile in $packageInfoFiles) {
+    $packageInfoJson = Get-Content $packageInfoFile -Raw
+    $packageInfo = ConvertFrom-Json $packageInfoJson
+    $ArtifactsList += New-Object PSObject -Property @{
+                          groupId = $packageInfo.Group
+                          name    = $packageInfo.ArtifactName
+                      }
+
+  }
+}
+
 Write-Host ""
 Write-Host "ArtifactsList:"
 $ArtifactsList | Format-Table -Property GroupId, Name | Out-String | Write-Host