[main] Update common Docker engineering infrastructure with latest

eng/common/templates/1es.yml

@@ -32,6 +32,13 @@ parameters:
     name: $(defaultSourceAnalysisPoolName)
     image: $(defaultSourceAnalysisPoolImage)
     os: windows
+# Container image SBOMs are generated manually during the build job. 1ESPT's
+# automatic SBOM generation only adds unnecessary steps and artifacts to
+# builds. SBOM is not needed for JSON outputs. If a pipeline outputs binary
+# artifacts that ship to customers, then set this parameter to true.
+- name: enableSbom
+  type: boolean
+  default: false
 
 resources:
   repositories:
@@ -47,10 +54,8 @@ extends:
     templateParameters:
       pool: ${{ parameters.pool }}
       sdl:
-        # Required for unofficial pipelines because we rely on the ManifestGeneratorTask that is
-        # automatically installed by 1ES pipeline templates
         sbom:
-          enabled: true
+          enabled: ${{ parameters.enableSbom }}
         binskim:
           enabled: true
         componentgovernance:

eng/common/templates/jobs/build-images.yml

@@ -10,7 +10,7 @@ parameters:
   noCache: false
   internalProjectName: null
   publicProjectName: null
-  isInternalServicingValidation: false
+  storageAccountServiceConnection: null
 
 jobs:
 - job: ${{ parameters.name }}
@@ -73,11 +73,11 @@ jobs:
         id: ${{ parameters.publishConfig.buildAcr.serviceConnection.id }}
         tenantId: ${{ parameters.publishConfig.buildAcr.serviceConnection.tenantId }}
         clientId: ${{ parameters.publishConfig.buildAcr.serviceConnection.clientId }}
-      - ${{ if eq(parameters.isInternalServicingValidation, true) }}:
+      - ${{ if parameters.storageAccountServiceConnection }}:
         - name: storage
-          id: $(dotnetstaging.serviceConnection.id)
-          tenantId: $(dotnetstaging.serviceConnection.tenantId)
-          clientId: $(dotnetstaging.serviceConnection.clientId)
+          id: ${{ parameters.storageAccountServiceConnection.id }}
+          tenantId: ${{ parameters.storageAccountServiceConnection.tenantId }}
+          clientId: ${{ parameters.storageAccountServiceConnection.clientId }}
       internalProjectName: ${{ parameters.internalProjectName }}
       dockerClientOS: ${{ parameters.dockerClientOS }}
       args: >-

eng/common/templates/jobs/publish.yml

@@ -236,7 +236,7 @@ jobs:
 
   - template: /eng/common/templates/steps/annotate-eol-digests.yml@self
     parameters:
-      publishConfig: ${{ parameters.publishConfig }}
+      acr: ${{ parameters.publishConfig.publishAcr }}
       dataFile: $(artifactsPath)/eol-annotation-data/eol-annotation-data.json
 
   - script: >
@@ -284,8 +284,8 @@ jobs:
       --task "🟪 Publish Image Info"
       --task "🟪 Ingest Kusto Image Info"
       --task "🟪 Generate EOL Annotation Data"
-      --task "🟪 Annotate EOL Images"
-      --task "🟪 Wait for Annotation Ingestion"
+      --task "🟪 Annotate EOL Images (${{ parameters.publishConfig.publishAcr.server }})"
+      --task "🟪 Wait for Annotation Ingestion (${{ parameters.publishConfig.publishAcr.server }})"
       $(dryRunArg)
       $(imageBuilder.commonCmdArgs)
     displayName: Post Publish Notification

eng/common/templates/stages/build-and-test.yml

@@ -25,7 +25,7 @@ parameters:
 
   versionsRepoRef: ""
 
-  isInternalServicingValidation: false
+  storageAccountServiceConnection: null
 
   linuxAmd64Pool:
     vmImage: $(defaultLinuxAmd64PoolImage)
@@ -113,7 +113,7 @@ stages:
       publishConfig: ${{ parameters.publishConfig }}
       internalProjectName: ${{ parameters.internalProjectName }}
       publicProjectName: ${{ parameters.publicProjectName }}
-      isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }}
+      storageAccountServiceConnection: ${{ parameters.storageAccountServiceConnection }}
   - template: /eng/common/templates/jobs/build-images.yml@self
     parameters:
       name: Linux_arm64
@@ -131,7 +131,7 @@ stages:
       publishConfig: ${{ parameters.publishConfig }}
       internalProjectName: ${{ parameters.internalProjectName }}
       publicProjectName: ${{ parameters.publicProjectName }}
-      isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }}
+      storageAccountServiceConnection: ${{ parameters.storageAccountServiceConnection }}
   - template: /eng/common/templates/jobs/build-images.yml@self
     parameters:
       name: Linux_arm32
@@ -149,7 +149,7 @@ stages:
       publishConfig: ${{ parameters.publishConfig }}
       internalProjectName: ${{ parameters.internalProjectName }}
       publicProjectName: ${{ parameters.publicProjectName }}
-      isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }}
+      storageAccountServiceConnection: ${{ parameters.storageAccountServiceConnection }}
   - template: /eng/common/templates/jobs/build-images.yml@self
     parameters:
       name: Windows1809_amd64
@@ -167,7 +167,7 @@ stages:
       publishConfig: ${{ parameters.publishConfig }}
       internalProjectName: ${{ parameters.internalProjectName }}
       publicProjectName: ${{ parameters.publicProjectName }}
-      isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }}
+      storageAccountServiceConnection: ${{ parameters.storageAccountServiceConnection }}
   - template: /eng/common/templates/jobs/build-images.yml@self
     parameters:
       name: Windows2022_amd64
@@ -185,7 +185,7 @@ stages:
       publishConfig: ${{ parameters.publishConfig }}
       internalProjectName: ${{ parameters.internalProjectName }}
       publicProjectName: ${{ parameters.publicProjectName }}
-      isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }}
+      storageAccountServiceConnection: ${{ parameters.storageAccountServiceConnection }}
   - template: /eng/common/templates/jobs/build-images.yml@self
     parameters:
       name: Windows2025_amd64
@@ -204,7 +204,7 @@ stages:
       internalProjectName: ${{ parameters.internalProjectName }}
       publicProjectName: ${{ parameters.publicProjectName }}
       versionsRepoRef: ${{ parameters.versionsRepoRef }}
-      isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }}
+      storageAccountServiceConnection: ${{ parameters.storageAccountServiceConnection }}
   - template: /eng/common/templates/jobs/build-images.yml@self
     parameters:
       name: WindowsLtsc2016_amd64
@@ -222,7 +222,7 @@ stages:
       publishConfig: ${{ parameters.publishConfig }}
       internalProjectName: ${{ parameters.internalProjectName }}
       publicProjectName: ${{ parameters.publicProjectName }}
-      isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }}
+      storageAccountServiceConnection: ${{ parameters.storageAccountServiceConnection }}
 
 ################################################################################
 # Post-Build

eng/common/templates/stages/dotnet/build-and-test.yml

@@ -3,7 +3,13 @@
 
 parameters:
   linuxAmd64Pool: ""
-  isInternalServicingValidation: false
+
+  # (Optional) This service connection should be an Azure Resource Manager
+  # service connection to a storage account that's needed during image builds.
+  # It can be used to build images with access to private/internal bits.
+  # If specified, this service connection will be used to pass a storage
+  # account access token as `--build-arg ACCESSTOKEN=***` to all image builds.
+  storageAccountServiceConnection: null
 
   # Parameters for pre-build jobs
   customGenerateMatrixInitSteps: []
@@ -40,7 +46,7 @@ stages:
     publishConfig: ${{ parameters.publishConfig }}
     internalProjectName: ${{ parameters.internalProjectName }}
     publicProjectName: ${{ parameters.publicProjectName }}
-    isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }}
+    storageAccountServiceConnection: ${{ parameters.storageAccountServiceConnection }}
     customGenerateMatrixInitSteps: ${{ parameters.customGenerateMatrixInitSteps }}
     buildMatrixCustomBuildLegGroupArgs: ${{ parameters.buildMatrixCustomBuildLegGroupArgs }}
     testMatrixCustomBuildLegGroupArgs: ${{ parameters.testMatrixCustomBuildLegGroupArgs }}

eng/common/templates/stages/dotnet/build-test-publish-repo.yml

@@ -2,7 +2,6 @@
 
 parameters:
   linuxAmd64Pool: ""
-  isInternalServicingValidation: false
 
   # Parameters for pre-build jobs
   customGenerateMatrixInitSteps: []
@@ -39,7 +38,6 @@ stages:
 - template: /eng/common/templates/stages/dotnet/build-and-test.yml@self
   parameters:
     linuxAmd64Pool: ${{ parameters.linuxAmd64Pool }}
-    isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }}
     # Pre-build
     customGenerateMatrixInitSteps: ${{ parameters.customGenerateMatrixInitSteps }}
     customCopyBaseImagesInitSteps: ${{ parameters.customCopyBaseImagesInitSteps }}
@@ -68,7 +66,6 @@ stages:
 - template: /eng/common/templates/stages/dotnet/publish.yml@self
   parameters:
     pool: ${{ parameters.linuxAmd64Pool }}
-    isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }}
     customPublishInitSteps: ${{ parameters.customPublishInitSteps }}
     internalProjectName: ${{ parameters.internalProjectName }}
     publicProjectName: ${{ parameters.publicProjectName }}

eng/common/templates/stages/dotnet/publish-config-nonprod.yml

@@ -61,6 +61,13 @@ stages:
 
       publicMirrorAcr:
         server: $(public-mirror.server)
+        resourceGroup: $(public-mirror.resourceGroup)
+        subscription: $(public-mirror.subscription)
+        serviceConnection:
+          name: $(public-mirror.serviceConnectionName)
+          id: $(public-mirror.serviceConnection.id)
+          tenantId: $(public-mirror.serviceConnection.tenantId)
+          clientId: $(public-mirror.serviceConnection.clientId)
 
       buildAcr:
         server: $(acr-staging-test.server)
@@ -73,6 +80,12 @@ stages:
           clientId: $(build-test.serviceConnection.clientId)
           tenantId: $(testTenant)
 
+      cleanServiceConnection:
+        name: $(clean-test.serviceConnectionName)
+        id: $(clean-test.serviceConnection.id)
+        clientId: $(clean-test.serviceConnection.clientId)
+        tenantId: $(testTenant)
+
       testServiceConnection:
         name: $(test-nonprod.serviceConnectionName)
         id: $(test-nonprod.serviceConnection.id)

eng/common/templates/stages/dotnet/publish-config-prod.yml

@@ -61,6 +61,13 @@ stages:
 
       publicMirrorAcr:
         server: $(public-mirror.server)
+        resourceGroup: $(public-mirror.resourceGroup)
+        subscription: $(public-mirror.subscription)
+        serviceConnection:
+          name: $(public-mirror.serviceConnectionName)
+          id: $(public-mirror.serviceConnection.id)
+          tenantId: $(public-mirror.serviceConnection.tenantId)
+          clientId: $(public-mirror.serviceConnection.clientId)
 
       buildAcr:
         server: $(acr-staging.server)
@@ -73,6 +80,12 @@ stages:
           clientId: $(build.serviceConnection.clientId)
           tenantId: $(build.serviceConnection.tenantId)
 
+      cleanServiceConnection:
+        name: $(clean.serviceConnectionName)
+        id: $(clean.serviceConnection.id)
+        clientId: $(clean.serviceConnection.clientId)
+        tenantId: $(clean.serviceConnection.tenantId)
+
       testServiceConnection:
         name: $(test.serviceConnectionName)
         id: $(test.serviceConnection.id)

eng/common/templates/stages/dotnet/publish.yml

@@ -6,7 +6,6 @@ parameters:
   publicProjectName: null
   publishConfig: null
   pool: ""
-  isInternalServicingValidation: false
   isStandalonePublish: false
   customPublishInitSteps: []
   sourceBuildPipelineDefinitionId: ''
@@ -20,7 +19,6 @@ stages:
     internalProjectName: ${{ parameters.internalProjectName }}
     publicProjectName: ${{ parameters.publicProjectName }}
     publishConfig: ${{ parameters.publishConfig }}
-    isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }}
     isStandalonePublish: ${{ parameters.isStandalonePublish }}
     sourceBuildPipelineDefinitionId: ${{ parameters.sourceBuildPipelineDefinitionId }}
     sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }}

eng/common/templates/stages/publish.yml

@@ -7,7 +7,6 @@ parameters:
 
   publishConfig: null
 
-  isInternalServicingValidation: false
   isStandalonePublish: false
 
   pool:
@@ -29,54 +28,53 @@ parameters:
 # Publish Images
 ################################################################################
 stages:
-- ${{ if eq(parameters.isInternalServicingValidation, 'false') }}:
-  - stage: Publish
-    ${{ if eq(parameters.isStandalonePublish, true) }}:
-      dependsOn: []
+- stage: Publish
+  ${{ if eq(parameters.isStandalonePublish, true) }}:
+    dependsOn: []
+  ${{ else }}:
+    ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}:
+      dependsOn: Test
     ${{ else }}:
-      ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}:
-        dependsOn: Test
-      ${{ else }}:
-        dependsOn: Post_Build
-    condition: "
+      dependsOn: Post_Build
+  condition: "
+    and(
+      not(canceled()),
       and(
-        not(canceled()),
-        and(
-          contains(variables['stages'], 'publish'),
+        contains(variables['stages'], 'publish'),
+        or(
           or(
+            and(
+              and(
+                contains(variables['stages'], 'build'),
+                succeeded('Post_Build')),
+              and(
+                contains(variables['stages'], 'test'),
+                in(dependencies.Test.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'))),
             or(
               and(
-                and(
-                  contains(variables['stages'], 'build'),
-                  succeeded('Post_Build')),
+                not(contains(variables['stages'], 'build')),
                 and(
                   contains(variables['stages'], 'test'),
                   in(dependencies.Test.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'))),
-              or(
-                and(
-                  not(contains(variables['stages'], 'build')),
-                  and(
-                    contains(variables['stages'], 'test'),
-                    in(dependencies.Test.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'))),
+              and(
+                not(contains(variables['stages'], 'test')),
                 and(
-                  not(contains(variables['stages'], 'test')),
-                  and(
-                    contains(variables['stages'], 'build'),
-                    succeeded('Post_Build'))))),
-            not(
-              or(
-                contains(variables['stages'], 'build'),
-                contains(variables['stages'], 'test'))))))"
-    jobs:
-    - template: /eng/common/templates/jobs/publish.yml@self
-      parameters:
-        pool: ${{ parameters.pool }}
-        internalProjectName: ${{ parameters.internalProjectName }}
-        publishConfig: ${{ parameters.publishConfig }}
-        customPublishVariables: ${{ parameters.customPublishVariables }}
-        customInitSteps: ${{ parameters.customPublishInitSteps }}
-        sourceBuildPipelineDefinitionId: ${{ parameters.sourceBuildPipelineDefinitionId }}
-        sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }}
-        versionsRepoRef: ${{ parameters.versionsRepoRef }}
-        versionsRepoPath: ${{ parameters.versionsRepoPath }}
-        overrideImageInfoCommit: ${{ parameters.overrideImageInfoCommit }}
+                  contains(variables['stages'], 'build'),
+                  succeeded('Post_Build'))))),
+          not(
+            or(
+              contains(variables['stages'], 'build'),
+              contains(variables['stages'], 'test'))))))"
+  jobs:
+  - template: /eng/common/templates/jobs/publish.yml@self
+    parameters:
+      pool: ${{ parameters.pool }}
+      internalProjectName: ${{ parameters.internalProjectName }}
+      publishConfig: ${{ parameters.publishConfig }}
+      customPublishVariables: ${{ parameters.customPublishVariables }}
+      customInitSteps: ${{ parameters.customPublishInitSteps }}
+      sourceBuildPipelineDefinitionId: ${{ parameters.sourceBuildPipelineDefinitionId }}
+      sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }}
+      versionsRepoRef: ${{ parameters.versionsRepoRef }}
+      versionsRepoPath: ${{ parameters.versionsRepoPath }}
+      overrideImageInfoCommit: ${{ parameters.overrideImageInfoCommit }}