Changes for core-sdk to move to yaml stages

joeloff · Sep 12, 2019 · c84ef86 · c84ef86
1 parent 118e430
commit c84ef86
Show file tree

Hide file tree

Showing 23 changed files with 392 additions and 322 deletions.
diff --git a/.vsts-ci.yml b/.vsts-ci.yml
diff --git a/build.sh b/build.sh
@@ -46,7 +46,7 @@ done
 
 dockerbuild()
 {
-    BUILD_COMMAND=/opt/code/run-build.sh $DIR/eng/dockerrun.sh --non-interactive "$@"
+    BUILD_COMMAND=$DIR/run-build.sh $DIR/eng/dockerrun.sh --non-interactive "$@"
 }
 
 # Check if we need to build in docker

diff --git a/eng/Publishing.props b/eng/Publishing.props
@@ -9,9 +9,7 @@
     <BlobStoragePartialRelativePath>$(Product)</BlobStoragePartialRelativePath>
     <BlobStoragePartialRelativePath Condition="'$(IsNotOrchestratedPublish)' == 'false'">assets/$(Product)</BlobStoragePartialRelativePath>
     <ChecksumsFeedUrl>$(DotnetPublishChecksumsBlobFeedUrl)</ChecksumsFeedUrl>
-    <ChecksumsAccountKey>$(DotNetPublishChecksumsBlobFeedKey)</ChecksumsAccountKey>
     <SdkAssetsFeedUrl>$(DotnetPublishSdkAssetsBlobFeedUrl)</SdkAssetsFeedUrl>
-    <SdkAssetsAzureAccountKey>$(DotNetPublishSdkAssetsBlobFeedKey)</SdkAssetsAzureAccountKey>
 
     <PublishSdkAssetsAndChecksumsToBlob>false</PublishSdkAssetsAndChecksumsToBlob>
     <PublishSdkAssetsAndChecksumsToBlob Condition=" '$(ChecksumsFeedUrl)' != '' and '$(SdkAssetsFeedUrl)' != '' ">true</PublishSdkAssetsAndChecksumsToBlob>
@@ -33,6 +31,9 @@
       <!-- Property AssetManifestFilePath will be reassigned by the Arcade SDK, so use a different name (DotNetAssetManifestFilePath) -->
       <DotNetAssetManifestFilePath>$(ArtifactsLogDir)AssetManifest\$(AssetManifestFileName).xml</DotNetAssetManifestFilePath>
       <ChecksumsAssetManifestFilePath>$(ArtifactsLogDir)AssetManifest\$(ChecksumsAssetManifestFileName).xml</ChecksumsAssetManifestFilePath>
+
+      <DotnetTempWorkingDirectory>$(ArtifactsDir)..\DotnetAssetsTmpDir\$([System.Guid]::NewGuid())</DotnetTempWorkingDirectory>
+      <ChecksumTempWorkingDirectory>$(ArtifactsDir)..\ChecksumAssetsTmpDir\$([System.Guid]::NewGuid())</ChecksumTempWorkingDirectory>
   </PropertyGroup>
 
   <ItemGroup>
@@ -45,9 +46,7 @@
     <SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)*.msi" />
     <SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)*.nupkg" />
     <SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)*.cab" />
-    <SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)*.svg" Condition=" '$(PublishBinariesAndBadge)' == 'true' ">
-      <ExcludeFromManifest>true</ExcludeFromManifest>
-    </SdkAssetsToPublish>
+    <SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)*.svg" Condition=" '$(PublishBinariesAndBadge)' == 'true' " />
     <SdkNonShippingAssetsToPublish Include="$(ArtifactsNonShippingPackagesDir)*.nupkg" />
     <SdkNonShippingAssetsToPublish Include="$(ArtifactsNonShippingPackagesDir)*.swr" />
     <CheckSumsToPublish Include="$(ArtifactsShippingPackagesDir)*.sha" />
@@ -74,28 +73,45 @@
       </ChecksumsToPushToBlobFeed>
     </ItemGroup>
 
-    <PushToBlobFeed
-      ExpectedFeedUrl="$(SdkAssetsFeedUrl)"
-      AccountKey="$(SdkAssetsAzureAccountKey)"
+    <MakeDir Directories="$(DotnetTempWorkingDirectory)"/>
+    <MakeDir Directories="$(ChecksumTempWorkingDirectory)"/>
+
+    <PushToAzureDevOpsArtifacts
       ItemsToPush="@(SdkAssetsToPushToBlobFeed)"
       ManifestBuildData="Location=$(SdkAssetsFeedUrl)"
       ManifestRepoUri="$(BUILD_REPOSITORY_URI)"
       ManifestBranch="$(BUILD_SOURCEBRANCH)"
       ManifestBuildId="$(BUILD_BUILDNUMBER)"
       ManifestCommit="$(BUILD_SOURCEVERSION)"
       AssetManifestPath="$(DotNetAssetManifestFilePath)"
-      PublishFlatContainer="true" />
+      PublishFlatContainer="true"
+      AssetsTemporaryDirectory="$(DotnetTempWorkingDirectory)" />
 
-    <PushToBlobFeed
-      ExpectedFeedUrl="$(ChecksumsFeedUrl)"
-      AccountKey="$(ChecksumsAccountKey)"
+    <PushToAzureDevOpsArtifacts
       ItemsToPush="@(ChecksumsToPushToBlobFeed)"
       ManifestBuildData="Location=$(ChecksumsFeedUrl)"
       ManifestRepoUri="$(BUILD_REPOSITORY_URI)"
       ManifestBranch="$(BUILD_SOURCEBRANCH)"
       ManifestBuildId="$(BUILD_BUILDNUMBER)"
       ManifestCommit="$(BUILD_SOURCEVERSION)"
       AssetManifestPath="$(ChecksumsAssetManifestFilePath)"
-      PublishFlatContainer="true"/>
+      PublishFlatContainer="true"
+      AssetsTemporaryDirectory="$(ChecksumTempWorkingDirectory)" />
+
+    <Copy
+      SourceFiles="$(DotNetAssetManifestFilePath)"
+      DestinationFolder="$(DotnetTempWorkingDirectory)\$(AssetManifestFileName)" />
+
+    <Copy
+      SourceFiles="$(ChecksumsAssetManifestFilePath)"
+      DestinationFolder="$(ChecksumTempWorkingDirectory)\$(ChecksumsAssetManifestFileName)" />
+
+    <Message
+      Text="##vso[artifact.upload containerfolder=AssetManifests;artifactname=AssetManifests]$(DotnetTempWorkingDirectory)/$(AssetManifestFileName)"
+      Importance="high" />
+
+    <Message
+      Text="##vso[artifact.upload containerfolder=AssetManifests;artifactname=AssetManifests]$(ChecksumTempWorkingDirectory)/$(ChecksumsAssetManifestFileName)"
+      Importance="high" />
   </Target>
 </Project>
diff --git a/eng/build.yml b/eng/build.yml
@@ -14,13 +14,19 @@ parameters:
   # Job timeout
   timeoutInMinutes: 180
 
+  # Publish using pipelines
+  enablePublishUsingPipelines: true
+
 phases:
 - template: /eng/common/templates/job/job.yml
   parameters:
     name: ${{ parameters.agentOs }}
     ${{ if eq(parameters.agentOs, 'Windows_NT') }}:
       enableMicrobuild: true
+    enablePublishBuildArtifacts: true
     enablePublishBuildAssets: true
+    enablePublishTestResults: true
+    enablePublishUsingPipelines: ${{parameters.enablePublishUsingPipelines}}
     enableTelemetry: true
     helixRepo: dotnet/core-sdk
     pool: ${{ parameters.pool }}
@@ -38,20 +44,22 @@ phases:
       - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
         - group: DotNet-Symbol-Server-PATs
         - group: DotNet-HelixApi-Access
+        - group: DotNet-Blob-Feed
         - _DotNetPublishToBlobFeed: true
         - _PushToVSFeed: true
         - _SignType: real
         - _BuildArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER)
                   /p:DotNetPublishToBlobFeed=$(_DotNetPublishToBlobFeed)
+                  /p:DotNetPublishBlobFeedKey=$(dotnetfeed-storage-access-key-1)
+                  /p:DotNetPublishBlobFeedUrl=$(PB_PublishBlobFeedUrl)
                   /p:DotnetPublishSdkAssetsBlobFeedUrl=$(_PublishBlobFeedUrl)
-                  /p:DotNetPublishSdkAssetsBlobFeedKey=$(dotnetcli-storage-key)
                   /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat)
                   /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat)
                   /p:PB_PublishType=blob
                   /p:DotnetPublishChecksumsBlobFeedUrl=$(_PublishChecksumsBlobFeedUrl)
-                  /p:DotNetPublishChecksumsBlobFeedKey=$(dotnetclichecksums-storage-key)
                   /p:DotNetSignType=real
                   /p:TeamName=$(_TeamName)
+                  /p:DotNetPublishUsingPipelines=$(_PublishUsingPipelines)
 
     steps:
     - checkout: self

diff --git a/eng/docker/alpine.3.6/Dockerfile b/eng/docker/alpine.3.6/Dockerfile
@@ -21,6 +21,7 @@ RUN chmod -R a+rwx /home
 # Set user to the one we just created
 USER ${USER_ID}
 
-# Set working directory
-WORKDIR /opt/code
+# Set working directory 
+ARG WORK_DIR
+WORKDIR ${WORK_DIR}
 
diff --git a/eng/docker/centos/Dockerfile b/eng/docker/centos/Dockerfile
@@ -21,5 +21,6 @@ RUN chmod -R 755 /usr/bin/sudo
 # Set user to the one we just created
 USER ${USER_ID}
 
-# Set working directory
-WORKDIR /opt/code
+# Set working directory 
+ARG WORK_DIR
+WORKDIR ${WORK_DIR}
diff --git a/eng/docker/debian/Dockerfile b/eng/docker/debian/Dockerfile
@@ -27,5 +27,6 @@ RUN chmod -R 755 /usr/lib/sudo
 # Set user to the one we just created
 USER ${USER_ID}
 
-# Set working directory
-WORKDIR /opt/code
+# Set working directory 
+ARG WORK_DIR
+WORKDIR ${WORK_DIR}
diff --git a/eng/docker/fedora.23/Dockerfile b/eng/docker/fedora.23/Dockerfile
@@ -26,5 +26,6 @@ RUN chmod -R a+rwx /home
 # Set user to the one we just created
 USER ${USER_ID}
 
-# Set working directory
-WORKDIR /opt/code
+# Set working directory 
+ARG WORK_DIR
+WORKDIR ${WORK_DIR}
diff --git a/eng/docker/fedora.24/Dockerfile b/eng/docker/fedora.24/Dockerfile
@@ -25,4 +25,5 @@ RUN chmod -R a+rwx /home
 USER ${USER_ID} 
 
 # Set working directory 
-WORKDIR /opt/code
+ARG WORK_DIR 
+WORKDIR ${WORK_DIR}
diff --git a/eng/docker/fedora.27/Dockerfile b/eng/docker/fedora.27/Dockerfile
@@ -29,4 +29,5 @@ RUN chmod -R a+rwx /home
 USER ${USER_ID} 
 
 # Set working directory 
-WORKDIR /opt/code
+ARG WORK_DIR 
+WORKDIR ${WORK_DIR}
diff --git a/eng/docker/fedora.28/Dockerfile b/eng/docker/fedora.28/Dockerfile
@@ -26,4 +26,5 @@ RUN chmod -R a+rwx /home
 USER ${USER_ID} 
 
 # Set working directory 
-WORKDIR /opt/code
+ARG WORK_DIR 
+WORKDIR ${WORK_DIR}
diff --git a/eng/docker/fedora.29/Dockerfile b/eng/docker/fedora.29/Dockerfile
@@ -26,4 +26,5 @@ RUN chmod -R a+rwx /home
 USER ${USER_ID} 
 
 # Set working directory 
-WORKDIR /opt/code
+ARG WORK_DIR 
+WORKDIR ${WORK_DIR}
diff --git a/eng/docker/opensuse.13.2/Dockerfile b/eng/docker/opensuse.13.2/Dockerfile
@@ -21,5 +21,6 @@ RUN chmod -R 755 /usr/lib/sudo
 # Set user to the one we just created
 USER ${USER_ID}
 
-# Set working directory
-WORKDIR /opt/code
+# Set working directory 
+ARG WORK_DIR
+WORKDIR ${WORK_DIR}
diff --git a/eng/docker/opensuse.42.1/Dockerfile b/eng/docker/opensuse.42.1/Dockerfile
@@ -46,5 +46,6 @@ RUN chmod -R 755 /usr/lib/sudo
 # Set user to the one we just created
 USER ${USER_ID}
 
-# Set working directory
-WORKDIR /opt/code
+# Set working directory 
+ARG WORK_DIR
+WORKDIR ${WORK_DIR}
diff --git a/eng/docker/opensuse.42.3/Dockerfile b/eng/docker/opensuse.42.3/Dockerfile
@@ -45,5 +45,6 @@ RUN chmod -R 755 /usr/lib/sudo
 # Set user to the one we just created
 USER ${USER_ID}
 
-# Set working directory
-WORKDIR /opt/code
+# Set working directory 
+ARG WORK_DIR
+WORKDIR ${WORK_DIR}
diff --git a/eng/docker/rhel.6/Dockerfile b/eng/docker/rhel.6/Dockerfile
@@ -26,6 +26,7 @@ USER ${USER_ID}
 # Set library path to make CURL and ICU libraries that are in /usr/local/lib visible
 ENV LD_LIBRARY_PATH /usr/local/lib
 
-# Set working directory
-WORKDIR /opt/code
+# Set working directory 
+ARG WORK_DIR
+WORKDIR ${WORK_DIR}
 
diff --git a/eng/docker/rhel/Dockerfile b/eng/docker/rhel/Dockerfile
@@ -19,5 +19,6 @@ RUN chown root:root /usr/bin/sudo && chmod 4755 /usr/bin/sudo
 # Set user to the one we just created
 USER ${USER_ID}
 
-# Set working directory
-WORKDIR /opt/code
+# Set working directory 
+ARG WORK_DIR
+WORKDIR ${WORK_DIR}
diff --git a/eng/docker/ubuntu.14.04/Dockerfile b/eng/docker/ubuntu.14.04/Dockerfile
@@ -59,5 +59,6 @@ RUN chmod -R 755 /usr/lib/sudo
 # Set user to the one we just created
 USER ${USER_ID}
 
-# Set working directory
-WORKDIR /opt/code
+# Set working directory 
+ARG WORK_DIR
+WORKDIR ${WORK_DIR}
diff --git a/eng/docker/ubuntu.16.04/Dockerfile b/eng/docker/ubuntu.16.04/Dockerfile
@@ -52,5 +52,6 @@ RUN chmod -R 755 /usr/lib/sudo
 # Set user to the one we just created
 USER ${USER_ID}
 
-# Set working directory
-WORKDIR /opt/code
+# Set working directory 
+ARG WORK_DIR
+WORKDIR ${WORK_DIR}
diff --git a/eng/docker/ubuntu.16.10/Dockerfile b/eng/docker/ubuntu.16.10/Dockerfile
@@ -59,5 +59,6 @@ RUN chmod -R 755 /usr/lib/sudo
 # Set user to the one we just created
 USER ${USER_ID}
 
-# Set working directory
-WORKDIR /opt/code
+# Set working directory 
+ARG WORK_DIR
+WORKDIR ${WORK_DIR}
diff --git a/eng/docker/ubuntu.18.04/Dockerfile b/eng/docker/ubuntu.18.04/Dockerfile
@@ -27,5 +27,6 @@ RUN chmod -R 755 /usr/lib/sudo
 # Set user to the one we just created
 USER ${USER_ID}
 
-# Set working directory
-WORKDIR /opt/code
+# Set working directory 
+ARG WORK_DIR
+WORKDIR ${WORK_DIR}
diff --git a/eng/dockerrun.ps1 b/eng/dockerrun.ps1
@@ -19,7 +19,7 @@ Write-Host "Additional args: $additionalArgs"
 
 $dockerFile = Resolve-Path (Join-Path $RepoRoot "eng\docker\$dockerImageName")
 
-docker build --build-arg USER_ID=1000 -t "$dockerContainerTag" $dockerFile
+docker build --build-arg WORK_DIR=$RepoRoot --build-arg USER_ID=1000 -t "$dockerContainerTag" $dockerFile
 
 $interactiveFlag = "-i"
 if ($noninteractive)
@@ -31,7 +31,7 @@ if ($noninteractive)
 
 docker run $interactiveFlag -t --rm --sig-proxy=true `
   --name "$dockerContainerName" `
-  -v "${RepoRoot}:/opt/code" `
+  -v "${RepoRoot}:${RepoRoot}" `
   -e DOTNET_CORESDK_IGNORE_TAR_EXIT_CODE=1 `
   -e CHANNEL `
   -e DOTNET_BUILD_SKIP_CROSSGEN `
@@ -57,4 +57,4 @@ docker run $interactiveFlag -t --rm --sig-proxy=true `
   -e EXTERNALRESTORESOURCES `
   -e ARCADE_PARTITION="${dockerImageName}" `
   $dockerContainerTag `
-  /opt/code/run-build.sh @additionalArgs
+  ${RepoRoot}/run-build.sh @additionalArgs
diff --git a/eng/dockerrun.sh b/eng/dockerrun.sh
@@ -39,7 +39,7 @@ while [[ $# > 0 ]]; do
             echo "Options:"
             echo "  <Dockerfile>    The path to the Dockerfile to use to create the build container"
             echo "  <ImageName>     The name of an existing Dockerfile folder under eng/docker to use as the Dockerfile"
-            echo "  <Command>  The command to run once inside the container (/opt/code is mapped to the repo root; defaults to nothing, which runs the default shell)"
+            echo "  <Command>  The command to run once inside the container (repo root is mapped to DOCKER_HOST_SHARE_DIR; defaults to nothing, which runs the default shell)"
             exit 0
             ;;
         *)
@@ -107,7 +107,7 @@ fi
 
 # Build the docker container (will be fast if it is already built)
 echo "Building Docker Container using Dockerfile: $DOCKERFILE"
-docker build --build-arg USER_ID=$(id -u) -t $DOTNET_BUILD_CONTAINER_TAG $DOCKERFILE
+docker build --build-arg WORK_DIR=$DOCKER_HOST_SHARE_DIR --build-arg USER_ID=$(id -u) -t $DOTNET_BUILD_CONTAINER_TAG $DOCKERFILE
 
 # Run the build in the container
 echo "Launching build in Docker Container"
@@ -118,7 +118,7 @@ echo "Using code from: $DOCKER_HOST_SHARE_DIR"
 # Note: passwords/keys should not be passed in the environment
 docker run $INTERACTIVE -t --rm --sig-proxy=true \
     --name $DOTNET_BUILD_CONTAINER_NAME \
-    -v $DOCKER_HOST_SHARE_DIR:/opt/code \
+    -v $DOCKER_HOST_SHARE_DIR:$DOCKER_HOST_SHARE_DIR \
     -e CHANNEL \
     -e DOTNET_BUILD_SKIP_CROSSGEN \
     -e PUBLISH_TO_AZURE_BLOB \