From 7cdc983dde9c0b4c15ec13f040345d2b8ea213ac Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Thu, 21 Mar 2024 09:17:29 -0700 Subject: [PATCH 01/43] Add SiteExtension.proj --- .gitignore | 2 +- build/common.props | 4 +- .../WebJobs.Script.SiteExtension.proj | 67 +++++++++++++++++++ .../applicationHost.xdt | 13 ++++ .../extension.xml | 3 + .../WebJobs.Script.WebHost.csproj | 1 + 6 files changed, 87 insertions(+), 3 deletions(-) create mode 100644 src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj create mode 100644 src/WebJobs.Script.SiteExtension/applicationHost.xdt create mode 100644 src/WebJobs.Script.SiteExtension/extension.xml diff --git a/.gitignore b/.gitignore index 0312f30722..f525ebecd1 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,6 @@ TestResults/ /tools/ExtensionsMetadataGenerator/test/ExtensionsMetadataGeneratorTests/runtimeAssemblies.txt local.settings.json -msbuild.binlog pp.xml +*.binlog BenchmarkDotNet.Artifacts/ \ No newline at end of file diff --git a/build/common.props b/build/common.props index 0628b2ca68..74706ab4e4 100644 --- a/build/common.props +++ b/build/common.props @@ -8,12 +8,12 @@ 6 0 - + <_VersionSuffix Condition="'$(PreviewVersion)' != ''">-preview.$(PreviewVersion).$(BuildNumber) <_VersionSuffix Condition="'$(PreviewVersion)' == '' and '$(VersionSuffix)' != ''">-$(VersionSuffix) - + $(MajorVersion).$(MinorVersion).$(PatchVersion) $(VersionPrefix)$(_VersionSuffix) $(MajorVersion).$(MinorVersion).0.0 diff --git a/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj b/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj new file mode 100644 index 0000000000..b9200cbfdf --- /dev/null +++ b/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj @@ -0,0 +1,67 @@ + + + + + netstandard2.1 + Functions + + + + + + + + + + + + + + + SiteExtension\$(Version)\ + $([MSBuild]::NormalizePath('$(PublishDir)$(SiteExtensionRelativeDir)')) + + + + + + + + + + <_ProjectReferenceWithRuntimes Include="@(ProjectReference)"> + + RuntimeIdentifier=%(PublishRuntimeIdentifier.Identity); + SelfContained=%(PublishRuntimeIdentifier.SelfContained); + PublishDir=$(SiteExtensionDir)%(PublishRuntimeIdentifier.PublishDir)/ + + false + + + + + + + + + + + + + + + <_SymbolIntermediate Include="$(SiteExtensionDir)**/*.pdb" Runtime="%(PublishRuntimeIdentifier.Identity)" PublishDir="%(PublishRuntimeIdentifier.PublishDir)" /> + <_Symbol + Include="@(_SymbolIntermediate)" + TargetPath="$(PublishDir)\Symbols\$(SiteExtensionName).Symbols.$(Version).%(Runtime)\$([System.String]::new('%(RecursiveDir)').Replace('%(PublishDir)', ''))%(Filename)%(Extension)" + Condition="$([System.String]::new('%(RecursiveDir)').StartsWith('%(PublishDir)'))" /> + + + + + diff --git a/src/WebJobs.Script.SiteExtension/applicationHost.xdt b/src/WebJobs.Script.SiteExtension/applicationHost.xdt new file mode 100644 index 0000000000..1344402c99 --- /dev/null +++ b/src/WebJobs.Script.SiteExtension/applicationHost.xdt @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/src/WebJobs.Script.SiteExtension/extension.xml b/src/WebJobs.Script.SiteExtension/extension.xml new file mode 100644 index 0000000000..882d91a302 --- /dev/null +++ b/src/WebJobs.Script.SiteExtension/extension.xml @@ -0,0 +1,3 @@ + + disabled + \ No newline at end of file diff --git a/src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj b/src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj index 7b0819c18b..cf207ef1f9 100644 --- a/src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj +++ b/src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj @@ -11,6 +11,7 @@ false NU5104 6.35.0 + win-x86;win-x64;linux-x64 true From f6781daf216b073b41f5fa379614983f0a80af91 Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Tue, 9 Apr 2024 15:16:25 -0700 Subject: [PATCH 02/43] Publish workers, hash hard links --- .../Tasks.targets | 127 ++++++++++++++++++ .../WebJobs.Script.SiteExtension.proj | 85 +++++++++--- 2 files changed, 195 insertions(+), 17 deletions(-) create mode 100644 src/WebJobs.Script.SiteExtension/Tasks.targets diff --git a/src/WebJobs.Script.SiteExtension/Tasks.targets b/src/WebJobs.Script.SiteExtension/Tasks.targets new file mode 100644 index 0000000000..b1d570c3d1 --- /dev/null +++ b/src/WebJobs.Script.SiteExtension/Tasks.targets @@ -0,0 +1,127 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj b/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj index b9200cbfdf..27c6be4f24 100644 --- a/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj +++ b/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj @@ -1,14 +1,21 @@ + netstandard2.1 Functions + + PublishProjectReferences; + RemoveUnneededRuntimes; + MoveSymbols; + WriteHardLinkHashes; + - - + + @@ -27,7 +34,7 @@ - + <_ProjectReferenceWithRuntimes Include="@(ProjectReference)"> @@ -42,26 +49,70 @@ + + - + + + + + + + + + + + + + + + + + <_PowershellRuntimesToKeep Include="win;win-x86;win10-x86;win-x64;win10-x64" /> + <_PowershellRuntimesToRemove Include="$([System.IO.Directory]::GetDirectories('$(SiteExtensionDir)workers/powershell'))" /> + + + + + + + <_RuntimesToRemove Include="@(PublishRuntimeIdentifier->'$(SiteExtensionDir)%(PublishDir)/runtimes/linux')" /> + <_RuntimesToRemove Include="@(PublishRuntimeIdentifier->'$(SiteExtensionDir)%(PublishDir)/runtimes/osx')" /> + + + + + + + + + + + <_FilesToHash Include="$(SiteExtensionDir)**" /> + + + + + + + - - - <_SymbolIntermediate Include="$(SiteExtensionDir)**/*.pdb" Runtime="%(PublishRuntimeIdentifier.Identity)" PublishDir="%(PublishRuntimeIdentifier.PublishDir)" /> - <_Symbol - Include="@(_SymbolIntermediate)" - TargetPath="$(PublishDir)\Symbols\$(SiteExtensionName).Symbols.$(Version).%(Runtime)\$([System.String]::new('%(RecursiveDir)').Replace('%(PublishDir)', ''))%(Filename)%(Extension)" - Condition="$([System.String]::new('%(RecursiveDir)').StartsWith('%(PublishDir)'))" /> + <_HashedFiles RelativePath=".$([System.IO.Path]::DirectorySeparatorChar)$([MSBuild]::MakeRelative('$(SiteExtensionDir)', '%(Identity)'))" /> - + From a83720f764b4de1099299d63181c42e903952b6b Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Tue, 16 Apr 2024 15:42:19 -0700 Subject: [PATCH 03/43] Add workers.props, avoid publishing worker files --- .gitignore | 1 + build/python.props | 5 ---- build/workers.props | 12 ++++++++ .../WebJobs.Script.WebHost.csproj | 29 +++++++++---------- src/WebJobs.Script/WebJobs.Script.csproj | 5 ---- 5 files changed, 27 insertions(+), 25 deletions(-) delete mode 100644 build/python.props create mode 100644 build/workers.props diff --git a/.gitignore b/.gitignore index f525ebecd1..255e3f18f6 100644 --- a/.gitignore +++ b/.gitignore @@ -31,4 +31,5 @@ TestResults/ local.settings.json pp.xml *.binlog +pp.xml BenchmarkDotNet.Artifacts/ \ No newline at end of file diff --git a/build/python.props b/build/python.props deleted file mode 100644 index f55ff8fb00..0000000000 --- a/build/python.props +++ /dev/null @@ -1,5 +0,0 @@ - - - - - diff --git a/build/workers.props b/build/workers.props new file mode 100644 index 0000000000..e2033905ad --- /dev/null +++ b/build/workers.props @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj b/src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj index cf207ef1f9..b953261236 100644 --- a/src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj +++ b/src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj @@ -1,6 +1,6 @@  - + net8.0 Microsoft.Azure.WebJobs.Script.WebHost @@ -58,10 +58,6 @@ - - - - @@ -136,23 +132,26 @@ - - - - - + - + <_WorkerPublishFiles Include="@(ResolvedFileToPublish)" Condition="$([System.String]::new('%(ResolvedFileToPublish.TargetPath)').StartsWith('workers'))" /> + - - - - + + + + + + + + + + - - - - - From d5355b54757dfe5bf49dd4103c8d8c0e935bc076 Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Fri, 19 Apr 2024 10:06:21 -0700 Subject: [PATCH 04/43] Refactor how worker is packaged --- .../Tasks.targets | 27 ------- .../WebJobs.Script.SiteExtension.proj | 70 ++++++++++--------- .../WebJobs.Script.WebHost.csproj | 2 +- 3 files changed, 38 insertions(+), 61 deletions(-) diff --git a/src/WebJobs.Script.SiteExtension/Tasks.targets b/src/WebJobs.Script.SiteExtension/Tasks.targets index b1d570c3d1..757ccba8a1 100644 --- a/src/WebJobs.Script.SiteExtension/Tasks.targets +++ b/src/WebJobs.Script.SiteExtension/Tasks.targets @@ -92,33 +92,6 @@ for (int i = 0; i < Directories.Length; i++) Directory.Move(directory, destination); } -]]> - - - - - - - - - - - - - - diff --git a/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj b/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj index 27c6be4f24..ffa9cd64dd 100644 --- a/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj +++ b/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj @@ -1,5 +1,11 @@ + + + false + + + @@ -14,8 +20,8 @@ - - + + @@ -34,13 +40,32 @@ + + + + <_PowershellRuntimesToRemoveRegex>^workers(/|\\)powershell(/|\\).*(/|\\)runtimes(/|\\)(?!win(/|\\)|win-x86(/|\\)|win10-x86(/|\\)|win-x64(/|\\)|win10-x64(/|\\)) + + + + <_PowershellRuntimeToRemove Include="@(None)" Condition="'%(TargetPath)' != '' AND $([System.Text.RegularExpressions.Regex]::IsMatch('%(TargetPath)', $(_PowershellRuntimesToRemoveRegex)))" /> + + + + + + $(SiteExtensionRelativeDir)%(TargetPath) + + + + <_ProjectReferenceWithRuntimes Include="@(ProjectReference)"> RuntimeIdentifier=%(PublishRuntimeIdentifier.Identity); SelfContained=%(PublishRuntimeIdentifier.SelfContained); - PublishDir=$(SiteExtensionDir)%(PublishRuntimeIdentifier.PublishDir)/ + PublishDir=$(SiteExtensionDir)%(PublishRuntimeIdentifier.PublishDir)/; + PublishWorkers=false false @@ -55,33 +80,6 @@ - - - - - - - - - - - - - - - - - <_PowershellRuntimesToKeep Include="win;win-x86;win10-x86;win-x64;win10-x64" /> - <_PowershellRuntimesToRemove Include="$([System.IO.Directory]::GetDirectories('$(SiteExtensionDir)workers/powershell'))" /> - - - - <_RuntimesToRemove Include="@(PublishRuntimeIdentifier->'$(SiteExtensionDir)%(PublishDir)/runtimes/linux')" /> @@ -91,9 +89,15 @@ - + + <_SymbolDirs Include="@(PublishRuntimeIdentifier->'$(SiteExtensionDir)%(PublishDir)')"> + $(PublishDir)Symbols\$(SiteExtensionName).Symbols.$(Version).%(Identity) + + <_WorkerSymbols Include="$(SiteExtensionDir)workers/**/*.pdb" Destination="$(PublishDir)Symbols\$(SiteExtensionName).Symbols.$(Version).%(PublishRuntimeIdentifier.Identity)/workers" /> + + + + diff --git a/src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj b/src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj index b953261236..34978ddffd 100644 --- a/src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj +++ b/src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj @@ -147,7 +147,7 @@ - + From 63572fca2623eeaedc02e3b66ee2e680d32d6348 Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Mon, 22 Apr 2024 15:35:32 -0700 Subject: [PATCH 05/43] Add PrivateSiteExtension --- build/workers.props | 3 ++ .../WebJobs.Script.PrivateSiteExtension.proj | 48 +++++++++++++++++++ .../applicationHost.xdt | 13 +++++ .../extension.xml | 3 ++ .../WebJobs.Script.SiteExtension.proj | 13 ++--- src/WebJobs.Script/WebJobs.Script.csproj | 1 - 6 files changed, 74 insertions(+), 7 deletions(-) create mode 100644 src/WebJobs.Script.PrivateSiteExtension/WebJobs.Script.PrivateSiteExtension.proj create mode 100644 src/WebJobs.Script.PrivateSiteExtension/applicationHost.xdt create mode 100644 src/WebJobs.Script.PrivateSiteExtension/extension.xml diff --git a/build/workers.props b/build/workers.props index e2033905ad..2270b8ec1d 100644 --- a/build/workers.props +++ b/build/workers.props @@ -1,6 +1,9 @@ + + + diff --git a/src/WebJobs.Script.PrivateSiteExtension/WebJobs.Script.PrivateSiteExtension.proj b/src/WebJobs.Script.PrivateSiteExtension/WebJobs.Script.PrivateSiteExtension.proj new file mode 100644 index 0000000000..6f6e061cb3 --- /dev/null +++ b/src/WebJobs.Script.PrivateSiteExtension/WebJobs.Script.PrivateSiteExtension.proj @@ -0,0 +1,48 @@ + + + + + netstandard2.1 + Functions.Private + + + + + + + + + + + + + + SiteExtension\$(Version)\ + $([MSBuild]::NormalizePath('$(PublishDir)$(SiteExtensionRelativeDir)')) + + + + + + + + + + <_ProjectReferenceWithRuntimes Include="@(ProjectReference)"> + + RuntimeIdentifier=%(PublishRuntimeIdentifier.Identity); + SelfContained=%(PublishRuntimeIdentifier.SelfContained); + PublishDir=$(SiteExtensionDir)%(PublishRuntimeIdentifier.PublishDir)/ + + false + + + + + + + + + + + diff --git a/src/WebJobs.Script.PrivateSiteExtension/applicationHost.xdt b/src/WebJobs.Script.PrivateSiteExtension/applicationHost.xdt new file mode 100644 index 0000000000..1344402c99 --- /dev/null +++ b/src/WebJobs.Script.PrivateSiteExtension/applicationHost.xdt @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/src/WebJobs.Script.PrivateSiteExtension/extension.xml b/src/WebJobs.Script.PrivateSiteExtension/extension.xml new file mode 100644 index 0000000000..882d91a302 --- /dev/null +++ b/src/WebJobs.Script.PrivateSiteExtension/extension.xml @@ -0,0 +1,3 @@ + + disabled + \ No newline at end of file diff --git a/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj b/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj index ffa9cd64dd..be6db1b58d 100644 --- a/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj +++ b/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj @@ -1,12 +1,6 @@ - - - false - - - netstandard2.1 @@ -19,6 +13,11 @@ + + + + + @@ -28,6 +27,8 @@ + + diff --git a/src/WebJobs.Script/WebJobs.Script.csproj b/src/WebJobs.Script/WebJobs.Script.csproj index e3f7a57dd7..d2427a66c7 100644 --- a/src/WebJobs.Script/WebJobs.Script.csproj +++ b/src/WebJobs.Script/WebJobs.Script.csproj @@ -49,7 +49,6 @@ - From c3a6504b6659430d9c06fb115948de489477f53f Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Fri, 10 May 2024 14:18:13 -0700 Subject: [PATCH 06/43] Fix duplicate build and file write error --- .../WebJobs.Script.SiteExtension.proj | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj b/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj index be6db1b58d..122faaaee3 100644 --- a/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj +++ b/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj @@ -64,10 +64,14 @@ <_ProjectReferenceWithRuntimes Include="@(ProjectReference)"> RuntimeIdentifier=%(PublishRuntimeIdentifier.Identity); - SelfContained=%(PublishRuntimeIdentifier.SelfContained); - PublishDir=$(SiteExtensionDir)%(PublishRuntimeIdentifier.PublishDir)/; - PublishWorkers=false + SelfContained=%(PublishRuntimeIdentifier.SelfContained) + + + PublishDir=$(SiteExtensionDir)%(PublishRuntimeIdentifier.PublishDir)/; + PublishWorkers=false; + _IsPublishing=true + false @@ -78,7 +82,12 @@ - + + <_PublishProjectReferenceExistent Include="@(_MSBuildProjectReferenceExistent)"> + %(AdditionalProperties);%(PublishAdditionalProperties) + + + From fd25b68d88535e9cd773a73bce986f796fdb426a Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Fri, 10 May 2024 14:37:45 -0700 Subject: [PATCH 07/43] Comment and split out powershell regex match --- .../WebJobs.Script.SiteExtension.proj | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj b/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj index 122faaaee3..54a1dc28ed 100644 --- a/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj +++ b/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj @@ -42,9 +42,18 @@ + + <_KeepPowerShellRuntime Include="win;win-x86;win10-x86;win-x64;win10-x64" /> + + - - <_PowershellRuntimesToRemoveRegex>^workers(/|\\)powershell(/|\\).*(/|\\)runtimes(/|\\)(?!win(/|\\)|win-x86(/|\\)|win10-x86(/|\\)|win-x64(/|\\)|win10-x64(/|\\)) + + <_PowershellRuntimesToKeepRegex>@(_KeepPowerShellRuntime->'%(Identity)(/|\\)', '|') + <_PowershellRuntimesToRemoveRegex>^workers(/|\\)powershell(/|\\).*(/|\\)runtimes(/|\\)(?!$(_PowershellRuntimesToKeepRegex)) From f3bcf1dad7ba12bbe5b1dcca157506628debf41f Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Fri, 10 May 2024 14:59:52 -0700 Subject: [PATCH 08/43] Fix MoveSymbols --- .../WebJobs.Script.SiteExtension.proj | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj b/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj index 54a1dc28ed..cda3b638c3 100644 --- a/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj +++ b/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj @@ -1,6 +1,6 @@ - - + + netstandard2.1 @@ -32,16 +32,16 @@ - SiteExtension\$(Version)\ + SiteExtension/$(Version)/ $([MSBuild]::NormalizePath('$(PublishDir)$(SiteExtensionRelativeDir)')) - + - + <_KeepPowerShellRuntime Include="win;win-x86;win10-x86;win-x64;win10-x64" /> @@ -60,7 +60,9 @@ <_PowershellRuntimeToRemove Include="@(None)" Condition="'%(TargetPath)' != '' AND $([System.Text.RegularExpressions.Regex]::IsMatch('%(TargetPath)', $(_PowershellRuntimesToRemoveRegex)))" /> + + $(SiteExtensionRelativeDir)%(TargetPath) @@ -72,11 +74,12 @@ <_ProjectReferenceWithRuntimes Include="@(ProjectReference)"> + RuntimeIdentifier=%(PublishRuntimeIdentifier.Identity); SelfContained=%(PublishRuntimeIdentifier.SelfContained) - + PublishDir=$(SiteExtensionDir)%(PublishRuntimeIdentifier.PublishDir)/; PublishWorkers=false; _IsPublishing=true @@ -110,11 +113,11 @@ <_SymbolDirs Include="@(PublishRuntimeIdentifier->'$(SiteExtensionDir)%(PublishDir)')"> - $(PublishDir)Symbols\$(SiteExtensionName).Symbols.$(Version).%(Identity) + $(PublishDir)Symbols/$(SiteExtensionName).Symbols.$(Version).%(Identity) - <_WorkerSymbols Include="$(SiteExtensionDir)workers/**/*.pdb" Destination="$(PublishDir)Symbols\$(SiteExtensionName).Symbols.$(Version).%(PublishRuntimeIdentifier.Identity)/workers" /> + <_WorkerSymbols Include="$(SiteExtensionDir)workers/**/*.pdb" Destination="$(PublishDir)Symbols/$(SiteExtensionName).Symbols.$(Version).%(PublishRuntimeIdentifier.Identity)/workers" /> - + From 493dfd15ba70c049b48597a327509e9724f6b1bd Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Fri, 10 May 2024 15:47:02 -0700 Subject: [PATCH 09/43] Publish PrivateSiteExtension --- build/workers.props | 22 ++++++- .../WebJobs.Script.PrivateSiteExtension.proj | 48 --------------- .../applicationHost.xdt | 13 ---- .../extension.xml | 3 - .../WebJobs.Script.SiteExtension.proj | 60 +++++++++++-------- 5 files changed, 57 insertions(+), 89 deletions(-) delete mode 100644 src/WebJobs.Script.PrivateSiteExtension/WebJobs.Script.PrivateSiteExtension.proj delete mode 100644 src/WebJobs.Script.PrivateSiteExtension/applicationHost.xdt delete mode 100644 src/WebJobs.Script.PrivateSiteExtension/extension.xml diff --git a/build/workers.props b/build/workers.props index 2270b8ec1d..21581452a2 100644 --- a/build/workers.props +++ b/build/workers.props @@ -1,7 +1,6 @@ - @@ -12,4 +11,25 @@ + + + <_KeepPowerShellRuntime Include="win;win-x86;win10-x86;win-x64;win10-x64" /> + + + + + <_PowershellRuntimesToKeepRegex>@(_KeepPowerShellRuntime->'%(Identity)(/|\\)', '|') + <_PowershellRuntimesToRemoveRegex>^workers(/|\\)powershell(/|\\).*(/|\\)runtimes(/|\\)(?!$(_PowershellRuntimesToKeepRegex)) + + + + <_PowershellRuntimeToRemove Include="@(None)" Condition="'%(None.TargetPath)' != '' AND $([System.Text.RegularExpressions.Regex]::IsMatch('%(None.TargetPath)', $(_PowershellRuntimesToRemoveRegex)))" /> + + + + diff --git a/src/WebJobs.Script.PrivateSiteExtension/WebJobs.Script.PrivateSiteExtension.proj b/src/WebJobs.Script.PrivateSiteExtension/WebJobs.Script.PrivateSiteExtension.proj deleted file mode 100644 index 6f6e061cb3..0000000000 --- a/src/WebJobs.Script.PrivateSiteExtension/WebJobs.Script.PrivateSiteExtension.proj +++ /dev/null @@ -1,48 +0,0 @@ - - - - - netstandard2.1 - Functions.Private - - - - - - - - - - - - - - SiteExtension\$(Version)\ - $([MSBuild]::NormalizePath('$(PublishDir)$(SiteExtensionRelativeDir)')) - - - - - - - - - - <_ProjectReferenceWithRuntimes Include="@(ProjectReference)"> - - RuntimeIdentifier=%(PublishRuntimeIdentifier.Identity); - SelfContained=%(PublishRuntimeIdentifier.SelfContained); - PublishDir=$(SiteExtensionDir)%(PublishRuntimeIdentifier.PublishDir)/ - - false - - - - - - - - - - - diff --git a/src/WebJobs.Script.PrivateSiteExtension/applicationHost.xdt b/src/WebJobs.Script.PrivateSiteExtension/applicationHost.xdt deleted file mode 100644 index 1344402c99..0000000000 --- a/src/WebJobs.Script.PrivateSiteExtension/applicationHost.xdt +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - diff --git a/src/WebJobs.Script.PrivateSiteExtension/extension.xml b/src/WebJobs.Script.PrivateSiteExtension/extension.xml deleted file mode 100644 index 882d91a302..0000000000 --- a/src/WebJobs.Script.PrivateSiteExtension/extension.xml +++ /dev/null @@ -1,3 +0,0 @@ - - disabled - \ No newline at end of file diff --git a/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj b/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj index cda3b638c3..4499713b5b 100644 --- a/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj +++ b/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj @@ -3,12 +3,16 @@ - netstandard2.1 + + net6.0 + win Functions PublishProjectReferences; + PublishPrivateProjectReferences; RemoveUnneededRuntimes; MoveSymbols; + DeletePrivateSymbols; WriteHardLinkHashes; @@ -19,8 +23,8 @@ - - + + @@ -34,34 +38,16 @@ SiteExtension/$(Version)/ $([MSBuild]::NormalizePath('$(PublishDir)$(SiteExtensionRelativeDir)')) + Private$(SiteExtensionRelativeDir) + $([MSBuild]::NormalizePath('$(PublishDir)$(PrivateSiteExtensionRelativeDir)')) + - - - <_KeepPowerShellRuntime Include="win;win-x86;win10-x86;win-x64;win10-x64" /> - - - - - <_PowershellRuntimesToKeepRegex>@(_KeepPowerShellRuntime->'%(Identity)(/|\\)', '|') - <_PowershellRuntimesToRemoveRegex>^workers(/|\\)powershell(/|\\).*(/|\\)runtimes(/|\\)(?!$(_PowershellRuntimesToKeepRegex)) - - - - <_PowershellRuntimeToRemove Include="@(None)" Condition="'%(TargetPath)' != '' AND $([System.Text.RegularExpressions.Regex]::IsMatch('%(TargetPath)', $(_PowershellRuntimesToRemoveRegex)))" /> - - - - @@ -84,6 +70,13 @@ PublishWorkers=false; _IsPublishing=true + + + PublishDir=$(PrivateSiteExtensionDir)%(PublishRuntimeIdentifier.PublishDir)/; + PublishWorkers=true; + _IsPublishing=true + + %(PublishRuntimeIdentifier.PrivateExtension) false @@ -102,10 +95,22 @@ + + + <_PublishPrivateProjectReferenceExistent Include="@(_MSBuildProjectReferenceExistent)" Condition="%(PublishPrivate)"> + %(AdditionalProperties);%(PublishPrivateAdditionalProperties) + + + + + + <_RuntimesToRemove Include="@(PublishRuntimeIdentifier->'$(SiteExtensionDir)%(PublishDir)/runtimes/linux')" /> <_RuntimesToRemove Include="@(PublishRuntimeIdentifier->'$(SiteExtensionDir)%(PublishDir)/runtimes/osx')" /> + <_RuntimesToRemove Include="@(PublishRuntimeIdentifier->'$(PrivateSiteExtensionDir)%(PublishDir)/runtimes/linux')" Condition="%(PrivateExtension)" /> + <_RuntimesToRemove Include="@(PublishRuntimeIdentifier->'$(PrivateSiteExtensionDir)%(PublishDir)/runtimes/osx')" Condition="%(PrivateExtension)" /> @@ -122,6 +127,13 @@ + + + <_PrivateSymbolsToRemove Include="$(PrivateSiteExtensionDir)/**/*.pdb" /> + + + + <_FilesToHash Include="$(SiteExtensionDir)**" /> From 362192a9a8b9735213720ff0526742ee38d4c419 Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Fri, 10 May 2024 16:05:29 -0700 Subject: [PATCH 10/43] Remove Python worker from private extension --- .../WebJobs.Script.SiteExtension.proj | 21 ++++++------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj b/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj index 4499713b5b..238042bd2a 100644 --- a/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj +++ b/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj @@ -38,7 +38,7 @@ SiteExtension/$(Version)/ $([MSBuild]::NormalizePath('$(PublishDir)$(SiteExtensionRelativeDir)')) - Private$(SiteExtensionRelativeDir) + PrivateSiteExtension/SiteExtensions/Functions/ $([MSBuild]::NormalizePath('$(PublishDir)$(PrivateSiteExtensionRelativeDir)')) @@ -64,18 +64,8 @@ RuntimeIdentifier=%(PublishRuntimeIdentifier.Identity); SelfContained=%(PublishRuntimeIdentifier.SelfContained) - - - PublishDir=$(SiteExtensionDir)%(PublishRuntimeIdentifier.PublishDir)/; - PublishWorkers=false; - _IsPublishing=true - - - - PublishDir=$(PrivateSiteExtensionDir)%(PublishRuntimeIdentifier.PublishDir)/; - PublishWorkers=true; - _IsPublishing=true - + $(SiteExtensionDir)%(PublishRuntimeIdentifier.PublishDir)/ + $(PrivateSiteExtensionDir)%(PublishRuntimeIdentifier.PublishDir)/ %(PublishRuntimeIdentifier.PrivateExtension) false @@ -89,7 +79,7 @@ <_PublishProjectReferenceExistent Include="@(_MSBuildProjectReferenceExistent)"> - %(AdditionalProperties);%(PublishAdditionalProperties) + %(AdditionalProperties);PublishDir=%(PublishDir);_IsPublishing=true;PublishWorkers=false @@ -98,10 +88,11 @@ <_PublishPrivateProjectReferenceExistent Include="@(_MSBuildProjectReferenceExistent)" Condition="%(PublishPrivate)"> - %(AdditionalProperties);%(PublishPrivateAdditionalProperties) + %(AdditionalProperties);PublishDir=%(PublishPrivateDir);_IsPublishing=true + From ded5dbcec6bbe19561559de0367e4ee020e5d87b Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Mon, 13 May 2024 10:21:36 -0700 Subject: [PATCH 11/43] Zip site extension artifacts --- eng/build/Engineering.targets | 1 + eng/build/ZipPublish.targets | 42 +++++++++++++++++++ .../WebJobs.Script.SiteExtension.proj | 33 ++++++++++++--- .../WebJobs.Script.WebHost.csproj | 19 --------- 4 files changed, 71 insertions(+), 24 deletions(-) create mode 100644 eng/build/ZipPublish.targets diff --git a/eng/build/Engineering.targets b/eng/build/Engineering.targets index 2e4906c2e9..5f8423e5f3 100644 --- a/eng/build/Engineering.targets +++ b/eng/build/Engineering.targets @@ -1,5 +1,6 @@ + diff --git a/eng/build/ZipPublish.targets b/eng/build/ZipPublish.targets new file mode 100644 index 0000000000..9422463ab4 --- /dev/null +++ b/eng/build/ZipPublish.targets @@ -0,0 +1,42 @@ + + + + + + + + + + + + + $([MSBuild]::NormalizePath($([MSBuild]::EnsureTrailingSlash('$(PublishDir)'))..))/ + $([MSBuild]::EnsureTrailingSlash('$(PublishZipDir)')) + + + + + + + + + + + + + $([MSBuild]::EnsureTrailingSlash('$(PublishZipDir)'))%(TargetName) + + + + + + + + + \ No newline at end of file diff --git a/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj b/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj index 238042bd2a..af7c998e9e 100644 --- a/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj +++ b/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj @@ -15,6 +15,7 @@ DeletePrivateSymbols; WriteHardLinkHashes; + PublishZip;PublishZipDir @@ -23,12 +24,12 @@ - - + - + + @@ -40,6 +41,7 @@ $([MSBuild]::NormalizePath('$(PublishDir)$(SiteExtensionRelativeDir)')) PrivateSiteExtension/SiteExtensions/Functions/ $([MSBuild]::NormalizePath('$(PublishDir)$(PrivateSiteExtensionRelativeDir)')) + $(PublishDir) @@ -49,6 +51,12 @@ + + + <_DotnetWorkerFiles Include="@(None)" Condition="'%(None.DetinationSubDirectory)' == 'workers\dotnet-isolated\bin\'" /> + + + $(SiteExtensionRelativeDir)%(TargetPath) @@ -64,6 +72,7 @@ RuntimeIdentifier=%(PublishRuntimeIdentifier.Identity); SelfContained=%(PublishRuntimeIdentifier.SelfContained) + %(PublishRuntimeIdentifier.Identity) $(SiteExtensionDir)%(PublishRuntimeIdentifier.PublishDir)/ $(PrivateSiteExtensionDir)%(PublishRuntimeIdentifier.PublishDir)/ %(PublishRuntimeIdentifier.PrivateExtension) @@ -81,8 +90,13 @@ <_PublishProjectReferenceExistent Include="@(_MSBuildProjectReferenceExistent)"> %(AdditionalProperties);PublishDir=%(PublishDir);_IsPublishing=true;PublishWorkers=false + - + @@ -90,8 +104,15 @@ <_PublishPrivateProjectReferenceExistent Include="@(_MSBuildProjectReferenceExistent)" Condition="%(PublishPrivate)"> %(AdditionalProperties);PublishDir=%(PublishPrivateDir);_IsPublishing=true + - + @@ -110,8 +131,10 @@ <_SymbolDirs Include="@(PublishRuntimeIdentifier->'$(SiteExtensionDir)%(PublishDir)')"> $(PublishDir)Symbols/$(SiteExtensionName).Symbols.$(Version).%(Identity) + $(SiteExtensionName).Symbols.$(Version).%(Identity).zip <_WorkerSymbols Include="$(SiteExtensionDir)workers/**/*.pdb" Destination="$(PublishDir)Symbols/$(SiteExtensionName).Symbols.$(Version).%(PublishRuntimeIdentifier.Identity)/workers" /> + diff --git a/src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj b/src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj index 34978ddffd..d1f81a4b25 100644 --- a/src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj +++ b/src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj @@ -153,23 +153,4 @@ - - - - $(PublishDir).. - $(PublishZipDir)\$(AssemblyName).$(Version).$(ArtifactsPivots).zip - - - - - - From d51de30da2e47fc4628340a26004c527f302e78b Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Mon, 13 May 2024 21:21:39 -0700 Subject: [PATCH 12/43] Clean dotnet worker files --- build/workers.props | 8 ++++++++ .../WebJobs.Script.SiteExtension.proj | 6 ------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/build/workers.props b/build/workers.props index 21581452a2..762ed650dd 100644 --- a/build/workers.props +++ b/build/workers.props @@ -32,4 +32,12 @@ + + + + <_DotnetWorkerFiles Include="@(None)" Condition="'%(None.DestinationSubDirectory)' == 'workers\dotnet-isolated\bin\'" /> + + + + diff --git a/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj b/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj index af7c998e9e..0273936225 100644 --- a/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj +++ b/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj @@ -51,12 +51,6 @@ - - - <_DotnetWorkerFiles Include="@(None)" Condition="'%(None.DetinationSubDirectory)' == 'workers\dotnet-isolated\bin\'" /> - - - $(SiteExtensionRelativeDir)%(TargetPath) From 0efa966d6e2266ef0f9ec6e06bcc8e74de7a4666 Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Tue, 14 May 2024 20:53:54 -0700 Subject: [PATCH 13/43] Default ZipArtifactsPath to PackageOutputPath --- eng/build/ZipPublish.targets | 44 ++++++----- .../official/jobs/build-artifacts-linux.yml | 11 +-- .../official/jobs/build-artifacts-windows.yml | 79 +++++++++---------- .../WebJobs.Script.SiteExtension.proj | 11 ++- 4 files changed, 76 insertions(+), 69 deletions(-) diff --git a/eng/build/ZipPublish.targets b/eng/build/ZipPublish.targets index 9422463ab4..43392636ec 100644 --- a/eng/build/ZipPublish.targets +++ b/eng/build/ZipPublish.targets @@ -2,41 +2,49 @@ - + - + - + - $([MSBuild]::NormalizePath($([MSBuild]::EnsureTrailingSlash('$(PublishDir)'))..))/ - $([MSBuild]::EnsureTrailingSlash('$(PublishZipDir)')) + + $(PackageOutputPath) + $([MSBuild]::EnsureTrailingSlash('$(ZipArtifactsPath)')) + <_RuntimeIdentifierWithPeriod Condition="'$(RuntimeIdentifier)' != ''">.$(RuntimeIdentifier.ToLowerInvariant()) - - - + + + - - - - $([MSBuild]::EnsureTrailingSlash('$(PublishZipDir)'))%(TargetName) - + + + + $([MSBuild]::EnsureTrailingSlash('$(ZipArtifactsPath)'))%(TargetName) + - - - + + <_ZipArtifactTargetPath Include="@(ZipArtifact->'%(TargetPath)')" /> + <_ZipArtifactTargetDirectories Include="@(_ZipArtifactTargetPath->'%(RootDir)%(Directory)'->Distinct())" /> + + + + + + \ No newline at end of file diff --git a/eng/ci/templates/official/jobs/build-artifacts-linux.yml b/eng/ci/templates/official/jobs/build-artifacts-linux.yml index 34448743d2..ca0036ebb8 100644 --- a/eng/ci/templates/official/jobs/build-artifacts-linux.yml +++ b/eng/ci/templates/official/jobs/build-artifacts-linux.yml @@ -6,18 +6,19 @@ jobs: project: src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj configuration: release runtime: linux-x64 - log_dir: $(Build.ArtifactStagingDirectory)/log + artifacts_path: $(Build.ArtifactStagingDirectory) + log_dir: $(artifacts_path)/log + zip_artifacts_path: $(artifacts_path)/Linux build_args: '-v m -c $(configuration) -r $(runtime) --self-contained true -p:BuildNumber=$(buildNumber) -p:MinorVersionPrefix=$(minorVersionPrefix) -p:IsPackable=false' - publish_zip_dir: $(Build.ArtifactStagingDirectory)/Linux templateContext: - outputParentDirectory: $(Build.ArtifactStagingDirectory) + outputParentDirectory: $(artifacts_path) outputs: # TODO: onboard to Azure Artifacts Drops to allow accessing this from docker linux pipeline in msazure # https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/azure-artifacts/artifact-services-onboarding - output: pipelineArtifact displayName: Publish linux artifacts - path: $(publish_zip_dir) + path: $(zip_artifacts_path) artifact: Linux - output: pipelineArtifact displayName: Publish logs @@ -58,4 +59,4 @@ jobs: publishWebProjects: false # we use our own publish logic zipAfterPublish: false # we use our own zip logic projects: $(project) - arguments: '$(build_args) --no-build -p:PublishZipDir=$(publish_zip_dir) -bl:$(log_dir)/publish.binlog' + arguments: '$(build_args) --no-build -p:ZipArtifactsPath=$(zip_artifacts_path) -bl:$(log_dir)/publish.binlog' diff --git a/eng/ci/templates/official/jobs/build-artifacts-windows.yml b/eng/ci/templates/official/jobs/build-artifacts-windows.yml index e06cd29f77..8df8197df6 100644 --- a/eng/ci/templates/official/jobs/build-artifacts-windows.yml +++ b/eng/ci/templates/official/jobs/build-artifacts-windows.yml @@ -2,61 +2,66 @@ jobs: - job: BuildArtifactsWindows displayName: Build Windows Artifacts + pool: + name: 1es-pool-azfunc + image: 1es-windows-2022 + os: windows + + variables: + ${{ if or( eq( variables['Build.Reason'], 'PullRequest' ), and( not( contains( variables['Build.SourceBranch'], 'release/inproc6/4.' ) ), not( contains( variables['Build.SourceBranch'], 'release/inproc8/4.' ) ), not( contains( variables['Build.SourceBranch'], 'release/4.' ) ), not( contains( variables['Build.SourceBranch'], 'release/ExtensionsMetadataGenerator/' ) ) ) ) }}: + packSuffixSwitchTemp: --version-suffix $(buildNumber) + emgSuffixSwitchTemp: --version-suffix ci$(buildNumber) + packSuffixSwitch: $[variables.packSuffixSwitchTemp] + emgSuffixSwitch: $[variables.emgSuffixSwitchTemp] + artifacts_path: $(Build.ArtifactStagingDirectory) + log_dir: $(artifacts_path)/log + nuget_package_path: $(artifacts_path)/NugetPackages + templateContext: - outputParentDirectory: $(Build.ArtifactStagingDirectory) + outputParentDirectory: $(artifacts_path) outputs: - output: pipelineArtifact displayName: Publish site extension - path: $(Build.ArtifactStagingDirectory)/SiteExtension + path: $(artifacts_path)/SiteExtension artifact: SiteExtension - output: pipelineArtifact displayName: Publish private site extension - path: $(Build.ArtifactStagingDirectory)/PrivateSiteExtension + path: $(artifacts_path)/PrivateSiteExtension artifact: PrivateSiteExtension - output: pipelineArtifact displayName: Publish site extension symbols - path: $(Build.ArtifactStagingDirectory)/Symbols + path: $(artifacts_path)/Symbols artifact: Symbols - output: pipelineArtifact displayName: Publish nuget packages - path: $(Build.ArtifactStagingDirectory)/NugetPackages + path: $(artifacts_path)/NugetPackages artifact: NugetPackages - - pool: - name: 1es-pool-azfunc - image: 1es-windows-2022 - os: windows - - variables: - ${{ if or( eq( variables['Build.Reason'], 'PullRequest' ), and( not( contains( variables['Build.SourceBranch'], 'release/inproc6/4.' ) ), not( contains( variables['Build.SourceBranch'], 'release/inproc8/4.' ) ), not( contains( variables['Build.SourceBranch'], 'release/4.' ) ), not( contains( variables['Build.SourceBranch'], 'release/ExtensionsMetadataGenerator/' ) ) ) ) }}: - suffixTemp: $(buildNumber) - packSuffixSwitchTemp: --version-suffix $(buildNumber) - emgSuffixSwitchTemp: --version-suffix ci$(buildNumber) - suffix: $[variables.suffixTemp] # this resolves to an empty string if it is missing - packSuffixSwitch: $[variables.packSuffixSwitchTemp] - emgSuffixSwitch: $[variables.emgSuffixSwitchTemp] + - output: pipelineArtifact + displayName: Publish logs + path: $(log_dir) + artifact: Windows_Log + sbomEnabled: false + condition: always() steps: - template: /eng/ci/templates/install-dotnet.yml@self - - task: PowerShell@2 - displayName: Build artifacts - inputs: - filePath: build/build-extensions.ps1 - arguments: '-buildNumber "$(buildNumber)" -suffix "$(suffix)" -minorVersionPrefix "$(minorVersionPrefix)"' - - - task: CopyFiles@2 + - task: DotNetCoreCLI@2 + displayName: Publish Site Extension inputs: - SourceFolder: out/pub/WebJobs.Script.WebHost - Contents: '**/*.zip' - TargetFolder: $(Build.ArtifactStagingDirectory) + command: custom + custom: publish + publishWebProjects: false # we use our own publish logic + zipAfterPublish: false # we use our own zip logic + projects: src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj + arguments: '-v m -c release -p:BuildNumber=$(buildNumber) -p:MinorVersionPrefix=$(minorVersionPrefix) -p:ZipArtifactsPath=$(artifacts_path) -bl:$(log_dir)/publish.binlog' - task: DotNetCoreCLI@2 displayName: Build host packages inputs: command: custom custom: pack - arguments: -p:BuildNumber=$(buildNumber) -c release $(packSuffixSwitch) + arguments: -p:BuildNumber=$(buildNumber) -c release $(packSuffixSwitch) -o $(nuget_package_path) projects: | **/WebJobs.Script.csproj **/WebJobs.Script.WebHost.csproj @@ -112,7 +117,7 @@ jobs: inputs: command: custom custom: pack - arguments: '--no-build -c release' + arguments: '--no-build -c release -o $(nuget_package_path)' projects: | **/WebJobs.Script.Abstractions.csproj @@ -120,7 +125,7 @@ jobs: displayName: Sign Abstractions package inputs: ConnectedServiceName: ESRP Service-internal - FolderPath: out/pkg/release + FolderPath: $(nuget_package_path) Pattern: Microsoft.Azure.WebJobs.Script.Abstractions*.nupkg signConfigType: inlineSignParams inlineOperation: | @@ -196,7 +201,7 @@ jobs: inputs: command: custom custom: pack - arguments: '--no-build -c release $(emgSuffixSwitch)' + arguments: '--no-build -c release $(emgSuffixSwitch) -o $(nuget_package_path)' projects: | **/ExtensionsMetadataGenerator.csproj @@ -204,7 +209,7 @@ jobs: displayName: Sign ExtensionsMetadataGenerator package inputs: ConnectedServiceName: ESRP Service-internal - FolderPath: out/pkg/release + FolderPath: $(nuget_package_path) Pattern: Microsoft.Azure.WebJobs.Script.ExtensionsMetadataGenerator*.nupkg signConfigType: inlineSignParams inlineOperation: | @@ -229,9 +234,3 @@ jobs: displayName: Delete CodeSignSummary files inputs: contents: '**/CodeSignSummary-*.md' - - - task: CopyFiles@2 - inputs: - SourceFolder: out/pkg/release - Contents: '**/*.nupkg' - TargetFolder: $(Build.ArtifactStagingDirectory)/NugetPackages diff --git a/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj b/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj index 0273936225..7de734935d 100644 --- a/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj +++ b/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj @@ -15,7 +15,7 @@ DeletePrivateSymbols; WriteHardLinkHashes; - PublishZip;PublishZipDir + ZipAfterPublish;ZipArtifactsPath @@ -41,7 +41,6 @@ $([MSBuild]::NormalizePath('$(PublishDir)$(SiteExtensionRelativeDir)')) PrivateSiteExtension/SiteExtensions/Functions/ $([MSBuild]::NormalizePath('$(PublishDir)$(PrivateSiteExtensionRelativeDir)')) - $(PublishDir) @@ -84,7 +83,7 @@ <_PublishProjectReferenceExistent Include="@(_MSBuildProjectReferenceExistent)"> %(AdditionalProperties);PublishDir=%(PublishDir);_IsPublishing=true;PublishWorkers=false - + %(AdditionalProperties);PublishDir=%(PublishPrivateDir);_IsPublishing=true - + TargetName="@(_PublishPrivateProjectReferenceExistent->'PrivateSiteExtension/$(SiteExtensionName).Private.$(Version).%(PublishRuntime).zip')" /> $(SiteExtensionName).Symbols.$(Version).%(Identity).zip <_WorkerSymbols Include="$(SiteExtensionDir)workers/**/*.pdb" Destination="$(PublishDir)Symbols/$(SiteExtensionName).Symbols.$(Version).%(PublishRuntimeIdentifier.Identity)/workers" /> - + From b4e13f78e27d5fa93625f812d348e204f561571c Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Wed, 15 May 2024 10:42:43 -0700 Subject: [PATCH 14/43] Add comments, update dotnet sdk --- eng/build/ZipPublish.targets | 10 ++++++---- global.json | 2 +- .../WebJobs.Script.SiteExtension.proj | 8 ++++++++ 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/eng/build/ZipPublish.targets b/eng/build/ZipPublish.targets index 43392636ec..5b052ba372 100644 --- a/eng/build/ZipPublish.targets +++ b/eng/build/ZipPublish.targets @@ -36,15 +36,17 @@ + <_ZipArtifactTargetPath Include="@(ZipArtifact->'%(TargetPath)')" /> <_ZipArtifactTargetDirectories Include="@(_ZipArtifactTargetPath->'%(RootDir)%(Directory)'->Distinct())" /> - - - - + + + + + \ No newline at end of file diff --git a/global.json b/global.json index 7932babe5b..ba12868df1 100644 --- a/global.json +++ b/global.json @@ -1,6 +1,6 @@ { "sdk": { - "version": "8.0.101", + "version": "8.0.204", "rollForward": "latestFeature" }, "msbuild-sdks": { diff --git a/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj b/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj index 7de734935d..03c149c534 100644 --- a/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj +++ b/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj @@ -28,6 +28,7 @@ + @@ -49,6 +50,7 @@ + @@ -78,6 +80,7 @@ + <_PublishProjectReferenceExistent Include="@(_MSBuildProjectReferenceExistent)"> @@ -92,6 +95,7 @@ RemoveProperties="$(PublishPropertiesToRemove)"/> + <_PublishPrivateProjectReferenceExistent Include="@(_MSBuildProjectReferenceExistent)" Condition="%(PublishPrivate)"> @@ -120,6 +124,7 @@ + <_SymbolDirs Include="@(PublishRuntimeIdentifier->'$(SiteExtensionDir)%(PublishDir)')"> @@ -134,6 +139,7 @@ + <_PrivateSymbolsToRemove Include="$(PrivateSiteExtensionDir)/**/*.pdb" /> @@ -141,6 +147,7 @@ + <_FilesToHash Include="$(SiteExtensionDir)**" /> @@ -150,6 +157,7 @@ + <_HashedFiles RelativePath=".$([System.IO.Path]::DirectorySeparatorChar)$([MSBuild]::MakeRelative('$(SiteExtensionDir)', '%(Identity)'))" /> From e7bfa3179afac290e90623253e79043a546e9483 Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Wed, 15 May 2024 11:07:50 -0700 Subject: [PATCH 15/43] Add explicit CI restore stage --- .../official/jobs/build-artifacts-windows.yml | 10 +++++++++- global.json | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/eng/ci/templates/official/jobs/build-artifacts-windows.yml b/eng/ci/templates/official/jobs/build-artifacts-windows.yml index 8df8197df6..35584c1e0d 100644 --- a/eng/ci/templates/official/jobs/build-artifacts-windows.yml +++ b/eng/ci/templates/official/jobs/build-artifacts-windows.yml @@ -46,6 +46,14 @@ jobs: steps: - template: /eng/ci/templates/install-dotnet.yml@self + - task: DotNetCoreCLI@2 + displayName: Publish Site Extension + inputs: + command: custom + custom: restore + projects: src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj + arguments: '-v m -p:PublishReadyToRun=true -bl:$(log_dir)/restore.binlog' + - task: DotNetCoreCLI@2 displayName: Publish Site Extension inputs: @@ -54,7 +62,7 @@ jobs: publishWebProjects: false # we use our own publish logic zipAfterPublish: false # we use our own zip logic projects: src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj - arguments: '-v m -c release -p:BuildNumber=$(buildNumber) -p:MinorVersionPrefix=$(minorVersionPrefix) -p:ZipArtifactsPath=$(artifacts_path) -bl:$(log_dir)/publish.binlog' + arguments: '-v m -c release --no-restore -p:BuildNumber=$(buildNumber) -p:MinorVersionPrefix=$(minorVersionPrefix) -p:ZipArtifactsPath=$(artifacts_path) -bl:$(log_dir)/publish.binlog' - task: DotNetCoreCLI@2 displayName: Build host packages diff --git a/global.json b/global.json index ba12868df1..2785548097 100644 --- a/global.json +++ b/global.json @@ -1,6 +1,6 @@ { "sdk": { - "version": "8.0.204", + "version": "8.0.205", "rollForward": "latestFeature" }, "msbuild-sdks": { From 933b779302e3362392a17c6c3121dce9eefd75b1 Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Wed, 15 May 2024 11:23:15 -0700 Subject: [PATCH 16/43] Split out build stage as well --- .../official/jobs/build-artifacts-windows.yml | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/eng/ci/templates/official/jobs/build-artifacts-windows.yml b/eng/ci/templates/official/jobs/build-artifacts-windows.yml index 35584c1e0d..d6c085b01f 100644 --- a/eng/ci/templates/official/jobs/build-artifacts-windows.yml +++ b/eng/ci/templates/official/jobs/build-artifacts-windows.yml @@ -16,6 +16,8 @@ jobs: artifacts_path: $(Build.ArtifactStagingDirectory) log_dir: $(artifacts_path)/log nuget_package_path: $(artifacts_path)/NugetPackages + site_ext_proj: src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj + build_args: '-v m -c release --no-restore -p:BuildNumber=$(buildNumber) -p:MinorVersionPrefix=$(minorVersionPrefix)' templateContext: outputParentDirectory: $(artifacts_path) @@ -47,13 +49,22 @@ jobs: - template: /eng/ci/templates/install-dotnet.yml@self - task: DotNetCoreCLI@2 - displayName: Publish Site Extension + displayName: Restore Site Extension inputs: command: custom custom: restore - projects: src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj + projects: $(site_ext_proj) arguments: '-v m -p:PublishReadyToRun=true -bl:$(log_dir)/restore.binlog' + + - task: DotNetCoreCLI@2 + displayName: Build Site Extension + inputs: + command: custom + custom: build + projects: $(site_ext_proj) + arguments: '--no-restore $(build_args) -bl:$(log_dir)/build.binlog' + - task: DotNetCoreCLI@2 displayName: Publish Site Extension inputs: @@ -61,8 +72,8 @@ jobs: custom: publish publishWebProjects: false # we use our own publish logic zipAfterPublish: false # we use our own zip logic - projects: src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj - arguments: '-v m -c release --no-restore -p:BuildNumber=$(buildNumber) -p:MinorVersionPrefix=$(minorVersionPrefix) -p:ZipArtifactsPath=$(artifacts_path) -bl:$(log_dir)/publish.binlog' + projects: $(site_ext_proj) + arguments: '--no-build $(build_args) -p:ZipArtifactsPath=$(artifacts_path) -bl:$(log_dir)/publish.binlog' - task: DotNetCoreCLI@2 displayName: Build host packages From f7fd676448cb9ec32e2d1ab0637405a854a006dd Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Wed, 15 May 2024 11:41:05 -0700 Subject: [PATCH 17/43] Add python.props back in with redirection message --- build/python.props | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 build/python.props diff --git a/build/python.props b/build/python.props new file mode 100644 index 0000000000..54d42b61a2 --- /dev/null +++ b/build/python.props @@ -0,0 +1,8 @@ + + + + + + + + From b7eb4e7bf5cc76bbe1420fc5448167f2faa68af9 Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Wed, 15 May 2024 11:41:31 -0700 Subject: [PATCH 18/43] Fix integration build issue --- eng/ci/templates/official/jobs/build-artifacts-windows.yml | 2 +- .../WebJobs.Script.Tests.Integration.csproj | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/eng/ci/templates/official/jobs/build-artifacts-windows.yml b/eng/ci/templates/official/jobs/build-artifacts-windows.yml index d6c085b01f..85f94ef49f 100644 --- a/eng/ci/templates/official/jobs/build-artifacts-windows.yml +++ b/eng/ci/templates/official/jobs/build-artifacts-windows.yml @@ -48,6 +48,7 @@ jobs: steps: - template: /eng/ci/templates/install-dotnet.yml@self + # Restore must be a separate step so we can pass in 'PublishReadyToRun=true' - task: DotNetCoreCLI@2 displayName: Restore Site Extension inputs: @@ -56,7 +57,6 @@ jobs: projects: $(site_ext_proj) arguments: '-v m -p:PublishReadyToRun=true -bl:$(log_dir)/restore.binlog' - - task: DotNetCoreCLI@2 displayName: Build Site Extension inputs: diff --git a/test/WebJobs.Script.Tests.Integration/WebJobs.Script.Tests.Integration.csproj b/test/WebJobs.Script.Tests.Integration/WebJobs.Script.Tests.Integration.csproj index 5b3546c5db..ff5cefceaa 100644 --- a/test/WebJobs.Script.Tests.Integration/WebJobs.Script.Tests.Integration.csproj +++ b/test/WebJobs.Script.Tests.Integration/WebJobs.Script.Tests.Integration.csproj @@ -1,6 +1,6 @@ - + net8.0 false @@ -42,9 +42,7 @@ - - From d7de7c52e989335c7ee181e81bd0499cd2db8b9f Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Wed, 15 May 2024 11:48:10 -0700 Subject: [PATCH 19/43] Fix symbols output --- eng/ci/templates/official/jobs/build-artifacts-windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/ci/templates/official/jobs/build-artifacts-windows.yml b/eng/ci/templates/official/jobs/build-artifacts-windows.yml index 85f94ef49f..f2c659cfae 100644 --- a/eng/ci/templates/official/jobs/build-artifacts-windows.yml +++ b/eng/ci/templates/official/jobs/build-artifacts-windows.yml @@ -32,7 +32,7 @@ jobs: artifact: PrivateSiteExtension - output: pipelineArtifact displayName: Publish site extension symbols - path: $(artifacts_path)/Symbols + path: $(artifacts_path)/SiteExtensionSymbols artifact: Symbols - output: pipelineArtifact displayName: Publish nuget packages From 100e017be909fe5236e74b649b17bd9458865438 Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Wed, 15 May 2024 12:52:08 -0700 Subject: [PATCH 20/43] Revert python.props changes --- build/python.props | 9 +- build/update-hostreferences.ps1 | 171 -------------------------------- build/workers.props | 3 +- 3 files changed, 5 insertions(+), 178 deletions(-) delete mode 100644 build/update-hostreferences.ps1 diff --git a/build/python.props b/build/python.props index 54d42b61a2..f55ff8fb00 100644 --- a/build/python.props +++ b/build/python.props @@ -1,8 +1,5 @@ - - - - - - + + + diff --git a/build/update-hostreferences.ps1 b/build/update-hostreferences.ps1 deleted file mode 100644 index 07fedd72a0..0000000000 --- a/build/update-hostreferences.ps1 +++ /dev/null @@ -1,171 +0,0 @@ -function WriteLog -{ - param ( - [Parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - [System.String] - $Message, - - [Switch] - $Throw - ) - - $Message = (Get-Date -Format G) + " -- $Message" - - if ($Throw) - { - throw $Message - } - - Write-Host $Message -} - -Class PackageInfo { - [string]$Name - [string]$Version -} - -function NewPackageInfo -{ - param ( - [Parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - [System.String] - $PackageInformation - ) - - $parts = $PackageInformation.Split(" ") - - if ($parts.Count -gt 2) - { - WriteLog "Invalid package format. The string should only contain 'nameversion'. Current value: '$PackageInformation'" - } - - $packageInfo = [PackageInfo]::New() - $packageInfo.Name = $parts[0] - $packageInfo.Version = $parts[1] - - return $packageInfo -} - -function GetPackageInfo -{ - param ( - [Parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - [System.String] - $Name, - - [Parameter(Mandatory=$false)] - [System.String] - $MajorVersion - ) - - $result = $null - $includeAllVersion = if (-not [string]::IsNullOrWhiteSpace($MajorVersion)) { "-AllVersions" } else { "" } - - $packageInfo = & { NuGet list $Name -Source $SOURCE -PreRelease $includeAllVersion } - - if ($packageInfo -like "*No packages found*") - { - WriteLog "Package name $Name not found in $SOURCE." -Throw - } - - if (-not $MajorVersion) - { - $result = NewPackageInfo -PackageInformation $packageInfo - } - else - { - foreach ($thisPackage in $packageInfo.Split([System.Environment]::NewLine)) - { - $package = NewPackageInfo -PackageInformation $thisPackage - - if ($package.Version.StartsWith($MajorVersion)) - { - $result = $package - break - } - } - } - - return $result -} - -WriteLog "Script started." - -# Make sure the project path exits -$path = "$PSScriptRoot\..\src\WebJobs.Script" -if (-not (Test-Path $path)) -{ - WriteLog "Failed to find '$path' to update package references" -Throw -} - -$URL = "https://raw.githubusercontent.com/Azure/azure-functions-integration-tests/main/integrationTestsBuild/V4/HostBuild.json" -$SOURCE = "https://azfunc.pkgs.visualstudio.com/e6a70c92-4128-439f-8012-382fe78d6396/_packaging/AzureFunctionsPreRelease/nuget/v3/index.json" - -WriteLog "Get the list of packages to update" - -$packagesToUpdate = Invoke-RestMethod -Uri $URL -ErrorAction Stop -if ($packagesToUpdate.Count -eq 0) -{ - WriteLog "There are no packages to update in '$URL'" -Throw -} - -# Update packages references -WriteLog "Package references to update: $($packagesToUpdate.Count)" - -$currentDirectory = Get-Location -try -{ - set-location $path - - foreach ($package in $packagesToUpdate) - { - $packageInfo = GetPackageInfo -Name $package.Name -MajorVersion $package.MajorVersion - - if ($package.Name -eq "Microsoft.Azure.Functions.PythonWorker") - { - # The PythonWorker is not defined in the src/WebJobs.Script/WebJobs.Script.csproj. It is defined in build/python.props. - # To update the package version, the xml file build/python.props needs to be updated directly. - $pythonPropsFilePath = "$PSScriptRoot\python.props" - - if (-not (Test-Path $pythonPropsFilePath)) - { - WriteLog "Python Props file '$pythonPropsFilePath' does not exist." -Throw - } - - WriteLog "Set Python package version in '$pythonPropsFilePath' to '$($packageInfo.Version)'" - - # Read the xml file - [xml]$xml = Get-Content $pythonPropsFilePath -Raw -ErrorAction Stop - - # Replace the package version - $xml.Project.ItemGroup.PackageReference.Version = $packageInfo.Version - - # Save the file - $xml.Save($pythonPropsFilePath) - - if ($LASTEXITCODE -ne 0) - { - WriteLog "Failed to update Python Props file" -Throw - } - } - else - { - WriteLog "Adding '$($packageInfo.Name)' '$($packageInfo.Version)' to project" - & { dotnet add package $packageInfo.Name -v $packageInfo.Version -s $SOURCE --no-restore } - - if ($LASTEXITCODE -ne 0) - { - WriteLog "dotnet add package '$($packageInfo.Name)' -v '$($packageInfo.Version)' -s $SOURCE --no-restore failed" -Throw - } - } - } -} -finally -{ - Set-Location $currentDirectory -} - -WriteLog "Script completed." \ No newline at end of file diff --git a/build/workers.props b/build/workers.props index 762ed650dd..ba49ec6760 100644 --- a/build/workers.props +++ b/build/workers.props @@ -1,5 +1,7 @@ + + @@ -8,7 +10,6 @@ - From 5a68e5c6685d24b373e9c84f5a80f1407862918e Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Wed, 15 May 2024 13:38:31 -0700 Subject: [PATCH 21/43] Revert global.json change --- global.json | 2 +- src/WebJobs.Script.SiteExtension/Tasks.targets | 2 +- src/WebJobs.Script.SiteExtension/extension.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/global.json b/global.json index 2785548097..7932babe5b 100644 --- a/global.json +++ b/global.json @@ -1,6 +1,6 @@ { "sdk": { - "version": "8.0.205", + "version": "8.0.101", "rollForward": "latestFeature" }, "msbuild-sdks": { diff --git a/src/WebJobs.Script.SiteExtension/Tasks.targets b/src/WebJobs.Script.SiteExtension/Tasks.targets index 757ccba8a1..67f3ef84c7 100644 --- a/src/WebJobs.Script.SiteExtension/Tasks.targets +++ b/src/WebJobs.Script.SiteExtension/Tasks.targets @@ -97,4 +97,4 @@ for (int i = 0; i < Directories.Length; i++) - \ No newline at end of file + diff --git a/src/WebJobs.Script.SiteExtension/extension.xml b/src/WebJobs.Script.SiteExtension/extension.xml index 882d91a302..46338aa98a 100644 --- a/src/WebJobs.Script.SiteExtension/extension.xml +++ b/src/WebJobs.Script.SiteExtension/extension.xml @@ -1,3 +1,3 @@  disabled - \ No newline at end of file + From bdef249c10c8eb679ca2b5513e948f4dab5c99ed Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Wed, 15 May 2024 15:17:16 -0700 Subject: [PATCH 22/43] Extract build site extension into template --- azure-pipelines.yml | 14 +++----- .../official/jobs/build-artifacts-windows.yml | 28 +--------------- eng/ci/templates/steps/build-site-ext.yml | 33 +++++++++++++++++++ 3 files changed, 38 insertions(+), 37 deletions(-) create mode 100644 eng/ci/templates/steps/build-site-ext.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 4b7c988caf..7fbcdfa603 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -67,20 +67,14 @@ jobs: - ImageOverride -equals MMS2019TLS steps: - template: build/install-dotnet.yml - - task: PowerShell@2 - displayName: "Build artifacts" - inputs: - filePath: '$(Build.Repository.LocalPath)\build\build-extensions.ps1' - arguments: '-buildNumber "$(buildNumber)" -suffix "$(suffix)"' + - task: PowerShell@2 displayName: "Check for security vulnerabilities" inputs: filePath: '$(Build.Repository.LocalPath)\build\check-vulnerabilities.ps1' - - task: CopyFiles@2 - inputs: - SourceFolder: 'out/pub/WebJobs.Script.WebHost' - Contents: '**/*.zip' - TargetFolder: '$(Build.ArtifactStagingDirectory)' + + - template: eng/ci/templates/steps/build-site-ext.yml + - task: DotNetCoreCLI@2 displayName: 'Build host packages' inputs: diff --git a/eng/ci/templates/official/jobs/build-artifacts-windows.yml b/eng/ci/templates/official/jobs/build-artifacts-windows.yml index f2c659cfae..4b1be96ca1 100644 --- a/eng/ci/templates/official/jobs/build-artifacts-windows.yml +++ b/eng/ci/templates/official/jobs/build-artifacts-windows.yml @@ -47,33 +47,7 @@ jobs: steps: - template: /eng/ci/templates/install-dotnet.yml@self - - # Restore must be a separate step so we can pass in 'PublishReadyToRun=true' - - task: DotNetCoreCLI@2 - displayName: Restore Site Extension - inputs: - command: custom - custom: restore - projects: $(site_ext_proj) - arguments: '-v m -p:PublishReadyToRun=true -bl:$(log_dir)/restore.binlog' - - - task: DotNetCoreCLI@2 - displayName: Build Site Extension - inputs: - command: custom - custom: build - projects: $(site_ext_proj) - arguments: '--no-restore $(build_args) -bl:$(log_dir)/build.binlog' - - - task: DotNetCoreCLI@2 - displayName: Publish Site Extension - inputs: - command: custom - custom: publish - publishWebProjects: false # we use our own publish logic - zipAfterPublish: false # we use our own zip logic - projects: $(site_ext_proj) - arguments: '--no-build $(build_args) -p:ZipArtifactsPath=$(artifacts_path) -bl:$(log_dir)/publish.binlog' + - template: /eng/ci/templates/steps/build-site-ext.yml@self - task: DotNetCoreCLI@2 displayName: Build host packages diff --git a/eng/ci/templates/steps/build-site-ext.yml b/eng/ci/templates/steps/build-site-ext.yml new file mode 100644 index 0000000000..954886ac6a --- /dev/null +++ b/eng/ci/templates/steps/build-site-ext.yml @@ -0,0 +1,33 @@ +steps: + # Not aware of a way to inject variables in the middle of a job from a template. Parameters seem unnecessary here as they should never be changed. + - pwsh: | + Write-Host "##vso[task.setvariable variable=site_ext_proj]src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj" + Write-Host "##vso[task.setvariable variable=log_dir]$(Build.ArtifactStagingDirectory)/log" + displayName: Update site ext variables + + # Restore must be a separate step so we can pass in 'PublishReadyToRun=true' + - task: DotNetCoreCLI@2 + displayName: Restore site extension + inputs: + command: custom + custom: restore + projects: $(site_ext_proj) + arguments: '-v m -p:PublishReadyToRun=true -bl:$(log_dir)/site_ext.restore.binlog' + + - task: DotNetCoreCLI@2 + displayName: Build site extension + inputs: + command: custom + custom: build + projects: $(site_ext_proj) + arguments: '--no-restore -v m -p:BuildNumber=$(buildNumber) -p:MinorVersionPrefix=$(minorVersionPrefix) -bl:$(log_dir)/site_ext.build.binlog' + + - task: DotNetCoreCLI@2 + displayName: Publish site extension + inputs: + command: custom + custom: publish + publishWebProjects: false # we use our own publish logic + zipAfterPublish: false # we use our own zip logic + projects: $(site_ext_proj) + arguments: '--no-build -v m -p:BuildNumber=$(buildNumber) -p:MinorVersionPrefix=$(minorVersionPrefix) -p:ZipArtifactsPath=$(Build.ArtifactStagingDirectory) -bl:$(log_dir)/site_ext.publish.binlog' From 06291ab5ed337e3ba242f22a31cba1c9b08d770f Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Wed, 15 May 2024 15:17:48 -0700 Subject: [PATCH 23/43] Remove build-extension.ps1 --- build/build-extensions.ps1 | 219 ------------------------------------- 1 file changed, 219 deletions(-) delete mode 100644 build/build-extensions.ps1 diff --git a/build/build-extensions.ps1 b/build/build-extensions.ps1 deleted file mode 100644 index ba83ca973c..0000000000 --- a/build/build-extensions.ps1 +++ /dev/null @@ -1,219 +0,0 @@ -param ( - [string]$buildNumber = "0", - [string]$suffix = "", - [ValidateSet("6", "8", "")][string]$minorVersionPrefix = "", - [string]$hashesForHardlinksFile = "hashesForHardlinks.txt" -) - -Import-Module "$PSScriptRoot\Get-AzureFunctionsVersion.psm1" -Force -$rootDir = Split-Path -Parent $PSScriptRoot -$outDir = "$rootDir\out" -$publishDir = "$outDir\pub\WebJobs.Script.WebHost" - -$extensionVersion = Get-AzureFunctionsVersion $buildNumber $suffix $minorVersionPrefix -Write-Host "Site extension version: $extensionVersion" - -function ZipContent([string] $sourceDirectory, [string] $target) { - $stopwatch = [System.Diagnostics.Stopwatch]::StartNew() - - Write-Host "======================================" - Write-Host "Zipping $sourceDirectory into $target" - - if (Test-Path $target) { - Remove-Item $target - } - - Add-Type -assembly "system.io.compression.filesystem" - [IO.Compression.ZipFile]::CreateFromDirectory($sourceDirectory, $target) - - Write-Host "Done zipping $target. Elapsed: $($stopwatch.Elapsed)" - Write-Host "======================================" - Write-Host "" -} - -function BuildRuntime([string] $targetRid, [bool] $isSelfContained) { - $stopwatch = [System.Diagnostics.Stopwatch]::StartNew() - - $publishTarget = "$publishDir\release_$targetRid" - $projectPath = "$rootDir\src\WebJobs.Script.WebHost\WebJobs.Script.WebHost.csproj" - if (-not (Test-Path $projectPath)) - { - throw "Project path '$projectPath' does not exist." - } - - $cmd = "publish", $projectPath , "-r", "$targetRid", "--self-contained", "$isSelfContained", "-v", "m", "-c", "Release", "-p:IsPackable=false", "-p:BuildNumber=$buildNumber", "-p:MinorVersionPrefix=$minorVersionPrefix" - - Write-Host "======================================" - Write-Host "Building $targetRid" - Write-Host " Self-Contained: $isSelfContained" - Write-Host " Publish Directory: $publishTarget" - Write-Host "" - Write-Host "dotnet $cmd" - Write-Host "" - - & dotnet $cmd - - if ($LASTEXITCODE -ne 0) - { - exit $LASTEXITCODE - } - - Write-Host "" - $symbols = Get-ChildItem -Path $publishTarget -Filter *.pdb - $symbols += Get-ChildItem -Path "$publishTarget\workers\dotnet-isolated\*" -Include "*.pdb", "*.dbg" -Recurse - Write-Host "Zipping symbols: $($symbols.Count) symbols found" - - $symbolsPath = "$publishDir\Symbols" - if (!(Test-Path -PathType Container $symbolsPath)) { - New-Item -ItemType Directory -Path $symbolsPath | Out-Null - } - - $symbols | Compress-Archive -DestinationPath "$symbolsPath\Functions.Symbols.$extensionVersion.$targetRid.zip" | Out-Null - $symbols | Remove-Item | Out-Null - - Write-Host "" - CleanOutput $publishTarget - Write-Host "" - Write-Host "Done building $targetRid. Elapsed: $($stopwatch.Elapsed)" - Write-Host "======================================" - Write-Host "" -} - -function GetFolderSizeInMb([string] $rootPath) { - return [math]::Round((Get-ChildItem $rootPath -Recurse | Measure-Object -Property Length -Sum -ErrorAction Stop).Sum / 1Mb, 2) -} - -function CleanOutput([string] $rootPath) { - Write-Host "Cleaning build output under $rootPath" - Write-Host " Current size: $(GetFolderSizeInMb $rootPath) Mb" - - Write-Host " Removing any linux and osx runtimes" - Remove-Item -Recurse -Force "$privateSiteExtensionPath\$bitness\runtimes\linux" -ErrorAction SilentlyContinue - Remove-Item -Recurse -Force "$privateSiteExtensionPath\$bitness\runtimes\osx" -ErrorAction SilentlyContinue - - Write-Host " Removing python worker" - Remove-Item -Recurse -Force "$rootPath\workers\python" -ErrorAction SilentlyContinue - - $keepRuntimes = @('win', 'win-x86', 'win10-x86', 'win-x64', 'win10-x64') - Write-Host " Removing all powershell runtimes except $keepRuntimes" - Get-ChildItem "$rootPath\workers\powershell" -Directory -ErrorAction SilentlyContinue | - ForEach-Object { Get-ChildItem "$($_.FullName)\runtimes" -Directory -Exclude $keepRuntimes } | - Remove-Item -Recurse -Force -ErrorAction SilentlyContinue - - Write-Host " Removing FunctionsNetHost(linux executable) and dependencies from dotnet-isolated worker" - $dotnetIsolatedBinPath = Join-Path $rootPath "workers\dotnet-isolated\bin" - if (Test-Path $dotnetIsolatedBinPath) { - Remove-Item -Path (Join-Path $dotnetIsolatedBinPath "FunctionsNetHost") -ErrorAction SilentlyContinue - Get-ChildItem -Path $dotnetIsolatedBinPath -Filter "*.so" | Remove-Item -ErrorAction SilentlyContinue - } - - Write-Host " Current size: $(GetFolderSizeInMb $rootPath) Mb" -} - -function CreateSiteExtensions() { - $stopwatch = [System.Diagnostics.Stopwatch]::StartNew() - $siteExtensionPath = "$publishDir\temp_extension" - - if (Test-Path $siteExtensionPath) { - Write-Host " Existing site extension path found. Deleting." - Remove-Item $siteExtensionPath -Recurse -Force | Out-Null - } - - # The official site extension needs to be nested inside a folder with its version. - # Not using the suffix (eg: '-ci') here as it may not work correctly in a private stamp - $officialSiteExtensionPath = "$siteExtensionPath\$extensionVersion" - - Write-Host "======================================" - Write-Host "Copying build to temp directory to prepare for zipping official site extension." - Copy-Item -Path $publishDir\release_win-x86\ -Destination $officialSiteExtensionPath\32bit -Force -Recurse > $null - Copy-Item -Path $publishDir\release_win-x64 -Destination $officialSiteExtensionPath\64bit -Force -Recurse > $null - Copy-Item -Path $officialSiteExtensionPath\32bit\applicationHost.xdt -Destination $officialSiteExtensionPath -Force > $null - Write-Host " Deleting workers directory: $officialSiteExtensionPath\32bit\workers" - Remove-Item -Recurse -Force "$officialSiteExtensionPath\32bit\workers" -ErrorAction SilentlyContinue - Write-Host " Moving workers directory: $officialSiteExtensionPath\64bit\workers to $officialSiteExtensionPath\workers" - Move-Item -Path "$officialSiteExtensionPath\64bit\workers" -Destination "$officialSiteExtensionPath\workers" - - # This goes in the root dir - Copy-Item $rootDir\src\WebJobs.Script.WebHost\extension.xml $siteExtensionPath > $null - - Write-Host "Done copying. Elapsed: $($stopwatch.Elapsed)" - Write-Host "======================================" - Write-Host "" - - Write-Host "======================================" - Write-Host "Generating hashes for hard links" - WriteHashesFile $siteExtensionPath/$extensionVersion - Write-Host "Done generating hashes for hard links into $siteExtensionPath/$extensionVersion" - Write-Host "======================================" - Write-Host - - $zipOutput = "$publishDir\SiteExtension" - $hashesForHardLinksPath = "$siteExtensionPath\$extensionVersion\$hashesForHardlinksFile" - New-Item -Itemtype directory -path $zipOutput -Force > $null - if ($minorVersionPrefix -eq "") { - ZipContent $siteExtensionPath "$zipOutput\Functions.$extensionVersion.zip" - } elseif ($minorVersionPrefix -eq "8") { - Write-Host "======================================" - # Only the "Functions" site extension supports hard links - Write-Host "MinorVersionPrefix is '8'. Removing $hashesForHardLinksPath before zipping." - Remove-Item -Force "$hashesForHardLinksPath" -ErrorAction Stop - # The .NET 8 host doesn't require any workers. Doing this to save space. - Write-Host "Removing workers before zipping." - # The host requires that this folder exists and it cannot be empty - Remove-Item -Recurse -Force "$siteExtensionPath\$extensionVersion\workers" -ErrorAction Stop - New-Item -Path "$siteExtensionPath\$extensionVersion" -Name "workers" -ItemType Directory -ErrorAction Stop | Out-Null - Set-Content -Force -Path "$siteExtensionPath\$extensionVersion\workers\this_folder_intentionally_empty.txt" -Value ".NET 8 builds do not have workers. However, this folder must contain at least one file." -ErrorAction Stop - Write-Host "======================================" - Write-Host - ZipContent $siteExtensionPath "$zipOutput\FunctionsInProc8.$extensionVersion.zip" - } elseif ($minorVersionPrefix -eq "6") { - # Only the "Functions" site extension supports hard links - Write-Host "======================================" - Write-Host "MinorVersionPrefix is '6'. Removing $hashesForHardLinksPath before zipping." - Remove-Item -Force "$hashesForHardLinksPath" -ErrorAction Stop - Write-Host "======================================" - Write-Host - ZipContent $siteExtensionPath "$zipOutput\FunctionsInProc.$extensionVersion.zip" - } - - Remove-Item $siteExtensionPath -Recurse -Force > $null - - Write-Host "======================================" - $stopwatch.Reset() - Write-Host "Copying build to temp directory to prepare for zipping private site extension." - Copy-Item -Path $publishDir\release_win-x86\ -Destination $siteExtensionPath\SiteExtensions\Functions\32bit -Force -Recurse > $null - Copy-Item -Path $siteExtensionPath\SiteExtensions\Functions\32bit\applicationHost.xdt -Destination $siteExtensionPath\SiteExtensions\Functions -Force > $null - Write-Host "Done copying. Elapsed: $($stopwatch.Elapsed)" - Write-Host "======================================" - Write-Host "" - - $zipOutput = "$publishDir\PrivateSiteExtension" - New-Item -Itemtype directory -path $zipOutput -Force > $null - ZipContent $siteExtensionPath "$zipOutput\Functions.Private.$extensionVersion.win-x32.inproc.zip" - - Remove-Item $siteExtensionPath -Recurse -Force > $null -} - -function WriteHashesFile([string] $directoryPath) { - New-Item -Path "$directoryPath/../temp_hashes" -ItemType Directory | Out-Null - $temp_current = (Get-Location) - Set-Location $directoryPath - Get-ChildItem -Recurse $directoryPath | Where-Object { $_.PsIsContainer -eq $false } | Foreach-Object { "Hash:" + [System.Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes((Get-FileHash -Algorithm MD5 $_.FullName).Hash)) + " FileName:" + (Resolve-Path -Relative -Path $_.FullName) } | Out-File -FilePath "$directoryPath\..\temp_hashes\$hashesForHardlinksFile" - Move-Item -Path "$directoryPath/../temp_hashes/$hashesForHardlinksFile" -Destination "$directoryPath" -Force - Set-Location $temp_current - Remove-Item "$directoryPath/../temp_hashes" -Recurse -Force > $null -} - -Write-Host "Output directory: $publishDir" -if (Test-Path $publishDir) { - Write-Host " Existing build output found. Deleting." - Remove-Item $publishDir -Recurse -Force -ErrorAction Stop -} - -Write-Host "Extensions version: $extensionVersion" -Write-Host "" - -BuildRuntime "win-x86" -BuildRuntime "win-x64" - -CreateSiteExtensions From 921856b48e4e79ef30e91c9a86a849963829f2e1 Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Wed, 15 May 2024 15:24:55 -0700 Subject: [PATCH 24/43] Add build-extensions.ps1 back, with error directing to replacement --- build/build-extensions.ps1 | 1 + 1 file changed, 1 insertion(+) create mode 100644 build/build-extensions.ps1 diff --git a/build/build-extensions.ps1 b/build/build-extensions.ps1 new file mode 100644 index 0000000000..479c6b3e0e --- /dev/null +++ b/build/build-extensions.ps1 @@ -0,0 +1 @@ +Write-Error "This script is no longer used. Instead, publish src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj directly." From 4181e94080aed2c4c4c6f2064cd86ebb345ec2ae Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Wed, 15 May 2024 15:42:23 -0700 Subject: [PATCH 25/43] Fix log publishing --- .../official/jobs/build-artifacts-windows.yml | 17 +++++++---------- eng/ci/templates/steps/build-site-ext.yml | 5 ++++- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/eng/ci/templates/official/jobs/build-artifacts-windows.yml b/eng/ci/templates/official/jobs/build-artifacts-windows.yml index 4b1be96ca1..8bed813dc8 100644 --- a/eng/ci/templates/official/jobs/build-artifacts-windows.yml +++ b/eng/ci/templates/official/jobs/build-artifacts-windows.yml @@ -13,30 +13,27 @@ jobs: emgSuffixSwitchTemp: --version-suffix ci$(buildNumber) packSuffixSwitch: $[variables.packSuffixSwitchTemp] emgSuffixSwitch: $[variables.emgSuffixSwitchTemp] - artifacts_path: $(Build.ArtifactStagingDirectory) - log_dir: $(artifacts_path)/log - nuget_package_path: $(artifacts_path)/NugetPackages - site_ext_proj: src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj - build_args: '-v m -c release --no-restore -p:BuildNumber=$(buildNumber) -p:MinorVersionPrefix=$(minorVersionPrefix)' + nuget_package_path: $(Build.ArtifactStagingDirectory)/NugetPackages + log_dir: $(Build.ArtifactStagingDirectory)/log templateContext: - outputParentDirectory: $(artifacts_path) + outputParentDirectory: $(Build.ArtifactStagingDirectory) outputs: - output: pipelineArtifact displayName: Publish site extension - path: $(artifacts_path)/SiteExtension + path: $(Build.ArtifactStagingDirectory)/SiteExtension artifact: SiteExtension - output: pipelineArtifact displayName: Publish private site extension - path: $(artifacts_path)/PrivateSiteExtension + path: $(Build.ArtifactStagingDirectory)/PrivateSiteExtension artifact: PrivateSiteExtension - output: pipelineArtifact displayName: Publish site extension symbols - path: $(artifacts_path)/SiteExtensionSymbols + path: $(Build.ArtifactStagingDirectory)/SiteExtensionSymbols artifact: Symbols - output: pipelineArtifact displayName: Publish nuget packages - path: $(artifacts_path)/NugetPackages + path: $(Build.ArtifactStagingDirectory)/NugetPackages artifact: NugetPackages - output: pipelineArtifact displayName: Publish logs diff --git a/eng/ci/templates/steps/build-site-ext.yml b/eng/ci/templates/steps/build-site-ext.yml index 954886ac6a..444c490e8a 100644 --- a/eng/ci/templates/steps/build-site-ext.yml +++ b/eng/ci/templates/steps/build-site-ext.yml @@ -2,7 +2,10 @@ steps: # Not aware of a way to inject variables in the middle of a job from a template. Parameters seem unnecessary here as they should never be changed. - pwsh: | Write-Host "##vso[task.setvariable variable=site_ext_proj]src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj" - Write-Host "##vso[task.setvariable variable=log_dir]$(Build.ArtifactStagingDirectory)/log" + + if (-not '$(log_dir)'){ + Write-Host "##vso[task.setvariable variable=log_dir]$(Build.ArtifactStagingDirectory)/log" + } displayName: Update site ext variables # Restore must be a separate step so we can pass in 'PublishReadyToRun=true' From be3c1207a0eab859bfdc360b97d89fb7ce340738 Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Wed, 15 May 2024 15:59:31 -0700 Subject: [PATCH 26/43] Fix check-vuln placement --- azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7fbcdfa603..38e1dd556b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -68,13 +68,13 @@ jobs: steps: - template: build/install-dotnet.yml + - template: eng/ci/templates/steps/build-site-ext.yml + - task: PowerShell@2 displayName: "Check for security vulnerabilities" inputs: filePath: '$(Build.Repository.LocalPath)\build\check-vulnerabilities.ps1' - - template: eng/ci/templates/steps/build-site-ext.yml - - task: DotNetCoreCLI@2 displayName: 'Build host packages' inputs: From 619fbb9f601407627889a36e494cd48db1f96d2d Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Thu, 16 May 2024 09:43:24 -0700 Subject: [PATCH 27/43] Fix deletion of code-sign summary files --- .../official/jobs/build-artifacts-windows.yml | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/eng/ci/templates/official/jobs/build-artifacts-windows.yml b/eng/ci/templates/official/jobs/build-artifacts-windows.yml index 8bed813dc8..04cd5dc238 100644 --- a/eng/ci/templates/official/jobs/build-artifacts-windows.yml +++ b/eng/ci/templates/official/jobs/build-artifacts-windows.yml @@ -97,11 +97,6 @@ jobs: } ] - - task: DeleteFiles@1 - displayName: Delete CodeSignSummary files - inputs: - contents: '**/CodeSignSummary-*.md' - - task: DotNetCoreCLI@2 displayName: Pack Abstractions inputs: @@ -181,11 +176,6 @@ jobs: } ] - - task: DeleteFiles@1 - displayName: Delete CodeSignSummary files - inputs: - contents: '**/CodeSignSummary-*.md' - - task: DotNetCoreCLI@2 displayName: Pack ExtensionsMetadataGenerator inputs: @@ -224,3 +214,8 @@ jobs: displayName: Delete CodeSignSummary files inputs: contents: '**/CodeSignSummary-*.md' + + - task: DeleteFiles@1 + displayName: Delete CodeSignSummary files + inputs: + contents: '$(nuget_package_path)/**/CodeSignSummary-*.md' From 3bd60b6bd605172b0f6d6b987d18cc9c032abcd3 Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Thu, 16 May 2024 10:53:37 -0700 Subject: [PATCH 28/43] Fix symbol SBOM and artifact --- azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 38e1dd556b..ec02b60060 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -263,9 +263,9 @@ jobs: - task: ManifestGeneratorTask@0 displayName: 'SBOM Generation Task - Symbols' inputs: - BuildDropPath: '$(Build.ArtifactStagingDirectory)\Symbols' + BuildDropPath: '$(Build.ArtifactStagingDirectory)\SiteExtensionSymbols' Verbosity: 'Information' - - publish: $(Build.ArtifactStagingDirectory)\Symbols + - publish: $(Build.ArtifactStagingDirectory)\SiteExtensionSymbols artifact: Symbols - task: ManifestGeneratorTask@0 displayName: 'SBOM Generation Task - NugetPackages' From a5d5fefe4d1263a7c6810f916101d7a90e3a44e8 Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Mon, 3 Jun 2024 15:37:20 -0700 Subject: [PATCH 29/43] Add single/multi-tfm support for SiteExtension.proj --- eng/build/ZipPublish.targets | 2 +- .../Publish.MultiTFM.targets | 23 +++ .../Publish.SingleTFM.targets | 160 ++++++++++++++++++ .../WebJobs.Script.SiteExtension.proj | 149 +--------------- 4 files changed, 188 insertions(+), 146 deletions(-) create mode 100644 src/WebJobs.Script.SiteExtension/Publish.MultiTFM.targets create mode 100644 src/WebJobs.Script.SiteExtension/Publish.SingleTFM.targets diff --git a/eng/build/ZipPublish.targets b/eng/build/ZipPublish.targets index 5b052ba372..a030ea4fb2 100644 --- a/eng/build/ZipPublish.targets +++ b/eng/build/ZipPublish.targets @@ -29,7 +29,7 @@ Zip with a pre-determined name to the provided location > `dotnet publish -p:ZipArtifactsPath={some_directory}` --> - + $([MSBuild]::EnsureTrailingSlash('$(ZipArtifactsPath)'))%(TargetName) diff --git a/src/WebJobs.Script.SiteExtension/Publish.MultiTFM.targets b/src/WebJobs.Script.SiteExtension/Publish.MultiTFM.targets new file mode 100644 index 0000000000..33fbb44d44 --- /dev/null +++ b/src/WebJobs.Script.SiteExtension/Publish.MultiTFM.targets @@ -0,0 +1,23 @@ + + + + + + + + + + + <_TargetFramework Include="$(TargetFrameworks)" /> + + <_TargetFrameworkNormalized Include="@(_TargetFramework->Trim()->Distinct())" /> + <_InnerBuildProjects Include="$(MSBuildProjectFile)"> + TargetFramework=%(_TargetFrameworkNormalized.Identity) + + + + + + \ No newline at end of file diff --git a/src/WebJobs.Script.SiteExtension/Publish.SingleTFM.targets b/src/WebJobs.Script.SiteExtension/Publish.SingleTFM.targets new file mode 100644 index 0000000000..aabd5f0446 --- /dev/null +++ b/src/WebJobs.Script.SiteExtension/Publish.SingleTFM.targets @@ -0,0 +1,160 @@ + + + + + + + + + ValidatePublishSettings; + PublishProjectReferences; + PublishPrivateProjectReferences; + RemoveUnneededRuntimes; + MoveSymbols; + DeletePrivateSymbols; + WriteHardLinkHashes; + + ZipAfterPublish;ZipArtifactsPath + + + + + + + + + + + SiteExtension/$(Version)/ + $([MSBuild]::NormalizePath('$(PublishDir)$(SiteExtensionRelativeDir)')) + PrivateSiteExtension/SiteExtensions/Functions/ + $([MSBuild]::NormalizePath('$(PublishDir)$(PrivateSiteExtensionRelativeDir)')) + + + + + + + + + + + + + $(SiteExtensionRelativeDir)%(TargetPath) + + + + + + + <_ProjectReferenceWithRuntimes Include="@(ProjectReference)"> + + + RuntimeIdentifier=%(PublishRuntimeIdentifier.Identity); + SelfContained=%(PublishRuntimeIdentifier.SelfContained) + + %(PublishRuntimeIdentifier.Identity) + $(SiteExtensionDir)%(PublishRuntimeIdentifier.PublishDir)/ + $(PrivateSiteExtensionDir)%(PublishRuntimeIdentifier.PublishDir)/ + %(PublishRuntimeIdentifier.PrivateExtension) + false + + + + + + + + + + + + <_PublishProjectReferenceExistent Include="@(_MSBuildProjectReferenceExistent)"> + %(AdditionalProperties);PublishDir=%(PublishDir);_IsPublishing=true;PublishWorkers=false + + + + + + + + + + <_PublishPrivateProjectReferenceExistent Include="@(_MSBuildProjectReferenceExistent)" Condition="%(PublishPrivate)"> + %(AdditionalProperties);PublishDir=%(PublishPrivateDir);_IsPublishing=true + + + + + + + + + + + <_RuntimesToRemove Include="@(PublishRuntimeIdentifier->'$(SiteExtensionDir)%(PublishDir)/runtimes/linux')" /> + <_RuntimesToRemove Include="@(PublishRuntimeIdentifier->'$(SiteExtensionDir)%(PublishDir)/runtimes/osx')" /> + <_RuntimesToRemove Include="@(PublishRuntimeIdentifier->'$(PrivateSiteExtensionDir)%(PublishDir)/runtimes/linux')" Condition="%(PrivateExtension)" /> + <_RuntimesToRemove Include="@(PublishRuntimeIdentifier->'$(PrivateSiteExtensionDir)%(PublishDir)/runtimes/osx')" Condition="%(PrivateExtension)" /> + + + + + + + + <_SymbolDirs Include="@(PublishRuntimeIdentifier->'$(SiteExtensionDir)%(PublishDir)')"> + $(PublishDir)Symbols/$(SiteExtensionName).Symbols.$(Version).%(Identity) + $(SiteExtensionName).Symbols.$(Version).%(Identity).zip + + <_WorkerSymbols Include="$(SiteExtensionDir)workers/**/*.pdb" Destination="$(PublishDir)Symbols/$(SiteExtensionName).Symbols.$(Version).%(PublishRuntimeIdentifier.Identity)/workers" /> + + + + + + + + + + + <_PrivateSymbolsToRemove Include="$(PrivateSiteExtensionDir)/**/*.pdb" /> + + + + + + + + <_FilesToHash Include="$(SiteExtensionDir)**" /> + + + + + + + + + + <_HashedFiles RelativePath=".$([System.IO.Path]::DirectorySeparatorChar)$([MSBuild]::MakeRelative('$(SiteExtensionDir)', '%(Identity)'))" /> + + + + + \ No newline at end of file diff --git a/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj b/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj index 03c149c534..59e24340ac 100644 --- a/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj +++ b/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj @@ -4,18 +4,12 @@ - net6.0 + net8.0 win Functions - - PublishProjectReferences; - PublishPrivateProjectReferences; - RemoveUnneededRuntimes; - MoveSymbols; - DeletePrivateSymbols; - WriteHardLinkHashes; - - ZipAfterPublish;ZipArtifactsPath + $(MSBuildThisFileDirectory)Publish.SingleTFM.targets + $(MSBuildThisFileDirectory)Publish.MultiTFM.targets + false @@ -33,139 +27,4 @@ - - - - - - SiteExtension/$(Version)/ - $([MSBuild]::NormalizePath('$(PublishDir)$(SiteExtensionRelativeDir)')) - PrivateSiteExtension/SiteExtensions/Functions/ - $([MSBuild]::NormalizePath('$(PublishDir)$(PrivateSiteExtensionRelativeDir)')) - - - - - - - - - - - - - $(SiteExtensionRelativeDir)%(TargetPath) - - - - - - - <_ProjectReferenceWithRuntimes Include="@(ProjectReference)"> - - - RuntimeIdentifier=%(PublishRuntimeIdentifier.Identity); - SelfContained=%(PublishRuntimeIdentifier.SelfContained) - - %(PublishRuntimeIdentifier.Identity) - $(SiteExtensionDir)%(PublishRuntimeIdentifier.PublishDir)/ - $(PrivateSiteExtensionDir)%(PublishRuntimeIdentifier.PublishDir)/ - %(PublishRuntimeIdentifier.PrivateExtension) - false - - - - - - - - - - - - <_PublishProjectReferenceExistent Include="@(_MSBuildProjectReferenceExistent)"> - %(AdditionalProperties);PublishDir=%(PublishDir);_IsPublishing=true;PublishWorkers=false - - - - - - - - - - <_PublishPrivateProjectReferenceExistent Include="@(_MSBuildProjectReferenceExistent)" Condition="%(PublishPrivate)"> - %(AdditionalProperties);PublishDir=%(PublishPrivateDir);_IsPublishing=true - - - - - - - - - - - <_RuntimesToRemove Include="@(PublishRuntimeIdentifier->'$(SiteExtensionDir)%(PublishDir)/runtimes/linux')" /> - <_RuntimesToRemove Include="@(PublishRuntimeIdentifier->'$(SiteExtensionDir)%(PublishDir)/runtimes/osx')" /> - <_RuntimesToRemove Include="@(PublishRuntimeIdentifier->'$(PrivateSiteExtensionDir)%(PublishDir)/runtimes/linux')" Condition="%(PrivateExtension)" /> - <_RuntimesToRemove Include="@(PublishRuntimeIdentifier->'$(PrivateSiteExtensionDir)%(PublishDir)/runtimes/osx')" Condition="%(PrivateExtension)" /> - - - - - - - - <_SymbolDirs Include="@(PublishRuntimeIdentifier->'$(SiteExtensionDir)%(PublishDir)')"> - $(PublishDir)Symbols/$(SiteExtensionName).Symbols.$(Version).%(Identity) - $(SiteExtensionName).Symbols.$(Version).%(Identity).zip - - <_WorkerSymbols Include="$(SiteExtensionDir)workers/**/*.pdb" Destination="$(PublishDir)Symbols/$(SiteExtensionName).Symbols.$(Version).%(PublishRuntimeIdentifier.Identity)/workers" /> - - - - - - - - - - - <_PrivateSymbolsToRemove Include="$(PrivateSiteExtensionDir)/**/*.pdb" /> - - - - - - - - <_FilesToHash Include="$(SiteExtensionDir)**" /> - - - - - - - - - - <_HashedFiles RelativePath=".$([System.IO.Path]::DirectorySeparatorChar)$([MSBuild]::MakeRelative('$(SiteExtensionDir)', '%(Identity)'))" /> - - - - From 1175ff42b76ee79cac247c3017c9206cd70c7624 Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Mon, 3 Jun 2024 15:44:15 -0700 Subject: [PATCH 30/43] Fix end-of-file line breaks --- eng/build/ZipPublish.targets | 2 +- src/WebJobs.Script.SiteExtension/Publish.MultiTFM.targets | 2 +- src/WebJobs.Script.SiteExtension/Publish.SingleTFM.targets | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/build/ZipPublish.targets b/eng/build/ZipPublish.targets index a030ea4fb2..2c1136bfb0 100644 --- a/eng/build/ZipPublish.targets +++ b/eng/build/ZipPublish.targets @@ -49,4 +49,4 @@ - \ No newline at end of file + diff --git a/src/WebJobs.Script.SiteExtension/Publish.MultiTFM.targets b/src/WebJobs.Script.SiteExtension/Publish.MultiTFM.targets index 33fbb44d44..e66bc0bc5d 100644 --- a/src/WebJobs.Script.SiteExtension/Publish.MultiTFM.targets +++ b/src/WebJobs.Script.SiteExtension/Publish.MultiTFM.targets @@ -20,4 +20,4 @@ - \ No newline at end of file + diff --git a/src/WebJobs.Script.SiteExtension/Publish.SingleTFM.targets b/src/WebJobs.Script.SiteExtension/Publish.SingleTFM.targets index aabd5f0446..f12158c383 100644 --- a/src/WebJobs.Script.SiteExtension/Publish.SingleTFM.targets +++ b/src/WebJobs.Script.SiteExtension/Publish.SingleTFM.targets @@ -157,4 +157,4 @@ Lines="@(_HashedFiles->'Hash: %(FileHash) FileName: %(RelativePath)')" /> - \ No newline at end of file + From 3b663d73b957bb8d82287e6b11d61d7714e08f77 Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Thu, 25 Jul 2024 15:18:08 -0700 Subject: [PATCH 31/43] Extract Workers to their own props files --- Directory.Build.props | 1 + Directory.Build.targets | 2 +- build/python.props | 7 ++++--- eng/build/Workers.Dotnet.props | 15 +++++++++++++++ eng/build/Workers.Java.props | 7 +++++++ eng/build/Workers.Node.props | 7 +++++++ .../build/Workers.Powershell.props | 17 +---------------- eng/build/Workers.Python.props | 15 +++++++++++++++ eng/build/Workers.props | 6 ++++++ .../WebJobs.Script.SiteExtension.proj | 4 +++- .../WebJobs.Script.WebHost.csproj | 10 +--------- .../WebJobs.Script.Tests.Integration.csproj | 4 +++- 12 files changed, 64 insertions(+), 31 deletions(-) create mode 100644 eng/build/Workers.Dotnet.props create mode 100644 eng/build/Workers.Java.props create mode 100644 eng/build/Workers.Node.props rename build/workers.props => eng/build/Workers.Powershell.props (60%) create mode 100644 eng/build/Workers.Python.props create mode 100644 eng/build/Workers.props diff --git a/Directory.Build.props b/Directory.Build.props index 7d45472b7e..054d6269e1 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -11,6 +11,7 @@ $(MSBuildThisFileDirectory) $(RepoRoot)eng/ + $(EngRoot)build/ diff --git a/Directory.Build.targets b/Directory.Build.targets index 9c9f2a6f8f..adc05e884c 100644 --- a/Directory.Build.targets +++ b/Directory.Build.targets @@ -1,5 +1,5 @@ - + diff --git a/build/python.props b/build/python.props index 1e1f189869..ebfd61c498 100644 --- a/build/python.props +++ b/build/python.props @@ -1,5 +1,6 @@ - - - + + + + diff --git a/eng/build/Workers.Dotnet.props b/eng/build/Workers.Dotnet.props new file mode 100644 index 0000000000..c9cc32cc20 --- /dev/null +++ b/eng/build/Workers.Dotnet.props @@ -0,0 +1,15 @@ + + + + + + + + + + <_DotnetWorkerFiles Include="@(None)" Condition="'%(None.DestinationSubDirectory)' == 'workers\dotnet-isolated\bin\'" /> + + + + + diff --git a/eng/build/Workers.Java.props b/eng/build/Workers.Java.props new file mode 100644 index 0000000000..3848f4c92b --- /dev/null +++ b/eng/build/Workers.Java.props @@ -0,0 +1,7 @@ + + + + + + + diff --git a/eng/build/Workers.Node.props b/eng/build/Workers.Node.props new file mode 100644 index 0000000000..2263ab17df --- /dev/null +++ b/eng/build/Workers.Node.props @@ -0,0 +1,7 @@ + + + + + + + diff --git a/build/workers.props b/eng/build/Workers.Powershell.props similarity index 60% rename from build/workers.props rename to eng/build/Workers.Powershell.props index 0bf1782833..fb75ddaafa 100644 --- a/build/workers.props +++ b/eng/build/Workers.Powershell.props @@ -1,18 +1,12 @@ - - - - - - - + <_KeepPowerShellRuntime Include="win;win-x86;win10-x86;win-x64;win10-x64" /> @@ -32,13 +26,4 @@ - - - - - <_DotnetWorkerFiles Include="@(None)" Condition="'%(None.DestinationSubDirectory)' == 'workers\dotnet-isolated\bin\'" /> - - - - diff --git a/eng/build/Workers.Python.props b/eng/build/Workers.Python.props new file mode 100644 index 0000000000..2d30a22e2b --- /dev/null +++ b/eng/build/Workers.Python.props @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff --git a/eng/build/Workers.props b/eng/build/Workers.props new file mode 100644 index 0000000000..e333ed3e89 --- /dev/null +++ b/eng/build/Workers.props @@ -0,0 +1,6 @@ + + + + + + diff --git a/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj b/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj index 59e24340ac..42ceba0a7d 100644 --- a/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj +++ b/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj @@ -1,6 +1,6 @@ + - @@ -12,6 +12,8 @@ false + + diff --git a/src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj b/src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj index 906ca2e8e5..e03bdecb7c 100644 --- a/src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj +++ b/src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj @@ -1,6 +1,6 @@  - + net8.0 Microsoft.Azure.WebJobs.Script.WebHost @@ -146,12 +146,4 @@ - - - - - - - - diff --git a/test/WebJobs.Script.Tests.Integration/WebJobs.Script.Tests.Integration.csproj b/test/WebJobs.Script.Tests.Integration/WebJobs.Script.Tests.Integration.csproj index 8f95fd45eb..3c7e103cca 100644 --- a/test/WebJobs.Script.Tests.Integration/WebJobs.Script.Tests.Integration.csproj +++ b/test/WebJobs.Script.Tests.Integration/WebJobs.Script.Tests.Integration.csproj @@ -1,6 +1,5 @@ - net8.0 false @@ -20,6 +19,9 @@ true + + + From 298419fdbfb47df485a8f6fc756524ef5568837c Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Fri, 26 Jul 2024 09:32:54 -0700 Subject: [PATCH 32/43] Fix ZipPublish when zip not specified --- CODEOWNERS | 1 + eng/build/Workers.Dotnet.props | 2 +- eng/build/Workers.props | 15 +++++++++++++++ eng/build/ZipPublish.targets | 5 ++++- .../WebJobs.Script.WebHost.csproj | 14 -------------- 5 files changed, 21 insertions(+), 16 deletions(-) diff --git a/CODEOWNERS b/CODEOWNERS index 05689a05d6..14ee763f60 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -9,6 +9,7 @@ # Adding codeowner for Python specific files such that GitHub automatically adds python folks as a reviewer. build/python.props @vrdmr @gavin-aguiar @YunchuWang @pdthummar @hallvictoria +eng/build/Workers.Python.props @vrdmr @gavin-aguiar @YunchuWang @pdthummar @hallvictoria # Deps.json validation file test/WebJobs.Script.Tests/Microsoft.Azure.WebJobs.Script.WebHost.deps.json @fabiocav @brettsam @mathewc diff --git a/eng/build/Workers.Dotnet.props b/eng/build/Workers.Dotnet.props index c9cc32cc20..212af9bf9e 100644 --- a/eng/build/Workers.Dotnet.props +++ b/eng/build/Workers.Dotnet.props @@ -4,7 +4,7 @@ - + <_DotnetWorkerFiles Include="@(None)" Condition="'%(None.DestinationSubDirectory)' == 'workers\dotnet-isolated\bin\'" /> diff --git a/eng/build/Workers.props b/eng/build/Workers.props index e333ed3e89..047a7a0860 100644 --- a/eng/build/Workers.props +++ b/eng/build/Workers.props @@ -3,4 +3,19 @@ + + + + <_WorkerPublishFiles Include="@(ResolvedFileToPublish)" Condition="$([System.String]::new('%(ResolvedFileToPublish.TargetPath)').StartsWith('workers'))" /> + + + + + + + + + + + diff --git a/eng/build/ZipPublish.targets b/eng/build/ZipPublish.targets index 2c1136bfb0..c89459077a 100644 --- a/eng/build/ZipPublish.targets +++ b/eng/build/ZipPublish.targets @@ -29,7 +29,10 @@ Zip with a pre-determined name to the provided location > `dotnet publish -p:ZipArtifactsPath={some_directory}` --> - + $([MSBuild]::EnsureTrailingSlash('$(ZipArtifactsPath)'))%(TargetName) diff --git a/src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj b/src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj index e03bdecb7c..8ac8c06548 100644 --- a/src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj +++ b/src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj @@ -132,18 +132,4 @@ - - - - <_WorkerPublishFiles Include="@(ResolvedFileToPublish)" Condition="$([System.String]::new('%(ResolvedFileToPublish.TargetPath)').StartsWith('workers'))" /> - - - - - - - - - - From 4b4983fbe8c82a6b2d3e204ed1aeccc8763c6168 Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Fri, 26 Jul 2024 13:14:08 -0700 Subject: [PATCH 33/43] Remove redundant python removal --- .../WebJobs.Script.SiteExtension.proj | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj b/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj index 42ceba0a7d..6d53ba16ce 100644 --- a/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj +++ b/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj @@ -14,11 +14,6 @@ - - - - - From c1dcc497dbb6af1023a4a47c158f09813c2e640d Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Fri, 26 Jul 2024 14:48:01 -0700 Subject: [PATCH 34/43] .proj -> .csproj --- src/WebJobs.Script.SiteExtension/Publish.SingleTFM.targets | 4 ++-- ...SiteExtension.proj => WebJobs.Script.SiteExtension.csproj} | 0 2 files changed, 2 insertions(+), 2 deletions(-) rename src/WebJobs.Script.SiteExtension/{WebJobs.Script.SiteExtension.proj => WebJobs.Script.SiteExtension.csproj} (100%) diff --git a/src/WebJobs.Script.SiteExtension/Publish.SingleTFM.targets b/src/WebJobs.Script.SiteExtension/Publish.SingleTFM.targets index f12158c383..52906eba4d 100644 --- a/src/WebJobs.Script.SiteExtension/Publish.SingleTFM.targets +++ b/src/WebJobs.Script.SiteExtension/Publish.SingleTFM.targets @@ -42,8 +42,8 @@ - - $(SiteExtensionRelativeDir)%(TargetPath) + + $(SiteExtensionRelativeDir)%(None.TargetPath) diff --git a/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj b/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.csproj similarity index 100% rename from src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj rename to src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.csproj From a1586732e8eb176d9e18d62c9786a20c16186560 Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Fri, 26 Jul 2024 15:19:56 -0700 Subject: [PATCH 35/43] Remove minorVersionPrefix --- CODEOWNERS | 2 +- eng/ci/templates/official/jobs/build-artifacts-windows.yml | 2 +- eng/ci/templates/steps/build-site-ext.yml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CODEOWNERS b/CODEOWNERS index 14ee763f60..17f02f644e 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -14,7 +14,7 @@ eng/build/Workers.Python.props @vrdmr @gavin-aguiar @YunchuWang @pdthummar @hall # Deps.json validation file test/WebJobs.Script.Tests/Microsoft.Azure.WebJobs.Script.WebHost.deps.json @fabiocav @brettsam @mathewc -src/WebJobs.Script.WebHost/PreJIT/* @vrdmr @pragnagopa @eliaslopezgt @VpOfEngineering @azure/azure-functions-core +src/WebJobs.Script.WebHost/PreJIT/* @vrdmr @pragnagopa @eliaslopezgt @azure/azure-functions-core # CI owners diff --git a/eng/ci/templates/official/jobs/build-artifacts-windows.yml b/eng/ci/templates/official/jobs/build-artifacts-windows.yml index d6205a1f17..5bc61ec085 100644 --- a/eng/ci/templates/official/jobs/build-artifacts-windows.yml +++ b/eng/ci/templates/official/jobs/build-artifacts-windows.yml @@ -8,7 +8,7 @@ jobs: os: windows variables: - ${{ if or( eq( variables['Build.Reason'], 'PullRequest' ), and(not( contains( variables['Build.SourceBranch'], 'release/in-proc.' ) ), not( contains( variables['Build.SourceBranch'], 'release/4.' ) ), not( contains( variables['Build.SourceBranch'], 'release/ExtensionsMetadataGenerator/' ) ) ) ) }}: + ${{ if or( eq( variables['Build.Reason'], 'PullRequest' ), and(not( contains( variables['Build.SourceBranch'], 'release/in-proc' ) ), not( contains( variables['Build.SourceBranch'], 'release/4.' ) ), not( contains( variables['Build.SourceBranch'], 'release/ExtensionsMetadataGenerator/' ) ) ) ) }}: packSuffixSwitchTemp: --version-suffix $(buildNumber) emgSuffixSwitchTemp: --version-suffix ci$(buildNumber) packSuffixSwitch: $[variables.packSuffixSwitchTemp] diff --git a/eng/ci/templates/steps/build-site-ext.yml b/eng/ci/templates/steps/build-site-ext.yml index 444c490e8a..13cca647d6 100644 --- a/eng/ci/templates/steps/build-site-ext.yml +++ b/eng/ci/templates/steps/build-site-ext.yml @@ -23,7 +23,7 @@ steps: command: custom custom: build projects: $(site_ext_proj) - arguments: '--no-restore -v m -p:BuildNumber=$(buildNumber) -p:MinorVersionPrefix=$(minorVersionPrefix) -bl:$(log_dir)/site_ext.build.binlog' + arguments: '--no-restore -v m -p:BuildNumber=$(buildNumber) -bl:$(log_dir)/site_ext.build.binlog' - task: DotNetCoreCLI@2 displayName: Publish site extension @@ -33,4 +33,4 @@ steps: publishWebProjects: false # we use our own publish logic zipAfterPublish: false # we use our own zip logic projects: $(site_ext_proj) - arguments: '--no-build -v m -p:BuildNumber=$(buildNumber) -p:MinorVersionPrefix=$(minorVersionPrefix) -p:ZipArtifactsPath=$(Build.ArtifactStagingDirectory) -bl:$(log_dir)/site_ext.publish.binlog' + arguments: '--no-build -v m -p:BuildNumber=$(buildNumber) -p:ZipArtifactsPath=$(Build.ArtifactStagingDirectory) -bl:$(log_dir)/site_ext.publish.binlog' From a3c35fa4198403cf292e800ee9a9f08e6100aab4 Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Fri, 26 Jul 2024 15:20:37 -0700 Subject: [PATCH 36/43] Update CODEOWNERS --- CODEOWNERS | 4 ---- 1 file changed, 4 deletions(-) diff --git a/CODEOWNERS b/CODEOWNERS index 17f02f644e..80e046649b 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -15,7 +15,3 @@ eng/build/Workers.Python.props @vrdmr @gavin-aguiar @YunchuWang @pdthummar @hall test/WebJobs.Script.Tests/Microsoft.Azure.WebJobs.Script.WebHost.deps.json @fabiocav @brettsam @mathewc src/WebJobs.Script.WebHost/PreJIT/* @vrdmr @pragnagopa @eliaslopezgt @azure/azure-functions-core - -# CI owners - -eng/ci/* @fabiocav @jviau @brettsam From 05e0bc2f7cfa81c736a38115e60d0cc0ff53b59e Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Fri, 26 Jul 2024 15:46:18 -0700 Subject: [PATCH 37/43] Add EnsureWorkersFolder target --- .../Publish.SingleTFM.targets | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/WebJobs.Script.SiteExtension/Publish.SingleTFM.targets b/src/WebJobs.Script.SiteExtension/Publish.SingleTFM.targets index 52906eba4d..92b1f39ceb 100644 --- a/src/WebJobs.Script.SiteExtension/Publish.SingleTFM.targets +++ b/src/WebJobs.Script.SiteExtension/Publish.SingleTFM.targets @@ -113,6 +113,14 @@ + + + + + From f7a982555a9ba71bd1805719e22a9c362df27e19 Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Mon, 29 Jul 2024 10:34:08 -0700 Subject: [PATCH 38/43] Fix windows build --- .../official/jobs/build-artifacts-linux.yml | 2 +- eng/ci/templates/steps/build-site-ext.yml | 20 ++++++++----------- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/eng/ci/templates/official/jobs/build-artifacts-linux.yml b/eng/ci/templates/official/jobs/build-artifacts-linux.yml index ca0036ebb8..51cbbf81b0 100644 --- a/eng/ci/templates/official/jobs/build-artifacts-linux.yml +++ b/eng/ci/templates/official/jobs/build-artifacts-linux.yml @@ -9,7 +9,7 @@ jobs: artifacts_path: $(Build.ArtifactStagingDirectory) log_dir: $(artifacts_path)/log zip_artifacts_path: $(artifacts_path)/Linux - build_args: '-v m -c $(configuration) -r $(runtime) --self-contained true -p:BuildNumber=$(buildNumber) -p:MinorVersionPrefix=$(minorVersionPrefix) -p:IsPackable=false' + build_args: '-v m -c $(configuration) -r $(runtime) --self-contained true -p:BuildNumber=$(buildNumber) -p:IsPackable=false' templateContext: outputParentDirectory: $(artifacts_path) diff --git a/eng/ci/templates/steps/build-site-ext.yml b/eng/ci/templates/steps/build-site-ext.yml index 13cca647d6..7230d7d92a 100644 --- a/eng/ci/templates/steps/build-site-ext.yml +++ b/eng/ci/templates/steps/build-site-ext.yml @@ -1,20 +1,16 @@ -steps: - # Not aware of a way to inject variables in the middle of a job from a template. Parameters seem unnecessary here as they should never be changed. - - pwsh: | - Write-Host "##vso[task.setvariable variable=site_ext_proj]src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.proj" - - if (-not '$(log_dir)'){ - Write-Host "##vso[task.setvariable variable=log_dir]$(Build.ArtifactStagingDirectory)/log" - } - displayName: Update site ext variables +parameters: + - name: project + type: string + default: src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.csproj +steps: # Restore must be a separate step so we can pass in 'PublishReadyToRun=true' - task: DotNetCoreCLI@2 displayName: Restore site extension inputs: command: custom custom: restore - projects: $(site_ext_proj) + projects: ${{ parameters.project }} arguments: '-v m -p:PublishReadyToRun=true -bl:$(log_dir)/site_ext.restore.binlog' - task: DotNetCoreCLI@2 @@ -22,7 +18,7 @@ steps: inputs: command: custom custom: build - projects: $(site_ext_proj) + projects: ${{ parameters.project }} arguments: '--no-restore -v m -p:BuildNumber=$(buildNumber) -bl:$(log_dir)/site_ext.build.binlog' - task: DotNetCoreCLI@2 @@ -32,5 +28,5 @@ steps: custom: publish publishWebProjects: false # we use our own publish logic zipAfterPublish: false # we use our own zip logic - projects: $(site_ext_proj) + projects: ${{ parameters.project }} arguments: '--no-build -v m -p:BuildNumber=$(buildNumber) -p:ZipArtifactsPath=$(Build.ArtifactStagingDirectory) -bl:$(log_dir)/site_ext.publish.binlog' From d74a7faa24714b09a99df1710098531dfa3ee98e Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Mon, 29 Jul 2024 11:05:07 -0700 Subject: [PATCH 39/43] Pack linux artifact --- eng/ci/templates/official/jobs/build-artifacts-linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/ci/templates/official/jobs/build-artifacts-linux.yml b/eng/ci/templates/official/jobs/build-artifacts-linux.yml index 51cbbf81b0..440ec3a814 100644 --- a/eng/ci/templates/official/jobs/build-artifacts-linux.yml +++ b/eng/ci/templates/official/jobs/build-artifacts-linux.yml @@ -9,7 +9,7 @@ jobs: artifacts_path: $(Build.ArtifactStagingDirectory) log_dir: $(artifacts_path)/log zip_artifacts_path: $(artifacts_path)/Linux - build_args: '-v m -c $(configuration) -r $(runtime) --self-contained true -p:BuildNumber=$(buildNumber) -p:IsPackable=false' + build_args: '-v m -c $(configuration) -r $(runtime) --self-contained true -p:BuildNumber=$(buildNumber)' templateContext: outputParentDirectory: $(artifacts_path) From 9e5bb6fe862c401f6c9b7e80ae8dfcfc1252dc13 Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Mon, 29 Jul 2024 12:56:59 -0700 Subject: [PATCH 40/43] Add release config --- eng/ci/templates/steps/build-site-ext.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/ci/templates/steps/build-site-ext.yml b/eng/ci/templates/steps/build-site-ext.yml index 7230d7d92a..fbdeb5df15 100644 --- a/eng/ci/templates/steps/build-site-ext.yml +++ b/eng/ci/templates/steps/build-site-ext.yml @@ -19,7 +19,7 @@ steps: command: custom custom: build projects: ${{ parameters.project }} - arguments: '--no-restore -v m -p:BuildNumber=$(buildNumber) -bl:$(log_dir)/site_ext.build.binlog' + arguments: '--no-restore -v m -c release -p:BuildNumber=$(buildNumber) -bl:$(log_dir)/site_ext.build.binlog' - task: DotNetCoreCLI@2 displayName: Publish site extension @@ -29,4 +29,4 @@ steps: publishWebProjects: false # we use our own publish logic zipAfterPublish: false # we use our own zip logic projects: ${{ parameters.project }} - arguments: '--no-build -v m -p:BuildNumber=$(buildNumber) -p:ZipArtifactsPath=$(Build.ArtifactStagingDirectory) -bl:$(log_dir)/site_ext.publish.binlog' + arguments: '--no-build -v m -c release -p:BuildNumber=$(buildNumber) -p:ZipArtifactsPath=$(Build.ArtifactStagingDirectory) -bl:$(log_dir)/site_ext.publish.binlog' From e116ada317d87025a3e2d4e434f8a203b75c7886 Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Mon, 29 Jul 2024 13:36:46 -0700 Subject: [PATCH 41/43] Fix nuget signing path --- eng/ci/templates/official/jobs/build-artifacts-windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/ci/templates/official/jobs/build-artifacts-windows.yml b/eng/ci/templates/official/jobs/build-artifacts-windows.yml index 5bc61ec085..439619025d 100644 --- a/eng/ci/templates/official/jobs/build-artifacts-windows.yml +++ b/eng/ci/templates/official/jobs/build-artifacts-windows.yml @@ -101,7 +101,7 @@ jobs: - template: ci/sign-files.yml@eng parameters: displayName: Sign NugetPackages - folderPath: out/pkg/release + folderPath: $(nuget_package_path) pattern: '*.nupkg' signType: nuget From e69c425c67491da38ad639d45f1ae658522a78d1 Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Tue, 30 Jul 2024 12:42:36 -0700 Subject: [PATCH 42/43] Add condition for hard link hashes --- src/WebJobs.Script.SiteExtension/Publish.SingleTFM.targets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/WebJobs.Script.SiteExtension/Publish.SingleTFM.targets b/src/WebJobs.Script.SiteExtension/Publish.SingleTFM.targets index 92b1f39ceb..7137585c65 100644 --- a/src/WebJobs.Script.SiteExtension/Publish.SingleTFM.targets +++ b/src/WebJobs.Script.SiteExtension/Publish.SingleTFM.targets @@ -155,7 +155,7 @@ - + <_HashedFiles RelativePath=".$([System.IO.Path]::DirectorySeparatorChar)$([MSBuild]::MakeRelative('$(SiteExtensionDir)', '%(Identity)'))" /> From 2db260c2c5c227cf750253b7b098ca888a1b2404 Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Wed, 13 Nov 2024 17:03:32 -0800 Subject: [PATCH 43/43] Add IsZippable check and readme --- eng/build/ZipPublish.targets | 2 +- .../WebJobs.Script.SiteExtension.csproj | 1 + src/WebJobs.Script.SiteExtension/readme.md | 24 +++++++++++++++++++ 3 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 src/WebJobs.Script.SiteExtension/readme.md diff --git a/eng/build/ZipPublish.targets b/eng/build/ZipPublish.targets index c89459077a..6dc234eb47 100644 --- a/eng/build/ZipPublish.targets +++ b/eng/build/ZipPublish.targets @@ -32,7 +32,7 @@ + Condition="'$(ZipArtifactsPath)' != '' AND '@(ZipArtifact)' != '' AND '$(IsCrossTargetingBuild)' != 'true' AND '$(IsZippable)' == 'true'"> $([MSBuild]::EnsureTrailingSlash('$(ZipArtifactsPath)'))%(TargetName) diff --git a/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.csproj b/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.csproj index 6d53ba16ce..03a21cb341 100644 --- a/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.csproj +++ b/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.csproj @@ -7,6 +7,7 @@ net8.0 win Functions + true $(MSBuildThisFileDirectory)Publish.SingleTFM.targets $(MSBuildThisFileDirectory)Publish.MultiTFM.targets false diff --git a/src/WebJobs.Script.SiteExtension/readme.md b/src/WebJobs.Script.SiteExtension/readme.md new file mode 100644 index 0000000000..1e01261a06 --- /dev/null +++ b/src/WebJobs.Script.SiteExtension/readme.md @@ -0,0 +1,24 @@ +# Site Extension + +This project is responsible for building the artifacts we ship to antares as a site extension. + +## Usage + +Like any MSBuild project, this can be restored, build, and published separately or together. + +``` shell +# Together +dotnet publish -c {config} + +# Separately +dotnet restore -p:PublishReadyToRun=true # needs to be set to true (fixed in .net9 SDK) +dotnet build -c {config} --no-restore +dotnet publish -c {config} --no-build +``` + +By default the outputs will not be zipped. To the zip the final outputs, add `-p:ZipAfterPublish=true` to the `publish` command. + + +## Outputs + +The output site extension can be found at `{repo_root}/out/pub/WebJobs.Script.SiteExtension/{config}_win`. When using `-p:ZipAfterPublish=true`, the zipped package is found at `{repo_root}/out/pkg/{config}`