[3.0] Streamline native builds (and build native libs for 3.0)

.gitattributes

@@ -53,6 +53,10 @@
 *.7z binary
 *.ttf binary
 *.stout binary
+*.so binary
+*.dll binary
+*.dylib binary
+*.jar binary
 
 # Verify
 *.verified.txt text eol=lf working-tree-encoding=UTF-8

.github/workflows/dotnet.yml

@@ -27,6 +27,8 @@ jobs:
               with:
                 dotnet-version: '9.0.x'
                 dotnet-quality: 'preview'
+            - name: Install Workloads
+              run: dotnet workload restore
             - name: Pack
               # TODO decide whether we want experimental builds to use Debug or Release - using Release for now...
               run: >-
@@ -70,6 +72,8 @@ jobs:
               with:
                 dotnet-version: '8.0.x'
                 dotnet-quality: 'preview'
+            - name: Install Workloads
+              run: dotnet workload restore
             - name: Restore
               run: dotnet restore --runtime ${{ matrix.rid }}
             - name: Test

.github/workflows/native.yml

@@ -0,0 +1,102 @@
+name: Native Builds
+on:
+  pull_request:
+    types: [opened, edited, synchronize]
+permissions:
+  pull-requests: write
+  contents: write
+env:
+  # A space-separated list of paths to native libraries to build.
+  NATIVE_LIBRARY_PATHS: "sources/SDL/Native"
+  # A space-separated list of submodule paths for each native library path. Use _ if a submodule is not used - this must
+  # match the number of spaces in NATIVE_LIBRARY_PATHS.
+  NATIVE_LIBRARY_SUBMODULE_PATHS: "eng/submodules/sdl"
+  # A space-separated list of shorthands to the native library paths that will build the native library for each native
+  # library path. This must match the number of spaces in NATIVE_LIBRARY_PATHS. If a shorthand builds multiple native
+  # binary paths, these will be deduplicated.
+  NATIVE_LIBRARY_SHORTHANDS: "SDL"
+jobs:
+  prerequisites:
+    name: PR Check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - id: parse
+        name: Determine which native libraries the user has asked to build, and where to look for changes.
+        run: ./workflow-stage1.sh
+        working-directory: eng/native/buildsystem
+        env:
+          PR_COMMENT: ${{ github.event.pull_request.body }}
+      - uses: dorny/paths-filter@v3
+        id: filter
+        name: Determine which natives need to be rebuilt.
+        with:
+          filters: ${{ steps.parse.outputs.workflow_filters }}
+      - name: Find a warning message from a previous run, if applicable.
+        uses: peter-evans/find-comment@v3
+        id: fc
+        with:
+          issue-number: ${{ github.event.number }}
+          comment-author: "github-actions[bot]"
+          body-includes: "Some of the native library builds modified in this PR were not referenced in the PR description."
+      - id: strat
+        name: Create the matrix strategy for the native builds.
+        run: ./workflow-stage2.sh
+        working-directory: eng/native/buildsystem
+        env:
+          NATIVE_LIBRARY_APPLICABLE_SHORTHANDS: ${{ join(fromJSON(steps.filter.outputs.changes), ' ') }}
+          NATIVE_LIBRARY_USER_REFERENCED_SHORTHANDS: ${{ steps.parse.outputs.targets_referenced }}
+          PR_EXISTING_NOTICE_ID: ${{ steps.fc.outputs.comment-id }}
+      - name: Write a comment if needed.
+        if: ${{ steps.strat.outputs.comment_to_write }}
+        uses: peter-evans/create-or-update-comment@v4
+        with:
+          comment-id: ${{ steps.fc.outputs.comment-id }}
+          issue-number: ${{ github.event.number }}
+          body: ${{ steps.strat.outputs.comment_to_write }}
+          edit-mode: replace
+    outputs:
+      matrix_strategy: ${{ steps.strat.outputs.matrix_strategy }}
+      targets_referenced: ${{ steps.parse.outputs.targets_referenced }}
+  build:
+    needs: [prerequisites]
+    strategy:
+      matrix:
+        flat: ${{ fromJSON(needs.prerequisites.outputs.matrix_strategy) }}
+    runs-on: ${{ (startsWith(matrix.flat.runtime, 'osx') || startsWith(matrix.flat.runtime, 'ios') || startsWith(matrix.flat.runtime, 'tvos')) && 'macos-latest' || startsWith(matrix.flat.runtime, 'win') && 'windows-latest' || 'ubuntu-latest' }}
+    name: ${{ matrix.flat.target }} / ${{ matrix.flat.runtime }}
+    steps:
+      - uses: actions/checkout@v3
+      - run: |
+          echo "https://github.com/actions/upload-artifact/issues/174" > .workaround-${{ matrix.flat.target }}-${{ matrix.flat.runtime }}.txt
+          ${{ format('.{0}{1}', startsWith(matrix.flat.runtime, 'win') && '\' || '/', matrix.flat.exec) }}
+        working-directory: ${{ matrix.flat.dir }}
+      - uses: actions/upload-artifact@v4
+        with:
+          name: natives-${{ matrix.flat.target }}-${{ matrix.flat.runtime }}
+          path: |
+            ./.workaround-*.txt
+            ./${{ matrix.flat.dir }}/runtimes/${{ format('{0}{1}', matrix.flat.runtime, !contains(matrix.flat.runtime, '-') && '*' || '') }}/**/*
+            ./${{ matrix.flat.dir }}/lib/*${{ matrix.flat.runtime }}*/**/*
+            ./${{ matrix.flat.dir }}/*${{ matrix.flat.runtime }}*/**/*
+  commit:
+    name: Commit Binaries
+    needs: [prerequisites, build]
+    runs-on: ubuntu-latest
+    if: ${{ needs.prerequisites.outputs.targets_referenced }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.head_ref }}
+      - name: Download All Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: natives-*
+          merge-multiple: true
+      - name: Commit Artifacts
+        run: |
+          git config --local user.email "[email protected]"
+          git config --local user.name "The Silk.NET Automaton"
+          git add .
+          git commit -m "Update native binaries for $(git rev-parse HEAD)"
+          git push

.gitignore

@@ -489,3 +489,12 @@ src/Website/Silk.NET.Statiq/cache
 
 # Verify
 *.received.*
+
+# Native Binaries
+!**/runtimes/*/native/*.dylib
+!**/runtimes/*/native/*.so
+!**/runtimes/*/native/*.dll
+!**/lib/**/*.aar
+build/
+!eng/build/
+.workaround*

Directory.Build.props

@@ -30,6 +30,7 @@
     <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
     <ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
     <CentralPackageTransitivePinningEnabled>true</CentralPackageTransitivePinningEnabled>
+    <ImplicitUsings>enable</ImplicitUsings>
   </PropertyGroup>
 
   <PropertyGroup Condition="'$(Configuration)' == 'Debug'">
@@ -64,10 +65,12 @@
     <PublishRepositoryUrl>true</PublishRepositoryUrl>
     <VersionDependsOn>SilkShippingControl;$(VersionDependsOn)</VersionDependsOn>
     <PackageVersionDependsOn>SilkShippingControl;$(PackageVersionDependsOn)</PackageVersionDependsOn>
+    <TargetsForTfmSpecificContentInPackage>SilkNativePackaging;$(TargetsForTfmSpecificContentInPackage)</TargetsForTfmSpecificContentInPackage>
+    <GenerateNuspecDependsOn>SilkShippingControl;$(GenerateNuspecDependsOn)</GenerateNuspecDependsOn>
   </PropertyGroup>
 
   <!-- SourceLink -->
-  <PropertyGroup Condition="'$(SilkSourceLinkExempt)' == ''">
+  <PropertyGroup Condition="'$(SilkSourceLinkExempt)' == '' and '$(SilkNativePackage)' != 'true' and '$(SilkMetapackage)' != 'true'">
     <DebugType>portable</DebugType>
     <EmbedAllSources>true</EmbedAllSources>
     <EmbedUntrackedSources>true</EmbedUntrackedSources>

Directory.Build.targets

@@ -139,14 +139,25 @@
   </UsingTask>
 
   <Target Name="SilkShippingControl" BeforeTargets="BeforeBuild">
+    <!-- Non-C# packages (i.e. metapackages, natives) -->
+    <PropertyGroup Condition="'$(SilkNativePackage)' == 'true' or '$(SilkMetapackage)' == 'true'">
+      <IncludeBuildOutput Condition="'$(SilkNativeHasAndroidJars)' != 'true'">false</IncludeBuildOutput>
+      <IncludeBuildOutput Condition="'$(TargetFramework)' != ''">false</IncludeBuildOutput>
+      <IncludeBuildOutput Condition="'$(SilkNativeHasAndroidJars)' == 'true' and $([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) == 'android'">true</IncludeBuildOutput>
+      <NoWarn>NU5128;1591;$(NoWarn)</NoWarn>
+      <GenerateDocumentationFile>false</GenerateDocumentationFile>
+    </PropertyGroup>
+
     <!-- Public API -->
     <PropertyGroup Condition="'$(SilkTrackPublicAPI)' != 'true' or '$(TargetFramework)' == ''">
       <SilkTrackPublicAPI>false</SilkTrackPublicAPI>
     </PropertyGroup>
     <ItemGroup Condition="'$(SilkTrackPublicAPI)' == 'true'">
       <PackageReference Include="Microsoft.CodeAnalysis.PublicApiAnalyzers" PrivateAssets="All" />
     </ItemGroup>
-    <ItemGroup Condition="'$(SilkSourceLinkExempt)' == ''">
+
+    <!-- SourceLink -->
+    <ItemGroup Condition="'$(SilkSourceLinkExempt)' == '' and '$(SilkNativePackage)' != 'true' and '$(SilkMetapackage)' != 'true'">
       <PackageReference Include="DotNet.ReproducibleBuilds" PrivateAssets="All"/>
       <PackageReference Include="DotNet.ReproducibleBuilds.Isolated" PrivateAssets="All"/>
     </ItemGroup>
@@ -161,6 +172,7 @@
     <WriteLinesToFile File="PublicAPI/$(TargetFramework)/PublicAPI.Unshipped.txt"
                       Lines="@(SilkNewPublicAPILines)"
                       Condition="'$(SilkTrackPublicAPI)' == 'true' and !Exists('PublicAPI/$(TargetFramework)/PublicAPI.Unshipped.txt')" />
+
     <!-- Versioning -->
     <PropertyGroup>
       <SilkChangelog>$([System.IO.File]::ReadAllText("$(MSBuildThisFileDirectory)docs/CHANGELOG.md"))</SilkChangelog>
@@ -171,19 +183,81 @@
       <Output PropertyName="SilkReleaseNotes" TaskParameter="SilkReleaseNotes" />
     </SilkGetVersionInfoTask>
     <PropertyGroup>
+      <SilkOriginalVersionSuffix>$(VersionSuffix)</SilkOriginalVersionSuffix>
       <Version>$(SilkVersion)</Version>
       <VersionSuffix Condition="'$(VersionSuffix)' == ''">$(SilkVersionSuffix)</VersionSuffix>
       <PackageReleaseNotes>$(SilkReleaseNotes)</PackageReleaseNotes>
       <PackageVersion Condition="'$(VersionSuffix)' == ''">$(SilkVersion)</PackageVersion>
       <PackageVersion Condition="'$(VersionSuffix)' != ''">$(SilkVersion)-$(VersionSuffix)</PackageVersion>
     </PropertyGroup>
-    <!-- Trimming -->
+
+    <!-- Native Packaging -->
     <PropertyGroup>
-      <IsTrimmable Condition="'$(SilkTrimmingExempt)' == ''">true</IsTrimmable>
-      <EnableTrimAnalyzer Condition="'$(SilkTrimmingExempt)' == ''">true</EnableTrimAnalyzer>
-      <EnableAotAnalyzer Condition="'$(SilkTrimmingExempt)' == ''">true</EnableAotAnalyzer>
-      <EnableSingleFileAnalyzer Condition="'$(SilkTrimmingExempt)' == ''">true</EnableSingleFileAnalyzer>
-      <TrimMode Condition="'$(SilkTrimmingExempt)' == ''">full</TrimMode>
+      <SilkVersionTxtPath>$(MSBuildProjectDirectory)/version.txt</SilkVersionTxtPath>
+      <PackageVersion Condition="Exists('$(SilkVersionTxtPath)')">$([System.IO.File]::ReadAllText("$(SilkVersionTxtPath)").Trim())</PackageVersion>
+      <PackageVersion Condition="'$(SilkOriginalVersionSuffix)' != '' and Exists('$(SilkVersionTxtPath)') and $(PackageVersion.Contains('-'))">$(PackageVersion)$(SilkOriginalVersionSuffix)</PackageVersion>
+      <PackageVersion Condition="'$(SilkOriginalVersionSuffix)' != '' and Exists('$(SilkVersionTxtPath)') and !$(PackageVersion.Contains('-'))">$(PackageVersion)-$(SilkOriginalVersionSuffix)</PackageVersion>
     </PropertyGroup>
+    <Error Text="Native packages should have a version.txt file! Read documentation/for-contributors/build-system.md for more info."
+           Condition="'$(SilkNativePackage)' == 'true' and !Exists('$(SilkVersionTxtPath)')" />
+    <ItemGroup Condition="'$(SilkNativePackage)' == 'true'">
+      <None Include="$(MSBuildProjectDirectory)/runtimes/**/*" Pack="true" PackagePath="runtimes" />
+      <!-- bundled in the aar instead -->
+      <None Remove="$(MSBuildProjectDirectory)/runtimes/android*/**/*" Condition="'$(SilkNativeHasAndroidJars)' == 'true'" />
+    </ItemGroup>
+    <ItemGroup Condition="'$(SilkNativePackage)' == 'true' and $([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) == 'android'">
+      <AndroidNativeLibrary Include="$(MSBuildProjectDirectory)/runtimes/android*/native/*.so" />
+      <ProguardConfiguration Include="$(MSBuildProjectDirectory)/android/proguard.txt" Condition="Exists('$(MSBuildProjectDirectory)/android/proguard.txt')" />
+      <AndroidLibrary Include="$(MSBuildProjectDirectory)/android/*.jar" Bind="true" Pack="true" />
+      <TransformFile Include="$(MSBuildProjectDirectory)/android/*.xml" />
+    </ItemGroup>
+
+    <!-- Trimming -->
+    <PropertyGroup Condition="'$(SilkTrimmingExempt)' == ''">
+      <IsTrimmable>true</IsTrimmable>
+      <EnableTrimAnalyzer>true</EnableTrimAnalyzer>
+      <EnableAotAnalyzer>true</EnableAotAnalyzer>
+      <EnableSingleFileAnalyzer>true</EnableSingleFileAnalyzer>
+      <TrimMode>full</TrimMode>
+    </PropertyGroup>
+  </Target>
+
+
+  <!-- Closest things to documentation on what the hell this voodoo does:
+       https://github.com/NuGet/Home/issues/10063#issuecomment-713083004
+       https://github.com/xamarin/xamarin-android/pull/5242
+       https://github.com/dotnet/sdk/issues/14042
+       https://github.com/dotnet/sdk/blob/main/src/Tasks/Microsoft.NET.Build.Tasks/targets/Microsoft.NET.PackTool.targets#L54
+       https://github.com/xamarin/xamarin-android/issues/5957
+       https://github.com/xamarin/xamarin-android/blob/main/Documentation/guides/OneDotNetEmbeddedResources.md
+
+       This is an adaptation of what we used in Silk.NET 2.X to add files to packages, which was originally focused on
+       Android. For 3.0, we're trying to lean as much as possible on the built-in stuff for e.g. jars as this is in a
+       much better state these days. However, some of the links above may only be relevant with that context. -->
+  <Target Name="SilkNativePackaging" Condition="'$(SilkNativePackage)' == 'true'">
+    <GetNuGetShortFolderName TargetFrameworkMoniker="$(TargetFrameworkMoniker)" TargetPlatformMoniker="$(TargetPlatformMoniker)">
+      <Output TaskParameter="NuGetShortFolderName" PropertyName="SilkNuGetShortFolderName" />
+    </GetNuGetShortFolderName>
+    <ReadLinesFromFile File="$(MSBuildThisFileDirectory)eng/native/nuget/NativeNuGetPackage.targets">
+      <Output TaskParameter="Lines" ItemName="SilkNativeNuGetPackageTargets" />
+    </ReadLinesFromFile>
+    <WriteLinesToFile File="$(IntermediateOutputPath)$(TargetFramework)/$(MSBuildProjectName).targets"
+                      Lines="@(SilkNativeNuGetPackageTargets->'%(Identity)'->Replace('TO_BE_REPLACED_MTOUCH_NATIVE_REFERENCE_LINKER_FLAGS', '$(SilkNativeiOSLinkerFlags)')->Replace('TO_BE_REPLACED_PROPERTY_PREFIX', '$(MSBuildProjectName.Replace('.', '_'))'))" />
+    <WriteLinesToFile File="$(IntermediateOutputPath)$(TargetFramework)/_._" Lines="@(SilkDummyItem)" />
+    <ItemGroup>
+      <TfmSpecificPackageFile Include="$(IntermediateOutputPath)$(TargetFramework)/$(MSBuildProjectName).targets"
+                              Link="build/$(SilkNuGetShortFolderName)/$(MSBuildProjectName).targets"
+                              Pack="true"
+                              PackagePath="build/$(SilkNuGetShortFolderName)/$(MSBuildProjectName).targets" />
+      <TfmSpecificPackageFile Include="$(IntermediateOutputPath)$(TargetFramework)/$(MSBuildProjectName).targets"
+                              Link="build/$(SilkNuGetShortFolderName)/$(MSBuildProjectName).targets"
+                              Pack="true"
+                              PackagePath="buildTransitive/$(SilkNuGetShortFolderName)/$(MSBuildProjectName).targets" />
+      <TfmSpecificPackageFile Include="$(IntermediateOutputPath)$(TargetFramework)/_._"
+                              Link="lib/$(SilkNuGetShortFolderName)/_._"
+                              Pack="true"
+                              PackagePath="lib/$(SilkNuGetShortFolderName)/_._"
+                              Condition="'$(IncludeBuildOutput)' == 'false'" />
+    </ItemGroup>
   </Target>
 </Project>

Silk.NET.sln

@@ -95,6 +95,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Silk.NET", "sources\Core\Si
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Win32", "Win32", "{6E739132-EEAB-43A5-83C7-EB58C50D03A1}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Silk.NET.SDL.Native", "sources\SDL\Native\Silk.NET.SDL.Native.csproj", "{F16C0AB9-DE7E-4C09-9EE9-DAA8B8E935A6}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -153,6 +155,10 @@ Global
 		{6FA628B8-9696-4847-89F9-E58F470AF4FB}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{6FA628B8-9696-4847-89F9-E58F470AF4FB}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{6FA628B8-9696-4847-89F9-E58F470AF4FB}.Release|Any CPU.Build.0 = Release|Any CPU
+		{F16C0AB9-DE7E-4C09-9EE9-DAA8B8E935A6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{F16C0AB9-DE7E-4C09-9EE9-DAA8B8E935A6}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{F16C0AB9-DE7E-4C09-9EE9-DAA8B8E935A6}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{F16C0AB9-DE7E-4C09-9EE9-DAA8B8E935A6}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -182,6 +188,7 @@ Global
 		{3CADD95A-179F-4ECF-A49D-4B753832C63C} = {475AEF7B-0154-4989-AF82-97E3A95A96AF}
 		{6FA628B8-9696-4847-89F9-E58F470AF4FB} = {5CD096DB-6C44-48F1-9093-AD4C84B6B7EC}
 		{6E739132-EEAB-43A5-83C7-EB58C50D03A1} = {DD29EA8F-B1A6-45AA-8D2E-B38DA56D9EF6}
+		{F16C0AB9-DE7E-4C09-9EE9-DAA8B8E935A6} = {EC4D7B06-D277-4411-BD7B-71A6D37683F0}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {78D2CF6A-60A1-43E3-837B-00B73C9DA384}