publish_pykotor.yml

name: Compile PyKotor

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

env:
  OS_RUNNERS_JSON: '["windows-2019", "ubuntu-20.04", "macos-12"]'
  PYTHON_VERSIONS_JSON: '["3.8", "3.9", "3.10", "3.11", "3.12"]'
  ARCHITECTURES_JSON: '["x86", "x64"]'
  INTERPRETERS_JSON: '["python"]'
  UPX_VERSION: '4.2.2'
  
on:
  - push
  - workflow_dispatch

permissions:
  contents: write

jobs:
  build:
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false  # Disable automatic cancellation of other jobs
      matrix:
        os: [windows-2019, ubuntu-20.04, macos-12]  # make sure you update the above one too.
        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
        architecture: ['x86', 'x64']
        python_pypy: ['python']
        #qt_version: ['pyqt5', 'pyqt6', 'pyside2', 'pyside6']
        include:
          - arch: x86
            vc_redist2015: "https://download.microsoft.com/download/9/3/F/93FCF1E7-E6A4-478B-96E7-D4B285925B00/vc_redist.x86.exe"
            vc_redist-latest: "https://aka.ms/vs/17/release/vc_redist.x86.exe"
            vc_redist2019: "https://aka.ms/vs/17/release/vc_redist.x86.exe"
          - arch: x64
            vc_redist2015: "https://download.microsoft.com/download/9/3/F/93FCF1E7-E6A4-478B-96E7-D4B285925B00/vc_redist.x64.exe"
            vc_redist-latest: "https://aka.ms/vs/17/release/vc_redist.x64.exe"
            vc_redist2019: "https://aka.ms/vs/17/release/vc_redist.x64.exe"
        exclude:
          # unix x86 is definitely not supported.
          - os: ubuntu-20.04
            architecture: x86
          - os: macos-12
            architecture: x86
          # latest PyPy version as of 17/03/2024 is 3.10.
          - python_pypy: 'pypy'
            python-version: '3.11'
          - python_pypy: 'pypy'
            python-version: '3.12'
          # PyQt5 doesn't seem to be compatible with PyPy
          #- python_pypy: "pypy"
          #  qt_version: 'pyqt5'
          #- python_pypy: "pypy"
          #  qt_version: 'pyside2'
          #- python_pypy: "pypy"
          #  qt_version: 'pyqt6'
          #- python_pypy: "pypy"
          #  python-version: '3.10'
          #  os: 'windows-2019'

    outputs:
      matrix-os: ${{ toJson(matrix.os) }}
      matrix-python-version: ${{ toJson(matrix.python-version) }}

    steps:
    - name: Determine Python version string
      id: set-python-version
      run: |
        if ( "${{ matrix.python_pypy }}" -eq "pypy" ) {
          Add-Content -Path $env:GITHUB_ENV -Value "PYTHON_VERSION=pypy-${{ matrix.python-version }}"
        } else {
          Add-Content -Path $env:GITHUB_ENV -Value "PYTHON_VERSION=${{ matrix.python-version }}"
        }
      shell: pwsh

    - uses: actions/checkout@v4
    - name: Test Powershell installer
      if: ${{ runner.os != 'Windows' }}
      id: setup-powershell
      run: |
        bash ./install_powershell.sh
      shell: bash

    - name: Set up ${{ matrix.python_pypy }} ${{ matrix.python-version }}
      if: ${{ runner.os != 'macOS' && (runner.os != 'Linux' || matrix.python_pypy == 'pypy') }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ env.PYTHON_VERSION }}
        architecture: ${{ matrix.architecture }}

    - name: Set up python 3.12 mac (stops brew erroring on linking)
      if: ${{ runner.os == 'macOS' && matrix.python-version != '3.12' }}
      run: |
        brew install python@3.12 --quiet || true
      shell: bash

    - name: Set up ${{ matrix.python_pypy }} ${{ matrix.python-version }} macOS
      if: ${{ runner.os == 'macOS' }}
      run: |  # warning: `brew link --overwrite python@ver` is unsafe on any non-virtualized macos.
        echo "NONINTERACTIVE DEFAULT: $NONINTERACTIVE"
        export NONINTERACTIVE=1
        echo "NONINTERACTIVE NEW: $NONINTERACTIVE"
        brew analytics on
        brew update
        brew install ${{ matrix.python_pypy }}@${{ matrix.python-version }} || brew link --overwrite ${{ matrix.python_pypy }}@${{ matrix.python-version }}

    - name: Set up ${{ matrix.python_pypy }} ${{ matrix.python-version }} ${{ runner.os }}  # macos too?
      if: ${{ runner.os == 'Linux' && matrix.python_pypy == 'python' }}
      run: |
        if ("${{ runner.os }}" -eq "Linux") {
          sudo apt-get update
          sudo apt-get install build-essential zlib1g-dev libncurses5-dev libgdbm-dev libssl-dev libreadline-dev libffi-dev libsqlite3-dev libbz2-dev tk-dev -y
        } elseif ("${{ runner.os }}" -eq "macOS") {
          # Ensure Xcode Command Line Tools are installed?
          # xcode-select --install 2>$null | Out-Null
        }
        $pyVersion = switch ("${{ matrix.python-version }}") {
          "3.7" { "3.7.17" }
          "3.8" { "3.8.18" }
          "3.9" { "3.9.18" }
          "3.10" { "3.10.13" }
          "3.11" { "3.11.8" }
          "3.12" { "3.12.2" }
        }

        Write-Output "Downloading ${{ matrix.python_pypy }} '$pyVersion'..."
        Invoke-WebRequest -Uri https://www.python.org/ftp/python/$pyVersion/Python-$pyVersion.tgz -OutFile Python-$pyVersion.tgz
        tar -xvf Python-$pyVersion.tgz
        $current_working_dir = (Get-Location).Path
        Set-Location -LiteralPath "Python-$pyVersion" -ErrorAction Stop
        $env:LDFLAGS=""
        sudo ./configure --enable-optimizations --with-ensurepip=install --enable-shared --disable-new-dtags
        sudo make -j $(nproc)
        sudo make altinstall
        Set-Location -LiteralPath $current_working_dir
      shell: pwsh
    
    - name: Reset APT sources to default
      if: ${{ runner.os == 'Linux' }}
      run: |
        echo "Resetting APT sources to default Ubuntu repositories"
        sudo rm /etc/apt/sources.list
        echo "deb http://archive.ubuntu.com/ubuntu $(lsb_release -cs) main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list
        echo "deb http://archive.ubuntu.com/ubuntu $(lsb_release -cs)-updates main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list
        echo "deb http://archive.ubuntu.com/ubuntu $(lsb_release -cs)-backports main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list
        echo "deb http://security.ubuntu.com/ubuntu $(lsb_release -cs)-security main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list
        sudo apt-get update -y
      shell: bash

    - name: Setup python venvs
      run: |  # create the venv early to work around an issue with the matrix runners' concurrency
        $pythonExeName = "${{ matrix.python_pypy }}"
        if ("${{ runner.os }}" -ne "Windows")
        {
          $pythonExeName = "${{ matrix.python_pypy }}${{ matrix.python-version }}"
          # LD_LIBRARY_PATH must be updated. However this won't be permanent, just long enough to create the venv.
          $env:LD_LIBRARY_PATH = "/usr/local/lib:$env:LD_LIBRARY_PATH"
        }
        & $pythonExeName -m venv .venv_holopatcher_${{ matrix.os }}_${{ matrix.python_pypy }}_${{ matrix.python-version }}_${{ matrix.architecture }}
        & $pythonExeName -m venv .venv_toolset_${{ matrix.os }}_${{ matrix.python_pypy }}_${{ matrix.python-version }}_${{ matrix.architecture }}
        & $pythonExeName -m venv .venv_guiduplicator_${{ matrix.os }}_${{ matrix.python_pypy }}_${{ matrix.python-version }}_${{ matrix.architecture }}
        & $pythonExeName -m venv .venv_kotordiff_${{ matrix.os }}_${{ matrix.python_pypy }}_${{ matrix.python-version }}_${{ matrix.architecture }}
        & $pythonExeName -m venv .venv_batchpatcher_${{ matrix.os }}_${{ matrix.python_pypy }}_${{ matrix.python-version }}_${{ matrix.architecture }}
      shell: pwsh

    - name: Set UPX download URL
      # upx docs express that crashes are happening on ventura and above with upx, don't use on mac.
      if: runner.os != 'macOS'
      id: upx_setup
      run: |
        $build = "no"
        $archiveName = ""
        if ("${{ runner.os }}" -eq "Windows") {
          if ("${{ matrix.architecture }}" -eq "x86") {
            $archiveName = "upx-${{ env.UPX_VERSION }}-win32.zip"
          } else {
            $archiveName = "upx-${{ env.UPX_VERSION }}-win64.zip"
          }
        } elseif ("${{ runner.os }}" -eq "Linux") {
          $archiveName = "upx-${{ env.UPX_VERSION }}-amd64_linux.tar.xz"
        } elseif ("${{ runner.os }}" -eq "macOS") {
          $build = "yes"
          $archiveName = "upx-${{ env.UPX_VERSION }}-src.tar.xz"
        }
        $url = "https://github.com/upx/upx/releases/download/v${{ env.UPX_VERSION }}/$archiveName"
    
        # Write to the GITHUB_OUTPUT environment file
        Add-Content -Path $env:GITHUB_OUTPUT -Value "build=$build"
        Add-Content -Path $env:GITHUB_OUTPUT -Value "url=$url"
        Add-Content -Path $env:GITHUB_OUTPUT -Value "archiveName=$archiveName"
      shell: pwsh

    - name: Download and prepare UPX
      if: runner.os != 'macOS'
      run: |        
        $ext = "${{ runner.os }}" -eq "Windows" ? "zip" : "tar.xz"
        $url = "${{ steps.upx_setup.outputs.url }}"
        $archiveName = "${{ steps.upx_setup.outputs.archiveName }}"
        $outputPath = "upx-dir"

        # Use Invoke-WebRequest or curl depending on the OS
        if ("${{ runner.os }}" -eq "Windows") {
          Invoke-WebRequest -Uri $url -OutFile $archiveName
        } elseif ("${{ runner.os }}" -eq "Linux") {
          curl -L $url -o $archiveName
        }
        
        New-Item -ItemType Directory -Force -Path "upx-dir" -ErrorAction SilentlyContinue
        if ("${{ runner.os }}" -ne "macOS") {
          if ($ext -eq "zip") {
            $fileNameWithoutExtension = [System.IO.Path]::GetFileNameWithoutExtension($archiveName)
            Expand-Archive -Path $archiveName -DestinationPath "temp_folder_upx"
            # Ensure upx-dir exists; create it if it doesn't
            if (-not (Test-Path -Path "upx-dir")) {
                New-Item -ItemType Directory -Path "upx-dir"
            }
            Get-ChildItem -Path "temp_folder_upx/$fileNameWithoutExtension" -Recurse | Move-Item -Destination "upx-dir"
            Remove-Item "temp_folder_upx" -Recurse -Force -ErrorAction SilentlyContinue
          } else {
            tar -xvf $archiveName --strip-components=1 -C "upx-dir"
          }
    
          Remove-Item $archiveName # Clean up downloaded archive
        }
      shell: pwsh

    - name: Set UPX directory path
      if: runner.os != 'macOS'
      id: upx_dir
      run: |
        $upx_dir = "./upx-dir"
        $upx_dir = $([System.IO.Path]::GetFullPath('./upx-dir'))
        Dir -Recurse $upx_dir | Get-Childitem
        echo "UPX_DIR=$upx_dir" | Out-File -FilePath $env:GITHUB_ENV -Append
        Write-Output "UPX_DIR set to '$upx_dir'"
      shell: pwsh

    - name: Install Visual Studio 2015 C++ Redistributable
      if: runner.os == 'Windows'
      run: |
        $url = "${{ matrix.vc_redist2015 }}"
        $output = "vc_redist.exe"
        Invoke-WebRequest -Uri $url -OutFile $output
        Start-Process $output -ArgumentList '/install', '/quiet', '/norestart' -Wait
        Remove-Item -Path $output
        #choco install vcredist2015 -y
      shell: pwsh

    - name: Install Visual Studio 2019 C++ Redistributable
      if: runner.os == 'Windows'
      run: |
        $url = "${{ matrix.vc_redist2019 }}"
        $output = "vc_redist.exe"
        Invoke-WebRequest -Uri $url -OutFile $output
        Start-Process $output -ArgumentList '/install', '/quiet', '/norestart' -Wait
        Remove-Item -Path $output
        #choco install vcredist2019 -y
      shell: pwsh

    - name: Install Visual Studio latest C++ Redistributable
      if: runner.os == 'Windows'
      run: |
        $url = "${{ matrix.vc_redist-latest }}"
        $output = "vc_redist.exe"
        Invoke-WebRequest -Uri $url -OutFile $output
        Start-Process $output -ArgumentList '/install', '/quiet', '/norestart' -Wait
        Remove-Item -Path $output
      shell: pwsh

    - name: Install Holocron Toolset dependencies
      if: ${{ success() || failure() }}
      id: toolset_deps
      run: |  # known pyinstaller versions that work with upx compressions, not all are available on all python versions.
        try {
          . ./install_python_venv.ps1 -noprompt -venv_name .venv_toolset_${{ matrix.os }}_${{ matrix.python_pypy }}_${{ matrix.python-version }}_${{ matrix.architecture }}
          Write-Host "Python executable path: $pythonExePath"
          & $pythonExePath -m pip install --upgrade pip
          if ("${{ runner.os }}" -eq "Windows") {
            if ("${{ matrix.python-version }}" -eq "3.12") {
              pip install pyinstaller --prefer-binary -U
            } elseif ("${{ matrix.python-version }}" -eq "3.11") {
              pip install "pyinstaller==5.13.2" --prefer-binary
            } else {
              pip install "pyinstaller==5.4" --prefer-binary
            }
          } else {
            pip install pyinstaller -U --prefer-binary
          }
          #$env:QT_API = if (-not [string]::IsNullOrEmpty($env:QT_API)) { $env:QT_API } else { "{{ matrix.qt_version }}" }
          $output = ""
          $errorLines = @()
          . ./compile/deps_toolset.ps1 -noprompt -venv_name .venv_toolset_${{ matrix.os }}_${{ matrix.python_pypy }}_${{ matrix.python-version }}_${{ matrix.architecture }} 2>&1 | ForEach-Object {
            Write-Output $_.ToString()
            $output += $_.ToString() + "`n"
            if($_ -match 'ERROR:') {
              $errorLines += $_.ToString()
            }
          }
          if ($errorLines.Count -gt 0) {
            $errorLines | ForEach-Object { Write-Error $_ }
            Add-Content -Path $env:GITHUB_OUTPUT -Value "success=false"
            exit 1
          } else {
            Add-Content -Path $env:GITHUB_OUTPUT -Value "success=true"
          }
        } catch {
          Write-Host -ForegroundColor Red "$($_.InvocationInfo.PositionMessage)`n$($_.Exception.Message)"
          Add-Content -Path $env:GITHUB_OUTPUT -Value "success=false"
          exit 1
        }
      shell: pwsh

    - name: HolocronToolset - Adjust RPATH for shared libraries in the virtual environment
      if: ${{ (success() || failure()) && runner.os == 'Disabled_Linux' && steps.toolset_deps.outputs.success == 'true' }}
      run: |
        . ./install_python_venv.ps1 -noprompt -venv_name .venv_toolset_${{ matrix.os }}_${{ matrix.python_pypy }}_${{ matrix.python-version }}_${{ matrix.architecture }}
        # Ensure patchelf is installed
        sudo apt-get update && sudo apt-get install -y patchelf

        # Find the virtual environment's site-packages directory
        $venvPath = $env:VIRTUAL_ENV
        Write-Host "Virtual environment path: $venvPath"
        
        # Dynamically find the Python version directory inside the venv
        $pythonLibPath = Get-ChildItem -Path "$venvPath/lib" | Where-Object { $_.PSIsContainer } | Select-Object -First 1
        $sitePackagesPath = Join-Path -Path $pythonLibPath.FullName -ChildPath "site-packages"
        
        Write-Host "Site-packages path: $sitePackagesPath"

        # Use patchelf to adjust RPATH for all shared libraries in the virtual environment
        Get-ChildItem -Path $sitePackagesPath -Filter *.so -Recurse | ForEach-Object {
          $libPath = $_.FullName
          Write-Host "Patching $libPath"
          sudo patchelf --set-rpath '$ORIGIN' $libPath
        }
      shell: pwsh

    - name: Compile Holocron Toolset
      if: ${{ (success() || failure()) && steps.toolset_deps.outputs.success == 'true' }}
      run: |
        $upxDir = $env:UPX_DIR
        Write-Host "Using UPX directory at '$upxDir'"
        $output = ""
        . ./compile/compile_toolset.ps1 -noprompt -venv_name .venv_toolset_${{ matrix.os }}_${{ matrix.python_pypy }}_${{ matrix.python-version }}_${{ matrix.architecture }} -upx_dir $upxDir 2>&1 | ForEach-Object {
          Write-Output $_.ToString()
          $output += $_.ToString() + "`n"
          if($_ -match 'ERROR:') {
            $errorLines += $_.ToString()
          }
        }
        $warningCount = 0
        $output -split "`n" | ForEach-Object {
            if ($_ -match 'WARNING: Library not found: could not resolve' -or
                $_ -match 'WARNING: Cannot find ' -or
                $_ -match 'WARNING: lib not found:' -or
                $_ -match 'WARNING: Tcl modules directory' -or
                $_ -match 'WARNING: Failed to upx strip') {
                $warningCount++
            }
        }
        if ($errorLines.Count -gt 0) {
          $errorLines | ForEach-Object { Write-Error $_ }
          Add-Content -Path $env:GITHUB_OUTPUT -Value "success=false"
          exit 1
        } elseif ($warningCount -ge 3) {
          Write-Output "Many 'library not found' warnings raised, pyinstaller was probably unsuccessful."
          Add-Content -Path $env:GITHUB_OUTPUT -Value "success=false"
          exit 1
        } else {
          Add-Content -Path $env:GITHUB_OUTPUT -Value "success=true"
        }
      shell: pwsh

    - name: HolocronToolset - Create Static Binary
      if: ${{ (success() || failure()) && runner.os == 'Disabled_Linux' && steps.toolset_deps.outputs.success == 'true' }}
      run: |
        . ./install_python_venv.ps1 -noprompt -venv_name .venv_toolset_${{ matrix.os }}_${{ matrix.python_pypy }}_${{ matrix.python-version }}_${{ matrix.architecture }}
        & $pythonExePath -m pip install staticx
        staticx ./dist/HolocronToolset ./dist/HolocronToolset-static
        # Rename the static binary for clarity and consistency
        Write-Output "./dist/HolocronToolset-static  --> ./dist/HolocronToolset"
        Move-Item -LiteralPath ./dist/HolocronToolset-static -Destination ./dist/HolocronToolset -Force
      shell: pwsh

    - name: Install HoloPatcher dependencies
      if: ${{ success() || failure() }}
      id: holopatcher_deps
      run: |  # known pyinstaller versions that work with upx compressions, not all are available on all python versions.
        try {
          . ./install_python_venv.ps1 -noprompt -venv_name .venv_holopatcher_${{ matrix.os }}_${{ matrix.python_pypy }}_${{ matrix.python-version }}_${{ matrix.architecture }}
          & $pythonExePath -m pip install --upgrade pip
          if ("${{ runner.os }}" -eq "Windows") {
            if ("${{ matrix.python-version }}" -eq "3.12") {
              pip install "pyinstaller==5.13.2" --prefer-binary
            } elseif ("${{ matrix.python-version }}" -eq "3.11") {
              pip install "pyinstaller==5.13.2" --prefer-binary
            } else {
              pip install "pyinstaller==5.4" --prefer-binary
            }
          } else {
            pip install pyinstaller -U --prefer-binary
          }
          $output = ""
          $errorLines = @()
          . ./compile/deps_holopatcher.ps1 -noprompt -venv_name .venv_holopatcher_${{ matrix.os }}_${{ matrix.python_pypy }}_${{ matrix.python-version }}_${{ matrix.architecture }} 2>&1 | ForEach-Object {
            Write-Output $_.ToString()
            $output += $_.ToString() + "`n"
            if($_ -match 'ERROR:') {
              $errorLines += $_.ToString()
            }
          }
          if ($errorLines.Count -gt 0) {
            $errorLines | ForEach-Object { Write-Error $_ }
            Add-Content -Path $env:GITHUB_OUTPUT -Value "success=false"
            exit 1
          } else {
            Add-Content -Path $env:GITHUB_OUTPUT -Value "success=true"
          }
        } catch {
          Write-Host -ForegroundColor Red "$($_.InvocationInfo.PositionMessage)`n$($_.Exception.Message)"
          Add-Content -Path $env:GITHUB_OUTPUT -Value "success=false"
          exit 1
        }
      shell: pwsh

    - name: HoloPatcher - Adjust RPATH for shared libraries in the virtual environment
      if: ${{ (success() || failure()) && runner.os == 'Disabled_Linux' && steps.holopatcher_deps.outputs.success == 'true' }}
      run: |
        . ./install_python_venv.ps1 -noprompt -venv_name .venv_holopatcher_${{ matrix.os }}_${{ matrix.python_pypy }}_${{ matrix.python-version }}_${{ matrix.architecture }}
        # Ensure patchelf is installed
        sudo apt-get update && sudo apt-get install -y patchelf

        # Find the virtual environment's site-packages directory
        $venvPath = $env:VIRTUAL_ENV
        Write-Host "Virtual environment path: $venvPath"
        
        # Dynamically find the Python version directory inside the venv
        $pythonLibPath = Get-ChildItem -Path "$venvPath/lib" | Where-Object { $_.PSIsContainer } | Select-Object -First 1
        $sitePackagesPath = Join-Path -Path $pythonLibPath.FullName -ChildPath "site-packages"
        
        Write-Host "Site-packages path: $sitePackagesPath"

        # Use patchelf to adjust RPATH for all shared libraries in the virtual environment
        Get-ChildItem -Path $sitePackagesPath -Filter *.so -Recurse | ForEach-Object {
          $libPath = $_.FullName
          Write-Host "Patching $libPath"
          sudo patchelf --set-rpath '$ORIGIN' $libPath
        }
      shell: pwsh
    
    - name: Compile HoloPatcher
      if: ${{ (success() || failure()) && steps.holopatcher_deps.outputs.success == 'true' }}
      run: |
        $upxDir = $env:UPX_DIR
        Write-Host "Using UPX directory at '$upxDir'"
        $output = ""
        . ./compile/compile_holopatcher.ps1 -noprompt -venv_name .venv_holopatcher_${{ matrix.os }}_${{ matrix.python_pypy }}_${{ matrix.python-version }}_${{ matrix.architecture }} -upx_dir $upxDir 2>&1 | ForEach-Object {
          Write-Output $_.ToString()
          $output += $_.ToString() + "`n"
          if($_ -match 'ERROR:') {
            $errorLines += $_.ToString()
          }
        }
        $warningCount = 0
        $output -split "`n" | ForEach-Object {
            if ($_ -match 'WARNING: Library not found: could not resolve' -or
                $_ -match 'WARNING: Cannot find ' -or
                $_ -match 'WARNING: lib not found:' -or
                $_ -match 'WARNING: Tcl modules directory' -or
                $_ -match 'WARNING: Failed to upx strip') {
                $warningCount++
            }
        }
        if ($errorLines.Count -gt 0) {
          $errorLines | ForEach-Object { Write-Error $_ }
          Add-Content -Path $env:GITHUB_OUTPUT -Value "success=false"
          exit 1
        } elseif ($warningCount -ge 3) {
          Write-Output "Many 'library not found' warnings raised, pyinstaller was probably unsuccessful."
          Add-Content -Path $env:GITHUB_OUTPUT -Value "success=false"
          exit 1
        } else {
          Add-Content -Path $env:GITHUB_OUTPUT -Value "success=true"
        }
      shell: pwsh

    - name: HoloPatcher - Create Static Binary
      if: ${{ (success() || failure()) && runner.os == 'Disabled_Linux' && steps.holopatcher_deps.outputs.success == 'true' }}
      run: |
        . ./install_python_venv.ps1 -noprompt -venv_name .venv_holopatcher_${{ matrix.os }}_${{ matrix.python_pypy }}_${{ matrix.python-version }}_${{ matrix.architecture }}
        & $pythonExePath -m pip install staticx
        staticx ./dist/HoloPatcher ./dist/HoloPatcher-static
        # Rename the static binary for clarity and consistency
        Write-Output "./dist/HoloPatcher-static  --> ./dist/HoloPatcher"
        Move-Item -LiteralPath ./dist/HoloPatcher-static -Destination ./dist/HoloPatcher -Force
      shell: pwsh

    - name: Install BatchPatcher dependencies
      if: ${{ success() || failure() }}
      id: batchpatcher_deps
      run: |  # known pyinstaller versions that work with upx compressions, not all are available on all python versions.
        try {
          . ./install_python_venv.ps1 -noprompt -venv_name .venv_batchpatcher_${{ matrix.os }}_${{ matrix.python_pypy }}_${{ matrix.python-version }}_${{ matrix.architecture }}
          & $pythonExePath -m pip install --upgrade pip
          if ("${{ runner.os }}" -eq "Windows") {
            if ("${{ matrix.python-version }}" -eq "3.12") {
              pip install pyinstaller --prefer-binary -U
            } elseif ("${{ matrix.python-version }}" -eq "3.11") {
              pip install "pyinstaller==5.13.2" --prefer-binary
            } else {
              pip install "pyinstaller==5.4" --prefer-binary
            }
          } else {
            pip install pyinstaller -U --prefer-binary
          }
          $output = ""
          $errorLines = @()
          . ./compile/deps_batchpatcher.ps1 -noprompt -venv_name .venv_batchpatcher_${{ matrix.os }}_${{ matrix.python_pypy }}_${{ matrix.python-version }}_${{ matrix.architecture }} 2>&1 | ForEach-Object {
            Write-Output $_.ToString()
            $output += $_.ToString() + "`n"
            if($_ -match 'ERROR:') {
              $errorLines += $_.ToString()
            }
          }
          if ($errorLines.Count -gt 0) {
            $errorLines | ForEach-Object { Write-Error $_ }
            Add-Content -Path $env:GITHUB_OUTPUT -Value "success=false"
            exit 1
          } else {
            Add-Content -Path $env:GITHUB_OUTPUT -Value "success=true"
          }
        } catch {
          Write-Host -ForegroundColor Red "$($_.InvocationInfo.PositionMessage)`n$($_.Exception.Message)"
          Add-Content -Path $env:GITHUB_OUTPUT -Value "success=false"
          exit 1
        }
      shell: pwsh

    - name: BatchPatcher - Adjust RPATH for shared libraries in the virtual environment
      if: ${{ (success() || failure()) && runner.os == 'Disabled_Linux' && steps.batchpatcher_deps.outputs.success == 'true' }}
      run: |
        . ./install_python_venv.ps1 -noprompt -venv_name .venv_batchpatcher_${{ matrix.os }}_${{ matrix.python_pypy }}_${{ matrix.python-version }}_${{ matrix.architecture }}
        # Ensure patchelf is installed
        sudo apt-get update && sudo apt-get install -y patchelf

        # Find the virtual environment's site-packages directory
        $venvPath = $env:VIRTUAL_ENV
        Write-Host "Virtual environment path: $venvPath"
        
        # Dynamically find the Python version directory inside the venv
        $pythonLibPath = Get-ChildItem -Path "$venvPath/lib" | Where-Object { $_.PSIsContainer } | Select-Object -First 1
        $sitePackagesPath = Join-Path -Path $pythonLibPath.FullName -ChildPath "site-packages"
        
        Write-Host "Site-packages path: $sitePackagesPath"

        # Use patchelf to adjust RPATH for all shared libraries in the virtual environment
        Get-ChildItem -Path $sitePackagesPath -Filter *.so -Recurse | ForEach-Object {
          $libPath = $_.FullName
          Write-Host "Patching $libPath"
          sudo patchelf --set-rpath '$ORIGIN' $libPath
        }
      shell: pwsh

    - name: Compile BatchPatcher
      if: ${{ (success() || failure()) && steps.batchpatcher_deps.outputs.success == 'true' }}
      run: |
        $upxDir = $env:UPX_DIR
        Write-Host "Using UPX directory at '$upxDir'"
        $output = ""
        . ./compile/compile_batchpatcher.ps1 -noprompt -venv_name .venv_batchpatcher_${{ matrix.os }}_${{ matrix.python_pypy }}_${{ matrix.python-version }}_${{ matrix.architecture }} -upx_dir $upxDir 2>&1 | ForEach-Object {
          Write-Output $_.ToString()
          $output += $_.ToString() + "`n"
          if($_ -match 'ERROR:') {
            $errorLines += $_.ToString()
          }
        }
        $warningCount = 0
        $output -split "`n" | ForEach-Object {
            if ($_ -match 'WARNING: Library not found: could not resolve' -or
                $_ -match 'WARNING: Cannot find ' -or
                $_ -match 'WARNING: lib not found:' -or
                $_ -match 'WARNING: Tcl modules directory' -or
                $_ -match 'WARNING: Failed to upx strip') {
                $warningCount++
            }
        }
        if ($errorLines.Count -gt 0) {
          $errorLines | ForEach-Object { Write-Error $_ }
          Add-Content -Path $env:GITHUB_OUTPUT -Value "success=false"
          exit 1
        } elseif ($warningCount -ge 3) {
          Write-Output "Many 'library not found' warnings raised, pyinstaller was probably unsuccessful."
          Add-Content -Path $env:GITHUB_OUTPUT -Value "success=false"
          exit 1
        } else {
          Add-Content -Path $env:GITHUB_OUTPUT -Value "success=true"
        }
      shell: pwsh

    - name: BatchPatcher - Create Static Binary
      if: ${{ (success() || failure()) && runner.os == 'Disabled_Linux' && steps.batchpatcher_deps.outputs.success == 'true' }}
      run: |
        . ./install_python_venv.ps1 -noprompt -venv_name .venv_batchpatcher_${{ matrix.os }}_${{ matrix.python_pypy }}_${{ matrix.python-version }}_${{ matrix.architecture }}
        & $pythonExePath -m pip install staticx
        staticx ./dist/K_BatchPatcher ./dist/K_BatchPatcher-static
        # Rename the static binary for clarity and consistency
        Write-Output "./dist/K_BatchPatcher-static  --> ./dist/K_BatchPatcher"
        Move-Item -LiteralPath ./dist/K_BatchPatcher-static -Destination ./dist/K_BatchPatcher -Force
      shell: pwsh

    - name: Compile KotorDiff
      if: ${{ success() || failure() }}
      run: |
        $upxDir = $env:UPX_DIR
        Write-Host "Using UPX directory at '$upxDir'"
        . ./install_python_venv.ps1 -noprompt -venv_name .venv_kotordiff_${{ matrix.os }}_${{ matrix.python_pypy }}_${{ matrix.python-version }}_${{ matrix.architecture }}
        & $pythonExePath -m pip install --upgrade pip
        if ("${{ runner.os }}" -eq "Windows") {
          if ("${{ matrix.python-version }}" -eq "3.12") {
            pip install "pyinstaller==5.13.2" --prefer-binary
          } elseif ("${{ matrix.python-version }}" -eq "3.11") {
            pip install "pyinstaller==5.13.2" --prefer-binary
          } else {
            pip install "pyinstaller==5.4" --prefer-binary
          }
        } else {
          pip install pyinstaller -U --prefer-binary
        }
        pip install -r Tools/KotorDiff/requirements.txt --prefer-binary
        $output = ""
        . ./compile/compile_kotordiff.ps1 -noprompt -venv_name .venv_kotordiff_${{ matrix.os }}_${{ matrix.python_pypy }}_${{ matrix.python-version }}_${{ matrix.architecture }} -upx_dir $upxDir 2>&1 | ForEach-Object {
          Write-Output $_.ToString()
          $output += $_.ToString() + "`n"
          if($_ -match 'ERROR:') {
            $errorLines += $_.ToString()
          }
        }
        $warningCount = 0
        $output -split "`n" | ForEach-Object {
            if ($_ -match 'WARNING: Library not found: could not resolve' -or
                $_ -match 'WARNING: Cannot find ' -or
                $_ -match 'WARNING: lib not found:' -or
                $_ -match 'WARNING: Tcl modules directory' -or
                $_ -match 'WARNING: Failed to upx strip') {
                $warningCount++
            }
        }
        if ($errorLines.Count -gt 0) {
          $errorLines | ForEach-Object { Write-Error $_ }
          Add-Content -Path $env:GITHUB_OUTPUT -Value "success=false"
          exit 1
        } elseif ($warningCount -ge 3) {
          Write-Output "Many 'library not found' warnings raised, pyinstaller was probably unsuccessful."
          Add-Content -Path $env:GITHUB_OUTPUT -Value "success=false"
          exit 1
        } else {
          Add-Content -Path $env:GITHUB_OUTPUT -Value "success=true"
        }
      shell: pwsh

    - name: KotorDiff - Create Static Binary
      if: ${{ (success() || failure()) && runner.os == 'Linux' }}
      run: |
        . ./install_python_venv.ps1 -noprompt -venv_name .venv_kotordiff_${{ matrix.os }}_${{ matrix.python_pypy }}_${{ matrix.python-version }}_${{ matrix.architecture }}
        & $pythonExePath -m pip install staticx
        staticx ./dist/KotorDiff ./dist/KotorDiff-static
        # Rename the static binary for clarity and consistency
        Write-Output "./dist/KotorDiff-static  --> ./dist/KotorDiff"
        Move-Item -LiteralPath ./dist/KotorDiff-static -Destination ./dist/KotorDiff -Force
      shell: pwsh

    - name: Compile GUIDuplicator
      if: ${{ success() || failure() }}
      run: |
        $upxDir = $env:UPX_DIR
        Write-Host "Using UPX directory at '$upxDir'"
        . ./install_python_venv.ps1 -noprompt -venv_name .venv_guiduplicator_${{ matrix.os }}_${{ matrix.python_pypy }}_${{ matrix.python-version }}_${{ matrix.architecture }}
        & $pythonExePath -m pip install --upgrade pip
        if ("${{ runner.os }}" -eq "Windows") {
          if ("${{ matrix.python-version }}" -eq "3.12") {
            pip install "pyinstaller==5.13.2" --prefer-binary
          } elseif ("${{ matrix.python-version }}" -eq "3.11") {
            pip install "pyinstaller==5.13.2" --prefer-binary
          } else {
            pip install "pyinstaller==5.4" --prefer-binary
          }
        } else {
          pip install pyinstaller -U --prefer-binary
        }
        pip install -r Tools/GuiDuplicator/requirements.txt --prefer-binary
        $output = ""
        . ./compile/compile_gui_duplicator.ps1 -noprompt -venv_name .venv_guiduplicator_${{ matrix.os }}_${{ matrix.python_pypy }}_${{ matrix.python-version }}_${{ matrix.architecture }} -upx_dir $upxDir 2>&1 | ForEach-Object {
          Write-Output $_.ToString()
          if($_ -match 'ERROR:') {
            $errorLines += $_.ToString()
          }
        }
        $warningCount = 0
        $output -split "`n" | ForEach-Object {
            if ($_ -match 'WARNING: Library not found: could not resolve' -or
                $_ -match 'WARNING: Cannot find ' -or
                $_ -match 'WARNING: lib not found:' -or
                $_ -match 'WARNING: Tcl modules directory' -or
                $_ -match 'WARNING: Failed to upx strip') {
                $warningCount++
            }
        }
        if ($errorLines.Count -gt 0) {
          $errorLines | ForEach-Object { Write-Error $_ }
          Add-Content -Path $env:GITHUB_OUTPUT -Value "success=false"
          exit 1
        } elseif ($warningCount -ge 3) {
          Write-Output "Many 'library not found' warnings raised, pyinstaller was probably unsuccessful."
          Add-Content -Path $env:GITHUB_OUTPUT -Value "success=false"
          exit 1
        } else {
          Add-Content -Path $env:GITHUB_OUTPUT -Value "success=true"
        }
      shell: pwsh

    - name: GUIDuplicator - Create Static Binary
      if: ${{ (success() || failure()) && runner.os == 'Linux' }}
      run: |
        . ./install_python_venv.ps1 -noprompt -venv_name .venv_guiduplicator_${{ matrix.os }}_${{ matrix.python_pypy }}_${{ matrix.python-version }}_${{ matrix.architecture }}
        & $pythonExePath -m pip install staticx
        staticx ./dist/GuiDuplicator ./dist/GuiDuplicator-static
        # Rename the static binary for clarity and consistency
        Write-Output "./dist/GuiDuplicator-static  --> ./dist/GuiDuplicator"
        Move-Item -LiteralPath ./dist/GuiDuplicator-static -Destination ./dist/GuiDuplicator -Force
      shell: pwsh

    - name: Upload compiled binaries attempt 1
      if: ${{ success() || failure() }}
      id: upload_attempt_1
      uses: actions/upload-artifact@v4
      with:
        name: publish_${{ matrix.os }}_${{ matrix.python_pypy }}_${{ matrix.python-version }}_${{ matrix.architecture }}
        path: ./dist/**
        retention-days: 90
      continue-on-error: true

    - name: Upload compiled binaries attempt 2
      id: upload_attempt_2
      if: ${{ (success() || failure()) && steps.upload_attempt_1.outcome == 'failure' }}
      uses: actions/upload-artifact@v4
      with:
        name: publish_${{ matrix.os }}_${{ matrix.python_pypy }}_${{ matrix.python-version }}_${{ matrix.architecture }}
        path: ./dist/**
        retention-days: 90
      continue-on-error: true

    - name: Upload compiled binaries attempt 3
      id: upload_attempt_3
      if: ${{ (success() || failure()) && steps.upload_attempt_2.outcome == 'failure' }}
      uses: actions/upload-artifact@v4
      with:
        name: publish_${{ matrix.os }}_${{ matrix.python_pypy }}_${{ matrix.python-version }}_${{ matrix.architecture }}
        path: ./dist/**
        retention-days: 90
      continue-on-error: true

    - name: Upload compiled binaries attempt 4
      id: upload_attempt_4
      if: ${{ (success() || failure()) && steps.upload_attempt_3.outcome == 'failure' }}
      uses: actions/upload-artifact@v4
      with:
        name: publish_${{ matrix.os }}_${{ matrix.python_pypy }}_${{ matrix.python-version }}_${{ matrix.architecture }}
        path: ./dist/**
        retention-days: 90
      continue-on-error: true

    - name: Upload compiled binaries attempt 5
      id: upload_attempt_5
      if: ${{ (success() || failure()) && steps.upload_attempt_4.outcome == 'failure' }}
      uses: actions/upload-artifact@v4
      with:
        name: publish_${{ matrix.os }}_${{ matrix.python_pypy }}_${{ matrix.python-version }}_${{ matrix.architecture }}
        path: ./dist/**
        retention-days: 90

  package:  # The goal of this job is to repackage by toolname, rather than all tools by os_pyversion_arch
    needs: build  # do not start this job until all 'build' jobs complete
    if: ${{ success() || failure() }}
    runs-on: ubuntu-latest
    outputs:
      filesToArchive: ${{ steps.set-matrix.outputs.matrixJson }}
    steps:
      - name: Download all artifacts
        uses: actions/download-artifact@v4
        with:
          path: published_workflow_builds/
          pattern: publish_*

      - name: Re-package artifacts by Python version and architecture
        shell: pwsh
        run: |
          $here_dirpath = (Get-Location)
          Write-Output "here: '$here_dirpath'"
          $packagedArtifactsPath = Join-Path -Path $here_dirpath -ChildPath "packaged_artifacts"
          New-Item -ItemType Directory -Force -Path $packagedArtifactsPath

          # Navigate to the directory with downloaded artifacts
          Set-Location -Path "published_workflow_builds"

          $artifactNames = New-Object 'System.Collections.Generic.HashSet[string]'
          # Iterate through each os_pythonversion_arch folder.
          Get-ChildItem -Filter "publish_*" | ForEach-Object {
            $matrixPackagePath = $_.FullName
            Write-Output "Found matrix package '$matrixPackagePath'"
            $os, $python_or_pypy, $pythonVersion, $architecture = $_.BaseName -replace '^publish_', '' -split '_'

            $shortOsName = $os
            if ($os -eq "ubuntu-20.04" -or $os -eq "ubuntu-22.04" -or $os -eq "ubuntu-latest") {
              $shortOsName = "Linux"
            } elseif ($os -eq "macos-11" -or $os -eq "macos-12" -or $os -eq "macos-latest") {
              $shortOsName = "Mac"
            } elseif ($os -eq "windows-2019" -or $os -eq "windows-2022" -or $os -eq "windows-latest") {
              $shortOsName = "Win"
            }

            # Iterate through each tool in the folder
            Get-ChildItem -Path $matrixPackagePath | ForEach-Object {
              $toolExePath = $_.FullName
              $toolFileName = $_.Name
              $fileBaseName = [IO.Path]::GetFileNameWithoutExtension($_.Name)
              $fileExtension = $_.Extension

              $outerArchiveName = "$fileBaseName`_$pythonVersion"
              $outerZipPath = Join-Path -Path $packagedArtifactsPath -ChildPath $outerArchiveName
              $osSpecificArchiveName = "$fileBaseName`_$shortOsName-$architecture"
              $osSpecificArchivePath = Join-Path -Path $outerZipPath -ChildPath $osSpecificArchiveName

              Write-Output "   ToolFileName: '$toolFileName'"
              Write-Output "   Tool original filepath: '$toolExePath'"
              Write-Output "outerArchiveName: '$outerArchiveName'"
              Write-Output " - osSpecificArchiveName: '$osSpecificArchiveName'"
              Write-Output "outerZipPath: '$outerZipPath'"
              Write-Output " - osSpecificArchivePath: '$osSpecificArchivePath'"

              New-Item -ItemType Directory -Force -Path $outerZipPath
              chmod 777 -R $toolExePath
              $toolExeParentDirPath = Split-Path -Parent $toolExePath
              Write-Output "Creating archive for '$toolFileName' at '$toolExeParentDirPath'..."

              Write-Output ("Push-Location to start in '$toolExeParentDirPath' (originally at '$(Get-Location)')")
              Push-Location -Path $toolExeParentDirPath
              # Archive the tool by os identifier.
              if ((-not $fileExtension) -or (-not $fileExtension.Trim()) -or ($fileExtension.ToLower().Trim() -eq ".app")) {
                $osSpecificArchivePath = "$osSpecificArchivePath.tar.gz"
                if (Test-Path $toolExePath -PathType Container -ErrorAction SilentlyContinue) {  # It's a directory
                  tar -czf "$osSpecificArchivePath" -C "$toolExePath" .
                } else {  # It's a file
                  tar -czf "$osSpecificArchivePath" -C "$toolExeParentDirPath" $toolFileName
                }
              } else {
                $osSpecificArchivePath = "$osSpecificArchivePath.zip"
                zip -r -9 "$osSpecificArchivePath" $toolFileName
              }
              Pop-Location
              Write-Output ("Pop-Location to return to $(Get-Location) (was pushed to '$toolExeParentDirPath')")

              Write-Output "Compressed archive saved to '$osSpecificArchivePath'"
              chmod 777 -R "$osSpecificArchivePath"

              $artifactNames.Add($outerArchiveName)
            }
          }

          # Save artifact names to a file
          $artifactNames | Out-File -FilePath "../artifact-names.txt" -Encoding UTF8

          # use for debug
          # Get-ChildItem -Force | ForEach-Object {
          #   $size = if ($_.PSIsContainer) { "N/A" } else { $_.Length }
          #   $attrs = $_.Attributes.ToString() -replace 'ReadOnly', 'RO' -replace 'Hidden', 'H' -replace 'System', 'S' -replace 'Archive', 'A' -replace 'Directory', 'D' -replace ', ', '|'
          #   [PSCustomObject]@{
          #       Name = $_.Name
          #       Size = $size
          #       Attributes = $attrs
          #   }
          # } | Format-Table -AutoSize

          # Navigate back to the root of the workspace
          Set-Location -Path "../"

      - name: Generate matrix for uploading
        if: ${{ success() || failure() }}
        id: set-matrix
        shell: pwsh
        run: |
          $artifactNames = Get-Content 'artifact-names.txt' -ReadCount 0
      
          # Initialize an array to hold the artifact names directly
          $matrixArray = @()
      
          foreach ($name in $artifactNames) {
            if (-not [string]::IsNullOrEmpty($name)) {
              Write-Host "Processing artifact name: $name"
              # Trim the name and add directly to the array
              $matrixArray += $name.trim()
            }
          }
      
          # Convert the array directly to JSON
          $jsonMatrix = $matrixArray | ConvertTo-Json -Depth 5 -Compress

          # Use a single line of JSON for the matrix to avoid issues
          $singleLineJsonMatrix = $jsonMatrix -replace "`r", ""
          $singleLineJsonMatrix = $singleLineJsonMatrix -replace "`n", ""
          Write-Host "Matrix JSON:"
          Write-Host $singleLineJsonMatrix
      
          echo "matrixJson<<EOF" >> $env:GITHUB_OUTPUT
          echo $singleLineJsonMatrix >> $env:GITHUB_OUTPUT
          echo "EOF" >> $env:GITHUB_OUTPUT

      - name: Upload all repackages for next job
        if: ${{ success() || failure() }}
        uses: actions/upload-artifact@v4
        with:
          name: all_tool_dists_onearchive
          path: packaged_artifacts/**
          retention-days: 1  # only used for the next job.
          compression-level: 0

  upload:
    needs: package
    if: ${{ success() || failure() }}
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        artifactb: ${{ fromJson(needs.package.outputs.filesToArchive) }}
    steps:
      - name: Download repackages from package job
        if: ${{ success() || failure() }}
        uses: actions/download-artifact@v4
        with:
          path: all_tool_dists_onearchive
          pattern: all_tool_dists*

      - name: Upload re-packaged artifact
        if: ${{ success() || failure() }}
        uses: actions/upload-artifact@v4
        with:
          name: ${{ matrix.artifactb }}
          path: all_tool_dists_onearchive/all_tool_dists_onearchive/${{ matrix.artifactb }}
          compression-level: 9
          if-no-files-found: error