diff --git a/.github/workflows/build-all.yaml b/.github/workflows/build-all.yaml index 072a3e86..572ed37a 100644 --- a/.github/workflows/build-all.yaml +++ b/.github/workflows/build-all.yaml @@ -7,40 +7,67 @@ on: - main jobs: + build-dawn-mac: + uses: ./.github/workflows/build-dawn-mac.yaml + + build-dawn-win: + uses: ./.github/workflows/build-dawn-win.yaml + build-win: + needs: [build-dawn-win] runs-on: windows-2022 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 with: python-version: '3.10' + - uses: actions/checkout@v4 + + - uses: actions/download-artifact@v4 + with: + path: artifact + + - name: Copy Dawn + shell: cmd + run: | + xcopy artifact\dawn-windows-x64 build\dawn.out\ /s /y + - name: Build shell: cmd run: | call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat" - orcadev.bat build --release --version ${{github.ref_name}} + orcadev.bat build --release --version ${{github.ref_name}} || exit 1 - name: Package shell: cmd run: | - orcadev.bat package-sdk orca-sdk-windows + orcadev.bat package-sdk orca-sdk-windows || exit 1 + tar --format=ustar -czf orca-sdk-windows.tar.gz orca-sdk-windows - uses: actions/upload-artifact@v4 with: name: orca-sdk-windows - path: orca-sdk-windows + path: orca-sdk-windows.tar.gz build-macos-x64: + needs: [build-dawn-mac] runs-on: macos-13 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 with: python-version: '3.10' + - uses: actions/checkout@v4 + + - uses: actions/download-artifact@v4 + with: + path: artifact + + - name: Copy Dawn + run: | + mkdir -p build + cp -r artifact/dawn-mac-x64 build/dawn.out + - name: Build run: | ./orcadev build --release --version ${{github.ref_name}} @@ -48,21 +75,32 @@ jobs: - name: Package run: | ./orcadev package-sdk orca-sdk-mac-x64 + tar --format=ustar -czf orca-sdk-mac-x64.tar.gz orca-sdk-mac-x64 - uses: actions/upload-artifact@v4 with: name: orca-sdk-mac-x64 - path: orca-sdk-mac-x64 + path: orca-sdk-mac-x64.tar.gz build-macos-arm64: + needs: [build-dawn-mac] runs-on: macos-14 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 with: python-version: '3.10' + - uses: actions/checkout@v4 + + - uses: actions/download-artifact@v4 + with: + path: artifact + + - name: Copy Dawn + run: | + mkdir -p build + cp -r artifact/dawn-mac-arm64 build/dawn.out + - name: Build run: | ./orcadev build --release --version ${{github.ref_name}} @@ -70,11 +108,12 @@ jobs: - name: Package run: | ./orcadev package-sdk orca-sdk-mac-arm64 + tar --format=ustar -czf orca-sdk-mac-arm64.tar.gz orca-sdk-mac-arm64 - uses: actions/upload-artifact@v4 with: name: orca-sdk-mac-arm64 - path: orca-sdk-mac-arm64 + path: orca-sdk-mac-arm64.tar.gz mac-make-universal: runs-on: macos-latest @@ -83,12 +122,13 @@ jobs: - uses: actions/download-artifact@v4 with: path: artifacts + merge-multiple: true - name: Make universal binary run: | - chmod +x artifacts/orca-sdk-mac-x64/bin/orca - chmod +x artifacts/orca-sdk-mac-arm64/bin/orca - lipo -create artifacts/orca-sdk-mac-x64/bin/orca artifacts/orca-sdk-mac-arm64/bin/orca -output orca + tar -xzf artifacts/orca-sdk-mac-x64.tar.gz + tar -xzf artifacts/orca-sdk-mac-arm64.tar.gz + lipo -create orca-sdk-mac-x64/bin/orca orca-sdk-mac-arm64/bin/orca -output orca - name: Tar run: | diff --git a/.github/workflows/build-dawn-mac.yaml b/.github/workflows/build-dawn-mac.yaml new file mode 100644 index 00000000..644ade00 --- /dev/null +++ b/.github/workflows/build-dawn-mac.yaml @@ -0,0 +1,42 @@ +name: build-dawn-mac + +on: + workflow_dispatch: + workflow_call: + +jobs: + build-mac: + strategy: + matrix: + os: [macos-13, macos-14] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Dawn version + id: dawn-version + run: | + echo "DAWN_COMMIT=$(cat deps/dawn-commit.txt)" >> $GITHUB_OUTPUT + ARCH=${{ runner.arch }} + echo "LOWERCASE_ARCH=$(echo $ARCH | tr '[:upper:]' '[:lower:]')" >> $GITHUB_OUTPUT + + - name: Cache + id: cache-dawn + uses: actions/cache@v3 + with: + path: build/dawn.out + key: dawn-${{ runner.os }}-${{ runner.arch }}-${{ steps.dawn-version.outputs.DAWN_COMMIT }} + + - name: Build Dawn + if: steps.cache-dawn.outputs.cache-hit != 'true' + run: | + ./orcadev build-dawn --release --parallel 4 + + - uses: actions/upload-artifact@v4 + with: + name: dawn-mac-${{steps.dawn-version.outputs.LOWERCASE_ARCH}} + path: build/dawn.out diff --git a/.github/workflows/build-dawn-win.yaml b/.github/workflows/build-dawn-win.yaml new file mode 100644 index 00000000..2a7855a7 --- /dev/null +++ b/.github/workflows/build-dawn-win.yaml @@ -0,0 +1,41 @@ +name: build-dawn-win + +on: + workflow_dispatch: + workflow_call: + +jobs: + build-win: + runs-on: windows-2022 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Dawn version + id: dawn-version + shell: cmd + run: | + @chcp 65001>nul + set /p DAWN_COMMIT=> %GITHUB_OUTPUT% + + - name: Cache + id: cache-dawn + uses: actions/cache@v3 + with: + path: build/dawn.out + key: dawn-${{ runner.os }}-${{ runner.arch }}-${{ steps.dawn-version.outputs.DAWN_COMMIT }} + + - name: Build Dawn + if: steps.cache-dawn.outputs.cache-hit != 'true' + shell: cmd + run: | + orcadev build-dawn --release --parallel 4 + + - uses: actions/upload-artifact@v4 + with: + name: dawn-windows-x64 + path: build/dawn.out diff --git a/.github/workflows/build-dawn.yaml b/.github/workflows/build-dawn.yaml deleted file mode 100644 index fb214bef..00000000 --- a/.github/workflows/build-dawn.yaml +++ /dev/null @@ -1,74 +0,0 @@ -name: build-dawn - -on: - workflow_dispatch: - workflow_call: - push: - branches: - - webgpu-canvas - -jobs: - build-mac: - runs-on: macos-13 - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-python@v4 - with: - python-version: '3.10' - - - name: Dawn version - id: dawn-version - run: | - echo "DAWN_COMMIT=$(cat deps/dawn-commit.txt)" >> $GITHUB_OUTPUT - - - name: Cache - id: cache-dawn - uses: actions/cache@v3 - with: - path: dawn.out - key: dawn-${{ runner.os }}-${{ runner.arch }}-${{ steps.dawn-version.outputs.DAWN_COMMIT }} - - - name: Build Dawn - if: steps.cache-dawn.outputs.cache-hit != 'true' - run: | - ./orcadev build-dawn --release --parallel 4 - - - uses: actions/upload-artifact@v4 - with: - name: dawn-mac-x64 - path: build/dawn.out - - build-win: - runs-on: windows-2022 - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-python@v4 - with: - python-version: '3.10' - - - name: Dawn version - id: dawn-version - shell: cmd - run: | - set /p DAWN_COMMIT=> $GITHUB_OUTPUT - - - name: Cache - id: cache-dawn - uses: actions/cache@v3 - with: - path: dawn.out - key: dawn-${{ runner.os }}-${{ runner.arch }}-${{ steps.dawn-version.outputs.DAWN_COMMIT }} - - - name: Build Dawn - if: steps.cache-dawn.outputs.cache-hit != 'true' - shell: cmd - run: | - orcadev build-dawn --release --parallel 4 - - - uses: actions/upload-artifact@v4 - with: - name: dawn-windows-x64 - path: build/dawn.out diff --git a/.github/workflows/create-release.yaml b/.github/workflows/create-release.yaml index 3666b81e..dd73081d 100644 --- a/.github/workflows/create-release.yaml +++ b/.github/workflows/create-release.yaml @@ -9,39 +9,48 @@ jobs: uses: ./.github/workflows/build-all.yaml release: - runs-on: ubuntu-latest - permissions: - contents: write - needs: [build-all] - steps: - - uses: actions/download-artifact@v4 - with: - # when name is not specified, all artifacts from this run will be downloaded - path: artifacts - - - name: Package CLI Tool - run: | - mkdir releases - cp artifacts/orca-sdk-windows/bin/orca.exe releases - cp artifacts/orca-mac/orca-mac.tar.gz releases - - - name: Package Windows SDK - run: | - find orca-sdk-windows -type f -exec sha1sum {} + | LC_ALL=C sort | sha1sum | cut -z -f 1 -d " " > artifacts/sha1.sum - tar --format=ustar -czf releases/orca-sdk-windows.tar.gz -C artifacts orca-sdk-windows sha1.sum - - - name: Package Mac x64 Release - run: | - chmod +x artifacts/orca-sdk-mac-x64/bin/orca - find orca-sdk-mac-x64 -type f -exec sha1sum {} + | LC_ALL=C sort | sha1sum | cut -z -f 1 -d " " > artifacts/sha1.sum - tar --format=ustar -czf releases/orca-sdk-mac-x64.tar.gz -C artifacts orca-sdk-mac-x64 sha1.sum - - - name: Package Mac arm64 Release - run: | - chmod +x artifacts/orca-sdk-mac-arm64/bin/orca - find orca-sdk-mac-arm64 -type f -exec sha1sum {} + | LC_ALL=C sort | sha1sum | cut -z -f 1 -d " " > artifacts/sha1.sum - tar --format=ustar -czf releases/orca-sdk-mac-arm64.tar.gz -C artifacts orca-sdk-mac-arm64 sha1.sum - - - uses: ncipollo/release-action@v1 - with: - artifacts: "releases/*" + runs-on: ubuntu-latest + permissions: + contents: write + needs: [build-all] + steps: + - uses: actions/download-artifact@v4 + with: + # when name is not specified, all artifacts from this run will be downloaded + path: artifacts + merge-multiple: true + + - name: Untar + run: | + tar -xzf artifacts/orca-sdk-windows.tar.gz + tar -xzf artifacts/orca-sdk-mac-x64.tar.gz + tar -xzf artifacts/orca-sdk-mac-arm64.tar.gz + + - name: Package CLI Tool + run: | + mkdir releases + cp orca-sdk-windows/bin/orca.exe releases + cp orca-mac.tar.gz releases + + - name: Package Windows SDK + run: | + find orca-sdk-windows -type f -exec sha1sum {} + | LC_ALL=C sort | sha1sum | cut -z -f 1 -d " " > sha1.sum + tar --format=ustar -czf releases/orca-sdk-windows.tar.gz orca-sdk-windows sha1.sum + + - name: Package Mac x64 Release + run: | + chmod +x orca-sdk-mac-x64/bin/orca + chmod +x orca-sdk-mac-x64/bin/orca_runtime + find orca-sdk-mac-x64 -type f -exec sha1sum {} + | LC_ALL=C sort | sha1sum | cut -z -f 1 -d " " > sha1.sum + tar --format=ustar -czf releases/orca-sdk-mac-x64.tar.gz orca-sdk-mac-x64 sha1.sum + + - name: Package Mac arm64 Release + run: | + chmod +x orca-sdk-mac-arm64/bin/orca + chmod +x orca-sdk-mac-arm64/bin/orca_runtime + find orca-sdk-mac-arm64 -type f -exec sha1sum {} + | LC_ALL=C sort | sha1sum | cut -z -f 1 -d " " > sha1.sum + tar --format=ustar -czf releases/orca-sdk-mac-arm64.tar.gz orca-sdk-mac-arm64 sha1.sum + + - uses: ncipollo/release-action@v1 + with: + artifacts: "releases/*" diff --git a/scripts/dev.py b/scripts/dev.py index 286b45ce..e31988aa 100644 --- a/scripts/dev.py +++ b/scripts/dev.py @@ -100,9 +100,9 @@ def dawn_required_commit(): def dawn_required_files(): artifacts = [] if platform.system() == "Windows": - artifacts = ["build/bin/webgpu.lib", "build/bin/webgpu.dll", "src/ext/dawn/include/webgpu.h"] + artifacts = ["bin/webgpu.lib", "bin/webgpu.dll", "include/webgpu.h"] else: - artifacts = ["build/bin/libwebgpu.dylib", "src/ext/dawn/include/webgpu.h"] + artifacts = ["bin/libwebgpu.dylib", "include/webgpu.h"] return artifacts @@ -112,28 +112,41 @@ def check_dawn(): artifacts = dawn_required_files() up_to_date = False + messages = [] - if os.path.exists("build/dawn.json"): - with open("build/dawn.json", "r") as f: - sums = json.loads(f.read()) - up_to_date = True + if os.path.exists("build/dawn.out/dawn.json"): + with pushd("build/dawn.out"): + with open("dawn.json", "r") as f: + sums = json.loads(f.read()) - for artifact in artifacts: - if artifact in sums: - if os.path.isfile(artifact): - s = checksum.filesum(artifact) - if sums[artifact]['commit'] != DAWN_COMMIT or s != sums[artifact]['sum']: + up_to_date = True + + for artifact in artifacts: + if artifact in sums: + if os.path.isfile(artifact): + s = checksum.filesum(artifact) + if sums[artifact]['commit'] != DAWN_COMMIT or s != sums[artifact]['sum']: + messages.append(f"build/dawn.out/{artifact} doesn't match checksum") + up_to_date = False + else: + messages.append(f"build/dawn.out/{artifact} not found") up_to_date = False break else: + messages.append(f"build/dawn.out/{artifact} is not listed in checksum file") up_to_date = False - break - else: - up_to_date = False - break + else: + messages = ["build/dawn.out/dawn.json not found"] + + if up_to_date: + os.makedirs("src/ext/dawn/include", exist_ok=True) + shutil.copytree("build/dawn.out/include", "src/ext/dawn/include/", dirs_exist_ok=True) + + os.makedirs("build/bin", exist_ok=True) + shutil.copytree("build/dawn.out/bin", "build/bin", dirs_exist_ok=True) - return up_to_date + return (up_to_date, messages) def build_dawn(args): @@ -151,7 +164,8 @@ def build_dawn_internal(release, jobs, force): # check if we already have the binary if not force: - if check_dawn(): + dawn_ok, _ = check_dawn() + if dawn_ok: print(" * already up to date") print("Done") @@ -254,43 +268,38 @@ def build_dawn_internal(release, jobs, force): os.makedirs("dawn.out/include", exist_ok=True) os.makedirs("dawn.out/bin", exist_ok=True) - os.makedirs("../src/ext/dawn/include", exist_ok=True) shutil.copy("dawn.build/gen/include/dawn/webgpu.h", "dawn.out/include/") - shutil.copytree("dawn.out/include", "../src/ext/dawn/include/", dirs_exist_ok=True) - sums['src/ext/dawn/include/webgpu.h'] = { + sums['include/webgpu.h'] = { "commit": DAWN_COMMIT, - "sum": checksum.filesum('../src/ext/dawn/include/webgpu.h') + "sum": checksum.filesum('dawn.out/include/webgpu.h') } if platform.system() == "Windows": shutil.copy(f"dawn.build/{mode}/webgpu.dll", "dawn.out/bin/") shutil.copy(f"dawn.build/src/dawn/native/{mode}/webgpu.lib", "dawn.out/bin/") - shutil.copytree("dawn.out/bin", "bin", dirs_exist_ok=True) - sums['build/bin/webgpu.dll'] = { + sums['bin/webgpu.dll'] = { "commit": DAWN_COMMIT, - "sum": checksum.filesum('bin/webgpu.dll') + "sum": checksum.filesum('dawn.out/bin/webgpu.dll') } - sums['build/bin/webgpu.lib'] = { + sums['bin/webgpu.lib'] = { "commit": DAWN_COMMIT, - "sum": checksum.filesum('bin/webgpu.lib') + "sum": checksum.filesum('dawn.out/bin/webgpu.lib') } else: shutil.copy("dawn.build/src/dawn/native/libwebgpu.dylib", "dawn.out/bin/") - shutil.copytree("dawn.out/bin", "bin", dirs_exist_ok=True) - sums['build/bin/libwebgpu.dylib'] = { + sums['bin/libwebgpu.dylib'] = { "commit": DAWN_COMMIT, - "sum": checksum.filesum('bin/libwebgpu.dylib') + "sum": checksum.filesum('dawn.out/bin/libwebgpu.dylib') } # save artifacts checksums - with open('build/dawn.json', 'w') as f: + with open('build/dawn.out/dawn.json', 'w') as f: json.dump(sums, f) - print("Done") #------------------------------------------------------ # build runtime @@ -457,22 +466,22 @@ def build_platform_layer(args): def build_platform_layer_internal(release): print("Building Orca platform layer...") - if not check_dawn(): + dawn_ok, dawn_messages = check_dawn() + if not dawn_ok: msg = log_error("Dawn files are not present or don't match required commit.") msg.more(f"Dawn commit: {dawn_required_commit()}") msg.more("Dawn required files:") - for artifact in dawn_required_files(): - msg.more(f" * {artifact}") + for entry in dawn_messages: + msg.more(f" * {entry}") cmd = "./orcadev" if platform.system() == "Darwin" else "orcadev.bat" msg.more("") msg.more(f"You can build the required files by running '{cmd} build-dawn'") msg.more("") msg.more("Alternatively you can trigger a CI run to build the binaries on github:") - msg.more(" * Go to https://github.com/orca-app/orca/actions/workflows/build-dawn.yaml") - msg.more(" * Click on \"Run workflow\"") - msg.more(" * Once the run is complete, download the artifacts") - msg.more(" * Put the contents of the artifacts 'bin' directory in './build/bin'") - msg.more(" * Put the contents of the artifacts 'include' directory in './src/ext/dawn/include'") + msg.more(" * For Windows, go to https://github.com/orca-app/orca/actions/workflows/build-dawn-win.yaml") + msg.more(" * For macOS, go to https://github.com/orca-app/orca/actions/workflows/build-dawn-mac.yaml") + msg.more(" * Click on \"Run workflow\" to tigger a new run, or download artifacts from a previous run") + msg.more(" * Put the contents of the artifacts folder in './build/dawn.out'") exit() os.makedirs("build/bin", exist_ok=True) @@ -520,13 +529,13 @@ def build_platform_layer_lib_win(release): "ole32.lib", "shell32.lib", "shlwapi.lib", - "/LIBPATH:src/ext/angle/lib", + "/LIBPATH:build/lib", + "/LIBPATH:build/bin", "libEGL.dll.lib", "libGLESv2.dll.lib", "/DELAYLOAD:libEGL.dll", "/DELAYLOAD:libGLESv2.dll", - "/LIBPATH:src/ext/dawn/bin", - "webgpu.dll.lib", + "webgpu.lib", "/DELAYLOAD:webgpu.dll" ] @@ -1023,7 +1032,45 @@ def package_sdk_internal(dest, target): shutil.copytree(os.path.join("build", "orca-libc"), libc_dir, dirs_exist_ok=True) shutil.copytree("resources", res_dir, dirs_exist_ok=True) - shutil.copytree("src", src_dir, dirs_exist_ok=True) + + ignore_patterns = shutil.ignore_patterns( + '*.[!h]', # exclude anything that is not a header + 'tool', + 'ext', + 'orca-libc' + ) + + def ignore(dirName, files): + # ignore everything that's not a header + result = [ x for x in files if os.path.isfile(os.path.join(dirName, x)) and x[-2:] != ".h" ] + + # exclude or include specific sub-directories + ignoreDirs = [] + onlyDirs = [] + + if dirName == 'src': + ignoreDirs = ['tool', 'orca-libc', 'wasm'] + elif dirName == 'src/ext/curl': + onlyDirs = ['include'] + elif dirName == 'src/ext/wasm3': + onlyDirs = ['source'] + elif dirName == 'src/ext/zlib': + ignoreDirs = ['build'] + + if len(ignoreDirs): + result += [x for x in files if x in ignoreDirs] + + if len(onlyDirs): + result += [x for x in files if os.path.isdir(os.path.join(dirName, x)) and x not in onlyDirs] + + return result + + shutil.copytree("src", src_dir, dirs_exist_ok=True, ignore=ignore) + + for dirpath, dirnames, filenames in os.walk(src_dir, topdown=False): + if not os.listdir(dirpath): + os.rmdir(dirpath) + def package_sdk(args): dest = args.dest diff --git a/scripts/log.py b/scripts/log.py index 613503a6..bd516e28 100644 --- a/scripts/log.py +++ b/scripts/log.py @@ -14,10 +14,8 @@ def more(self, *msgs): if len(msgs) == 0: msgs = [""] for msg in msgs: - print(msg) self.msgs.append(msg) - def log_error(msg): msg = f"ERROR: {msg}" entry = Entry(msg) @@ -79,5 +77,7 @@ def shellfunc(*args, **kwargs): exitcode = 1 finally: log_finish(exitcode == 0) + if exitcode == 0 and len(errors): + exitcode = 1 exit(exitcode) return shellfunc