Merge pull request #1 from livepeer/master

syncing with the main

.github/discord-embed-webhook.py

@@ -0,0 +1,87 @@
+import argparse
+import os
+import json
+import pathlib
+from typing import Any
+
+from requests import Response
+from discord_webhook import DiscordEmbed, DiscordWebhook
+
+
+GITHUB_CONTEXT_JSON = os.environ.get("GITHUB_CONTEXT_JSON", "{}")
+DOWNLOAD_BASE_URL_TEMPLATE = (
+    "https://build.livepeer.live/go-livepeer/{version}/{filename}"
+)
+
+
+def get_github_context_vars(context: dict[str, Any]) -> dict[str, str]:
+    context_vars = {}
+    event: dict[str, Any] = context["event"]
+    if context.get("event_name") == "pull_request":
+        pull_request_context: dict[str, Any] = event["pull_request"]
+        head = pull_request_context.get("head", {})
+        repo = head.get("repo", {})
+        sha = head.get("sha")
+        context_vars["title"] = head.get("ref")
+        context_vars["sha"] = sha
+        context_vars["commit_url"] = f'{repo.get("html_url")}/commit/{sha}'
+    elif context.get("event_name") == "push":
+        sha = event["after"]
+        context_vars["title"] = context["ref_name"]
+        context_vars["sha"] = sha
+        context_vars["commit_url"] = event["compare"]
+    return context_vars
+
+
+def populate_embeds(embed: DiscordEmbed, ref_name: str, checksums: list[str]):
+    for line in checksums:
+        _, filename = line.split()
+        download_url = DOWNLOAD_BASE_URL_TEMPLATE.format(
+            version=ref_name,
+            filename=filename,
+        )
+        title = filename.removeprefix("livepeer-").split(".")[0]
+        print(f"Adding embed field name={title} value={download_url}")
+        embed.add_embed_field(name=title, value=download_url, inline=False)
+
+
+def main(args):
+    checksums = []
+    github_context: dict[str, Any] = json.loads(GITHUB_CONTEXT_JSON)
+    context_vars = get_github_context_vars(github_context)
+    checksums_file = pathlib.Path("releases") / f"{args.ref_name}_checksums.txt"
+    checksums = checksums_file.read_text().splitlines()
+    webhook = DiscordWebhook(
+        url=args.discord_url,
+        content=":white_check_mark: Build succeeded for go-livepeer :white_check_mark:",
+        username="[BOT] Livepeer builder",
+    )
+    webhook.add_file(filename=checksums_file.name, file=checksums_file.read_bytes())
+    embed = DiscordEmbed(
+        title=context_vars.get("title"),
+        description=args.git_commit,
+        color=2928914,
+        url=context_vars.get("commit_url"),
+    )
+    embed.add_embed_field(name="Commit SHA", value=context_vars.get("sha"))
+    embed.set_author(name=args.git_committer)
+    populate_embeds(embed, args.ref_name, checksums)
+    embed.set_timestamp()
+    webhook.add_embed(embed)
+    response: Response = webhook.execute()
+    print("sending webhook with content:")
+    print(webhook.json)
+    # Fail the script if discord returns anything except OK status
+    assert (
+        response.ok
+    ), f"Discord webhook failed {response.status_code} {response.content}"
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser("Discord embed content generator for build system")
+    parser.add_argument("--discord-url", help="Discord webhook URL")
+    parser.add_argument("--ref-name", help="Tag/branch/commit for current build")
+    parser.add_argument("--git-commit", help="git commit message")
+    parser.add_argument("--git-committer", help="git commit author name")
+    args = parser.parse_args()
+    main(args)

.github/requirements.txt

@@ -0,0 +1 @@
+discord-webhook

.github/workflows/build.yaml

@@ -15,7 +15,7 @@ concurrency:
 jobs:
   linux-build:
     name: Build binaries for ${{ matrix.target.GOOS }}-${{ matrix.target.type }}-${{ matrix.target.GOARCH }}
-    runs-on: ${{ matrix.target.runner }}
+    runs-on: ubuntu-20.04
     container:
       image: ${{ matrix.target.container }}
       env:
@@ -26,31 +26,26 @@ jobs:
         target:
           - GOOS: linux
             GOARCH: amd64
-            runner: ubuntu-20.04
             container: ubuntu:20.04
             type: cpu
 
           - GOOS: linux
             GOARCH: arm64
-            runner: ubuntu-20.04
             container: ubuntu:20.04
             type: cpu
 
           - GOOS: linux
             GOARCH: amd64
-            runner: ubuntu-20.04
             container: livepeerci/cuda:12.0.0-cudnn8-devel-ubuntu20.04
             type: gpu
 
           - GOOS: linux
             GOARCH: arm64
-            runner: ubuntu-20.04
             container: livepeerci/cuda:12.0.0-cudnn8-devel-ubuntu20.04
             type: gpu
 
           - GOOS: windows
             GOARCH: amd64
-            runner: ubuntu-20.04
             container: ubuntu:22.04
             type: cpu
 
@@ -63,7 +58,7 @@ jobs:
           apt update && apt install -yqq git zip unzip zlib1g-dev zlib1g yasm
 
       - name: Check out code
-        uses: actions/[email protected].1
+        uses: actions/[email protected].7
         with:
           fetch-depth: 0
           # Check https://github.com/livepeer/go-livepeer/pull/1891
@@ -78,19 +73,14 @@ jobs:
           cache: true
           cache-dependency-path: go.sum
 
-      - name: Cache ffmpeg
-        id: cache-ffmpeg
-        uses: actions/cache@v3
-        with:
-          path: /home/runner/compiled
-          key: ${{ runner.os }}-${{ matrix.target.GOOS }}-${{ matrix.target.GOARCH }}-${{ matrix.target.type }}-ffmpeg-${{ hashFiles('**/install_ffmpeg.sh') }}
-
       - name: Set build environment
         run: |
           echo "GOARCH=${{ matrix.target.GOARCH }}" >> $GITHUB_ENV
           echo "GOOS=${{ matrix.target.GOOS }}" >> $GITHUB_ENV
           echo "GO_BUILD_DIR=lp-builds/" >> $GITHUB_ENV
           echo "PKG_CONFIG_PATH=/github/home/compiled/lib/pkgconfig" >> $GITHUB_ENV
+          echo "LP_BUILD_DIR=livepeer-${{ matrix.target.GOOS }}-${{ matrix.target.GOARCH }}" >> $GITHUB_ENV
+          mkdir -p lp-builds/ releases/
 
       - name: Set GPU build environment
         if: matrix.target.type == 'gpu'
@@ -99,6 +89,7 @@ jobs:
           echo "LIBRARY_PATH=/usr/local/cuda_${{ matrix.target.GOARCH }}/lib64" >> $GITHUB_ENV
           echo "CGO_LDFLAGS=-L/usr/local/cuda_${{ matrix.target.GOARCH }}/lib64" >> $GITHUB_ENV
           echo "RELEASE_TAG=gpu" >> $GITHUB_ENV
+          echo "LP_BUILD_DIR=livepeer-${{ matrix.target.GOOS }}-gpu-${{ matrix.target.GOARCH }}" >> $GITHUB_ENV
 
       - name: Install dependencies
         run: |
@@ -120,7 +111,6 @@ jobs:
         run: go mod download
 
       - name: Install ffmpeg
-        if: steps.cache-ffmpeg.outputs.cache-hit != 'true'
         run: ./install_ffmpeg.sh
 
       - name: Build binaries
@@ -130,24 +120,28 @@ jobs:
           git config --global --add safe.directory '*'
           ./ci_env.sh make
 
-      - name: Upload build
-        if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository
-        env:
-          GHA_REF: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || github.ref }}
-          GCLOUD_KEY: ${{ secrets.GCLOUD_KEY }}
-          GCLOUD_SECRET: ${{ secrets.GCLOUD_SECRET }}
-          DISCORD_URL: ${{ secrets.DISCORD_URL }}
-        run: ./upload_build.sh
+      - name: Archive binaries for windows
+        if: matrix.target.GOOS == 'windows'
+        run: |
+          mkdir -p "${GO_BUILD_DIR}/${LP_BUILD_DIR}/"
+          cd "$GO_BUILD_DIR/"
+          find . -type f -exec mv '{}' "$LP_BUILD_DIR/" \;
+          zip -9rq "../releases/$LP_BUILD_DIR.zip" ./
+
+      - name: Archive binaries
+        if: matrix.target.GOOS != 'windows'
+        run: |
+          mkdir -p "${GO_BUILD_DIR}/${LP_BUILD_DIR}/"
+          cd "$GO_BUILD_DIR/"
+          find . -type f -exec mv '{}' "$LP_BUILD_DIR" \;
+          tar -czvf "../releases/$LP_BUILD_DIR.tar.gz" ./
 
       - name: Upload artifacts for cutting release
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: release-artifacts
+          name: release-artifacts-${{ matrix.target.GOOS }}-${{ matrix.target.type }}-${{ matrix.target.GOARCH }}
           path: releases/
 
-      - name: Notify new build upload
-        run: curl -X POST https://holy-bread-207a.livepeer.workers.dev
-
   macos-build:
     name: Build binaries for ${{ matrix.target.GOOS }}-${{ matrix.target.GOARCH }}
     runs-on: ${{ matrix.target.runner }}
@@ -161,11 +155,11 @@ jobs:
 
           - GOOS: darwin
             GOARCH: arm64
-            runner: macos-latest
+            runner: macos-14-xlarge
 
     steps:
       - name: Check out code
-        uses: actions/[email protected].1
+        uses: actions/[email protected].7
         with:
           fetch-depth: 0
           # Check https://github.com/livepeer/go-livepeer/pull/1891
@@ -185,10 +179,12 @@ jobs:
           echo "GOARCH=${{ matrix.target.GOARCH }}" >> $GITHUB_ENV
           echo "GOOS=${{ matrix.target.GOOS }}" >> $GITHUB_ENV
           echo "GO_BUILD_DIR=lp-builds/" >> $GITHUB_ENV
+          echo "LP_BUILD_DIR=livepeer-${{ matrix.target.GOOS }}-${{ matrix.target.GOARCH }}" >> $GITHUB_ENV
+          mkdir -p lp-builds/ releases/
 
       - name: Cache ffmpeg
         id: cache-ffmpeg
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/compiled
           key: ${{ runner.os }}-${{ matrix.target.GOOS }}-${{ matrix.target.GOARCH }}-ffmpeg-${{ hashFiles('**/install_ffmpeg.sh') }}
@@ -230,45 +226,60 @@ jobs:
           app-notarization-team-id: ${{ secrets.CI_MACOS_NOTARIZATION_TEAM_ID }}
           binary-path: "lp-builds/"
 
-      - name: Upload build
-        if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository
-        env:
-          GHA_REF: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || github.ref }}
-          GCLOUD_KEY: ${{ secrets.GCLOUD_KEY }}
-          GCLOUD_SECRET: ${{ secrets.GCLOUD_SECRET }}
-          DISCORD_URL: ${{ secrets.DISCORD_URL }}
-        run: ./upload_build.sh
+      - name: Archive binaries
+        if: matrix.platform.name != 'windows'
+        run: |
+          mkdir -p "${GO_BUILD_DIR}/${LP_BUILD_DIR}/"
+          cd "$GO_BUILD_DIR/"
+          find . -type f -exec mv '{}' "$LP_BUILD_DIR" \;
+          tar -czvf "../releases/${LP_BUILD_DIR}.tar.gz" .
 
       - name: Upload artifacts for cutting release
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: release-artifacts
+          name: release-artifacts-${{ matrix.target.GOOS }}-${{ matrix.target.GOARCH }}
           path: releases/
 
-      - name: Notify new build upload
-        run: curl -X POST https://holy-bread-207a.livepeer.workers.dev
-
   upload:
     name: Upload artifacts to google bucket
+    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository
     permissions:
       contents: "read"
       id-token: "write"
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     needs:
       - macos-build
       - linux-build
     steps:
+      - name: Check out code
+        uses: actions/[email protected]
+        with:
+          fetch-depth: 0
+          # Check https://github.com/livepeer/go-livepeer/pull/1891
+          # for ref value discussion
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Set up python
+        id: python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.12
+          cache: pip
+          cache-dependency-path: .github/requirements.txt
+          update-environment: true
+
       - name: Download artifacts
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
-          name: release-artifacts
+          pattern: release-artifacts-*
           path: releases/
+          merge-multiple: true
 
       - name: Generate sha256 checksum and gpg signatures for release artifacts
         uses: livepeer/action-gh-checksum-and-gpg-sign@latest
         with:
           artifacts-dir: releases
-          release-name: ${{ (github.ref_type == 'tag' && github.ref_name) || github.sha }}
+          release-name: ${{ (github.ref_type == 'tag' && github.ref_name) || (github.event.pull_request.head.sha || github.sha) }}
           gpg-key: ${{ secrets.CI_GPG_SIGNING_KEY }}
           gpg-key-passphrase: ${{ secrets.CI_GPG_SIGNING_PASSPHRASE }}
 
@@ -289,16 +300,33 @@ jobs:
 
       - name: Upload release archives to Google Cloud
         id: upload-archives
-        uses: google-github-actions/upload-cloud-storage@v1
+        uses: google-github-actions/upload-cloud-storage@v2
         with:
           path: "releases"
-          destination: "build.livepeer.live/${{ github.event.repository.name }}/${{ (github.ref_type == 'tag' && github.ref_name) || github.sha }}"
+          destination: "build.livepeer.live/${{ github.event.repository.name }}/${{ (github.ref_type == 'tag' && github.ref_name) || (github.event.pull_request.head.sha || github.sha) }}"
           parent: false
+          process_gcloudignore: false
 
       - name: Upload branch manifest file
         id: upload-manifest
-        uses: google-github-actions/upload-cloud-storage@v1
+        uses: google-github-actions/upload-cloud-storage@v2
         with:
           path: ${{ steps.branch-manifest.outputs.manifest-file }}
           destination: "build.livepeer.live/${{ github.event.repository.name }}/"
           parent: false
+          process_gcloudignore: false
+
+      - name: Trigger discord webhook
+        shell: bash
+        env:
+          GITHUB_CONTEXT_JSON: ${{ toJson(github) }}
+        run: |
+          pip install -r .github/requirements.txt
+          python .github/discord-embed-webhook.py \
+            --ref-name="${{ (github.ref_type == 'tag' && github.ref_name) || (github.event.pull_request.head.sha || github.sha) }}" \
+            --discord-url="${{ secrets.DISCORD_URL }}" \
+            --git-commit="$(git log -1 --pretty=format:'%s')" \
+            --git-committer="$(git log -1 --pretty=format:'%an')"
+
+      - name: Notify new build upload
+        run: curl -X POST https://holy-bread-207a.livepeer.workers.dev

.github/workflows/docker.yaml

@@ -22,7 +22,7 @@ jobs:
     runs-on: oxford
     steps:
       - name: Check out code
-        uses: actions/[email protected].1
+        uses: actions/[email protected].7
         with:
           fetch-depth: 0
           # Check https://github.com/livepeer/go-livepeer/pull/1891
@@ -106,7 +106,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: Check out code
-        uses: actions/[email protected].1
+        uses: actions/[email protected].7
         with:
           fetch-depth: 0
           # Check https://github.com/livepeer/go-livepeer/pull/1891

.github/workflows/git.yaml

@@ -9,10 +9,10 @@ concurrency:
 
 jobs:
   block-fixup:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - name: Check out code
-        uses: actions/[email protected].1
+        uses: actions/[email protected].7
 
       - name: Block fixup commit merge
         uses: 13rac1/[email protected]