chore(CI): test building release packages on PRs

tensorlakeai · Dec 18, 2024 · 408ef7c · 408ef7c
1 parent 63e982c
commit 408ef7c
Show file tree

Hide file tree

Showing 9 changed files with 67 additions and 215 deletions.
diff --git a/.github/workflows/generate.yml b/.github/workflows/generate.yml
diff --git a/.github/workflows/publish_executor_containers.yaml b/.github/workflows/publish_executor_containers.yaml
@@ -3,11 +3,6 @@ name: Publish Example Executor Containers
 on:
   workflow_dispatch:
 
-permissions:
-  contents: write
-  actions: write
-  packages: write
-
 jobs:
   build-and-push-docker-images:
     name: Build and Push Docker Executor 

diff --git a/.github/workflows/publish_indexify_server.yaml b/.github/workflows/publish_indexify_server.yaml
@@ -27,97 +27,8 @@ defaults:
     working-directory: ./server
 
 jobs:
-  build-linux-amd64-package:
-    name: Build x86_64 Linux Package
-    runs-on: ubuntu-latest-xlarge
-    steps:
-      - uses: actions/checkout@v4
-      - run: rustup toolchain install nightly --component rustfmt
-      - run: cargo +nightly fmt --check
-      - run: cargo install cargo-deb
-      - run: make build-release
-      - run: cargo deb --no-build --no-strip --target x86_64-unknown-linux-gnu
-      - uses: actions/upload-artifact@v4
-        with:
-          name: indexify-server-linux-amd64
-          path: server/target/x86_64-unknown-linux-gnu/release/indexify-server
-          if-no-files-found: error
-      - uses: actions/upload-artifact@v4
-        with:
-          name: indexify-server-deb-linux-amd64
-          path: server/target/x86_64-unknown-linux-gnu/debian/indexify-server_*.deb
-          if-no-files-found: error
-
-  build-linux-arm64-package:
-    name: Build Aarch64 Linux Package
-    runs-on: ubuntu-latest-xlarge
-    steps:
-      - uses: actions/checkout@v4
-      - run: rustup toolchain install nightly --component rustfmt
-      - run: cargo +nightly fmt --check
-      - run: make build-release-aarch64
-      - run: cargo install cargo-deb
-      - run: cargo deb --no-build --no-strip --target aarch64-unknown-linux-gnu
-      - uses: actions/upload-artifact@v4
-        with:
-          name: indexify-server-linux-aarch64
-          path: server/target/aarch64-unknown-linux-gnu/release/indexify-server
-          if-no-files-found: error
-      - uses: actions/upload-artifact@v4
-        with:
-          name: indexify-server-deb-linux-aarch64
-          path: server/target/aarch64-unknown-linux-gnu/debian/indexify-server_*.deb
-          if-no-files-found: error
-
-  build-macos-package:
-    name: Build macOS Package
-    runs-on: macos-14-xlarge
-    steps:
-      - uses: actions/checkout@v4
-      - run: rustup update
-      - run: rustup toolchain install nightly --component rustfmt
-      - run: rustup target add aarch64-apple-darwin
-      - run: cargo +nightly fmt --check
-      - run: make package-ui
-      - run: cargo build --release
-      - uses: actions/upload-artifact@v4
-        with:
-          name: indexify-server-darwin-arm64
-          path: server/target/release/indexify-server
-          if-no-files-found: error
-
-  # build-macos-package-x86:
-  #   name: Build macOS Package
-  #   runs-on: macos-12
-  #   steps:
-  #     - uses: actions/checkout@v4
-  #     - run: rustup update
-  #     - run: rustup toolchain install nightly --component rustfmt
-  #     - run: cargo +nightly fmt --check
-  #     - run: make package-ui
-  #     - run: cargo build --release
-  #     - uses: actions/upload-artifact@v4
-  #       with:
-  #         name: indexify-server-darwin-amd64
-  #         path: server/target/release/indexify-server
-  #         if-no-files-found: error
-
-  # build-windows-package:
-  #   name: Build Windows Package
-  #   runs-on: windows-latest-large
-  #   steps:
-  #     - uses: actions/checkout@v4
-  #     - uses: ilammy/setup-nasm@v1
-  #     - run: rustup toolchain install nightly --component rustfmt
-  #     - run: cargo +nightly fmt --check
-  #     - run: choco install protoc
-  #     - run: make package-ui
-  #     - run: cargo build --release
-  #     - uses: actions/upload-artifact@v4
-  #       with:
-  #         name: indexify-server-windows-amd64.exe
-  #         path: server/target/release/indexify-server.exe
-  #         if-no-files-found: error
+  build-release-packages:
+    uses: ./.github/workflows/wf_build_indexify_server_release_packages.yaml
 
   extract-version:
     name: Extract Version Number
@@ -133,11 +44,7 @@ jobs:
     name: Create GitHub Release
     runs-on: ubuntu-latest
     needs:
-      - build-linux-amd64-package
-      - build-linux-arm64-package
-     # - build-windows-package
-      - build-macos-package
-      # - build-macos-package-x86
+      - build-release-packages
       - extract-version
     steps:
       - uses: actions/checkout@v4

diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
@@ -288,4 +288,9 @@ jobs:
           if pgrep -f indexify; then
             echo "Error: Some Indexify processes are still running."
             exit 1
-          fi
+          fi
+
+  build_release_packages:
+    name: Test Building Release Packages
+    needs: [build_server]
+    uses: ./.github/workflows/wf_build_indexify_server_release_packages.yaml
diff --git a/...uild_docker_indexify_server_workflow.yaml → ...ild_indexify_server_release_packages.yaml b/...uild_docker_indexify_server_workflow.yaml → ...ild_indexify_server_release_packages.yaml
@@ -1,15 +1,7 @@
-name: Test - Build Docker Indexify Server
+name: Build Indexify Server Release Packages
 
 on:
-  push:
-    branches:
-      - 'main'
-  pull_request:
-    branches:
-      - 'main'
-    paths:
-      - 'server/**'
-      - '.github/workflows/test_build_docker_indexify_server_workflow.yaml'
+  workflow_call:
 
 env:
   CARGO_TERM_COLOR: always
@@ -76,4 +68,36 @@ jobs:
         with:
           name: indexify-server-darwin-arm64
           path: server/target/release/indexify-server
-          if-no-files-found: error
+          if-no-files-found: error
+
+  # build-macos-package-x86:
+  #   name: Build macOS Package
+  #   runs-on: macos-12
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #     - run: rustup update
+  #     - run: rustup toolchain install nightly --component rustfmt
+  #     - run: cargo +nightly fmt --check
+  #     - run: make package-ui
+  #     - run: cargo build --release
+  #     - uses: actions/upload-artifact@v4
+  #       with:
+  #         name: indexify-server-darwin-amd64
+  #         path: server/target/release/indexify-server
+  #         if-no-files-found: error
+
+  # build-windows-package:
+  #   name: Build Windows Package
+  #   runs-on: windows-latest-large
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #     - uses: ilammy/setup-nasm@v1
+  #     - run: rustup toolchain install nightly --component rustfmt
+  #     - run: cargo +nightly fmt --check
+  #     - run: make package-ui
+  #     - run: cargo build --release
+  #     - uses: actions/upload-artifact@v4
+  #       with:
+  #         name: indexify-server-windows-amd64.exe
+  #         path: server/target/release/indexify-server.exe
+  #         if-no-files-found: error
diff --git a/server/Cross.toml b/server/Cross.toml
@@ -1,7 +1,6 @@
 [target.aarch64-unknown-linux-gnu]
 dockerfile = "dockerfiles/Dockerfile.builder_linux_aarch64"
 pre-build = [
-    "apt-get update && apt-get install --assume-yes build-essential npm curl protobuf-compiler",
     "dpkg --add-architecture $CROSS_DEB_ARCH",
     "mkdir -p /.npm",
     "chown -R 1001:128 /.npm"

diff --git a/server/dockerfiles/Dockerfile.builder_linux_aarch64 b/server/dockerfiles/Dockerfile.builder_linux_aarch64
@@ -1,22 +1,19 @@
-FROM --platform=linux/arm64/v8 python:3.11 AS python
+FROM ubuntu:24.04
 
-FROM ubuntu:22.04
-ARG DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && \
+    apt-get install --assume-yes --no-install-recommends \
+    ca-certificates \
+    build-essential pkg-config clang \
+    g++-aarch64-linux-gnu libc6-dev-arm64-cross \
+    git curl
 
-RUN apt-get update && apt-get install --assume-yes --no-install-recommends \
-    g++-aarch64-linux-gnu \
-    libc6-dev-arm64-cross git clang
+RUN curl -fsSL https://deb.nodesource.com/setup_22.x -o nodesource_setup.sh && \
+    bash -E nodesource_setup.sh && \
+    apt-get install --assume-yes --no-install-recommends nodejs
 
 ENV CROSS_TOOLCHAIN_PREFIX=aarch64-linux-gnu-
 ENV CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER="$CROSS_TOOLCHAIN_PREFIX"gcc \
     AR_aarch64_unknown_linux_gnu="$CROSS_TOOLCHAIN_PREFIX"ar \
     CC_aarch64_unknown_linux_gnu="$CROSS_TOOLCHAIN_PREFIX"gcc \
     CXX_aarch64_unknown_linux_gnu="$CROSS_TOOLCHAIN_PREFIX"g++ \
-    RUST_TEST_THREADS=1 \
-    PKG_CONFIG_PATH="/usr/lib/aarch64-linux-gnu/pkgconfig/:${PKG_CONFIG_PATH}"
-
-RUN apt install -y python3.11 python3.11-dev
-
-COPY --from=python /usr/local/lib/ /opt/sysroot/usr/lib/
-
-ENV RUSTFLAGS="-L /opt/sysroot/usr/lib"
+    PKG_CONFIG_PATH="/usr/lib/aarch64-linux-gnu/pkgconfig/:${PKG_CONFIG_PATH}"
diff --git a/server/dockerfiles/Dockerfile.builder_linux_ubuntu_20.04 b/server/dockerfiles/Dockerfile.builder_linux_ubuntu_20.04
diff --git a/server/dockerfiles/Dockerfile.builder_linux_x86 b/server/dockerfiles/Dockerfile.builder_linux_x86
@@ -1,28 +1,12 @@
-FROM ubuntu:20.04
-#LABEL stage=builder
-
-WORKDIR /indexify-build
-ENV DEBIAN_FRONTEND=noninteractive
-
-RUN apt-get update
-RUN apt -y install software-properties-common unzip \
-    build-essential make cmake ca-certificates \
-    curl pkg-config git \
-    sqlite3 clang gcc-10 g++-10
-
-RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --default-toolchain stable -y
-
-RUN curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v25.1/protoc-25.1-linux-x86_64.zip
-
-RUN unzip protoc-25.1-linux-x86_64.zip -d /usr/local
-
-ENV PATH="/${HOME}/.cargo/bin:/{HOME}/.local/bin:${PATH}"
-
-ENV CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse
-
-RUN curl -sL https://deb.nodesource.com/setup_22.x | bash
-
-RUN apt install -y nodejs
-
-RUN apt remove -y gcc-9 g++-9
-RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 20 --slave /usr/bin/g++ g++ /usr/bin/g++-10
+FROM ubuntu:24.04
+
+RUN apt-get update && \
+    apt-get install --assume-yes --no-install-recommends \
+    ca-certificates \
+    build-essential pkg-config clang \
+    g++ \
+    git curl
+
+RUN curl -fsSL https://deb.nodesource.com/setup_22.x -o nodesource_setup.sh && \
+    bash -E nodesource_setup.sh && \
+    apt-get install --assume-yes --no-install-recommends nodejs