From f4646283e84664837e50544f73eafc215ae05171 Mon Sep 17 00:00:00 2001 From: Benjamin Boudreau Date: Tue, 17 Dec 2024 16:56:49 -0500 Subject: [PATCH] chore(CI): test building release packages on PRs --- .github/workflows/generate.yml | 55 ----------- .../workflows/publish_indexify_server.yaml | 99 +------------------ .github/workflows/tests.yaml | 7 +- ...ild_indexify_server_release_packages.yaml} | 46 ++++++--- server/Cross.toml | 1 - .../Dockerfile.builder_linux_aarch64 | 15 ++- .../Dockerfile.builder_linux_ubuntu_20.04 | 4 - .../dockerfiles/Dockerfile.builder_linux_x86 | 27 ++--- 8 files changed, 57 insertions(+), 197 deletions(-) delete mode 100644 .github/workflows/generate.yml rename .github/workflows/{test_build_docker_indexify_server_workflow.yaml => wf_build_indexify_server_release_packages.yaml} (65%) delete mode 100644 server/dockerfiles/Dockerfile.builder_linux_ubuntu_20.04 diff --git a/.github/workflows/generate.yml b/.github/workflows/generate.yml deleted file mode 100644 index d90ee49ec..000000000 --- a/.github/workflows/generate.yml +++ /dev/null @@ -1,55 +0,0 @@ -name: Generate Download Commands - -on: - release: - types: [published] - workflow_dispatch: - -jobs: - commit-json: - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Set up Node.js - uses: actions/setup-node@v4 - with: - node-version: '20' - - - name: Install dependencies - run: | - npm install @octokit/core - npm install fs - - - name: Generate release.json - run: node ./scripts/generate-curl-commands.js ${{ github.repository_owner }} ${{ github.event.repository.name }} - env: - GITHUB_PERSONAL_ACCESS_TOKEN: ${{ secrets.TARGET_REPO_PAT }} - - - name: Checkout target repository - uses: actions/checkout@v3 - with: - repository: tensorlake/hub - token: ${{ secrets.TARGET_REPO_PAT }} - path: target-repo - - - name: Copy release.json to target repository - run: cp release.json target-repo/ - - - name: Commit and push JSON file to target repository - run: | - cd target-repo - git config --global user.name 'github-actions[bot]' - git config --global user.email 'github-actions[bot]@users.noreply.github.com' - git add release.json - git commit -m "Add release.json for ${{ github.event.release.tag_name }}" - git push origin HEAD:main - env: - GITHUB_TOKEN: ${{ secrets.TARGET_REPO_PAT }} - - - diff --git a/.github/workflows/publish_indexify_server.yaml b/.github/workflows/publish_indexify_server.yaml index 7ff96b055..6f9102641 100644 --- a/.github/workflows/publish_indexify_server.yaml +++ b/.github/workflows/publish_indexify_server.yaml @@ -27,97 +27,8 @@ defaults: working-directory: ./server jobs: - build-linux-amd64-package: - name: Build x86_64 Linux Package - runs-on: ubuntu-latest-xlarge - steps: - - uses: actions/checkout@v4 - - run: rustup toolchain install nightly --component rustfmt - - run: cargo +nightly fmt --check - - run: cargo install cargo-deb - - run: make build-release - - run: cargo deb --no-build --no-strip --target x86_64-unknown-linux-gnu - - uses: actions/upload-artifact@v4 - with: - name: indexify-server-linux-amd64 - path: server/target/x86_64-unknown-linux-gnu/release/indexify-server - if-no-files-found: error - - uses: actions/upload-artifact@v4 - with: - name: indexify-server-deb-linux-amd64 - path: server/target/x86_64-unknown-linux-gnu/debian/indexify-server_*.deb - if-no-files-found: error - - build-linux-arm64-package: - name: Build Aarch64 Linux Package - runs-on: ubuntu-latest-xlarge - steps: - - uses: actions/checkout@v4 - - run: rustup toolchain install nightly --component rustfmt - - run: cargo +nightly fmt --check - - run: make build-release-aarch64 - - run: cargo install cargo-deb - - run: cargo deb --no-build --no-strip --target aarch64-unknown-linux-gnu - - uses: actions/upload-artifact@v4 - with: - name: indexify-server-linux-aarch64 - path: server/target/aarch64-unknown-linux-gnu/release/indexify-server - if-no-files-found: error - - uses: actions/upload-artifact@v4 - with: - name: indexify-server-deb-linux-aarch64 - path: server/target/aarch64-unknown-linux-gnu/debian/indexify-server_*.deb - if-no-files-found: error - - build-macos-package: - name: Build macOS Package - runs-on: macos-14-xlarge - steps: - - uses: actions/checkout@v4 - - run: rustup update - - run: rustup toolchain install nightly --component rustfmt - - run: rustup target add aarch64-apple-darwin - - run: cargo +nightly fmt --check - - run: make package-ui - - run: cargo build --release - - uses: actions/upload-artifact@v4 - with: - name: indexify-server-darwin-arm64 - path: server/target/release/indexify-server - if-no-files-found: error - - # build-macos-package-x86: - # name: Build macOS Package - # runs-on: macos-12 - # steps: - # - uses: actions/checkout@v4 - # - run: rustup update - # - run: rustup toolchain install nightly --component rustfmt - # - run: cargo +nightly fmt --check - # - run: make package-ui - # - run: cargo build --release - # - uses: actions/upload-artifact@v4 - # with: - # name: indexify-server-darwin-amd64 - # path: server/target/release/indexify-server - # if-no-files-found: error - - # build-windows-package: - # name: Build Windows Package - # runs-on: windows-latest-large - # steps: - # - uses: actions/checkout@v4 - # - uses: ilammy/setup-nasm@v1 - # - run: rustup toolchain install nightly --component rustfmt - # - run: cargo +nightly fmt --check - # - run: choco install protoc - # - run: make package-ui - # - run: cargo build --release - # - uses: actions/upload-artifact@v4 - # with: - # name: indexify-server-windows-amd64.exe - # path: server/target/release/indexify-server.exe - # if-no-files-found: error + build-release-packages: + uses: ./.github/workflows/wf_build_indexify_server_release_packages.yaml extract-version: name: Extract Version Number @@ -133,11 +44,7 @@ jobs: name: Create GitHub Release runs-on: ubuntu-latest needs: - - build-linux-amd64-package - - build-linux-arm64-package - # - build-windows-package - - build-macos-package - # - build-macos-package-x86 + - build-release-packages - extract-version steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index ad26e9f3a..adcf91217 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -288,4 +288,9 @@ jobs: if pgrep -f indexify; then echo "Error: Some Indexify processes are still running." exit 1 - fi \ No newline at end of file + fi + + build_release_packages: + name: Test Building Release Packages + needs: [acceptance_tests, build_server] + uses: ./.github/workflows/wf_build_indexify_server_release_packages.yaml \ No newline at end of file diff --git a/.github/workflows/test_build_docker_indexify_server_workflow.yaml b/.github/workflows/wf_build_indexify_server_release_packages.yaml similarity index 65% rename from .github/workflows/test_build_docker_indexify_server_workflow.yaml rename to .github/workflows/wf_build_indexify_server_release_packages.yaml index 85dbd5b06..f40dc9259 100644 --- a/.github/workflows/test_build_docker_indexify_server_workflow.yaml +++ b/.github/workflows/wf_build_indexify_server_release_packages.yaml @@ -1,15 +1,7 @@ -name: Test - Build Docker Indexify Server +name: Build Indexify Server Release Packages on: - push: - branches: - - 'main' - pull_request: - branches: - - 'main' - paths: - - 'server/**' - - '.github/workflows/test_build_docker_indexify_server_workflow.yaml' + workflow_call: env: CARGO_TERM_COLOR: always @@ -76,4 +68,36 @@ jobs: with: name: indexify-server-darwin-arm64 path: server/target/release/indexify-server - if-no-files-found: error \ No newline at end of file + if-no-files-found: error + + # build-macos-package-x86: + # name: Build macOS Package + # runs-on: macos-12 + # steps: + # - uses: actions/checkout@v4 + # - run: rustup update + # - run: rustup toolchain install nightly --component rustfmt + # - run: cargo +nightly fmt --check + # - run: make package-ui + # - run: cargo build --release + # - uses: actions/upload-artifact@v4 + # with: + # name: indexify-server-darwin-amd64 + # path: server/target/release/indexify-server + # if-no-files-found: error + + # build-windows-package: + # name: Build Windows Package + # runs-on: windows-latest-large + # steps: + # - uses: actions/checkout@v4 + # - uses: ilammy/setup-nasm@v1 + # - run: rustup toolchain install nightly --component rustfmt + # - run: cargo +nightly fmt --check + # - run: make package-ui + # - run: cargo build --release + # - uses: actions/upload-artifact@v4 + # with: + # name: indexify-server-windows-amd64.exe + # path: server/target/release/indexify-server.exe + # if-no-files-found: error \ No newline at end of file diff --git a/server/Cross.toml b/server/Cross.toml index f2edef017..2d933b392 100644 --- a/server/Cross.toml +++ b/server/Cross.toml @@ -1,7 +1,6 @@ [target.aarch64-unknown-linux-gnu] dockerfile = "dockerfiles/Dockerfile.builder_linux_aarch64" pre-build = [ - "apt-get update && apt-get install --assume-yes build-essential npm curl protobuf-compiler", "dpkg --add-architecture $CROSS_DEB_ARCH", "mkdir -p /.npm", "chown -R 1001:128 /.npm" diff --git a/server/dockerfiles/Dockerfile.builder_linux_aarch64 b/server/dockerfiles/Dockerfile.builder_linux_aarch64 index e6dcfbcdb..bd6affa08 100644 --- a/server/dockerfiles/Dockerfile.builder_linux_aarch64 +++ b/server/dockerfiles/Dockerfile.builder_linux_aarch64 @@ -1,22 +1,19 @@ -FROM --platform=linux/arm64/v8 python:3.11 AS python - -FROM ubuntu:22.04 -ARG DEBIAN_FRONTEND=noninteractive +FROM ubuntu:24.04 RUN apt-get update && apt-get install --assume-yes --no-install-recommends \ + build-essential npm curl \ g++-aarch64-linux-gnu \ libc6-dev-arm64-cross git clang +RUN curl -fsSL https://deb.nodesource.com/setup_22.x -o nodesource_setup.sh && \ + bash -E nodesource_setup.sh && \ + apt-get install --assume-yes --no-install-recommends nodejs + ENV CROSS_TOOLCHAIN_PREFIX=aarch64-linux-gnu- ENV CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER="$CROSS_TOOLCHAIN_PREFIX"gcc \ AR_aarch64_unknown_linux_gnu="$CROSS_TOOLCHAIN_PREFIX"ar \ CC_aarch64_unknown_linux_gnu="$CROSS_TOOLCHAIN_PREFIX"gcc \ CXX_aarch64_unknown_linux_gnu="$CROSS_TOOLCHAIN_PREFIX"g++ \ - RUST_TEST_THREADS=1 \ PKG_CONFIG_PATH="/usr/lib/aarch64-linux-gnu/pkgconfig/:${PKG_CONFIG_PATH}" -RUN apt install -y python3.11 python3.11-dev - -COPY --from=python /usr/local/lib/ /opt/sysroot/usr/lib/ - ENV RUSTFLAGS="-L /opt/sysroot/usr/lib" diff --git a/server/dockerfiles/Dockerfile.builder_linux_ubuntu_20.04 b/server/dockerfiles/Dockerfile.builder_linux_ubuntu_20.04 deleted file mode 100644 index 35dc11463..000000000 --- a/server/dockerfiles/Dockerfile.builder_linux_ubuntu_20.04 +++ /dev/null @@ -1,4 +0,0 @@ -FROM ubuntu:20.04 -ARG DEBIAN_FRONTEND=noninteractive - -RUN apt-get update && apt-get install --assume-yes --no-install-recommends git clang build-essential npm curl protobuf-compiler diff --git a/server/dockerfiles/Dockerfile.builder_linux_x86 b/server/dockerfiles/Dockerfile.builder_linux_x86 index c28a8de2c..0199b3e38 100644 --- a/server/dockerfiles/Dockerfile.builder_linux_x86 +++ b/server/dockerfiles/Dockerfile.builder_linux_x86 @@ -1,28 +1,15 @@ -FROM ubuntu:20.04 -#LABEL stage=builder +FROM ubuntu:24.04 -WORKDIR /indexify-build -ENV DEBIAN_FRONTEND=noninteractive - -RUN apt-get update -RUN apt -y install software-properties-common unzip \ +RUN apt-get update && \ + apt-get install --assume-yes --no-install-recommends \ + software-properties-common unzip \ build-essential make cmake ca-certificates \ curl pkg-config git \ sqlite3 clang gcc-10 g++-10 -RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --default-toolchain stable -y - -RUN curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v25.1/protoc-25.1-linux-x86_64.zip - -RUN unzip protoc-25.1-linux-x86_64.zip -d /usr/local - -ENV PATH="/${HOME}/.cargo/bin:/{HOME}/.local/bin:${PATH}" - -ENV CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse - -RUN curl -sL https://deb.nodesource.com/setup_22.x | bash - -RUN apt install -y nodejs +RUN curl -fsSL https://deb.nodesource.com/setup_22.x -o nodesource_setup.sh && \ + bash -E nodesource_setup.sh && \ + apt-get install --assume-yes --no-install-recommends nodejs RUN apt remove -y gcc-9 g++-9 RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 20 --slave /usr/bin/g++ g++ /usr/bin/g++-10