diff --git a/.github/workflows/prestissimo-worker-dep-image-build.yml b/.github/workflows/prestissimo-worker-dep-image-build.yml new file mode 100644 index 0000000000000..884e98d1be67a --- /dev/null +++ b/.github/workflows/prestissimo-worker-dep-image-build.yml @@ -0,0 +1,88 @@ +name: "prestissimo-worker-dep-image-build" + +on: + pull_request: + push: + +jobs: + filter-relevant-changes: + name: "filter-relevant-changes" + runs-on: "ubuntu-24.04" + outputs: + prestissimo_dependency_changed: "${{steps.filter.outputs.prestissimo_dependency}}" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - name: "Filter relevant changes" + uses: "dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36" + id: "filter" + with: + base: "release-0.293-clp-connector" + filters: | + prestissimo_dependency: + - "presto-native-execution/scripts/**/*" + - "presto-native-execution/velox/scripts/**/*" + + prestissimo-worker-dep-image-build: + name: "prestissimo-worker-dep-image-build" + runs-on: "ubuntu-22.04" + env: + SHOULD_PUSH: ${{ github.event_name != 'pull_request' && github.ref == 'refs/heads/release-0.293-clp-connector' }} + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Login to image registry" + uses: "docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772" + with: + registry: "ghcr.io" + username: "${{github.actor}}" + password: "${{secrets.GITHUB_TOKEN}}" + + - name: "Set up Docker Buildx" + uses: "docker/setup-buildx-action@v3" + + - name: "Set up metadata for dependency image" + id: "metadata-deps-image" + uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804" + with: + images: "ghcr.io/${{github.repository}}/prestissimo-worker-dev-env" + tags: "type=raw,value=dev" + + - name: "Build and push dependency image" + if: "'true' == needs.filter-relevant-changes.outputs.prestissimo_dependency_changed" + uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" + with: + context: "./presto-native-execution" + file: "./presto-native-execution/scripts/dockerfiles/ubuntu-22.04-dependency.dockerfile" + push: ${{ 'true' == env.SHOULD_PUSH }} + tags: "${{steps.metadata-deps-image.outputs.tags}}" + labels: "${{steps.metadata-deps-image.outputs.labels}}" + + - name: "Pack dependency image as tar" + run: | + if [[ 'true' == "${{ needs.filter-relevant-changes.outputs.prestissimo_dependency_changed }}" ]]; then + SRC_IMAGE="${{ steps.metadata-deps-image.outputs.tags }}" + else + SRC_IMAGE="ghcr.io/y-scope/presto/prestissimo-worker-dev-env:dev" + docker pull "$SRC_IMAGE" + fi + + TAR="prestissimo-worker-dev-env-image.tar" + docker save "$SRC_IMAGE" -o "$TAR" + echo "Saved $SRC_IMAGE to $TAR" + + - name: "Upload dependency image artifact" + uses: "actions/upload-artifact@v4" + with: + name: prestissimo-worker-dev-env-image + path: prestissimo-worker-dev-env-image.tar + if-no-files-found: error + retention-days: 1 diff --git a/.github/workflows/prestissimo-worker-image-build.yml b/.github/workflows/prestissimo-worker-image-build.yml new file mode 100644 index 0000000000000..07e39c91e2e06 --- /dev/null +++ b/.github/workflows/prestissimo-worker-image-build.yml @@ -0,0 +1,92 @@ +name: "prestissimo-worker-image-build" + +on: + workflow_run: + workflows: [ "prestissimo-worker-dep-image-build" ] # name of the producer workflow + types: [ completed ] + pull_request: + push: + +jobs: + prestissimo-worker-image-build: + name: "prestissimo-worker-image-build" + runs-on: "ubuntu-22.04" + env: + SHOULD_PUSH: ${{ github.event_name != 'pull_request' && github.ref == 'refs/heads/release-0.293-clp-connector' }} + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Login to image registry" + uses: "docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772" + with: + registry: "ghcr.io" + username: "${{github.actor}}" + password: "${{secrets.GITHUB_TOKEN}}" + + - name: "Set up Docker Buildx" + uses: "docker/setup-buildx-action@v3" + + - name: "Download dependency image artifact" + uses: "actions/download-artifact@v4" + with: + name: "prestissimo-worker-dev-env-image" + path: ./_deps_image + run-id: "${{github.event.workflow_run.id}}" + github-token: "${{secrets.GITHUB_TOKEN}}" + + - name: "Load dependency image into Docker" + id: "load-deps" + shell: bash + run: | + set -euo pipefail + TAR="./_deps_image/prestissimo-worker-dev-env-image.tar" + OUT="$(docker load -i "$TAR")" + echo "$OUT" + IMAGE_REF="$(echo "$OUT" | awk -F': ' '/^Loaded image:/ {print $2}' | tail -n1)" + + # Fallback if the tar had no tag and you only get an image ID: + if [[ -z "${IMAGE_REF:-}" ]]; then + ID="$(echo "$OUT" | sed -n 's/^Loaded image ID: //p' | tail -n1)" + IMAGE_REF="$(docker image inspect --format='{{index .RepoTags 0}}' "$ID" 2>/dev/null || true)" + fi + + if [[ -z "${IMAGE_REF:-}" ]]; then + echo "Could not determine image tag from docker load output" >&2 + exit 1 + fi + + echo "image_ref=$IMAGE_REF" >> "$GITHUB_OUTPUT" + echo "Loaded ${IMAGE_REF} image." + + - name: "Set up metadata for runtime image" + id: "metadata-runtime-image" + uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804" + with: + images: "ghcr.io/${{github.repository}}/prestissimo-worker" + tags: "type=raw,value=dev" + + - name: "Get number of cores" + id: "get-cores" + run: |- + echo "num_cores=$(nproc)" >> $GITHUB_OUTPUT + + - name: "Build and push runtime image" + uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" + with: + build-args: |- + BASE_IMAGE=ubuntu:22.04 + DEPENDENCY_IMAGE=${{steps.load-deps.outputs.image_ref}} + EXTRA_CMAKE_FLAGS=-DPRESTO_ENABLE_TESTING=OFF \ + -DPRESTO_ENABLE_PARQUET=ON \ + -DPRESTO_ENABLE_S3=ON + NUM_THREADS=${{steps.get-cores.outputs.num_cores}} + OSNAME=ubuntu + context: "./presto-native-execution" + file: "./presto-native-execution/scripts/dockerfiles/prestissimo-runtime.dockerfile" + push: >- + ${{github.event_name != 'pull_request' + && github.ref == 'refs/heads/release-0.293-clp-connector'}} + tags: "${{steps.metadata-runtime-image.outputs.tags}}" + labels: "${{steps.metadata-runtime-image.outputs.labels}}" diff --git a/.github/workflows/prestissimo-worker-images-build.yml b/.github/workflows/prestissimo-worker-images-build.yml deleted file mode 100644 index b36dcb71949be..0000000000000 --- a/.github/workflows/prestissimo-worker-images-build.yml +++ /dev/null @@ -1,70 +0,0 @@ -name: "prestissimo-worker-images-build" - -on: - pull_request: - push: - -jobs: - prestissimo-worker-images-build: - name: "prestissimo-worker-images-build" - runs-on: "ubuntu-22.04" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Login to image registry" - uses: "docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772" - with: - registry: "ghcr.io" - username: "${{github.actor}}" - password: "${{secrets.GITHUB_TOKEN}}" - - - name: "Set up metadata for dependency image" - id: "metadata-deps-image" - uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804" - with: - images: "ghcr.io/${{github.repository}}/prestissimo-worker-dev-env" - tags: "type=raw,value=dev" - - - name: "Build and push dependency image" - uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" - with: - context: "./presto-native-execution" - file: "./presto-native-execution/scripts/dockerfiles/ubuntu-22.04-dependency.dockerfile" - push: >- - ${{github.event_name != 'pull_request' - && github.ref == 'refs/heads/release-0.293-clp-connector'}} - tags: "${{steps.metadata-deps-image.outputs.tags}}" - labels: "${{steps.metadata-deps-image.outputs.labels}}" - - - name: "Set up metadata for runtime image" - id: "metadata-runtime-image" - uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804" - with: - images: "ghcr.io/${{github.repository}}/prestissimo-worker" - tags: "type=raw,value=dev" - - - name: "Get number of cores" - id: "get-cores" - run: |- - echo "num_cores=$(nproc)" >> $GITHUB_OUTPUT - - - name: "Build and push runtime image" - uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" - with: - build-args: |- - BASE_IMAGE=ubuntu:22.04 - DEPENDENCY_IMAGE=${{steps.metadata-deps-image.outputs.tags}} - EXTRA_CMAKE_FLAGS=-DPRESTO_ENABLE_TESTING=OFF \ - -DPRESTO_ENABLE_PARQUET=ON \ - -DPRESTO_ENABLE_S3=ON - NUM_THREADS=${{steps.get-cores.outputs.num_cores}} - OSNAME=ubuntu - context: "./presto-native-execution" - file: "./presto-native-execution/scripts/dockerfiles/prestissimo-runtime.dockerfile" - push: >- - ${{github.event_name != 'pull_request' - && github.ref == 'refs/heads/release-0.293-clp-connector'}} - tags: "${{steps.metadata-runtime-image.outputs.tags}}" - labels: "${{steps.metadata-runtime-image.outputs.labels}}" diff --git a/presto-native-execution/presto_cpp/main/types/tests/PrestoToVeloxConnectorTest.cpp b/presto-native-execution/presto_cpp/main/types/tests/PrestoToVeloxConnectorTest.cpp index fcb6d52c865bc..68d6fa2c7ab92 100644 --- a/presto-native-execution/presto_cpp/main/types/tests/PrestoToVeloxConnectorTest.cpp +++ b/presto-native-execution/presto_cpp/main/types/tests/PrestoToVeloxConnectorTest.cpp @@ -18,7 +18,7 @@ using namespace facebook::presto; using namespace facebook::velox; -class PrestoToVeloxConnectorTest : public ::testing::Test {}; +class PrestoToVeloxConnectorTest : public ::testing::Test {}; TEST_F(PrestoToVeloxConnectorTest, registerVariousConnectors) { std::vector>>