Skip to content

[CH] Fix load cache missing columns #7192 #2

[CH] Fix load cache missing columns #7192

[CH] Fix load cache missing columns #7192 #2

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
name: Velox backend Dependency Cache
on:
push:
branches:
- 'main'
env:
ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
concurrency:
group: ${{ github.repository }}-${{ github.workflow }}
cancel-in-progress: false
jobs:
cache-native-lib-centos-7:
runs-on: ubuntu-20.04
container: apache/gluten:vcpkg-centos-7
steps:
- uses: actions/checkout@v2
- name: Generate cache key
run: |
echo ${{ hashFiles('./ep/build-velox/src/**', './dev/**', './cpp/*', './.github/workflows/*') }} > cache-key
- name: Check existing caches
id: check-cache
uses: actions/cache/restore@v3
with:
lookup-only: true
path: |
./cpp/build/releases/
key: cache-velox-build-centos-7-${{ hashFiles('./cache-key') }}
- name: Build Gluten native libraries
if: steps.check-cache.outputs.cache-hit != 'true'
run: |
df -a
bash dev/ci-velox-buildstatic-centos-7.sh
- name: Cache
if: steps.check-cache.outputs.cache-hit != 'true'
id: cache
uses: actions/cache/save@v3
with:
path: |
./cpp/build/releases/
key: cache-velox-build-centos-7-${{ hashFiles('./cache-key') }}
cache-native-lib-centos-8:
runs-on: ubuntu-20.04
container: ghcr.io/facebookincubator/velox-dev:centos8
steps:
- uses: actions/checkout@v2
- name: Generate cache key
run: |
echo ${{ hashFiles('./ep/build-velox/src/**', './dev/**', './cpp/*', './.github/workflows/*') }} > cache-key
- name: Check existing caches
id: check-cache
uses: actions/cache/restore@v3
with:
lookup-only: true
path: |
./cpp/build/releases/
./cpp/build/velox/udf/examples/
./cpp/build/velox/benchmarks/
/root/.m2/repository/org/apache/arrow/
key: cache-velox-build-centos-8-${{ hashFiles('./cache-key') }}
- name: Setup java and maven
if: steps.check-cache.outputs.cache-hit != 'true'
run: |
sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true
sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-* || true
yum install sudo patch java-1.8.0-openjdk-devel wget -y
bash .github/workflows/util/setup_helper.sh install_maven
- name: Build Gluten native libraries
if: steps.check-cache.outputs.cache-hit != 'true'
run: |
df -a
bash dev/ci-velox-buildshared-centos-8.sh
- name: Cache
if: steps.check-cache.outputs.cache-hit != 'true'
id: cache
uses: actions/cache/save@v3
with:
path: |
./cpp/build/releases/
./cpp/build/velox/udf/examples/
./cpp/build/velox/benchmarks/
/root/.m2/repository/org/apache/arrow/
key: cache-velox-build-centos-8-${{ hashFiles('./cache-key') }}
# ccache-native-lib-ubuntu-velox-ut:
# runs-on: ubuntu-20.04
# env:
# CCACHE_DIR: "${{ github.workspace }}/.ccache"
# container: ghcr.io/facebookincubator/velox-dev:amd64-ubuntu-22.04-avx
# steps:
# - uses: actions/checkout@v2
# - name: Get Ccache
# uses: actions/cache/restore@v3
# with:
# path: '${{ env.CCACHE_DIR }}'
# key: ccache-ubuntu-release-default
# - name: Ensure Cache Dirs Exists
# working-directory: ${{ github.workspace }}
# run: |
# mkdir -p '${{ env.CCACHE_DIR }}'
# - name: Build Gluten native libraries
# run: |
# rm -rf /opt/miniconda-for-velox/
# cd ep/build-velox/src && \
# ./get_velox.sh
# cd ../build/velox_ep/
# make EXTRA_CMAKE_FLAGS="-DVELOX_ENABLE_PARQUET=ON -DVELOX_BUILD_TESTING=ON -DVELOX_BUILD_TEST_UTILS=ON"
# - name: CCache after
# run: |
# ccache -vs
# - uses: actions/cache/save@v3
# with:
# path: '${{ env.CCACHE_DIR }}'
# key: ccache-ubuntu-release-default
# ccache-native-lib-centos-velox-ut:
# runs-on: ubuntu-20.04
# env:
# CCACHE_DIR: "${{ github.workspace }}/.ccache"
# container: ghcr.io/facebookincubator/velox-dev:centos8
# steps:
# - uses: actions/checkout@v2
# - name: Setup java and maven
# run: |
# yum install sudo patch java-1.8.0-openjdk-devel wget -y && \
# wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
# tar -xvf apache-maven-3.8.8-bin.tar.gz
# mv apache-maven-3.8.8 /usr/lib/maven
# - name: Get Ccache
# uses: actions/cache/restore@v3
# with:
# path: '${{ env.CCACHE_DIR }}'
# key: ccache-centos-release-default
# - name: Ensure Cache Dirs Exists
# working-directory: ${{ github.workspace }}
# run: |
# mkdir -p '${{ env.CCACHE_DIR }}'
# - name: Build Gluten native libraries
# run: |
# rm -rf /opt/miniconda-for-velox/
# cd ep/build-velox/src && \
# ./get_velox.sh
# cd ../build/velox_ep/
# source /opt/rh/gcc-toolset-9/enable
# make EXTRA_CMAKE_FLAGS="-DVELOX_ENABLE_PARQUET=ON -DVELOX_BUILD_TESTING=ON -DVELOX_BUILD_TEST_UTILS=ON"
#
# - name: CCache after
# run: |
# ccache -s
#
# - uses: actions/cache/save@v3
# with:
# path: '${{ env.CCACHE_DIR }}'
# key: ccache-centos-release-default