Merge pull request containerd#200 from yuchen0cc/main

workflow: support multi-distro & multi-arch build

.github/workflows/release.yml

@@ -0,0 +1,87 @@
+name: Release
+
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - "v*"
+
+jobs:
+  build:
+    name: "Build Release"
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        images: [ubuntu-18.04, ubuntu-20.04, ubuntu-22.04, centos-7, centos-8]
+        platforms: [linux/amd64, linux/arm64]
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+      with:
+        submodules: true
+    - name: Setup buildx instance
+      uses: docker/setup-buildx-action@v2
+      with:
+        use: true
+    - name: Build
+      shell: bash
+      run: |
+        BASE_IMAGE=${IMAGE/-/:}
+        echo ${BASE_IMAGE}
+        docker buildx build --build-arg BUILD_IMAGE=${BASE_IMAGE} -f .github/workflows/release/Dockerfile --platform=${{ matrix.platforms }} -o releases/ .
+        # remove unused package
+        if [[ "${IMAGE}" =~ "ubuntu" ]]; then
+          rm -f releases/overlaybd-*.rpm
+        elif [[ "${IMAGE}" =~ "centos" ]]; then
+          rm -f releases/overlaybd-*.deb
+        fi
+        ls -l releases/
+      env:
+        IMAGE: ${{ matrix.images }}
+    - name: Upload
+      uses: actions/upload-artifact@v3
+      with:
+        name: releases
+        path: releases/overlaybd-*.*
+
+  dev-release:
+    name: "Development Release"
+    if: github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/v')
+    runs-on: ubuntu-latest
+    needs: [build]
+    steps:
+      - name: Download builds and release notes
+        uses: actions/download-artifact@v3
+      - name: Display downloaded files
+        shell: bash
+        run: ls -l releases
+      - name: Create Release
+        uses: "marvinpinto/action-automatic-releases@latest"
+        with:
+          repo_token: "${{ secrets.GITHUB_TOKEN }}"
+          automatic_release_tag: "latest"
+          prerelease: true
+          title: "Development Build"
+          files: |
+            releases/overlaybd-*.*
+
+  release:
+    name: "Tagged Release"
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
+    runs-on: ubuntu-latest
+    needs: [build]
+    steps:
+      - name: Download builds and release notes
+        uses: actions/download-artifact@v3
+      - name: Display downloaded files
+        shell: bash
+        run: ls -l releases
+      - name: Create Release
+        uses: "marvinpinto/action-automatic-releases@latest"
+        with:
+          repo_token: "${{ secrets.GITHUB_TOKEN }}"
+          prerelease: false
+          files: |
+            releases/overlaybd-*.*

.github/workflows/release/Dockerfile

@@ -0,0 +1,23 @@
+#   Copyright The containerd Authors.
+
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+
+#       http://www.apache.org/licenses/LICENSE-2.0
+
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+ARG BUILD_IMAGE
+FROM ${BUILD_IMAGE} AS builder
+WORKDIR /src
+COPY . .
+ARG BUILD_IMAGE
+RUN ls -l /src && chmod 755 .github/workflows/release/build.sh && .github/workflows/release/build.sh ${BUILD_IMAGE}
+
+FROM scratch AS release
+COPY --from=builder /src/build/overlaybd-*.* /

.github/workflows/release/build.sh

@@ -0,0 +1,71 @@
+#!/bin/bash -x
+
+#   Copyright The containerd Authors.
+
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+
+#       http://www.apache.org/licenses/LICENSE-2.0
+
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+
+OS=${1}
+ARCH=`uname -m`
+BUILD_TYPE="Release"
+COMPILER=""
+PACKAGE_RELEASE=""
+CMAKE="cmake"
+CPACK="cpack"
+
+# Install Dependencies
+if [[ ${OS} =~ "ubuntu" ]]; then
+    export DEBIAN_FRONTEND="noninteractive"
+    export TZ="Etc/UTC"
+    apt-get update -y
+    apt-get install -y libgflags-dev libcurl4-openssl-dev libssl-dev libaio-dev libnl-3-dev libnl-genl-3-dev rpm wget make g++ git dpkg-dev sudo
+    apt-get install -y uuid-dev libjson-c-dev libkmod-dev libsystemd-dev autoconf automake libtool libpci-dev nasm libzstd-dev libext2fs-dev zlib1g-dev
+
+    DISTRO=${OS/:/1~}
+    PACKAGE_RELEASE="-DPACKAGE_RELEASE=0${DISTRO}"
+elif [[ ${OS} =~ "centos" ]]; then
+    if [[ ${OS} == "centos:7" ]]; then
+        yum install -y centos-release-scl
+        yum install -y devtoolset-7-gcc-c++
+
+        export PATH="/opt/rh/devtoolset-7/root/usr/bin:$PATH"
+        PACKAGE_RELEASE="-DPACKAGE_RELEASE=1.el7"
+        COMPILER="-DCMAKE_C_COMPILER=/opt/rh/devtoolset-7/root/usr/bin/gcc -DCMAKE_CXX_COMPILER=/opt/rh/devtoolset-7/root/usr/bin/g++"
+    elif [[ ${OS} == "centos:8" ]]; then
+        rm -rf /etc/yum.repos.d/* && curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-vault-8.5.2111.repo
+        yum install -y gcc gcc-c++
+
+        PACKAGE_RELEASE="-DPACKAGE_RELEASE=1.el8"
+    fi
+
+    yum install -y epel-release libaio-devel libcurl-devel openssl-devel libnl3-devel e2fsprogs-devel
+    yum install -y rpm-build make git wget sudo
+    yum install --skip-broken -y libzstd-static libzstd-devel
+fi
+
+if [[ ${ARCH} == "x86_64" ]]; then
+    wget https://cmake.org/files/v3.15/cmake-3.15.0-Linux-x86_64.tar.gz
+    tar -zxf cmake-3.15.0-Linux-x86_64.tar.gz -C /usr/local/
+    export PATH="/usr/local/cmake-3.15.0-Linux-x86_64/bin:$PATH"
+else
+    wget https://cmake.org/files/v3.20/cmake-3.20.6-linux-aarch64.tar.gz
+    tar -zxf cmake-3.20.6-linux-aarch64.tar.gz -C /usr/local/
+    export PATH="/usr/local/cmake-3.20.6-linux-aarch64/bin:$PATH"
+fi
+
+# Build
+mkdir build
+cd build
+${CMAKE} .. -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DBUILD_TESTING=0 -DENABLE_DSA=0 -DENABLE_ISAL=0 ${PACKAGE_RELEASE} ${COMPILER}
+make -j8
+${CPACK} --verbose

CMake/Findext2fs.cmake

@@ -1,21 +1,21 @@
 include(FetchContent)
 set(FETCHCONTENT_QUIET false)
 
-if (${ARCH} STREQUAL x86_64)
-    set(LIB_EXT2FS_URL "https://github.com/data-accelerator/e2fsprogs/releases/download/latest/libext2fs.tar.gz")
-elseif (${ARCH} STREQUAL aarch64)
-    set(LIB_EXT2FS_URL "https://github.com/data-accelerator/e2fsprogs/releases/download/latest/libext2fs.aarch64.tar.gz")
-endif ()
-
 FetchContent_Declare(
-  ext2fs
-  URL ${LIB_EXT2FS_URL}
+  e2fsprogs
+  GIT_REPOSITORY https://github.com/data-accelerator/e2fsprogs.git
+  GIT_TAG b4d19a0ec5cb535f13009d910833c62aa70d69a5
 )
 
-FetchContent_GetProperties(ext2fs)
-if (NOT ext2fs_POPULATED)
-  FetchContent_Populate(ext2fs)
-endif()
+FetchContent_MakeAvailable(e2fsprogs)
+
+set(LIBEXT2FS_INSTALL_DIR ${e2fsprogs_SOURCE_DIR}/build/libext2fs)
+add_custom_command(
+  OUTPUT ${LIBEXT2FS_INSTALL_DIR}/lib
+  WORKING_DIRECTORY ${e2fsprogs_SOURCE_DIR}
+  COMMAND chmod 755 build.sh && ./build.sh
+)
+add_custom_target(libext2fs ALL DEPENDS ${LIBEXT2FS_INSTALL_DIR}/lib)
 
-set(EXT2FS_INCLUDE_DIR ${ext2fs_SOURCE_DIR}/include)
-find_library(EXT2FS_LIBRARY ext2fs HINTS ${ext2fs_SOURCE_DIR}/lib/)
+set(EXT2FS_INCLUDE_DIR ${LIBEXT2FS_INSTALL_DIR}/include)
+set(EXT2FS_LIBRARY ${LIBEXT2FS_INSTALL_DIR}/lib/libext2fs.so)

CMake/Findphoton.cmake

@@ -4,7 +4,7 @@ set(FETCHCONTENT_QUIET false)
 FetchContent_Declare(
   photon
   GIT_REPOSITORY https://github.com/alibaba/PhotonLibOS.git
-  GIT_TAG v0.5.6
+  GIT_TAG v0.5.7
 )
 
 if(BUILD_TESTING)

CMakeLists.txt

@@ -21,6 +21,10 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/CMake")
 set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fpic")
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fpic")
 
+if (${ARCH} STREQUAL aarch64)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsigned-char")
+endif ()
+
 set(CMAKE_CXX_STANDARD_LIBRARIES "-static-libgcc ${CMAKE_CXX_STANDARD_LIBRARIES}")
 find_library(STATIC_LIBSTDC++ libstdc++.a PATHS /usr/lib/gcc/*/*)
 if(NOT ${STATIC_LIBSTDC++} STREQUAL "STATIC_LIBSTDC++-NOTFOUND")

src/overlaybd/extfs/CMakeLists.txt

@@ -3,10 +3,11 @@ add_library(extfs_lib STATIC ${SOURCE_TAR})
 
 if(NOT ORIGIN_EXT2FS)
   find_package(ext2fs REQUIRED)
+  add_dependencies(extfs_lib libext2fs)
   target_include_directories(extfs_lib PUBLIC ${EXT2FS_INCLUDE_DIR})
   set(LIB_EXT2FS ${EXT2FS_LIBRARY})
 
-  install(DIRECTORY ${ext2fs_SOURCE_DIR}/lib DESTINATION /opt/overlaybd/ USE_SOURCE_PERMISSIONS)
+  install(DIRECTORY ${LIBEXT2FS_INSTALL_DIR}/lib DESTINATION /opt/overlaybd/ USE_SOURCE_PERMISSIONS)
 else()
   set(LIB_EXT2FS ext2fs)
 endif()

src/overlaybd/zfile/CMakeLists.txt

@@ -17,7 +17,7 @@ target_include_directories(crc32_lib PUBLIC
 if (${ARCH} STREQUAL x86_64)
     target_compile_options(crc32_lib PUBLIC -msse4.2 -mcrc32)
 else()
-    target_compile_options(crc32_lib PUBLIC -march=native)
+    target_compile_options(crc32_lib PUBLIC -march=native -mcpu=generic+crc)
 endif()
 
 if(ENABLE_DSA OR ENABLE_ISAL)