Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Update CMake to build under library #56

Merged
merged 9 commits into from
Sep 13, 2023
Merged

Update CMake to build under library #56

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
e103aa4
Update CMake to build under library
mattjala Aug 29, 2023
2b9be27
Remove vol-tests submodule
mattjala Aug 29, 2023
797e5f1
Remove deprecated build methods
mattjala Aug 29, 2023
46f16ef
Update README for building
mattjala Aug 29, 2023
d91bff0
Remove redundant HDF5 libs from exports
mattjala Aug 30, 2023
8de2a29
Explicitly disable static VOL build
mattjala Aug 30, 2023
ac8b487
Remove references to HDF5_DIR
mattjala Aug 30, 2023
a876d6f
Restore other build methods, add workflows
mattjala Sep 11, 2023
972a689
Rework HDF5_LIBRARIES_TO_EXPORT in CMake
mattjala Sep 12, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
112 changes: 44 additions & 68 deletions .github/workflows/main.yml
View file
Open in desktop
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should avoid any of the steps in this workflow file using valgrind to run tests since for the most part that will just make them take significantly longer. We can setup another workflow in the future that uses sanitizers to check for this stuff without affecting the runtime much.

Original file line number Diff line number Diff line change
Expand Up @@ -16,18 +16,16 @@ jobs:
strategy:
fail-fast: false
matrix:
#os: [ubuntu-latest, mac-latest]
os: [ubuntu-latest]
python-version: ["3.8", "3.9", "3.10"]
test-path: ["/rest-vol/build/bin/test_rest_vol-shared"]
#test-path: ["/rest-vol/build/bin/test_rest_vol-shared", "/rest-vol/build/bin/h5vl_test"]
endpoint: ["http+unix://%2Ftmp%2Fhs%2Fsn_1.sock"]

runs-on: ${{matrix.os}}
steps:
- name: Create environment variable setup script
working-directory: ${{github.workspace}}
run: |
echo "#! /bin/sh" >> setup_env_vars.sh
printf '%s\n' 'echo "ADMIN_PASSWORD=admin" >> $GITHUB_ENV' >> setup_env_vars.sh
printf '%s\n' 'export ADMIN_PASSWORD=admin' >> setup_env_vars.sh
printf '%s\n' 'echo "ADMIN_USERNAME=admin" >> $GITHUB_ENV' >> setup_env_vars.sh
Expand All @@ -50,8 +48,8 @@ jobs:
printf '%s\n' 'export HSDS_ENDPOINT=${{matrix.endpoint}}' >> setup_env_vars.sh
printf '%s\n' 'echo "HDF5_VOL_CONNECTOR=REST" >> $GITHUB_ENV' >> setup_env_vars.sh
printf '%s\n' 'export HDF5_VOL_CONNECTOR=REST ' >> setup_env_vars.sh
printf '%s\n' 'echo "HDF5_PLUGIN_PATH=${{github.workspace}}/rest-vol/build/bin/" >> $GITHUB_ENV' >> setup_env_vars.sh
printf '%s\n' 'export HDF5_PLUGIN_PATH=${{github.workspace}}/rest-vol/build/bin/' >> setup_env_vars.sh
printf '%s\n' 'echo "HDF5_PLUGIN_PATH=${{github.workspace}}/hdf5/build/bin/" >> $GITHUB_ENV' >> setup_env_vars.sh
printf '%s\n' 'export HDF5_PLUGIN_PATH=${{github.workspace}}/hdf5/build/bin/' >> setup_env_vars.sh
printf '%s\n' 'echo "ROOT_DIR=${{github.workspace}}/hsdsdata" >> $GITHUB_ENV' >> setup_env_vars.sh
printf '%s\n' 'export ROOT_DIR=${{github.workspace}}/hsdsdata' >> setup_env_vars.sh
printf '%s\n' 'echo "BUCKET_NAME=hsdstest" >> $GITHUB_ENV' >> setup_env_vars.sh
Expand Down Expand Up @@ -81,93 +79,70 @@ jobs:
with:
python-version: ${{ matrix.python-version }}

- name: Install dependencies
- name: Install HSDS dependencies
shell: bash
run: |
python -m pip install --upgrade pip
python -m pip install pytest
if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
- name: Install package

- name: Install HSDS package
shell: bash
run: |
cd ${{github.workspace}}/hsds
pip install -e .

- name: Run unit tests
- name: Run HSDS unit tests
shell: bash
run: |
cd ${{github.workspace}}/hsds
pytest

- name: Install Autotools Dependencies (Linux, serial)
run: |
sudo apt update
sudo apt install automake autoconf libtool libtool-bin
sudo apt install gcc-11 g++-11 gfortran-11
echo "CC=gcc-11" >> $GITHUB_ENV
echo "CXX=g++-11" >> $GITHUB_ENV
echo "FC=gfortran-11" >> $GITHUB_ENV
sudo apt install libaec0 libaec-dev

- name: Get REST VOL
uses: actions/checkout@v3
with:
path: ${{github.workspace}}/rest-vol

- uses: actions/cache/restore@v3
id: hdf5-restore
with:
key: hdf5
path: ${{github.workspace}}/hdf5install

- uses: actions/checkout@v3
if: ${{!steps.hdf5-restore.outputs.cache-hit}}
with:
repository: HDFGroup/hdf5
repository: mattjala/hdf5 # TODO: For workflow testing
ref: vol_cmake # TODO: For workflow testing
path: ${{github.workspace}}/hdf5

- name: Get REST-VOL dependencies
run: |
sudo apt-get install libcurl4-openssl-dev
sudo apt-get install libyajl-dev
sudo apt-get install libcurl4-openssl-dev
sudo apt-get install libyajl-dev

- name: Autotools Configure HDF5
if: ${{!steps.hdf5-restore.outputs.cache-hit}}
- name: CMake Configure HDF5
run: |
sh ./autogen.sh
${{github.workspace}}/hdf5/configure --enable-shared --prefix=${{github.workspace}}/hdf5install
mkdir ./build
cd ./build
cmake -DHDF5_BUILD_HL_LIB=OFF -DHDF5_VOL_REST_ENABLE_EXAMPLES=OFF \
-DBUILD_STATIC_LIBS=OFF \
-DBUILD_SHARED_LIBS=ON -DHDF5_ENABLE_SZIP_SUPPORT=OFF \
-DHDF5_TEST_API=ON -DHDF5_VOL_ALLOW_EXTERNAL=ON \
-DHDF5_VOL_URL01=https://github.com/mattjala/vol-rest.git \
-DHDF5_VOL_VOL-REST_BRANCH="build_under_library" \
-DHDF5_VOL_VOL-REST_NAME="REST" -DHDF5_VOL_VOL-REST_TEST_PARALLEL=OFF \
-DHDF5_ENABLE_Z_LIB_SUPPORT=OFF -DHDF5_ALLOW_EXTERNAL_SUPPORT="GIT" \
-DCMAKE_BUILD_TYPE=RelWithDebInfo -DHDF5_ENABLE_THREADSAFE=OFF \
-DCMAKE_INSTALL_PREFIX=${{github.workspace}}/hdf5install \
..
shell: bash
working-directory: ${{github.workspace}}/hdf5

- name: Autotools Build HDF5
if: ${{!steps.hdf5-restore.outputs.cache-hit}}
- name: Build HDF5 + REST VOL
run: make
working-directory: ${{ github.workspace }}/hdf5
working-directory: ${{ github.workspace }}/hdf5/build

- name: Autotools Install
if: ${{!steps.hdf5-restore.outputs.cache-hit}}
- name: Install HDF + REST VOL
run: make install
working-directory: ${{ github.workspace }}/hdf5

- if: ${{!steps.hdf5-restore.outputs.cache-hit}}
name: Cache hdf5 install
uses: actions/cache/save@v3
with:
key: hdf5
path: ${{github.workspace}}/hdf5install

- name: Clone vol-tests repo
uses: actions/checkout@v3
with:
path: ${{github.workspace}}/rest-vol/test/vol-tests/
repository: mattjala/vol-tests

- name: Prepare directory contents
run: |
cd ${{github.workspace}}/rest-vol
mkdir ${{github.workspace}}/rest-vol/build
cd ${{github.workspace}}/rest-vol/src/hdf5
cp -r ${{github.workspace}}/hdf5install/* .
working-directory: ${{ github.workspace }}/hdf5/build

- name: Install valgrind
run: sudo apt install valgrind
working-directory: ${{ github.workspace }}

- name: Start HSDS
if: ${{ matrix.endpoint != 'http://127.0.0.1:5101'}}
Expand All @@ -190,15 +165,16 @@ jobs:
${{github.workspace}}/setup_env_vars.sh
python tests/integ/setup_test.py
working-directory: ${{github.workspace}}/hsds

- name: Build REST VOL (Cmake)
run: |
cd ${{github.workspace}}/rest-vol/build
sudo CFLAGS="-D_POSIX_C_SOURCE=199506L" cmake -DPREBUILT_HDF5_DIR=${{github.workspace}}/hdf5install -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/rest-vol/build/install ..
sudo make -j
sudo make install

- name: Tests at ${{matrix.test-path}}

- name: Test REST VOL
working-directory: ${{github.workspace}}/hdf5/build/
run: |
${{github.workspace}}/setup_env_vars.sh
${{github.workspace}}${{matrix.test-path}}
valgrind --leak-check=full -s ctest -R "test_rest_vol" -VV

# TODO: Attribute, dataset, link, and testhdf5 tests currently fail
# - name: Test REST VOL with API
# run: |
# ${{github.workspace}}/setup_env_vars.sh
# valgrind --leak-check=full -s ctest -R "vol-rest" -VV
# working-directory: ${{github.workspace}}/hdf5/build/
6 changes: 0 additions & 6 deletions .gitmodules
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +0,0 @@
[submodule "src/hdf5"]
path = src/hdf5
url = https://github.com/HDFGroup/hdf5.git
[submodule "test/vol-tests"]
path = test/vol-tests
url = https://github.com/HDFGroup/vol-tests.git
100 changes: 36 additions & 64 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,28 +100,19 @@ set (HDF5_VOL_REST_UTIL_DIR ${HDF5_VOL_REST_SOURCE_DIR}/src/util)
set (HDF5_VOL_REST_TEST_SRC_DIR ${HDF5_VOL_REST_SOURCE_DIR}/test)
set (HDF5_VOL_REST_EXAMPLES_DIR ${HDF5_VOL_REST_SOURCE_DIR}/examples)
set (HDF5_DIR_NAME "hdf5")
set (HDF5_DIR ${HDF5_VOL_REST_SRC_DIR}/${HDF5_DIR_NAME})
set (HDF5_BINARY_DIR ${HDF5_VOL_REST_BINARY_DIR}/${HDF5_DIR_NAME})


#-----------------------------------------------------------------------------
# Define a CMake variable which the user can override to use
# a pre-built HDF5 distribution for building the REST VOL connector
#-----------------------------------------------------------------------------
set (PREBUILT_HDF5_DIR CACHE STRING "Directory of pre-built HDF5 distribution")
set (HDF5_HL_DIR_NAME "hl")
#set (HDF5_DIR ${HDF5_VOL_REST_SRC_DIR}/${HDF5_DIR_NAME})
#set (HDF5_BINARY_DIR ${HDF5_VOL_REST_BINARY_DIR}/${HDF5_DIR_NAME})

#-----------------------------------------------------------------------------
# Find HDF5, cURL and YAJL before building
#-----------------------------------------------------------------------------
set (HDF5_FOUND FALSE)
if (NOT PREBUILT_HDF5_DIR)
find_package(HDF5 MODULE COMPONENTS C HL)
if (HDF5_ENABLE_THREADSAFE AND NOT Threads_FOUND)
find_package(Threads)
endif ()
if (NOT HDF5_FOUND)
set (HDF5_DIR ${HDF5_VOL_REST_SRC_DIR}/${HDF5_DIR_NAME})
endif ()
if (HDF5_FOUND STREQUAL "")
set (HDF5_FOUND FALSE)
endif()

if (HDF5_ENABLE_THREADSAFE AND NOT Threads_FOUND)
find_package(Threads)
endif ()

find_package (CURL REQUIRED)
Expand Down Expand Up @@ -288,7 +279,7 @@ macro (HDF5_VOL_REST_SET_LIB_OPTIONS libtarget libname libtype)
endmacro (HDF5_VOL_REST_SET_LIB_OPTIONS)

#-------------------------------------------------------------------------------
macro (TARGET_C_PROPERTIES wintarget libtype addcompileflags addlinkflags)
macro (RV_TARGET_C_PROPERTIES wintarget libtype addcompileflags addlinkflags)
if (MSVC)
TARGET_MSVC_PROPERTIES (${wintarget} ${libtype} "${addcompileflags} ${WIN_COMPILE_FLAGS}" "${addlinkflags} ${WIN_LINK_FLAGS}")
else ()
Expand Down Expand Up @@ -459,39 +450,34 @@ configure_file (${HDF5_VOL_REST_RESOURCES_DIR}/rv_cmake_config.h.in ${HDF5_VOL_R
#-----------------------------------------------------------------------------
# Include the main src directory and HDF5 distribution directory
#-----------------------------------------------------------------------------
if (PREBUILT_HDF5_DIR)
set (HDF5_VOL_REST_INCLUDE_DIRECTORIES
${HDF5_VOL_REST_SRC_DIR}
${HDF5_VOL_REST_BINARY_DIR}
${CMAKE_RUNTIME_OUTPUT_DIRECTORY}
${PREBUILT_HDF5_DIR}/include
CACHE
INTERNAL
"Include directories for HDF5 REST VOL connector"
)
elseif (HDF5_FOUND)
set (HDF5_VOL_REST_INCLUDE_DIRECTORIES

set (HDF5_VOL_REST_INCLUDE_DIRECTORIES
${HDF5_VOL_REST_SRC_DIR}
${HDF5_VOL_REST_BINARY_DIR}
${CMAKE_RUNTIME_OUTPUT_DIRECTORY}
)

if (HDF5_FOUND)
list(APPEND HDF5_VOL_REST_INCLUDE_DIRECTORIES
${HDF5_INCLUDE_DIRS}
CACHE
INTERNAL
"Include directories for HDF5 REST VOL connector"
${HDF5_BINARY_DIR}/src
${HDF5_SOURCE_DIR}/${HDF5_HL_DIR_NAME}/src
)
else ()
set (HDF5_VOL_REST_INCLUDE_DIRECTORIES
${HDF5_VOL_REST_SRC_DIR}
${HDF5_VOL_REST_BINARY_DIR}
${CMAKE_RUNTIME_OUTPUT_DIRECTORY}
${HDF5_DIR}
${HDF5_BINARY_DIR}
CACHE
INTERNAL
"Include directories for HDF5 REST VOL connector"
)
message(SEND_ERROR "HDF5 not found!")
#list(APPEND HDF5_VOL_REST_INCLUDE_DIRECTORIES
#${HDF5_DIR}
#${HDF5_BINARY_DIR}
#)
endif ()

set(HDF5_VOL_REST_INCLUDE_DIRECTORIES
${HDF5_VOL_REST_INCLUDE_DIRECTORIES}
CACHE
INTERNAL
"Include directories for HDF5 REST VOL connector"
)

INCLUDE_DIRECTORIES (${HDF5_VOL_REST_INCLUDE_DIRECTORIES})


Expand Down Expand Up @@ -587,27 +573,13 @@ include (${HDF5_VOL_REST_RESOURCES_DIR}/HDFCompilerFlags.cmake)
#endif ()

#-----------------------------------------------------------------------------
# Build HDF5, unless the user has specified to use
# a pre-built HDF5 distribution
# Include HDF5 Directories
#-----------------------------------------------------------------------------
if (NOT PREBUILT_HDF5_DIR AND NOT HDF5_FOUND)
include_directories(${HDF5_VOL_REST_SRC_DIR}/${HDF5_DIR_NAME}/src)
include_directories(${HDF5_VOL_REST_SRC_DIR}/${HDF5_DIR_NAME}/src/H5FDsubfiling)
add_subdirectory(${HDF5_DIR} ${PROJECT_BINARY_DIR}/${HDF5_DIR_NAME})
elseif (NOT PREBUILT_HDF5_DIR AND HDF5_FOUND)
set (HDF5_LIBRARIES_TO_EXPORT
${HDF5_C_HL_LIBRARIES}
${HDF5_C_LIBRARIES}
)
else ()
link_directories(${PREBUILT_HDF5_DIR}/lib)
set (HDF5_LIBRARIES_TO_EXPORT
libhdf5.so
libhdf5_hl.so
libhdf5.a
libhdf5_hl.a
)
endif ()
include_directories(${HDF5_INCLUDE_DIRS})

set (HDF5_LIBRARIES_TO_EXPORT
${HDF5_C_HL_LIBRARIES}
mattjala marked this conversation as resolved.
Show resolved Hide resolved
)

#-----------------------------------------------------------------------------
# Build the REST VOL
Expand Down
Loading