Update dependencies (#132)

* fix install and tests for latest pytorch

* set minimum python version to 3.7

* fix macOS GIL issue and add environment printing in github actions

* update github actions

* remove MacOS test from github actions

.github/workflows/build_and_test.yml

@@ -19,29 +19,24 @@ jobs:
       matrix:
         config:
         - {
-            name: "Ubuntu Latest GCC", artifact: "Linux.7z",
-            os: ubuntu-latest,
+            name: "Ubuntu 20.04 GCC", artifact: "Linux.7z",
+            os: ubuntu-20.04,
             cc: "gcc", cxx: "g++"
           }
-        - {
-            name: "macOS Latest Clang", artifact: "macOS.7z",
-            os: macos-latest,
-            cc: "clang", cxx: "clang++"
-          }
-
     steps:
     - uses: actions/checkout@v2
-    
+
     - name: Install dependencies
       working-directory: ${{github.workspace}}
       shell: bash
       run:   |
+        
+        python3 --version
+      
         sudo python3 -m pip install pyarrow
 
         if [ "$RUNNER_OS" == "Linux" ]; then
-             sudo python3 -m pip install torch --extra-index-url https://download.pytorch.org/whl/cpu
-        elif [ "$RUNNER_OS" == "macOS" ]; then
-             sudo python3 -m pip install torch
+             sudo pip3 install torch --extra-index-url https://download.pytorch.org/whl/cpu
         else
              echo "$RUNNER_OS not supported"
              exit 1
@@ -50,7 +45,9 @@ jobs:
     - name: Install Marius
       working-directory: ${{github.workspace}}
       shell: bash
-      run: sudo python3 -m pip install .[tests] --verbose
+      run: |
+        sudo pip3 install .[tests] --verbose
+        marius_env_info
 
     - name: Run Tests
       working-directory: ${{github.workspace}}

.github/workflows/db2graph_test_postgres.yml

@@ -14,7 +14,7 @@ jobs:
     container: ${{ matrix.python_container }}
     strategy:
       matrix:
-        python_container: ["python:3.6", "python:3.7", "python:3.8", "python:3.9", "python:3.10"]
+        python_container: ["python:3.7", "python:3.8", "python:3.9", "python:3.10"]
 
     services:
       postgres:

CMakeLists.txt

@@ -141,7 +141,12 @@ add_library(${PROJECT_NAME}
             ${project_CUDA_THIRD_PARTY_HEADERS}
             ${project_CUDA_THIRD_PARTY_SOURCES})
 
-target_link_libraries(${PROJECT_NAME} ${Python3_LIBRARIES})
+if(NOT APPLE)
+    target_link_libraries(${PROJECT_NAME} PUBLIC ${PYTHON_LIBRARIES})
+else()
+    set_target_properties(${PROJECT_NAME} PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
+endif()
+
 target_link_libraries(${PROJECT_NAME} ${TORCH_LIBRARIES})
 target_link_libraries(${PROJECT_NAME} spdlog)
 set_target_properties(${PROJECT_NAME} PROPERTIES PUBLIC_HEADER "${project_HEADERS}")

README.md

@@ -21,7 +21,7 @@ utilizes the data movement optimizations from Marius and adds support for scalab
 * CUDA >= 10.1
 * CuDNN >= 7 
 * pytorch >= 1.8
-* python >= 3.6
+* python >= 3.7
 * GCC >= 7 (On Linux) or Clang >= 11.0 (On MacOS)
 * cmake >= 3.12
 * make >= 3.8

docs/quickstart.rst

@@ -11,7 +11,7 @@ Requirements
 * CUDA >= 10.1
 * CuDNN >= 7
 * pytorch >= 1.8
-* python >= 3.6
+* python >= 3.7
 * GCC >= 7 (On Linux) or Clang 12.0 (On MacOS)
 * cmake >= 3.12
 * make >= 3.8

pyproject.toml

@@ -11,5 +11,5 @@ pythonpath = [
 ]
 
 [build-system]
-requires = ["setuptools==44.0"]
+requires = ["setuptools==44.0", "torch>=1.7.1"]
 build-backend = "setuptools.build_meta"

setup.cfg

@@ -43,7 +43,7 @@ install_requires =
     importlib_metadata>=4.0.0
 
 zip_safe = false
-python_requires = >=3.6
+python_requires = >=3.7
 include_package_data = true
 
 packages =

src/cpp/include/common/datatypes.h

@@ -53,11 +53,6 @@ typedef at::cuda::CUDAEvent CudaEvent;
 typedef DummyCuda CudaEvent;
 #endif
 
-// TODO enable direct IO support
-//#if __linux__
-//    #define IO_FLAGS O_DIRECT
-//#endif
-
 #ifndef IO_FLAGS
 #define IO_FLAGS 0
 #endif

src/cpp/src/data/graph.cpp

@@ -213,8 +213,8 @@ std::tuple<torch::Tensor, torch::Tensor> MariusGraph::getNeighborsForNodeIds(tor
 }
 
 void MariusGraph::sortAllEdges(EdgeList all_edges) {
-    all_src_sorted_edges_ = all_edges.index_select(0, all_edges.select(1, 0).argsort(0)).to(torch::kInt64);
-    all_dst_sorted_edges_ = all_edges.index_select(0, all_edges.select(1, -1).argsort(0)).to(torch::kInt64);
+    all_src_sorted_edges_ = all_edges.index_select(0, all_edges.select(1, 0).argsort(0, false)).to(torch::kInt64);
+    all_dst_sorted_edges_ = all_edges.index_select(0, all_edges.select(1, -1).argsort(0, false)).to(torch::kInt64);
 }
 
 DENSEGraph::DENSEGraph(){};

src/cpp/src/storage/graph_storage.cpp

@@ -7,7 +7,7 @@
 #include <algorithm>
 #include <random>
 
-//#include <omp.h>
+// #include <omp.h>
 
 #include "data/ordering.h"
 #include "reporting/logger.h"

src/python/distribution/marius_env_info.py

@@ -62,12 +62,11 @@ def get_python_info():
         try:
             imported_dep = __import__(dep)
             py_deps_version[dep + "_version"] = imported_dep.__version__
-        except RuntimeError:
+        except ModuleNotFoundError:
             py_deps_version[dep + "_version"] = "N/A"
 
     pytorch_info = {
-        "version": "{}.{}.{}".format(sys.version_info[0], sys.version_info[1], sys.version_info[2]),
-        "compiler": re.sub(r"\[|\]", "", sys.version.split("\n")[1]),
+        "version": sys.version,
         "deps": py_deps_version,
     }
     return pytorch_info

tox.ini

@@ -9,7 +9,7 @@ commands =
     rm -rf docs_html
     rm -rf build
 description = Clean up build directories
-whitelist_externals =
+allowlist_externals =
     /bin/rm
     /usr/bin/rm
 
@@ -28,7 +28,7 @@ commands =
     mv docs_build/docs/html docs_html
     rm -rf docs_build
 description = Build docs with sphinx
-whitelist_externals =
+allowlist_externals =
     /bin/cd
     /bin/mv
     /bin/mkdir
@@ -48,9 +48,10 @@ skip_install = true
 commands =
     black --extend-exclude third_party --preview .
     isort src/python/ examples/ test/ setup.py
-    bash -c 'find src/cpp/include src/cpp/src src/cpp/python_bindings test/cpp -iname *.h -o -iname *.cpp | xargs clang-format -i'
+    /bin/bash -c 'find src/cpp/include src/cpp/src src/cpp/python_bindings test/cpp -iname *.h -o -iname *.cpp | xargs clang-format -i'
 description = Run black, isort, and clang-format autoformatters.
-whitelist_externals =
+allowlist_externals =
+    /bin/bash
     /bin/find
     /bin/xargs
     /usr/bin/find
@@ -71,9 +72,9 @@ deps =
     pep8-naming==0.13.0
 commands =
     flake8
-    bash -c 'find src/cpp/include src/cpp/src src/cpp/python_bindings test/cpp -iname *.h -o -iname *.cpp | xargs clang-format --dry-run -Werror'
+    /bin/bash -c 'find src/cpp/include src/cpp/src src/cpp/python_bindings test/cpp -iname *.h -o -iname *.cpp | xargs clang-format --dry-run -Werror'
 description = Run flake8 and clang-tidy to check formatting
-whitelist_externals =
+allowlist_externals =
     /bin/find
     /bin/xargs
     /bin/bash