Merge pull request #34 from jorisv/topic/minimize_build_memory

Avoid to consume too much memory while building

.github/workflows/ci-ubuntu-macos.yml

@@ -22,42 +22,39 @@ jobs:
         os: ["ubuntu-latest", "macos-latest"]
 
     steps:
-    - uses: actions/checkout@v2
-
-    - name: Checkout submodules
-      run: |
-        git submodule update --init
+    - uses: actions/checkout@v4
+      with:
+        submodules: recursive
 
-    - uses: conda-incubator/setup-miniconda@v2
+    - uses: conda-incubator/setup-miniconda@v3
       with:
         activate-environment: pycppad
         auto-update-conda: true
         environment-file: .github/workflows/conda/conda-env.yml
         python-version: 3.8
-
-    - name: Install cmake and update conda
-      shell: bash -l {0}
-      run: |
-        conda activate pycppad
-        conda install cmake -c main
+        auto-activate-base: false
 
     - name: Build PyCppAD
-      shell: bash -l {0}
+      shell: bash -el {0}
       run: |
-        conda activate pycppad
+        conda list
         echo $CONDA_PREFIX
         
         mkdir build
         cd build
 
-        cmake .. -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DPYTHON_EXECUTABLE=$(which python3) -DBUILD_WITH_CPPAD_CODEGEN_BINDINGS=ON
-        make
-        export CTEST_OUTPUT_ON_FAILURE=1
-        make test
-        make install
+        cmake .. \
+          -G "Ninja" \
+          -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX \
+          -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} \
+          -DPYTHON_EXECUTABLE=$(which python3) \
+          -DBUILD_WITH_CPPAD_CODEGEN_BINDINGS=ON
+        ninja -j1
+        ctest --output-on-failure -C Release -V
+        ninja install
 
     - name: Uninstall PyCppAD
-      shell: bash -l {0}
+      shell: bash -el {0}
       run: |
         cd build
-        make uninstall
+        ninja uninstall

.github/workflows/ci-windows.yml

@@ -0,0 +1,70 @@
+name: PyCppAD CI for Windows
+on:
+  pull_request:
+  push:
+
+env:
+  # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.)
+  BUILD_TYPE: Release
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: true
+      matrix:
+        os: [windows-2019]
+        compiler:
+          - cmd: cl
+            cppad_codegen: OFF
+          - cmd: clang-cl
+            cppad_codegen: ON
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: recursive
+
+    - uses: conda-incubator/setup-miniconda@v3
+      with:
+        activate-environment: pycppad
+        auto-update-conda: true
+        environment-file: .github/workflows/conda/conda-env.yml
+        python-version: 3.8
+        auto-activate-base: false
+
+    - name: Build PyCppAD
+      shell: cmd /C CALL {0}
+      run: |
+        :: Set compiler to use.
+        :: This must be done here and not in env since
+        :: cxx-compiler activation script set this variable
+        :: and is called before.
+        set CC=${{ matrix.compiler.cmd }}
+        set CXX=${{ matrix.compiler.cmd }}
+
+        :: Create build directory
+        mkdir build
+        pushd build
+
+        :: Configure
+        cmake ^
+          -G "Ninja" ^
+          -DCMAKE_BUILD_TYPE=Release ^
+          -DCMAKE_INSTALL_PREFIX=%CONDA_PREFIX%\Library ^
+          -DPYTHON_SITELIB=%CONDA_PREFIX%\Lib\site-packages ^
+          -DPYTHON_EXECUTABLE=%CONDA_PREFIX%\python.exe ^
+          -DBUILD_WITH_CPPAD_CODEGEN_BINDINGS=${{ matrix.compiler.cppad_codegen }} ^
+          ..
+
+        :: Build
+        ninja -j1
+
+        :: Testing
+        ctest --output-on-failure -C Release -V
+
+        :: Test Python import
+        ninja install
+        cd ..
+        python -c "import pycppad"

.github/workflows/conda/conda-env.yml

@@ -1,10 +1,14 @@
 name: pycppad
 channels:
   - conda-forge
-  - nodefaults
 dependencies:
   - boost
   - eigenpy
   - python
   - cppad
-  - cppadcodegen
+  - cppadcodegen
+  - ninja
+  - cmake
+  - pkg-config
+  - ninja
+  - cxx-compiler

CMakeLists.txt

@@ -67,6 +67,8 @@ SET(${PROJECT_NAME}_HEADERS
 
 SET(${PROJECT_NAME}_SOURCES
   src/cppad.cpp
+  src/expose-type-double.cpp
+  src/expose-type-double-row-major.cpp
   )
 
 IF(BUILD_WITH_CPPAD_CODEGEN_BINDINGS)

python/CMakeLists.txt

@@ -36,6 +36,8 @@ SET_TARGET_PROPERTIES(${PYWRAP}
   SUFFIX ${PYTHON_EXT_SUFFIX}
   OUTPUT_NAME "${PYWRAP}"
   LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/python/${PROJECT_NAME}"
+  # On Windows, shared library are treated as binary
+  RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/python/${PROJECT_NAME}"
   )
 
 IF(UNIX AND NOT APPLE)
@@ -47,6 +49,7 @@ INSTALL(TARGETS ${PYWRAP} DESTINATION ${${PYWRAP}_INSTALL_DIR})
 # --- INSTALL SCRIPTS
 SET(PYTHON_FILES
   __init__.py
+  windows_dll_manager.py
   )
 
 FOREACH(python ${PYTHON_FILES})

python/pycppad/__init__.py

@@ -4,3 +4,30 @@
 
 from .pycppad_pywrap import *
 from .pycppad_pywrap import __version__, __raw_version__
+
+# On Windows, if pycppad.dll is not in the same directory than
+# the .pyd, it will not be loaded.
+# We first try to load pycppad, then, if it fail and we are on Windows:
+#  1. We add all paths inside PYCPPAD_WINDOWS_DLL_PATH to DllDirectory
+#  2. If PYCPPAD_WINDOWS_DLL_PATH we add the relative path from the
+#     package directory to the bin directory to DllDirectory
+# This solution is inspired from:
+#  - https://github.com/PixarAnimationStudios/OpenUSD/pull/1511/files
+#  - https://stackoverflow.com/questions/65334494/python-c-extension-packaging-dll-along-with-pyd
+# More resources on https://github.com/diffpy/pyobjcryst/issues/33
+try:
+    from .pycppad_pywrap import *
+    from .pycppad_pywrap import __version__, __raw_version__
+except ImportError:
+    import platform
+
+    if platform.system() == "Windows":
+        from .windows_dll_manager import get_dll_paths, build_directory_manager
+
+        with build_directory_manager() as dll_dir_manager:
+            for p in get_dll_paths():
+                dll_dir_manager.add_dll_directory(p)
+            from .pycppad_pywrap import *
+            from .pycppad_pywrap import __version__, __raw_version__
+    else:
+        raise

python/pycppad/windows_dll_manager.py

@@ -0,0 +1,65 @@
+import os
+import sys
+import contextlib
+
+
+def get_dll_paths():
+    pycppad_paths = os.getenv("PYCPPAD_WINDOWS_DLL_PATH")
+    if pycppad_paths is None:
+        # From https://peps.python.org/pep-0250/#implementation
+        # lib/python-version/site-packages/package
+        RELATIVE_DLL_PATH1 = "..\\..\\..\\..\\bin"
+        # lib/site-packages/package
+        RELATIVE_DLL_PATH2 = "..\\..\\..\\bin"
+        # For unit test
+        RELATIVE_DLL_PATH3 = "..\\..\\bin"
+        return [
+            os.path.join(os.path.dirname(__file__), RELATIVE_DLL_PATH1),
+            os.path.join(os.path.dirname(__file__), RELATIVE_DLL_PATH2),
+            os.path.join(os.path.dirname(__file__), RELATIVE_DLL_PATH3),
+        ]
+    else:
+        return pycppad_paths.split(os.pathsep)
+
+
+class PathManager(contextlib.AbstractContextManager):
+    """Restore PATH state after importing Python module"""
+
+    def add_dll_directory(self, dll_dir: str):
+        os.environ["PATH"] += os.pathsep + dll_dir
+
+    def __enter__(self):
+        self.old_path = os.environ["PATH"]
+        return self
+
+    def __exit__(self, *exc_details):
+        os.environ["PATH"] = self.old_path
+
+
+class DllDirectoryManager(contextlib.AbstractContextManager):
+    """Restore DllDirectory state after importing Python module"""
+
+    def add_dll_directory(self, dll_dir: str):
+        # add_dll_directory can fail on relative path and non
+        # existing path.
+        # Since we don't know all the fail criterion we just ignore
+        # thrown exception
+        try:
+            self.dll_dirs.append(os.add_dll_directory(dll_dir))
+        except OSError:
+            pass
+
+    def __enter__(self):
+        self.dll_dirs = []
+        return self
+
+    def __exit__(self, *exc_details):
+        for d in self.dll_dirs:
+            d.close()
+
+
+def build_directory_manager():
+    if sys.version_info >= (3, 8):
+        return DllDirectoryManager()
+    else:
+        return PathManager()

src/cppad.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright 2021 INRIA
+ * Copyright 2021-2024 INRIA
  */
 
 #ifdef PYCPPAD_WITH_CPPAD_CODEGEN_BINDINGS
@@ -9,6 +9,14 @@
 #include "pycppad/cppad.hpp"
 #include "pycppad/cppad-scalar.hpp"
 
+
+// Use explicit template instantiation to build these
+// function in different translation unit and avoid consuming
+// too much memory on Windows
+typedef ::CppAD::AD<double> ADScalar;
+extern template void eigenpy::exposeType<ADScalar>();
+extern template void eigenpy::exposeType<ADScalar,Eigen::RowMajor>();
+
 namespace pycppad
 {
 

src/expose-type-double-row-major.cpp

@@ -0,0 +1,9 @@
+/*
+ * Copyright 2024 INRIA
+ */
+
+#include "pycppad/cppad.hpp"
+#include "pycppad/cppad-scalar.hpp"
+
+typedef ::CppAD::AD<double> ADScalar;
+template void eigenpy::exposeType<ADScalar, Eigen::RowMajor>();

src/expose-type-double.cpp

@@ -0,0 +1,9 @@
+/*
+ * Copyright 2024 INRIA
+ */
+
+#include "pycppad/cppad.hpp"
+#include "pycppad/cppad-scalar.hpp"
+
+typedef ::CppAD::AD<double> ADScalar;
+template void eigenpy::exposeType<ADScalar>();