Unify daal4py and sklearnex building (uxlfoundation#1955)

* Unify daal4py and sklearnex setup scripts * Update public CI scripts and INSTALL.md * Linting * Return wrongly deleted setup line * Update installation docs * Change d4p_version define in setup.py * Remove extra variables setting * Revert "Remove extra variables setting" This reverts commit 3b61622. * Remove extra DALROOT setting * Update INSTALL.md Co-authored-by: Ben Moore <[email protected]> --------- Co-authored-by: Ben Moore <[email protected]>
icfaust · Jul 30, 2024 · 38e42e3 · 38e42e3
1 parent 53159f9
commit 38e42e3
Show file tree

Hide file tree

Showing 12 changed files with 66 additions and 305 deletions.
diff --git a/.ci/pipeline/build-and-test-lnx.yml b/.ci/pipeline/build-and-test-lnx.yml
@@ -40,7 +40,6 @@ steps:
       conda activate CB
       export DALROOT=$CONDA_PREFIX
       ./conda-recipe/build.sh
-      python setup_sklearnex.py install --single-version-externally-managed --record=record_sklearnex.txt
     displayName: "Build daal4py/sklearnex"
   - script: |
       . /usr/share/miniconda/etc/profile.d/conda.sh

diff --git a/.ci/pipeline/build-and-test-mac.yml b/.ci/pipeline/build-and-test-mac.yml
@@ -35,7 +35,6 @@ steps:
       source activate CB
       export DALROOT=$CONDA_PREFIX
       ./conda-recipe/build.sh
-      python setup_sklearnex.py install --single-version-externally-managed --record=record_sklearnex.txt
     displayName: Conda build
   - script: |
       source activate CB

diff --git a/.ci/pipeline/build-and-test-win.yml b/.ci/pipeline/build-and-test-win.yml
@@ -37,8 +37,6 @@ steps:
       set PYTHON=python
       call conda-recipe\bld.bat
       IF %ERRORLEVEL% neq 0 EXIT /b %ERRORLEVEL%
-      set DALROOT=%CONDA_PREFIX%
-      python setup_sklearnex.py install --single-version-externally-managed --record=record_sklearnex.txt
     displayName: 'Build daal4py/sklearnex'
   - script: |
       set PATH=C:\msys64\usr\bin;%PATH%

diff --git a/.ci/pipeline/docs.yml b/.ci/pipeline/docs.yml
@@ -67,7 +67,6 @@ jobs:
       export PREFIX=$(dirname $(dirname $(which python)))
       export DALROOT=$PREFIX
       ./conda-recipe/build.sh
-      python setup_sklearnex.py install --single-version-externally-managed --record=record_sklearnex.txt
     displayName: 'Build daal4py/sklearnex'
   - script: |
       export LD_LIBRARY_PATH=$(dirname $(dirname $(which python)))/lib:$LD_LIBRARY_PATH

diff --git a/.ci/pipeline/nightly.yml b/.ci/pipeline/nightly.yml
@@ -82,7 +82,6 @@ jobs:
       conda activate CB
       export DALROOT=$CONDA_PREFIX
       ./conda-recipe/build.sh
-      python setup_sklearnex.py install --single-version-externally-managed --record=record.txt
     displayName: 'Build daal4py/sklearnex'
   - script: |
       . /usr/share/miniconda/etc/profile.d/conda.sh

diff --git a/INSTALL.md b/INSTALL.md
@@ -139,41 +139,60 @@ Intel(R) Extension for Scikit-learn* is easily built from the sources with the m
 
 The package is available for Windows* OS, Linux* OS, and macOS*.
 
+Multi-node (distributed) and streaming support can be disabled if needed.
+
+The build-process (using setup.py) happens in 4 stages:
+1. Creating C++ and Cython sources from oneDAL C++ headers
+2. Building oneDAL Python interfaces via cmake and pybind11
+3. Running Cython on generated sources
+4. Compiling and linking them
+
 ### Prerequisites
 * Python version >= 3.8, <= 3.12
-* daal4py >= 2024.3
-
-**TIP:** [Build daal4py from sources](https://github.com/intel/scikit-learn-intelex/blob/main/daal4py/INSTALL.md) or get it from [distribution channels](https://intelpython.github.io/daal4py/#getting-daal4py).
+* Jinja2
+* Cython
+* Numpy
+* cmake and pybind11
+* A C++ compiler with C++11 support
+* Clang-Format
+* [Intel® oneAPI Data Analytics Library (oneDAL)](https://github.com/oneapi-src/oneDAL) version 2021.1 or later
+  * You can use the pre-built `dal-devel` conda package from conda-forge channel
+* MPI (optional, needed for distributed mode)
+  * You can use the pre-built `impi-devel` conda package from conda-forge channel
+* A DPC++ compiler (optional, needed for DPC++ interfaces)
 
 ### Configure the Build with Environment Variables
 * ``SKLEARNEX_VERSION``: sets the package version
 * ``DALROOT``: sets the oneAPI Data Analytics Library path
+* ``NO_DIST``: set to '1', 'yes' or alike to build without support for distributed mode
+* ``NO_STREAM``: set to '1', 'yes' or alike to build without support for streaming mode
+* ``OFF_ONEDAL_IFACE``: set to '1' to build without the support of oneDAL interfaces
 
 ### Build Intel(R) Extension for Scikit-learn
 
 - To install the package:
 
    ```bash
    cd <checkout-dir>
-   python setup_sklearnex.py install
+   python setup.py install
    ```
 
 - To install the package in the development mode:
 
    ```bash
    cd <checkout-dir>
-   python setup_sklearnex.py develop
+   python setup.py develop
    ```
 
-- To install scikit-learn-intelex without downloading daal4py:
+- To install scikit-learn-intelex without checking for dependencies:
 
    ```bash
    cd <checkout-dir>
-   python setup_sklearnex.py install --single-version-externally-managed --record=record.txt
+   python setup.py install --single-version-externally-managed --record=record.txt
    ```
    ```bash
    cd <checkout-dir>
-   python setup_sklearnex.py develop --no-deps
+   python setup.py develop --no-deps
    ```
 
 Where: 

diff --git a/daal4py/INSTALL.md b/daal4py/INSTALL.md
diff --git a/daal4py/README.md b/daal4py/README.md
@@ -110,7 +110,7 @@ conda install impi_rt -c intel
 
 </details>
 
-You can [build daal4py from sources](https://github.com/intel/scikit-learn-intelex/blob/main/daal4py/INSTALL.md) as well.
+You can [build daal4py from sources](https://github.com/intel/scikit-learn-intelex/blob/main/INSTALL.md) as well.
 
 
 # ⚠️ Scikit-learn patching

diff --git a/setup.cfg b/setup.cfg
@@ -2,6 +2,7 @@
 license_files =
    LICENSE
    doc/daal4py/third-party-programs.txt
+   doc/third-party-programs-sklearnex.txt
 
 [flake8]
 ignore = E265, E722, E402, F401, F403, W503

diff --git a/setup.py b/setup.py
@@ -80,9 +80,9 @@
     os.environ.get("OFF_ONEDAL_IFACE", "0") == "0" and ONEDAL_VERSION >= ONEDAL_2021_3
 )
 
-d4p_version = (
-    os.environ["DAAL4PY_VERSION"]
-    if "DAAL4PY_VERSION" in os.environ
+sklearnex_version = (
+    os.environ["SKLEARNEX_VERSION"]
+    if "SKLEARNEX_VERSION" in os.environ
     else time.strftime("%Y%m%d.%H%M%S")
 )
 
@@ -261,7 +261,7 @@ def get_build_options():
     ]
     eca = [
         "-DPY_ARRAY_UNIQUE_SYMBOL=daal4py_array_API",
-        '-DD4P_VERSION="' + d4p_version + '"',
+        '-DD4P_VERSION="' + sklearnex_version + '"',
         "-DNPY_ALLOW_THREADS=1",
     ]
     ela = []
@@ -398,7 +398,7 @@ def gen_pyx(odir):
     odir = os.path.abspath(odir)
     if not os.path.isdir(odir):
         os.mkdir(odir)
-    gen_daal4py(dal_root, odir, d4p_version, no_dist=no_dist, no_stream=no_stream)
+    gen_daal4py(dal_root, odir, sklearnex_version, no_dist=no_dist, no_stream=no_stream)
 
 
 gen_pyx(os.path.abspath("./build"))
@@ -555,6 +555,25 @@ def run(self):
     "onedal.primitives",
     "onedal.svm",
     "onedal.utils",
+    "sklearnex",
+    "sklearnex.basic_statistics",
+    "sklearnex.cluster",
+    "sklearnex.covariance",
+    "sklearnex.decomposition",
+    "sklearnex.ensemble",
+    "sklearnex.glob",
+    "sklearnex.linear_model",
+    "sklearnex.manifold",
+    "sklearnex.metrics",
+    "sklearnex.model_selection",
+    "sklearnex.neighbors",
+    "sklearnex.preview",
+    "sklearnex.preview.covariance",
+    "sklearnex.preview.cluster",
+    "sklearnex.preview.decomposition",
+    "sklearnex.preview.linear_model",
+    "sklearnex.svm",
+    "sklearnex.utils",
 ]
 
 if ONEDAL_VERSION >= 20230100:
@@ -569,24 +588,32 @@ def run(self):
         "onedal.spmd.covariance",
         "onedal.spmd.decomposition",
         "onedal.spmd.ensemble",
+        "sklearnex.spmd",
+        "sklearnex.spmd.covariance",
+        "sklearnex.spmd.decomposition",
+        "sklearnex.spmd.ensemble",
     ]
     if ONEDAL_VERSION >= 20230100:
         packages_with_tests += [
             "onedal.spmd.basic_statistics",
             "onedal.spmd.linear_model",
             "onedal.spmd.neighbors",
+            "sklearnex.spmd.basic_statistics",
+            "sklearnex.spmd.linear_model",
+            "sklearnex.spmd.neighbors",
         ]
     if ONEDAL_VERSION >= 20230200:
-        packages_with_tests += ["onedal.spmd.cluster"]
+        packages_with_tests += ["onedal.spmd.cluster", "sklearnex.spmd.cluster"]
 
 setup(
-    name="daal4py",
-    description="A convenient Python API to Intel(R) oneAPI Data Analytics Library",
+    name="scikit-learn-intelex",
+    description="Intel(R) Extension for Scikit-learn is a "
+    "seamless way to speed up your Scikit-learn application.",
     long_description=long_description,
     long_description_content_type="text/markdown",
     license="Apache-2.0",
     author="Intel Corporation",
-    version=d4p_version,
+    version=sklearnex_version,
     url="https://github.com/intel/scikit-learn-intelex",
     author_email="[email protected]",
     maintainer_email="[email protected]",
@@ -606,6 +633,7 @@ def run(self):
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
         "Topic :: Scientific/Engineering",
         "Topic :: System",
         "Topic :: Software Development",

diff --git a/setup_sklearnex.cfg b/setup_sklearnex.cfg