diff --git a/.ci/pipeline/build-and-test-lnx.yml b/.ci/pipeline/build-and-test-lnx.yml index c503b942fc..d6898360db 100644 --- a/.ci/pipeline/build-and-test-lnx.yml +++ b/.ci/pipeline/build-and-test-lnx.yml @@ -40,7 +40,6 @@ steps: conda activate CB export DALROOT=$CONDA_PREFIX ./conda-recipe/build.sh - python setup_sklearnex.py install --single-version-externally-managed --record=record_sklearnex.txt displayName: "Build daal4py/sklearnex" - script: | . /usr/share/miniconda/etc/profile.d/conda.sh diff --git a/.ci/pipeline/build-and-test-mac.yml b/.ci/pipeline/build-and-test-mac.yml index 0df12bc5d0..030d60f665 100644 --- a/.ci/pipeline/build-and-test-mac.yml +++ b/.ci/pipeline/build-and-test-mac.yml @@ -35,7 +35,6 @@ steps: source activate CB export DALROOT=$CONDA_PREFIX ./conda-recipe/build.sh - python setup_sklearnex.py install --single-version-externally-managed --record=record_sklearnex.txt displayName: Conda build - script: | source activate CB diff --git a/.ci/pipeline/build-and-test-win.yml b/.ci/pipeline/build-and-test-win.yml index 13b7986aaa..0db4207670 100644 --- a/.ci/pipeline/build-and-test-win.yml +++ b/.ci/pipeline/build-and-test-win.yml @@ -37,8 +37,6 @@ steps: set PYTHON=python call conda-recipe\bld.bat IF %ERRORLEVEL% neq 0 EXIT /b %ERRORLEVEL% - set DALROOT=%CONDA_PREFIX% - python setup_sklearnex.py install --single-version-externally-managed --record=record_sklearnex.txt displayName: 'Build daal4py/sklearnex' - script: | set PATH=C:\msys64\usr\bin;%PATH% diff --git a/.ci/pipeline/docs.yml b/.ci/pipeline/docs.yml index ba6e923511..e0eb865d0a 100644 --- a/.ci/pipeline/docs.yml +++ b/.ci/pipeline/docs.yml @@ -67,7 +67,6 @@ jobs: export PREFIX=$(dirname $(dirname $(which python))) export DALROOT=$PREFIX ./conda-recipe/build.sh - python setup_sklearnex.py install --single-version-externally-managed --record=record_sklearnex.txt displayName: 'Build daal4py/sklearnex' - script: | export LD_LIBRARY_PATH=$(dirname $(dirname $(which python)))/lib:$LD_LIBRARY_PATH diff --git a/.ci/pipeline/nightly.yml b/.ci/pipeline/nightly.yml index 56a58eba78..b6f32c0ff4 100644 --- a/.ci/pipeline/nightly.yml +++ b/.ci/pipeline/nightly.yml @@ -82,7 +82,6 @@ jobs: conda activate CB export DALROOT=$CONDA_PREFIX ./conda-recipe/build.sh - python setup_sklearnex.py install --single-version-externally-managed --record=record.txt displayName: 'Build daal4py/sklearnex' - script: | . /usr/share/miniconda/etc/profile.d/conda.sh diff --git a/INSTALL.md b/INSTALL.md index 9717d497c0..b3f56c1ec8 100755 --- a/INSTALL.md +++ b/INSTALL.md @@ -139,15 +139,34 @@ Intel(R) Extension for Scikit-learn* is easily built from the sources with the m The package is available for Windows* OS, Linux* OS, and macOS*. +Multi-node (distributed) and streaming support can be disabled if needed. + +The build-process (using setup.py) happens in 4 stages: +1. Creating C++ and Cython sources from oneDAL C++ headers +2. Building oneDAL Python interfaces via cmake and pybind11 +3. Running Cython on generated sources +4. Compiling and linking them + ### Prerequisites * Python version >= 3.8, <= 3.12 -* daal4py >= 2024.3 - -**TIP:** [Build daal4py from sources](https://github.com/intel/scikit-learn-intelex/blob/main/daal4py/INSTALL.md) or get it from [distribution channels](https://intelpython.github.io/daal4py/#getting-daal4py). +* Jinja2 +* Cython +* Numpy +* cmake and pybind11 +* A C++ compiler with C++11 support +* Clang-Format +* [Intel® oneAPI Data Analytics Library (oneDAL)](https://github.com/oneapi-src/oneDAL) version 2021.1 or later + * You can use the pre-built `dal-devel` conda package from conda-forge channel +* MPI (optional, needed for distributed mode) + * You can use the pre-built `impi-devel` conda package from conda-forge channel +* A DPC++ compiler (optional, needed for DPC++ interfaces) ### Configure the Build with Environment Variables * ``SKLEARNEX_VERSION``: sets the package version * ``DALROOT``: sets the oneAPI Data Analytics Library path +* ``NO_DIST``: set to '1', 'yes' or alike to build without support for distributed mode +* ``NO_STREAM``: set to '1', 'yes' or alike to build without support for streaming mode +* ``OFF_ONEDAL_IFACE``: set to '1' to build without the support of oneDAL interfaces ### Build Intel(R) Extension for Scikit-learn @@ -155,25 +174,25 @@ The package is available for Windows* OS, Linux* OS, and macOS*. ```bash cd - python setup_sklearnex.py install + python setup.py install ``` - To install the package in the development mode: ```bash cd - python setup_sklearnex.py develop + python setup.py develop ``` -- To install scikit-learn-intelex without downloading daal4py: +- To install scikit-learn-intelex without checking for dependencies: ```bash cd - python setup_sklearnex.py install --single-version-externally-managed --record=record.txt + python setup.py install --single-version-externally-managed --record=record.txt ``` ```bash cd - python setup_sklearnex.py develop --no-deps + python setup.py develop --no-deps ``` Where: diff --git a/daal4py/INSTALL.md b/daal4py/INSTALL.md deleted file mode 100755 index b559225bfd..0000000000 --- a/daal4py/INSTALL.md +++ /dev/null @@ -1,128 +0,0 @@ -# Building from sources -daal4py is easily built from sources with the majority of the necessary prerequisites available on conda. The instructions below detail how to gather the prerequisites, set your build environment, and finally build and install the completed package. daal4py can be built for all three major platforms (Windows, Linux, macOS). Multi-node (distributed) and streaming support can be disabled if needed. - -The build-process (using setup.py) happens in 3 stages: -1. Creating C++ and Cython sources from oneDAL C++ headers -2. Building oneDAL Python interfaces via cmake and pybind11 -3. Running Cython on generated sources -4. Compiling and linking them - -To build with oneAPI support, additional steps are required: -1. Point to DPC++ compiler by defining ``DPCPPROOT`` variable. - -```bash -export DPCPPROOT=/opt/intel/oneapi/compiler/latest -``` -2. Install Intel(R) oneAPI Data Analytics Library with oneAPI support: - - - From Conda channel. - - From oneAPI packages repository (pass the path to oneDAL via ``DALROOT`` variable) - -```bash -export DALROOT=/opt/intel/oneapi/dal/latest -``` - -## Building daal4py using conda-build -The easiest way to build daal4py is using the conda-build with the provided recipe. - -### Prerequisites -* Python version >= 3.6 -* conda-build version >= 3 -* C++ compiler with C++11 support - -For oneAPI support: -* A DPC++ compiler -* Intel(R) oneAPI Data Analytics Library version 2021.1 or later (https://github.com/oneapi-src/oneDAL) - * You can use the pre-built conda package from Intel channel on anaconda.org - -### Building daal4py -Library build command: -```bash -cd -conda build conda-recipe -c intel -c conda-forge -``` -This will build the conda package and tell you where to find it (```.../daal4py*.tar.bz2```). - -### Installing the built daal4py conda package -``` -conda install -``` -To actually use your daal4py, dependent packages need to be installed. To ensure, do - -Linux and Windows: -``` -conda install -c intel impi_rt dal numpy -``` -OsX: -``` -conda install -c intel -c conda-forge mpich dal numpy -``` - -## Building daal4py without conda-build -Without conda-build you have to manually setup your environment before building daal4py. - -### Prerequisites -* Python version >= 3.6 -* Jinja2 -* Cython -* Numpy -* cmake and pybind11 - * Needed for oneDAL interfaces. You can disable the support for oneDAL interfaces by setting OFF_ONEDAL_IFACE to '1'. -* A C++ compiler with C++11 support -* Clang-Format -* [Intel® oneAPI Data Analytics Library (oneDAL)](https://github.com/oneapi-src/oneDAL) version 2021.1 or later - * You can use the pre-built conda package from Intel channel on anaconda.org -* MPI - * You can use the pre-built conda package intel or conda-forge channel on anaconda.org - * Needed for distributed mode. You can disable support for distributed mode by setting NO_DIST to '1' or 'yes' -For oneAPI support -* A DPC++ compiler -* [Intel® oneAPI Data Analytics Library (oneDAL)](https://github.com/oneapi-src/oneDAL) version 2021.1 or later - * You can use the [pre-built conda package](https://anaconda.org/intel/dal) from Intel channel on anaconda.org. - -### Setting up a build environment -The easiest path for getting cython, oneDAL, MPI etc. is by creating a conda environment and setting environment variables: -``` -conda create -n DAAL4PY python=3.7 impi-devel dal dal-include cython jinja2 numpy clang-tools cmake pybind11 -c intel -c conda-forge -conda activate DAAL4PY -export DALROOT=$CONDA_PREFIX -export MPIROOT=$CONDA_PREFIX -``` - -### Configuring the build with environment variables -* DAAL4PY_VERSION: sets package version -* NO_DIST: set to '1', 'yes' or alike to build without support for distributed mode -* NO_STREAM: set to '1', 'yes' or alike to build without support for streaming mode -* OFF_ONEDAL_IFACE: set to '1' to build without the support of oneDAL interfaces - -### Notes on building for macOS -If building in High Sierra or higher, one may have to run into C++ build errors related to platform targets. Utilize ```export MACOSX_DEPLOYMENT_TARGET="10.9"``` if running into platform target issues. - -### Building daal4py -Requires Intel(R) oneAPI Data Analytics Library and Intel(R) MPI Library being properly set up, meaning you have to set DALROOT and MPIROOT variables. - -For install everything from build directory - -```bash -cd -python setup.py install --single-version-externally-managed --record=record.txt -``` - -For install package in development mode - -```bash -cd -python setup.py develop --no-deps -``` - -⚠️ Keys `--single-version-externally-managed` and `--no-deps` are required so that release packages are not downloads after installation of daal4py - -## Building documentation for daal4py -### Prerequisites for creating documentation -* sphinx -* sphinx_rtd_theme - -### Building documentation -1. Install daal4py into your python environment -2. ```cd doc/daal4py && make html``` -3. The documentation will be in ```doc/daal4py/_build/html``` diff --git a/daal4py/README.md b/daal4py/README.md index 4c4a64bd15..ae90188314 100755 --- a/daal4py/README.md +++ b/daal4py/README.md @@ -110,7 +110,7 @@ conda install impi_rt -c intel -You can [build daal4py from sources](https://github.com/intel/scikit-learn-intelex/blob/main/daal4py/INSTALL.md) as well. +You can [build daal4py from sources](https://github.com/intel/scikit-learn-intelex/blob/main/INSTALL.md) as well. # ⚠️ Scikit-learn patching diff --git a/setup.cfg b/setup.cfg index 0538c4d848..57ccf419a9 100755 --- a/setup.cfg +++ b/setup.cfg @@ -2,6 +2,7 @@ license_files = LICENSE doc/daal4py/third-party-programs.txt + doc/third-party-programs-sklearnex.txt [flake8] ignore = E265, E722, E402, F401, F403, W503 diff --git a/setup.py b/setup.py index 5e40c63d3e..1c5a815598 100644 --- a/setup.py +++ b/setup.py @@ -80,9 +80,9 @@ os.environ.get("OFF_ONEDAL_IFACE", "0") == "0" and ONEDAL_VERSION >= ONEDAL_2021_3 ) -d4p_version = ( - os.environ["DAAL4PY_VERSION"] - if "DAAL4PY_VERSION" in os.environ +sklearnex_version = ( + os.environ["SKLEARNEX_VERSION"] + if "SKLEARNEX_VERSION" in os.environ else time.strftime("%Y%m%d.%H%M%S") ) @@ -261,7 +261,7 @@ def get_build_options(): ] eca = [ "-DPY_ARRAY_UNIQUE_SYMBOL=daal4py_array_API", - '-DD4P_VERSION="' + d4p_version + '"', + '-DD4P_VERSION="' + sklearnex_version + '"', "-DNPY_ALLOW_THREADS=1", ] ela = [] @@ -398,7 +398,7 @@ def gen_pyx(odir): odir = os.path.abspath(odir) if not os.path.isdir(odir): os.mkdir(odir) - gen_daal4py(dal_root, odir, d4p_version, no_dist=no_dist, no_stream=no_stream) + gen_daal4py(dal_root, odir, sklearnex_version, no_dist=no_dist, no_stream=no_stream) gen_pyx(os.path.abspath("./build")) @@ -555,6 +555,25 @@ def run(self): "onedal.primitives", "onedal.svm", "onedal.utils", + "sklearnex", + "sklearnex.basic_statistics", + "sklearnex.cluster", + "sklearnex.covariance", + "sklearnex.decomposition", + "sklearnex.ensemble", + "sklearnex.glob", + "sklearnex.linear_model", + "sklearnex.manifold", + "sklearnex.metrics", + "sklearnex.model_selection", + "sklearnex.neighbors", + "sklearnex.preview", + "sklearnex.preview.covariance", + "sklearnex.preview.cluster", + "sklearnex.preview.decomposition", + "sklearnex.preview.linear_model", + "sklearnex.svm", + "sklearnex.utils", ] if ONEDAL_VERSION >= 20230100: @@ -569,24 +588,32 @@ def run(self): "onedal.spmd.covariance", "onedal.spmd.decomposition", "onedal.spmd.ensemble", + "sklearnex.spmd", + "sklearnex.spmd.covariance", + "sklearnex.spmd.decomposition", + "sklearnex.spmd.ensemble", ] if ONEDAL_VERSION >= 20230100: packages_with_tests += [ "onedal.spmd.basic_statistics", "onedal.spmd.linear_model", "onedal.spmd.neighbors", + "sklearnex.spmd.basic_statistics", + "sklearnex.spmd.linear_model", + "sklearnex.spmd.neighbors", ] if ONEDAL_VERSION >= 20230200: - packages_with_tests += ["onedal.spmd.cluster"] + packages_with_tests += ["onedal.spmd.cluster", "sklearnex.spmd.cluster"] setup( - name="daal4py", - description="A convenient Python API to Intel(R) oneAPI Data Analytics Library", + name="scikit-learn-intelex", + description="Intel(R) Extension for Scikit-learn is a " + "seamless way to speed up your Scikit-learn application.", long_description=long_description, long_description_content_type="text/markdown", license="Apache-2.0", author="Intel Corporation", - version=d4p_version, + version=sklearnex_version, url="https://github.com/intel/scikit-learn-intelex", author_email="onedal.maintainers@intel.com", maintainer_email="onedal.maintainers@intel.com", @@ -606,6 +633,7 @@ def run(self): "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Topic :: Scientific/Engineering", "Topic :: System", "Topic :: Software Development", diff --git a/setup_sklearnex.cfg b/setup_sklearnex.cfg deleted file mode 100755 index e0d2558251..0000000000 --- a/setup_sklearnex.cfg +++ /dev/null @@ -1,4 +0,0 @@ -[metadata] -license_files = - LICENSE - doc/third-party-programs-sklearnex.txt diff --git a/setup_sklearnex.py b/setup_sklearnex.py deleted file mode 100755 index 3a62a9ea1a..0000000000 --- a/setup_sklearnex.py +++ /dev/null @@ -1,149 +0,0 @@ -#! /usr/bin/env python -# ============================================================================== -# Copyright 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -# System imports -import os -import shutil -import sys -import time - -from setuptools import setup - -from scripts.package_helpers import get_packages_with_tests -from scripts.version import get_onedal_version - -sklearnex_version = ( - os.environ["SKLEARNEX_VERSION"] - if "SKLEARNEX_VERSION" in os.environ - else time.strftime("%Y%m%d.%H%M%S") -) - -IS_WIN = False -IS_MAC = False -IS_LIN = False - -if "linux" in sys.platform: - IS_LIN = True -elif sys.platform == "darwin": - IS_MAC = True -elif sys.platform in ["win32", "cygwin"]: - IS_WIN = True -else: - assert False, sys.platform + " not supported" - -dal_root = os.environ.get("DALROOT") - -if dal_root is None: - raise RuntimeError("Not set DALROOT variable") - -trues = ["true", "True", "TRUE", "1", "t", "T", "y", "Y", "Yes", "yes", "YES"] -no_dist = True if "NO_DIST" in os.environ and os.environ["NO_DIST"] in trues else False -dpcpp = shutil.which("icpx") is not None - -build_distribute = dpcpp and not no_dist and IS_LIN - -ONEDAL_VERSION = get_onedal_version(dal_root) - -project_urls = { - "Bug Tracker": "https://github.com/intel/scikit-learn-intelex/issues", - "Documentation": "https://intel.github.io/scikit-learn-intelex/", - "Source Code": "https://github.com/intel/scikit-learn-intelex", -} - -with open("README.md", "r", encoding="utf8") as f: - long_description = f.read() - -packages_with_tests = [ - "sklearnex", - "sklearnex.basic_statistics", - "sklearnex.cluster", - "sklearnex.covariance", - "sklearnex.decomposition", - "sklearnex.ensemble", - "sklearnex.glob", - "sklearnex.linear_model", - "sklearnex.manifold", - "sklearnex.metrics", - "sklearnex.model_selection", - "sklearnex.neighbors", - "sklearnex.preview", - "sklearnex.preview.covariance", - "sklearnex.preview.cluster", - "sklearnex.preview.decomposition", - "sklearnex.preview.linear_model", - "sklearnex.svm", - "sklearnex.utils", -] - -if build_distribute: - packages_with_tests += [ - "sklearnex.spmd", - "sklearnex.spmd.covariance", - "sklearnex.spmd.decomposition", - "sklearnex.spmd.ensemble", - ] - if ONEDAL_VERSION >= 20230100: - packages_with_tests += [ - "sklearnex.spmd.basic_statistics", - "sklearnex.spmd.linear_model", - "sklearnex.spmd.neighbors", - ] - if ONEDAL_VERSION >= 20230200: - packages_with_tests += ["sklearnex.spmd.cluster"] - -# sklearnex setup -setup( - name="scikit-learn-intelex", - description="Intel(R) Extension for Scikit-learn is a " - "seamless way to speed up your Scikit-learn application.", - long_description=long_description, - long_description_content_type="text/markdown", - license="Apache-2.0", - author="Intel Corporation", - version=sklearnex_version, - url="https://github.com/intel/scikit-learn-intelex", - author_email="onedal.maintainers@intel.com", - maintainer_email="onedal.maintainers@intel.com", - project_urls=project_urls, - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Environment :: Console", - "Intended Audience :: Developers", - "Intended Audience :: Other Audience", - "Intended Audience :: Science/Research", - "License :: OSI Approved :: Apache Software License", - "Operating System :: Microsoft :: Windows", - "Operating System :: POSIX :: Linux", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Topic :: Scientific/Engineering", - "Topic :: System", - "Topic :: Software Development", - ], - python_requires=">=3.8", - install_requires=["daal4py>=2024.3", "scikit-learn>=1.0"], - keywords=[ - "machine learning", - "scikit-learn", - "data science", - "data analytics", - ], - packages=get_packages_with_tests(packages_with_tests), -)