diff --git a/build/build_container.sh b/build/build_container.sh index 966d0e2..8556eb9 100755 --- a/build/build_container.sh +++ b/build/build_container.sh @@ -10,6 +10,7 @@ RAPIDS_VERSION="23.08" CUDA_VERSION="12.0" UBUNTU_VERSION="20.04" PYTHON_VERSION="3.10" +REPO_BRANCH="main" DOCKERFILE_DIR=docker/ RECIPE_ONLY=0 @@ -29,6 +30,7 @@ function print_help() { echo " --cuda-version CUDA_VERSION Defines which version of CUDA will be used (default='$CUDA_VERSION')." echo " --os-version OS_VERSION Defines which version of the container will be used (default='$UBUNTU_VERSION')." echo " --python-version PYTHON_VERSION Defines which version of the python interpreter will be used (default='$PYTHON_VERSION')." + echo " --repo-branch REPO_BRANCH Defines which branch from the DASF repositary to install (default='$REPO_BRANCH')." echo " --format FORMAT Select the container backend for this build." echo " Use 'docker' for Docker images." echo " Use 'singularity' for SIF images (default='$FORMAT')." @@ -70,6 +72,11 @@ while [[ $# -gt 0 ]]; do shift shift ;; + --repo-branch) + REPO_BRANCH="$2" + shift + shift + ;; --format) FORMAT="$2" shift @@ -151,6 +158,7 @@ hpccm --recipe hpccm/build_docker.py \ cuda-version=$CUDA_VERSION \ ubuntu-version=$UBUNTU_VERSION \ python-version=$PYTHON_VERSION \ + repo-branch=$REPO_BRANCH \ --format $FORMAT > $DOCKERFILE_DIR/$OUTPUT_FILE if [ $RECIPE_ONLY -eq 1 ]; then diff --git a/build/hpccm/build_docker.py b/build/hpccm/build_docker.py index 1e64821..93000ac 100644 --- a/build/hpccm/build_docker.py +++ b/build/hpccm/build_docker.py @@ -19,6 +19,7 @@ def str2bool(string): rapidsai_version = USERARG.get('rapids-version', '23.08') ubuntu_version = USERARG.get('ubuntu-version', '20.04') python_version = USERARG.get('python-version', '3.10') +repo_branch = USERARG.get('repo-branch', 'main') if python_version: python_version = f"-py{python_version}" @@ -54,7 +55,7 @@ def str2bool(string): # Install NVIDIA NSight packages package_list += ["nsight-compute"] -pip_package_install = "pip3 install --extra-index-url https://test.pypi.org/simple/ XPySom-dask git+https://github.com/discovery-unicamp/dasf-core.git" +pip_package_install = f"pip3 install -U git+https://github.com/discovery-unicamp/dasf-core.git@{repo_branch}" if device_target.lower() == "cpu": packages_list.extend(["python3-pip"]) @@ -70,22 +71,18 @@ def str2bool(string): Stage0 += apt_get(keys=apt_keys, ospackages=packages_list) Stage0 += apt_get(ospackages=packages_list) - pip_package_install = ("%s cupy_xarray" % pip_package_install) + Stage0 += shell(commands=["pip install --no-dependencies cupy_xarray==0.1.3"]) # this avoids CuPy being installed twice and taking too long (installation process doesn't find CuPy because its named cupy_cuda12x) if is_devel: pip_package_install = ("%s %s" % (pip_package_install, "git+https://github.com/cupy/cupy.git")) else: - pip_package_install = ("%s %s" % (pip_package_install, "cupy==13.0.0b1")) - Stage0 += shell(commands=["rm -r /usr/local/lib/python3.10/dist-packages/cupy_cuda12x-12.0.0b3.dist-info"]) # not the best solution but it works + pip_package_install = ("%s %s" % (pip_package_install, "cupy-cuda12x==13.2.0")) Stage0 += shell(commands=["pip3 install pip --upgrade"]) Stage0 += shell(commands=[pip_package_install]) -# TODO: fix numpy issue with version 1.24 and other fixed reqs -Stage0 += shell(commands=["pip install \"numpy<1.24\" \"pandas==1.5.2\" bokeh==2.4.3 \"protobuf<=3.20.1\" \"charset-normalizer<3.0\" \"tornado<6.2\""]) - if is_devel: Stage0 += shell(commands=["wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && bash Miniconda3-latest-Linux-x86_64.sh -b && cp /root/miniconda3/bin/conda /usr/bin/conda"]) diff --git a/pyproject.toml b/pyproject.toml index 4309bb8..c3c51b8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ classifiers = [ "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", "Operating System :: OS Independent", "Development Status :: 1 - Planning", "Environment :: GPU :: NVIDIA CUDA", @@ -22,37 +23,43 @@ readme="README.md" exclude = ["tests/*", "docs/*"] [tool.poetry.dependencies] -dacite = "*" -dask = "*" -dask_cuda = "*" -dask_jobqueue = "*" -dask_memusage = "*" -dask_ml = "*" -dask-pytorch-ddp = "*" -GPUtil = "*" +bokeh = "2.4.3" +charset-normalizer = "<3.0" +dacite = "1.8.1" +dask = "2023.3.2" +dask_cuda = "23.4.0" +dask_jobqueue = "0.8.2" +dask_memusage = "1.1" +dask_ml = "2023.3.24" +dask-pytorch-ddp = "0.2.2" +GPUtil = "1.4.0" gdown = "^4.6" -graphviz = "*" -h5py = "*" -hdbscan = "*" -ipympl = "*" -matplotlib = "*" -memray = "*" -networkx = "*" -ormsgpack = "*" -packaging = "*" -portalocker = "*" -protobuf = "~=3.20.1" -psutil = "*" -pyarrow = "*" +graphviz = "0.20.1" +h5py = "3.10.0" +hdbscan = "0.8.33" +ipympl = "0.9.3" +joblib = "1.1.1" +matplotlib = "3.7.1" +memray = "1.10.0" +networkx = "3.2.1" +numpy = "1.23.5" +ormsgpack = "1.4.1" +packaging = "23.1" +pandas = "1.5.2" +portalocker = "2.8.2" +protobuf = "<=3.20.1" +psutil = "5.9.4" +pyarrow = "10.0.1.dev0+ga6eabc2b.d20230609" python = "^3.8" pytorch-lightning = "*" -scikit-learn = "*" -torchvision = "*" -xarray = "*" -xgboost = "*" -zarr = "*" +scikit-learn = "1.2.0" +torchvision = "0.16.0a0" +tornado = "<6.2" +xarray = "2023.11.0" +xgboost = "1.7.5" +xpysom-dask = { git = "https://github.com/jcfaracco/xpysom-dask.git", branch = "dask" } +zarr = "2.16.1" -#xpysom = { git = "https://github.com/jcfaracco/xpysom/", branch = "dask2" } [tool.poetry.group.dev.dependencies] black = "*"