From da5086bed8f0a17dfeb7a9c38caf3c89e079f358 Mon Sep 17 00:00:00 2001 From: Saurav Maheshkar Date: Tue, 9 Apr 2024 13:51:19 +0100 Subject: [PATCH 1/8] chore(build): migrate packaging to pyproject --- MANIFEST.in | 4 -- pyproject.toml | 65 ++++++++++++++++++++++++++++++ readthedocs.yml | 8 ++-- requirements.txt | 3 -- requirements_docs.txt | 11 ------ requirements_examples.txt | 1 - requirements_tests.txt | 3 -- setup.py | 83 --------------------------------------- 8 files changed, 70 insertions(+), 108 deletions(-) delete mode 100644 MANIFEST.in create mode 100644 pyproject.toml delete mode 100644 requirements.txt delete mode 100644 requirements_docs.txt delete mode 100644 requirements_examples.txt delete mode 100644 requirements_tests.txt delete mode 100644 setup.py diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 0c57ac1..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1,4 +0,0 @@ -include README.md -include LICENSE -include requirements*.txt -include dm_pix/py.typed diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..e75ff1d --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,65 @@ +[build-system] +requires = ["flit_core >=3.2,<4"] +build-backend = "flit_core.buildapi" + +[project] +name = "dm_pix" +dynamic = ["version"] +description = 'PIX is an image processing library in JAX, for JAX.' +readme = "README.md" +license = { file = "LICENSE" } +requires-python = ">=3.8" +authors = [ + {name = "Google DeepMind", email = "pix-dev@google.com"}, +] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Intended Audience :: Education", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Topic :: Scientific/Engineering :: Image Processing", + "Topic :: Scientific/Engineering :: Mathematics", + "Topic :: Software Development :: Libraries", + "Topic :: Software Development :: Libraries :: Python Modules", +] +dependencies = [ + "chex>=0.0.6", +] + +[project.optional-dependencies] +extras = [ + "jax>=0.2.17", + "jaxlib>=0.1.69", +] + +tests = [ + "scipy", + "tensorflow", + "pytest-xdist", +] + +docs = [ + "sphinx==4.5.0", + "sphinx_rtd_theme==1.0.0", + "sphinxcontrib-katex==0.9.0", + "sphinxcontrib-bibtex==2.4.2", + "sphinx-autodoc-typehints==1.11.1", + "IPython==8.10.0", + "ipykernel==5.3.4", + "pandoc==1.1.0", + "myst_nb==0.13.1", + "docutils==0.16", + "matplotlib==3.5.0", +] + +examples = [ + "Pillow==10.3.0", +] + +[tool.setuptools.packages.find] +include=["dm_pix/py.typed"] +exclude = ["*_test.py", "examples"] diff --git a/readthedocs.yml b/readthedocs.yml index 0b22755..827ed76 100644 --- a/readthedocs.yml +++ b/readthedocs.yml @@ -15,7 +15,9 @@ sphinx: python: install: - - requirements: requirements_docs.txt - - requirements: requirements.txt - - method: setuptools + - method: pip path: . + - method: pip + path: . + extra_requirements: + - docs diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index aaea523..0000000 --- a/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -chex>=0.0.6 -# jax>=0.2.17 -# jaxlib>=0.1.69 diff --git a/requirements_docs.txt b/requirements_docs.txt deleted file mode 100644 index 1fbb409..0000000 --- a/requirements_docs.txt +++ /dev/null @@ -1,11 +0,0 @@ -sphinx==4.5.0 -sphinx_rtd_theme==1.0.0 -sphinxcontrib-katex==0.9.0 -sphinxcontrib-bibtex==2.4.2 -sphinx-autodoc-typehints==1.11.1 -IPython==8.10.0 -ipykernel==5.3.4 -pandoc==1.1.0 -myst_nb==0.13.1 -docutils==0.16 -matplotlib==3.5.0 diff --git a/requirements_examples.txt b/requirements_examples.txt deleted file mode 100644 index 7f361a8..0000000 --- a/requirements_examples.txt +++ /dev/null @@ -1 +0,0 @@ -Pillow==10.3.0 diff --git a/requirements_tests.txt b/requirements_tests.txt deleted file mode 100644 index aab4d65..0000000 --- a/requirements_tests.txt +++ /dev/null @@ -1,3 +0,0 @@ -pytest-xdist -tensorflow -scipy diff --git a/setup.py b/setup.py deleted file mode 100644 index 9f276e0..0000000 --- a/setup.py +++ /dev/null @@ -1,83 +0,0 @@ -# Copyright 2021 DeepMind Technologies Limited. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Setup for pip package.""" - -import os -from setuptools import find_namespace_packages -from setuptools import setup - -_CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) - - -def _get_version(): - with open(os.path.join(_CURRENT_DIR, 'dm_pix', '__init__.py')) as fp: - for line in fp: - if line.startswith('__version__') and '=' in line: - version = line[line.find('=') + 1:].strip(' \'"\n') - if version: - return version - raise ValueError('`__version__` not defined in `dm_pix/__init__.py`') - - -def _parse_requirements(requirements_txt_path): - with open(requirements_txt_path) as f: - return [ - line.rstrip() - for line in f - if not (line.isspace() or line.startswith('#')) - ] - - -_VERSION = _get_version() -_EXTRA_PACKAGES = { - 'jax': ['jax>=0.2.17'], - 'jaxlib': ['jaxlib>=0.1.69'], -} - -setup( - name='dm_pix', - version=_VERSION, - url='https://github.com/deepmind/dm_pix', - license='Apache 2.0', - author='DeepMind', - description='PIX is an image processing library in JAX, for JAX.', - long_description=open(os.path.join(_CURRENT_DIR, 'README.md')).read(), - long_description_content_type='text/markdown', - author_email='pix-dev@google.com', - # Contained modules and scripts. - packages=find_namespace_packages(exclude=['*_test.py', 'examples']), - install_requires=_parse_requirements( - os.path.join(_CURRENT_DIR, 'requirements.txt')), - extras_require=_EXTRA_PACKAGES, - tests_require=_parse_requirements( - os.path.join(_CURRENT_DIR, 'requirements_tests.txt')), - requires_python='>=3.8', - include_package_data=True, - zip_safe=False, - # PyPI package information. - classifiers=[ - 'Development Status :: 4 - Beta', - 'Intended Audience :: Developers', - 'Intended Audience :: Education', - 'Intended Audience :: Science/Research', - 'License :: OSI Approved :: Apache Software License', - 'Programming Language :: Python', - 'Programming Language :: Python :: 3', - 'Topic :: Scientific/Engineering :: Artificial Intelligence', - 'Topic :: Scientific/Engineering :: Image Processing', - 'Topic :: Scientific/Engineering :: Mathematics', - 'Topic :: Software Development :: Libraries', - 'Topic :: Software Development :: Libraries :: Python Modules', - ], -) From 657736c09e68e8168eba32603c7bdbe862e50ed8 Mon Sep 17 00:00:00 2001 From: Saurav Maheshkar Date: Tue, 9 Apr 2024 13:51:44 +0100 Subject: [PATCH 2/8] chore(build): adapt ci to pyproject --- .github/workflows/ci.yml | 4 +++- .github/workflows/pypi-publish.yml | 7 ++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7ffef83..49a4108 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,9 +18,11 @@ jobs: steps: - uses: "actions/checkout@v2" - - uses: "actions/setup-python@v1" + - uses: "actions/setup-python@v4" with: python-version: "${{ matrix.python-version }}" + cache: "pip" + cache-dependency-path: "pyproject.toml" - name: Run CI tests run: bash test.sh shell: bash diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml index 238420b..5828639 100644 --- a/.github/workflows/pypi-publish.yml +++ b/.github/workflows/pypi-publish.yml @@ -16,11 +16,12 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install setuptools wheel twine + pip install setuptools wheel twine build - name: Check consistency between the package version and release tag run: | + pip install . RELEASE_VER=${GITHUB_REF#refs/*/} - PACKAGE_VER="v`python setup.py --version`" + PACKAGE_VER="v`python -c 'import dm_pix; print(dm_pix.__version__)'`" if [ $RELEASE_VER != $PACKAGE_VER ] then echo "package ver. ($PACKAGE_VER) != release ver. ($RELEASE_VER)"; exit 1 @@ -30,5 +31,5 @@ jobs: TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} run: | - python setup.py sdist bdist_wheel + python -m build twine upload dist/* From 44b17728a937ae2777d17d2b52c2ae3ba03f2e16 Mon Sep 17 00:00:00 2001 From: Saurav Maheshkar Date: Tue, 9 Apr 2024 13:57:10 +0100 Subject: [PATCH 3/8] chore(build): adapt test script to pyproject --- test.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test.sh b/test.sh index d89cdb6..7e89870 100755 --- a/test.sh +++ b/test.sh @@ -26,20 +26,20 @@ python --version # Install JAX. python -m pip install --upgrade pip setuptools -python -m pip install -r requirements.txt -python -c 'import jax; print(jax.__version__)' # Run setup.py, this installs the python dependencies python -m pip install . +python -c 'import jax; print(jax.__version__)' + # Python test dependencies. -python -m pip install -r requirements_tests.txt +python -m pip install -e ".[test]" # CPU count on macos or linux if [ "$(uname)" == "Darwin" ]; then - N_JOBS=$(sysctl -n hw.logicalcpu) + N_JOBS=$(sysctl -n hw.logicalcpu) else - N_JOBS=$(grep -c ^processor /proc/cpuinfo) + N_JOBS=$(grep -c ^processor /proc/cpuinfo) fi # Run tests using pytest. From 495b8a80624f4da429fb1777d5bbdc9e6099c606 Mon Sep 17 00:00:00 2001 From: Saurav Maheshkar Date: Tue, 9 Apr 2024 14:26:40 +0100 Subject: [PATCH 4/8] fix: rename dependency group to `test` --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e75ff1d..518c850 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,7 @@ extras = [ "jaxlib>=0.1.69", ] -tests = [ +test = [ "scipy", "tensorflow", "pytest-xdist", From 47df7d61ee4b2439d8e6841c349f2564339fb270 Mon Sep 17 00:00:00 2001 From: Saurav Maheshkar Date: Thu, 18 Apr 2024 10:52:22 +0100 Subject: [PATCH 5/8] feat: add transitive `jax` deps --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 518c850..3ae52ec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,8 @@ classifiers = [ ] dependencies = [ "chex>=0.0.6", + # jax>=0.2.17 + # jaxlib>=0.1.69 ] [project.optional-dependencies] From 12d754d8d0a0aa942c206951b692e2491f83aa52 Mon Sep 17 00:00:00 2001 From: Saurav Maheshkar Date: Thu, 18 Apr 2024 10:54:19 +0100 Subject: [PATCH 6/8] docs: update comments in `test.sh` --- test.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/test.sh b/test.sh index 7e89870..d0df45f 100755 --- a/test.sh +++ b/test.sh @@ -24,12 +24,11 @@ python -m venv "${VENV_DIR}" source "${VENV_DIR}/bin/activate" python --version -# Install JAX. +# Update pip + setuptools and install dependencies specified in pyproject.toml python -m pip install --upgrade pip setuptools - -# Run setup.py, this installs the python dependencies python -m pip install . +# print jax version python -c 'import jax; print(jax.__version__)' # Python test dependencies. @@ -37,9 +36,9 @@ python -m pip install -e ".[test]" # CPU count on macos or linux if [ "$(uname)" == "Darwin" ]; then - N_JOBS=$(sysctl -n hw.logicalcpu) + N_JOBS=$(sysctl -n hw.logicalcpu) else - N_JOBS=$(grep -c ^processor /proc/cpuinfo) + N_JOBS=$(grep -c ^processor /proc/cpuinfo) fi # Run tests using pytest. From 2a2d52689593bd934db30ad5137242cc58f0cb5c Mon Sep 17 00:00:00 2001 From: Saurav Maheshkar Date: Tue, 23 Apr 2024 10:07:23 +0100 Subject: [PATCH 7/8] docs: update README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0e6f0b3..b1cb348 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ through [`jax.jit`][jit], [`jax.vmap`][vmap] and [`jax.pmap`][pmap]. PIX is written in pure Python, but depends on C++ code via JAX. Because JAX installation is different depending on your CUDA version, PIX does -not list JAX as a dependency in [`requirements.txt`], although it is technically +not list JAX as a dependency in [`pyproject.toml`], although it is technically listed for reference, but commented. First, follow [JAX installation instructions] to install JAX with the relevant @@ -110,7 +110,7 @@ If you already have PIX installed, you just need to install some extra dependencies and run `pytest` as follows: ```bash -$ pip install -r requirements_tests.txt +$ pip install -e ".[test]" $ python -m pytest [-n ] dm_pix ``` From 2fac0163ad4c6c5fe456dec6db858463f5f65005 Mon Sep 17 00:00:00 2001 From: Saurav Maheshkar Date: Tue, 23 Apr 2024 10:15:26 +0100 Subject: [PATCH 8/8] docs: fix link in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b1cb348..2161dd2 100644 --- a/README.md +++ b/README.md @@ -147,4 +147,4 @@ Please read our [contributing guidelines](./CONTRIBUTING.md) and send us PRs! [`examples/`]: ./examples/ [JAX logo]: ./examples/assets/jax_logo.jpg [JAX logo left-right]: ./examples/assets/flip_left_right_jax_logo.jpg -[`requirements.txt`]: ./requirements.txt +[`pyproject.toml`]: ./pyproject.toml