From 06649c4f595698616bc909ab4e2070002f4a541f Mon Sep 17 00:00:00 2001 From: Hannes Hapke Date: Thu, 6 Jul 2023 08:50:19 -0700 Subject: [PATCH] allow tfma > 0.42 and < 0.45 (#289) * allow tfma > 0.42 and < 0.45 * drop Python 3.7 support * bump CI Python version from 3.7 to 3.9 * updated python_requires in setup.py * updated deps * removed introduced line * CI with min version --- .github/workflows/ci.yml | 10 +-- model_card_toolkit/dependencies.py | 134 +++++++++++++++++++++++++++++ setup.py | 53 +++++------- 3 files changed, 160 insertions(+), 37 deletions(-) create mode 100644 model_card_toolkit/dependencies.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a0a0757..f7cebe7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,7 @@ on: - main - r* -concurrency: +concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true @@ -30,11 +30,11 @@ jobs: timeout-minutes: 60 steps: - - uses: actions/checkout@v2 - - name: Set up Python 3.7 - uses: actions/setup-python@v2 + - uses: actions/checkout@v3 + - name: Set up Python 3.8 + uses: actions/setup-python@v4 with: - python-version: 3.7 + python-version: 3.8 - name: Cache python environment uses: actions/cache@v2 with: diff --git a/model_card_toolkit/dependencies.py b/model_card_toolkit/dependencies.py new file mode 100644 index 0000000..2bb2977 --- /dev/null +++ b/model_card_toolkit/dependencies.py @@ -0,0 +1,134 @@ +# Copyright 2023 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Package dependencies for model-card-toolkit.""" + +import importlib +from typing import Dict, List + +_VERSIONS = { + 'absl': 'absl-py>=0.9,<1.1', + 'importlib_resources': 'importlib-resources>=1.3.0; python_version<"3.9"', + 'isort': 'isort', + 'jinja2': 'jinja2>=3.1,<3.2', + 'jsonschema': 'jsonschema>=3.2.0,<4', + 'matplotlib': 'matplotlib>=3.2.0,<4', + 'ml_metadata': 'ml-metadata>=1.5.0,<2.0.0', + 'pre-commit': 'pre-commit', + 'protobuf': 'protobuf>=3.19.0,<4', + 'pylint': 'pylint', + 'pytest': 'pytest', + 'tensorflow_data_validation': 'tensorflow-data-validation>=1.5.0,<2.0.0', + 'tensorflow_datasets': 'tensorflow-datasets>=4.8.2', + 'tensorflow_metadata': 'tensorflow-metadata>=1.5.0,<2.0.0', + 'tensorflow_model_analysis': 'tensorflow-model-analysis>=0.36.0,<0.45.0', + 'yapf': 'yapf', +} + +_REQUIRED_DEPS = [ + 'importlib_resources', # reading resource files, e.g. model card templates + 'jinja2', # rendering model card templates + 'jsonschema', # validating JSON schema + 'matplotlib', # plotting + 'protobuf', # working with model card protos +] + +_EXAMPLES_EXTRA_DEPS = [ + # Required for model_card_toolkit.documentation.examples.cats_vs_dogs + 'tensorflow_datasets', +] + +_TENSORFLOW_EXTRA_DEPS = [ + 'ml_metadata', + 'tensorflow_data_validation', + 'tensorflow_metadata', + 'tensorflow_model_analysis', +] + +_TEST_EXTRA_DEPS = ['absl', 'isort', 'pre-commit', 'pylint', 'pytest', 'yapf'] + +TENSORFLOW_EXTRA_IMPORT_ERROR_MSG = """ +This functionaliy requires `tensorflow` extra dependencies but they were not +found in your environment. You can install them with: +``` +pip install model-card-toolkit[tensorflow] +``` +""" + + +def _make_deps_list(package_names: List[str]) -> List[str]: + """Returns a list of dependencies with their constraints. + + Raises: ValueError if a `package_name` is not in the list of known dependencies. + """ + deps = [] + for package_name in package_names: + if package_name not in _VERSIONS: + raise ValueError( + f'Package {package_name} is not in the list of known dependencies: ' + f'{_VERSIONS.keys()}' + ) + deps.append(_VERSIONS[package_name]) + return deps + + +def make_required_install_packages() -> List[str]: + """Returns the list of required packages.""" + return _make_deps_list(_REQUIRED_DEPS) + + +def make_extra_packages_examples() -> List[str]: + """Returns the list of packages needed for running examples.""" + return _make_deps_list(_EXAMPLES_EXTRA_DEPS) + + +def make_extra_packages_tensorflow() -> List[str]: + """Returns the list of packages needed to use TensorFlow utils.""" + return _make_deps_list(_TENSORFLOW_EXTRA_DEPS) + + +def has_tensorflow_extra_deps() -> bool: + """Returns True if all tensorflow extra dependencies are installed.""" + return all(importlib.util.find_spec(name) for name in _TENSORFLOW_EXTRA_DEPS) + + +def ensure_tensorflow_extra_deps_installed(): + """Raises ImportError if tensorflow extra dependencies are not installed. + """ + if not has_tensorflow_extra_deps(): + raise ImportError(TENSORFLOW_EXTRA_IMPORT_ERROR_MSG) + + +def make_extra_packages_test() -> List[str]: + """Returns the list of packages needed for running tests.""" + return _make_deps_list(_TEST_EXTRA_DEPS) + + +def make_extra_packages_all() -> List[str]: + """Returns the list of all optional packages.""" + return [ + *make_extra_packages_examples(), + *make_extra_packages_tensorflow(), + *make_extra_packages_test(), + ] + + +def make_required_extra_packages() -> Dict[str, List[str]]: + """Returns the dict of required extra packages.""" + return { + 'examples': make_extra_packages_examples(), + 'tensorflow': make_extra_packages_tensorflow(), + 'test': make_extra_packages_test(), + 'all': make_extra_packages_all(), + } diff --git a/setup.py b/setup.py index ab967e9..2a89975 100644 --- a/setup.py +++ b/setup.py @@ -12,10 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Setup to install the Model Card Toolkit. - -Run with `python3 setup.py sdist bdist_wheel`. -""" +"""Setup to install the Model Card Toolkit.""" import platform import shutil @@ -25,23 +22,13 @@ from setuptools import Command, setup -REQUIRED_PACKAGES = [ - 'absl-py>=0.9,<1.1', - 'jinja2>=3.1,<3.2', - 'matplotlib>=3.2.0,<4', - 'jsonschema>=3.2.0,<4', - 'tensorflow-data-validation>=1.5.0,<2.0.0', - 'tensorflow-model-analysis>=0.36.0,<0.42.0', - 'tensorflow-metadata>=1.5.0,<2.0.0', - 'ml-metadata>=1.5.0,<2.0.0', -] - -TESTS_REQUIRE = [ - 'pytest', - 'tensorflow-datasets>=4.8.2', -] - -EXTRAS_REQUIRE = {'test': TESTS_REQUIRE} +# Get dependency lists. +with open('model_card_toolkit/dependencies.py') as fp: + globals_dict = {} + exec(fp.read(), globals_dict) # pylint: disable=exec-used +make_required_install_packages = globals_dict['make_required_install_packages'] +make_required_extra_packages = globals_dict['make_required_extra_packages'] +make_extra_packages_test = globals_dict['make_extra_packages_test'] # Get version from version module. with open('model_card_toolkit/version.py') as fp: @@ -49,6 +36,7 @@ exec(fp.read(), globals_dict) # pylint: disable=exec-used __version__ = globals_dict['__version__'] +# Get long description. with open('README.md', 'r', encoding='utf-8') as fh: _LONG_DESCRIPTION = fh.read() @@ -100,7 +88,7 @@ def run(self): [ self._bazel_cmd, 'run', '--verbose_failures', *self._additional_build_options, - 'model_card_toolkit:move_generated_files' + '//proto_build:move_generated_files' ] ) @@ -109,11 +97,14 @@ def run(self): name='model-card-toolkit', version=__version__, description='Model Card Toolkit', + author='The TensorFlow Authors', long_description=_LONG_DESCRIPTION, long_description_content_type='text/markdown', url='https://github.com/tensorflow/model-card-toolkit', - author='Google LLC', - author_email='tensorflow-extended-dev@googlegroups.com', + project_urls={ + 'Bug Tracker': 'https://github.com/tensorflow/model-card-toolkit/issues', + 'Documentation': 'https://www.tensorflow.org/responsible_ai/model_card_toolkit/guide', + }, packages=[ 'model_card_toolkit', 'model_card_toolkit.documentation', @@ -125,11 +116,10 @@ def run(self): package_data={ 'model_card_toolkit': ['schema/**/*.json', 'template/**/*.jinja'] }, - python_requires='>=3.7,<4', - install_requires=REQUIRED_PACKAGES, - tests_require=TESTS_REQUIRE, - extras_require=EXTRAS_REQUIRE, - # PyPI package information. + python_requires='>=3.8,<4', + install_requires=make_required_install_packages(), + tests_require=make_extra_packages_test(), + extras_require=make_required_extra_packages(), classifiers=[ 'Development Status :: 4 - Beta', 'Intended Audience :: Developers', @@ -138,7 +128,6 @@ def run(self): 'License :: OSI Approved :: Apache Software License', 'Operating System :: OS Independent', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Topic :: Scientific/Engineering', @@ -149,9 +138,9 @@ def run(self): 'Topic :: Software Development :: Libraries :: Python Modules', ], license='Apache 2.0', - keywords='model card toolkit ml metadata machine learning', + keywords=['model card toolkit', 'ml metadata', 'machine learning'], cmdclass={ 'build': _BuildCommand, 'bazel_build': _BazelBuildCommand, } -) +) # yapf: disable