From 921e20d298ea0b2795f173e152472f7298084af8 Mon Sep 17 00:00:00 2001 From: cdmikechen Date: Sun, 12 Nov 2023 18:53:09 +0800 Subject: [PATCH] SUBMARINE-1391. Remove Python3.7 and Tensorflow 2.6-2.10 support ### What is this PR for? Python 3.7 has reached its end of life at the end of June 2023. Meanwhile, we should keep support for the latest 3 versions of tensoflow (2.12-2.14). https://github.com/tensorflow/tensorflow/releases ### What type of PR is it? Improvement ### Todos * [x] - Remove python 3.7 and add python 3.11 * [x] - Remove tf 1.15 and 2.6-2.10 ### What is the Jira issue? https://issues.apache.org/jira/browse/SUBMARINE-1391 ### How should this be tested? ci test ### Screenshots (if appropriate) ### Questions: * Do the license files need updating? No * Are there breaking changes for older versions? Yes * Does this need new documentation? No Author: cdmikechen Signed-off-by: cdmikechen Closes #1116 from cdmikechen/tensorflow-support and squashes the following commits: 0a82f39d [cdmikechen] update test-requirement 683c0049 [cdmikechen] update require 9b4d30dc [cdmikechen] update python and tf version --- .github/workflows/python.yml | 51 +++++-------------- .../github-actions/test-requirements.txt | 4 +- submarine-sdk/pysubmarine/setup.py | 16 ++---- 3 files changed, 18 insertions(+), 53 deletions(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index a6ca7dade..974276790 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -42,19 +42,12 @@ jobs: timeout-minutes: 30 strategy: matrix: - python-version: ['3.7', '3.8', '3.9', '3.10'] - tf-version: ['1.15.0', '2.6.0', '2.7.0', '2.8.0', '2.9.0'] + python-version: ['3.8', '3.9', '3.10', '3.11'] + tf-version: ['2.12.0', '2.13.0', '2.14.0'] exclude: + # Support for Python 3.8 has been removed starting with TF 2.14. - python-version: '3.8' - tf-version: '1.15.0' - - python-version: '3.9' - tf-version: '1.15.0' - - python-version: '3.10' - tf-version: '1.15.0' - - python-version: '3.10' - tf-version: '2.6.0' - - python-version: '3.10' - tf-version: '2.7.0' + tf-version: '2.14.0' fail-fast: false steps: - uses: actions/checkout@v3 @@ -66,15 +59,10 @@ jobs: run: | pip install --upgrade pip setuptools wheel pip install -r ./submarine-sdk/pysubmarine/github-actions/test-requirements.txt - - name: Install pysubmarine with tf1 and pytorch - if: ${{ matrix.tf-version == '1.15.0' }} - run: | - pip install --no-cache-dir -e ./submarine-sdk/pysubmarine/.[tf,pytorch] - name: Install pysubmarine with tf2 and pytorch - if: startsWith(matrix.tf-version, '2.') run: | - sed -i "s/tensorflow>=2.6.0,<2.10.0/tensorflow~=${{ matrix.tf-version }}/" ./submarine-sdk/pysubmarine/setup.py - sed -i "s/tensorflow-estimator>=2.6.0,<2.10.0/tensorflow-estimator~=${{ matrix.tf-version }}/" ./submarine-sdk/pysubmarine/setup.py + sed -i "s/tensorflow>=2.12.0,<2.15.0/tensorflow~=${{ matrix.tf-version }}/" ./submarine-sdk/pysubmarine/setup.py + sed -i "s/tensorflow-estimator>=2.12.0,<2.15.0/tensorflow-estimator~=${{ matrix.tf-version }}/" ./submarine-sdk/pysubmarine/setup.py pip install --no-cache-dir -e ./submarine-sdk/pysubmarine/.[tf2,pytorch] - name: List installed packages run: pip list && pipdeptree @@ -92,19 +80,11 @@ jobs: timeout-minutes: 60 strategy: matrix: - python-version: ['3.7', '3.8', '3.9', '3.10'] - tf-version: ['1.15.0', '2.6.0', '2.7.0', '2.8.0', '2.9.0'] + python-version: ['3.8', '3.9', '3.10', '3.11'] + tf-version: ['2.12.0', '2.13.0', '2.14.0'] exclude: - python-version: '3.8' - tf-version: '1.15.0' - - python-version: '3.9' - tf-version: '1.15.0' - - python-version: '3.10' - tf-version: '1.15.0' - - python-version: '3.10' - tf-version: '2.6.0' - - python-version: '3.10' - tf-version: '2.7.0' + tf-version: '2.14.0' fail-fast: false steps: - uses: actions/checkout@v3 @@ -116,18 +96,11 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - - name: Install tf1 dependencies - if: ${{ matrix.tf-version == '1.15.0' }} - run: | - pip install --upgrade pip setuptools wheel - pip install --no-cache-dir -e ./submarine-sdk/pysubmarine/.[tf,pytorch] - pip install -r ./submarine-sdk/pysubmarine/github-actions/test-requirements.txt - name: Install tf2 dependencies - if: startsWith(matrix.tf-version, '2.') run: | pip install --upgrade pip setuptools wheel - sed -i "s/tensorflow>=2.6.0,<2.10.0/tensorflow~=${{ matrix.tf-version }}/" ./submarine-sdk/pysubmarine/setup.py - sed -i "s/tensorflow-estimator>=2.6.0,<2.10.0/tensorflow-estimator~=${{ matrix.tf-version }}/" ./submarine-sdk/pysubmarine/setup.py + sed -i "s/tensorflow>=2.12.0,<2.15.0/tensorflow~=${{ matrix.tf-version }}/" ./submarine-sdk/pysubmarine/setup.py + sed -i "s/tensorflow-estimator>=2.12.0,<2.15.0/tensorflow-estimator~=${{ matrix.tf-version }}/" ./submarine-sdk/pysubmarine/setup.py pip install --no-cache-dir -e ./submarine-sdk/pysubmarine/.[tf2,pytorch] pip install -r ./submarine-sdk/pysubmarine/github-actions/test-requirements.txt pip cache purge @@ -209,4 +182,4 @@ jobs: args: > -Dsonar.organization=apache -Dsonar.projectKey=apache_submarine_python_sdk - -Dsonar.python.version=3.7,3.8,3.9,3.10 + -Dsonar.python.version=3.8,3.9,3.10,3.11 diff --git a/submarine-sdk/pysubmarine/github-actions/test-requirements.txt b/submarine-sdk/pysubmarine/github-actions/test-requirements.txt index cf767b7eb..3e0470154 100644 --- a/submarine-sdk/pysubmarine/github-actions/test-requirements.txt +++ b/submarine-sdk/pysubmarine/github-actions/test-requirements.txt @@ -16,14 +16,14 @@ certifi>=14.05.14 freezegun==1.2.2 moto[s3]==4.0.5 -pandas>=1.2.0,<=1.3.5 # 1.4.x does not support cp37 +pandas>=1.2.0 pylint==2.15.2 PyMySQL==1.0.2 pytest==7.1.3 pytest-cov==3.0.0 pytest-localserver==0.7.0 python_dateutil>=2.5.3 -scikit-learn>=0.24.2,<=1.0.2 # 1.1.x does not support cp37 +scikit-learn>=0.24.2 setuptools>=21.0.0 urllib3>=1.15.1 pipdeptree==2.5.2 diff --git a/submarine-sdk/pysubmarine/setup.py b/submarine-sdk/pysubmarine/setup.py index 69320be65..3fd516a55 100644 --- a/submarine-sdk/pysubmarine/setup.py +++ b/submarine-sdk/pysubmarine/setup.py @@ -45,35 +45,27 @@ "pyaml", ], extras_require={ - "tf": ["tensorflow==1.15.5", "numpy>=1.16.0,<1.19.0", "protobuf>=3.6.1,<3.20"], "tf2": [ - "tensorflow>=2.6.0,<2.10.0", + "tensorflow>=2.12.0,<2.15.0", "numpy>=1.14.5", "keras>=2.6.0", - "protobuf>=3.9.2,<3.20", - "tensorflow-addons==0.17.0", - "tensorflow-estimator>=2.6.0,<2.10.0", + "tensorflow-addons>=0.17.0", + "tensorflow-estimator>=2.12.0,<2.15.0", "tf_slim==1.1.0", # todo(cdmikechen): Based on SUBMARINE-1372, typeguard has recently been upgraded to version 3.0, # which will restrict some python syntax and types more tightly. # We are not upgrading this in submarine 0.8.0 for now, # and will fix version compatibility issues in 0.8.1 or 0.9.0. "typeguard<3.0.0", - # todo(cdmikechen): SUBMARINE-1389. From scipy 1.11.0 - # (https://github.com/scipy/scipy/releases/tag/v1.11.0), - # scipy need numpy 1.21.6 or geater in python 3.9. - # So that we need to restrict scipy < 1.11.0 to support tf2.6. - # From submarine 0.8.1 or 0.9.0, we may no longer support tensorflow 2.6 - "scipy<1.11.0", ], "pytorch": ["torch>=1.5.0", "torchvision>=0.6.0"], }, classifiers=[ "Intended Audience :: Developers", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", ], entry_points={ "console_scripts": [