diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index f5b6b64a3d..137e200fbe 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -19,13 +19,26 @@ name: python-sdk on: [push, pull_request] jobs: + check-style: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v2 + - name: Install dependencies + run: | + pip install --upgrade pip + pip install -r ./dev-support/style-check/python/lint-requirements.txt + pip install -r ./dev-support/style-check/python/mypy-requirements.txt + - name: Check python sdk code style + run: ./dev-support/style-check/python/lint.sh + unit: runs-on: ubuntu-latest timeout-minutes: 30 strategy: matrix: python-version: [3.6, 3.7] - tf-version: [1.14.0, 1.15.0, 2.6.0] + tf-version: [1.15.0, 2.6.0] fail-fast: false steps: - uses: actions/checkout@v2 @@ -33,19 +46,16 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - name: Install dependencies + - name: Install requirements run: | - pip install --upgrade pip - pip install --no-cache-dir tensorflow==${{ matrix.tf-version }} - pip install --no-cache-dir torch==1.5.0 - pip install --no-cache-dir tensorflow-addons - pip install --no-cache-dir tf_slim - pip install --no-cache-dir ./submarine-sdk/pysubmarine/. + pip install --upgrade pip pip install -r ./submarine-sdk/pysubmarine/github-actions/test-requirements.txt - pip install -r ./dev-support/style-check/python/lint-requirements.txt - pip install -r ./dev-support/style-check/python/mypy-requirements.txt - - name: Check python sdk code style - run: ./dev-support/style-check/python/lint.sh + - name: Install pysubmarine with tf1 and pytorch + if: matrix.tf-version == '1.15.0' + run: pip install --no-cache-dir -e ./submarine-sdk/pysubmarine/.[tf,pytorch] + - name: Install pysubmarine with tf2 and pytorch + if: matrix.tf-version == '2.6.0' + run: pip install --no-cache-dir -e ./submarine-sdk/pysubmarine/.[tf2,pytorch] - name: Run unit test run: pytest --cov=submarine -vs -m "not e2e" diff --git a/submarine-sdk/pysubmarine/setup.py b/submarine-sdk/pysubmarine/setup.py index eb2a6877c2..1935c01327 100644 --- a/submarine-sdk/pysubmarine/setup.py +++ b/submarine-sdk/pysubmarine/setup.py @@ -28,7 +28,7 @@ packages=find_packages(exclude=["tests", "tests.*"]), install_requires=[ "six>=1.10.0", - "numpy==1.18.5", + "numpy==1.19.2", "pandas", "sqlalchemy>=1.4.0", "sqlparse", @@ -42,8 +42,13 @@ "boto3>=1.17.58", ], extras_require={ - "tf": ["tensorflow>=1.14.0,<2.0.0"], - "tf2": ["tensorflow==2.6.0", "tf_slim==1.1.0", "tensorflow-addons==0.14.0"], + "tf": ["tensorflow==1.15.0"], + "tf2": [ + "tensorflow==2.6.0", + "tf_slim==1.1.0", + "tensorflow-addons==0.14.0", + "tensorflow-estimator==2.6.0", + ], "pytorch": ["torch>=1.5.0", "torchvision>=0.6.0"], }, classifiers=[ diff --git a/submarine-sdk/pysubmarine/tests/ml/tensorflow/model/test_base_tf_model.py b/submarine-sdk/pysubmarine/tests/ml/tensorflow/model/test_base_tf_model.py index 8bf79157f9..17978e2ec6 100644 --- a/submarine-sdk/pysubmarine/tests/ml/tensorflow/model/test_base_tf_model.py +++ b/submarine-sdk/pysubmarine/tests/ml/tensorflow/model/test_base_tf_model.py @@ -16,11 +16,11 @@ import pytest import tensorflow as tf -from submarine.ml.tensorflow.model.base_tf_model import BaseTFModel - @pytest.mark.skipif(tf.__version__ >= "2.0.0", reason="requires tf1") def test_create_base_tf_model(): + from submarine.ml.tensorflow.model.base_tf_model import BaseTFModel + params = {"learning rate": 0.05} with pytest.raises(AssertionError, match="Does not define any input parameters"): BaseTFModel(params) diff --git a/submarine-sdk/pysubmarine/tests/ml/tensorflow/model/test_ccpm.py b/submarine-sdk/pysubmarine/tests/ml/tensorflow/model/test_ccpm.py index bc0fa876fc..b3422c157e 100644 --- a/submarine-sdk/pysubmarine/tests/ml/tensorflow/model/test_ccpm.py +++ b/submarine-sdk/pysubmarine/tests/ml/tensorflow/model/test_ccpm.py @@ -16,11 +16,11 @@ import pytest import tensorflow as tf -from submarine.ml.tensorflow.model import CCPM - @pytest.mark.skipif(tf.__version__ >= "2.0.0", reason="requires tf1") def test_run_ccpm(get_model_param): + from submarine.ml.tensorflow.model import CCPM + params = get_model_param model = CCPM(model_params=params) diff --git a/submarine-sdk/pysubmarine/tests/ml/tensorflow/model/test_deepfm.py b/submarine-sdk/pysubmarine/tests/ml/tensorflow/model/test_deepfm.py index 2a6bd8da8f..e9aff62b0d 100644 --- a/submarine-sdk/pysubmarine/tests/ml/tensorflow/model/test_deepfm.py +++ b/submarine-sdk/pysubmarine/tests/ml/tensorflow/model/test_deepfm.py @@ -16,11 +16,12 @@ import pytest import tensorflow as tf -from submarine.ml.tensorflow.model import DeepFM - @pytest.mark.skipif(tf.__version__ >= "2.0.0", reason="requires tf1") def test_run_deepfm(get_model_param): + + from submarine.ml.tensorflow.model import DeepFM + params = get_model_param model = DeepFM(model_params=params) diff --git a/submarine-sdk/pysubmarine/tests/ml/tensorflow/model/test_fm.py b/submarine-sdk/pysubmarine/tests/ml/tensorflow/model/test_fm.py index fb5abb8908..200c0a7b1a 100644 --- a/submarine-sdk/pysubmarine/tests/ml/tensorflow/model/test_fm.py +++ b/submarine-sdk/pysubmarine/tests/ml/tensorflow/model/test_fm.py @@ -16,11 +16,11 @@ import pytest import tensorflow as tf -from submarine.ml.tensorflow.model import FM - @pytest.mark.skipif(tf.__version__ >= "2.0.0", reason="requires tf1") def test_run_fm(get_model_param): + from submarine.ml.tensorflow.model import FM + params = get_model_param model = FM(model_params=params) diff --git a/submarine-sdk/pysubmarine/tests/ml/tensorflow/model/test_nfm.py b/submarine-sdk/pysubmarine/tests/ml/tensorflow/model/test_nfm.py index 5000810064..980aa4434e 100644 --- a/submarine-sdk/pysubmarine/tests/ml/tensorflow/model/test_nfm.py +++ b/submarine-sdk/pysubmarine/tests/ml/tensorflow/model/test_nfm.py @@ -16,11 +16,11 @@ import pytest import tensorflow as tf -from submarine.ml.tensorflow.model import NFM - @pytest.mark.skipif(tf.__version__ >= "2.0.0", reason="requires tf1") def test_run_nfm(get_model_param): + from submarine.ml.tensorflow.model import NFM + params = get_model_param model = NFM(model_params=params) diff --git a/submarine-sdk/pysubmarine/tests/ml/tensorflow/test_optimizer.py b/submarine-sdk/pysubmarine/tests/ml/tensorflow/test_optimizer.py index 776ed49eec..4841dcdd41 100644 --- a/submarine-sdk/pysubmarine/tests/ml/tensorflow/test_optimizer.py +++ b/submarine-sdk/pysubmarine/tests/ml/tensorflow/test_optimizer.py @@ -16,11 +16,11 @@ import pytest import tensorflow as tf -from submarine.ml.tensorflow.optimizer import get_optimizer - @pytest.mark.skipif(tf.__version__ >= "2.0.0", reason="requires tf1") def test_get_optimizer(): + from submarine.ml.tensorflow.optimizer import get_optimizer + optimizer_keys = ["adam", "adagrad", "momentum", "ftrl"] invalid_optimizer_keys = ["adddam"] diff --git a/submarine-sdk/pysubmarine/tests/ml/tensorflow_v2/model/test_base_tf_model.py b/submarine-sdk/pysubmarine/tests/ml/tensorflow_v2/model/test_base_tf_model.py index e706abbc57..74556ac630 100644 --- a/submarine-sdk/pysubmarine/tests/ml/tensorflow_v2/model/test_base_tf_model.py +++ b/submarine-sdk/pysubmarine/tests/ml/tensorflow_v2/model/test_base_tf_model.py @@ -16,11 +16,11 @@ import pytest import tensorflow as tf -from submarine.ml.tensorflow_v2.model.base_tf_model import BaseTFModel - @pytest.mark.skipif(tf.__version__ < "2.0.0", reason="requires tf2") def test_create_base_tf_model(): + from submarine.ml.tensorflow_v2.model.base_tf_model import BaseTFModel + params = {"learning rate": 0.05} with pytest.raises(AssertionError, match="Does not define any input parameters"): BaseTFModel(params) diff --git a/submarine-sdk/pysubmarine/tests/ml/tensorflow_v2/model/test_ccpm.py b/submarine-sdk/pysubmarine/tests/ml/tensorflow_v2/model/test_ccpm.py index 858b6bb4c3..99d78dfba7 100644 --- a/submarine-sdk/pysubmarine/tests/ml/tensorflow_v2/model/test_ccpm.py +++ b/submarine-sdk/pysubmarine/tests/ml/tensorflow_v2/model/test_ccpm.py @@ -16,11 +16,11 @@ import pytest import tensorflow as tf -from submarine.ml.tensorflow_v2.model import CCPM - @pytest.mark.skipif(tf.__version__ < "2.0.0", reason="requires tf2") def test_run_ccpm(get_model_param): + from submarine.ml.tensorflow_v2.model import CCPM + params = get_model_param model = CCPM(model_params=params) diff --git a/submarine-sdk/pysubmarine/tests/ml/tensorflow_v2/model/test_deepfm.py b/submarine-sdk/pysubmarine/tests/ml/tensorflow_v2/model/test_deepfm.py index c6f2d4dd01..bcaecf217a 100644 --- a/submarine-sdk/pysubmarine/tests/ml/tensorflow_v2/model/test_deepfm.py +++ b/submarine-sdk/pysubmarine/tests/ml/tensorflow_v2/model/test_deepfm.py @@ -16,11 +16,11 @@ import pytest import tensorflow as tf -from submarine.ml.tensorflow_v2.model import DeepFM - @pytest.mark.skipif(tf.__version__ < "2.0.0", reason="requires tf2") def test_run_deepfm(get_model_param): + from submarine.ml.tensorflow_v2.model import DeepFM + params = get_model_param model = DeepFM(model_params=params) diff --git a/submarine-sdk/pysubmarine/tests/ml/tensorflow_v2/model/test_fm.py b/submarine-sdk/pysubmarine/tests/ml/tensorflow_v2/model/test_fm.py index 1d95304630..e8e5717c4f 100644 --- a/submarine-sdk/pysubmarine/tests/ml/tensorflow_v2/model/test_fm.py +++ b/submarine-sdk/pysubmarine/tests/ml/tensorflow_v2/model/test_fm.py @@ -16,11 +16,12 @@ import pytest import tensorflow as tf -from submarine.ml.tensorflow_v2.model import FM - @pytest.mark.skipif(tf.__version__ < "2.0.0", reason="requires tf2") def test_run_fm(get_model_param): + + from submarine.ml.tensorflow_v2.model import FM + params = get_model_param model = FM(model_params=params) diff --git a/submarine-sdk/pysubmarine/tests/ml/tensorflow_v2/model/test_nfm.py b/submarine-sdk/pysubmarine/tests/ml/tensorflow_v2/model/test_nfm.py index 9de05f3bee..301c255f67 100644 --- a/submarine-sdk/pysubmarine/tests/ml/tensorflow_v2/model/test_nfm.py +++ b/submarine-sdk/pysubmarine/tests/ml/tensorflow_v2/model/test_nfm.py @@ -16,11 +16,11 @@ import pytest import tensorflow as tf -from submarine.ml.tensorflow_v2.model import NFM - @pytest.mark.skipif(tf.__version__ < "2.0.0", reason="requires tf2") def test_run_nfm(get_model_param): + from submarine.ml.tensorflow_v2.model import NFM + params = get_model_param model = NFM(model_params=params) diff --git a/submarine-sdk/pysubmarine/tests/ml/tensorflow_v2/test_optimizer.py b/submarine-sdk/pysubmarine/tests/ml/tensorflow_v2/test_optimizer.py index 1ef8fa112d..e539812586 100644 --- a/submarine-sdk/pysubmarine/tests/ml/tensorflow_v2/test_optimizer.py +++ b/submarine-sdk/pysubmarine/tests/ml/tensorflow_v2/test_optimizer.py @@ -16,11 +16,12 @@ import pytest import tensorflow as tf -from submarine.ml.tensorflow_v2.optimizer import get_optimizer - @pytest.mark.skipif(tf.__version__ < "2.0.0", reason="requires tf2") def test_get_optimizer(): + + from submarine.ml.tensorflow_v2.optimizer import get_optimizer + optimizer_keys = ["adam", "adagrad", "momentum", "ftrl"] invalid_optimizer_keys = ["adddam"]