From c0fe073627dd737d3a8df15cf1aadf408f984087 Mon Sep 17 00:00:00 2001 From: featherchen Date: Sat, 3 Jul 2021 11:31:50 +0800 Subject: [PATCH] SUBMARINE-861. Trim the size of pysubmarine ### What is this PR for? Trim the size of pysubmarine, in other words,we let the user install the package such as pytorch and Tensorflow by themself instead of pre-installation from pysubmarine ### What type of PR is it? Improvement ### Todos [ ] - Deal with Jupiter Lab ### What is the Jira issue? https://issues.apache.org/jira/browse/SUBMARINE-861 ### How should this be tested? ### Screenshots (if appropriate) ### Questions: * Do the license files need updating? No * Are there breaking changes for older versions? No * Does this need new documentation? No Author: featherchen Signed-off-by: byronhsu Closes #619 from featherchen/SUBMARINE-861 and squashes the following commits: 70807f59 [featherchen] add package in IT environment 8245e4a8 [featherchen] add tf-latest 42d10090 [featherchen] SUBMARINE-861. Add extras_reqire 4ed0f42f [featherchen] SUBMARINE-861. Add extras and modifile Jupiter dockerfile eb07371c [featherchen] SUBMARINE-861. Delete installation of pytorch and Tensorflow --- .github/workflows/python.yml | 6 +++--- dev-support/docker-images/jupyter-gpu/Dockerfile | 2 +- dev-support/docker-images/jupyter/Dockerfile | 2 +- dev-support/pysubmarine/install-conda.sh | 2 +- submarine-sdk/pysubmarine/setup.py | 10 ++++++---- 5 files changed, 12 insertions(+), 10 deletions(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 90d3c1fa36..70617ea105 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -43,7 +43,7 @@ jobs: - name: Set up JDK 1.8 uses: actions/setup-java@v1 with: - java-version: '1.8' + java-version: "1.8" - name: Set up Maven 3.6.3 uses: stCarolas/setup-maven@v4 with: @@ -130,7 +130,7 @@ jobs: - name: Install dependencies run: | pip install --upgrade pip - pip install --no-cache-dir ./submarine-sdk/pysubmarine/. + pip install --no-cache-dir -e ./submarine-sdk/pysubmarine/.[tf,pytorch] pip install -r ./submarine-sdk/pysubmarine/github-actions/test-requirements.txt - name: Run integration test run: | @@ -141,4 +141,4 @@ jobs: kubectl get pods kubectl -n default get events --sort-by='{.lastTimestamp}' kubectl describe nodes - if: ${{ failure() }} \ No newline at end of file + if: ${{ failure() }} diff --git a/dev-support/docker-images/jupyter-gpu/Dockerfile b/dev-support/docker-images/jupyter-gpu/Dockerfile index ca58f5abeb..63497600c9 100644 --- a/dev-support/docker-images/jupyter-gpu/Dockerfile +++ b/dev-support/docker-images/jupyter-gpu/Dockerfile @@ -96,7 +96,7 @@ RUN wget --quiet https://repo.continuum.io/miniconda/Miniconda3-py37_${MINICONDA # Install latest sumbarine python sdk and notebook RUN git clone https://github.com/apache/submarine && \ - pip install submarine/submarine-sdk/pysubmarine && \ + pip install -e submarine/submarine-sdk/pysubmarine[tf,pytorch] && \ conda install nodejs && \ conda install -c conda-forge jupyterlab jupyterlab-git && \ jupyter lab build diff --git a/dev-support/docker-images/jupyter/Dockerfile b/dev-support/docker-images/jupyter/Dockerfile index f66e4cfcee..93b44697fa 100644 --- a/dev-support/docker-images/jupyter/Dockerfile +++ b/dev-support/docker-images/jupyter/Dockerfile @@ -96,7 +96,7 @@ RUN wget --quiet https://repo.continuum.io/miniconda/Miniconda3-py37_${MINICONDA # Install latest sumbarine python sdk and notebook RUN git clone https://github.com/apache/submarine && \ - pip install submarine/submarine-sdk/pysubmarine && \ + pip install -e submarine/submarine-sdk/pysubmarine[tf,pytorch] && \ conda install nodejs && \ conda install -c conda-forge jupyterlab jupyterlab-git && \ jupyter lab build diff --git a/dev-support/pysubmarine/install-conda.sh b/dev-support/pysubmarine/install-conda.sh index be6a7cd4e1..de8bb204d0 100644 --- a/dev-support/pysubmarine/install-conda.sh +++ b/dev-support/pysubmarine/install-conda.sh @@ -36,6 +36,6 @@ pip install --upgrade pip # Install pysubmarine git clone https://github.com/apache/submarine.git cd submarine/submarine-sdk/pysubmarine -pip install . +pip install -e .[tf,pytorch] pip install -r github-actions/lint-requirements.txt pip install -r github-actions/test-requirements.txt diff --git a/submarine-sdk/pysubmarine/setup.py b/submarine-sdk/pysubmarine/setup.py index ad16512bdf..e3754e3c73 100644 --- a/submarine-sdk/pysubmarine/setup.py +++ b/submarine-sdk/pysubmarine/setup.py @@ -33,17 +33,19 @@ 'sqlalchemy', 'sqlparse', 'pymysql', - 'tensorflow>=1.14.0,<2.0.0', 'requests', 'urllib3 >= 1.15.1', 'certifi >= 14.05.14', 'python-dateutil >= 2.5.3', 'pyarrow==0.17.0', - 'torch>=1.5.0', - 'torchvision>=0.6.0', 'mlflow>=1.15.0', - 'boto3>=1.17.58 ' + 'boto3>=1.17.58', ], + extras_require={ + 'tf':['tensorflow>=1.14.0,<2.0.0'], + 'tf-latest':['tensorflow'], + 'pytorch':['torch>=1.5.0','torchvision>=0.6.0'], + }, classifiers=[ 'Intended Audience :: Developers', 'Programming Language :: Python :: 3.5',