fix(backend): update requirements scripts (kubeflow#10009)

* fix: update requirements scripts

Signed-off-by: David van der Spek <[email protected]>

* fix: visualization dependency hell

Signed-off-by: David van der Spek <[email protected]>

* bump snapshottest

Signed-off-by: David van der Spek <[email protected]>

* fix: pin jinja2

Signed-off-by: David van der Spek <[email protected]>

* fix: pin markupsafe to fix ci

Signed-off-by: David van der Spek <[email protected]>

---------

Signed-off-by: David van der Spek <[email protected]>

backend/Dockerfile.visualization

@@ -18,7 +18,7 @@
 # and exporter.py files in the directory specified above.
 
 # This image should be in sync with image in backend/src/apiserver/visualization/update_requirements.sh.
-FROM tensorflow/tensorflow:2.5.1
+FROM tensorflow/tensorflow:2.10.1
 
 RUN apt-get update \
   && apt-get install -y wget curl tar openssl

backend/README.md

@@ -55,6 +55,13 @@ need to be regenerated and checked-in. Refer to [backend/api](./api/README.md) f
     make all
     ```
 
+## Updating python dependencies
+
+[pip-tools](https://github.com/jazzband/pip-tools) is used to manage python
+dependencies. To update dependencies, edit [requirements.in](requirements.in)
+and run `./update_requirements.sh` to update and pin the transitive
+dependencies.
+
 # Visualization Server Instructions
 
 ## Updating python dependencies

backend/metadata_writer/Dockerfile

@@ -1,5 +1,5 @@
 # ml-metadata package depends on tensorflow package
-FROM python:3.7
+FROM python:3.8
 COPY backend/metadata_writer/requirements.txt /kfp/metadata_writer/
 RUN python3 -m pip install -r /kfp/metadata_writer/requirements.txt
 

backend/metadata_writer/README.md

@@ -4,6 +4,5 @@
 
 [pip-tools](https://github.com/jazzband/pip-tools) is used to manage python
 dependencies. To update dependencies, edit [requirements.in](requirements.in)
-and run `../update_requirements.sh python:3.7 <requirements.in >requirements.txt` to update and pin the transitive
+and run `./update_requirements.sh` to update and pin the transitive
 dependencies.
-

backend/metadata_writer/requirements.txt

@@ -1,32 +1,67 @@
 #
-# This file is autogenerated by pip-compile
-# To update, run:
+# This file is autogenerated by pip-compile with Python 3.8
+# by the following command:
 #
 #    pip-compile --output-file=- -
 #
-absl-py==0.12.0           # via ml-metadata
-attrs==20.3.0             # via ml-metadata
-cachetools==5.0.0         # via google-auth
-certifi==2021.10.8        # via kubernetes, requests
-charset-normalizer==2.0.10  # via requests
-google-auth==2.4.1        # via kubernetes
-grpcio==1.43.0            # via ml-metadata
-idna==3.3                 # via requests
-kubernetes==10.1.0        # via -r -
-lru-dict==1.1.7           # via -r -
-ml-metadata==1.5.0        # via -r -
-oauthlib==3.1.1           # via requests-oauthlib
-protobuf==3.19.3          # via ml-metadata
-pyasn1-modules==0.2.8     # via google-auth
-pyasn1==0.4.8             # via pyasn1-modules, rsa
-python-dateutil==2.8.2    # via kubernetes
-pyyaml==3.13              # via kubernetes
-requests-oauthlib==1.3.0  # via kubernetes
-requests==2.27.1          # via kubernetes, requests-oauthlib
-rsa==4.8                  # via google-auth
-six==1.16.0               # via absl-py, google-auth, grpcio, kubernetes, ml-metadata, python-dateutil
-urllib3==1.26.8           # via kubernetes, requests
-websocket-client==1.2.3   # via kubernetes
+absl-py==1.4.0
+    # via ml-metadata
+attrs==21.4.0
+    # via ml-metadata
+cachetools==5.3.1
+    # via google-auth
+certifi==2023.7.22
+    # via
+    #   kubernetes
+    #   requests
+charset-normalizer==3.2.0
+    # via requests
+google-auth==2.23.0
+    # via kubernetes
+grpcio==1.58.0
+    # via ml-metadata
+idna==3.4
+    # via requests
+kubernetes==10.1.0
+    # via -r -
+lru-dict==1.2.0
+    # via -r -
+ml-metadata==1.14.0
+    # via -r -
+oauthlib==3.2.2
+    # via requests-oauthlib
+protobuf==3.20.3
+    # via ml-metadata
+pyasn1==0.5.0
+    # via
+    #   pyasn1-modules
+    #   rsa
+pyasn1-modules==0.3.0
+    # via google-auth
+python-dateutil==2.8.2
+    # via kubernetes
+pyyaml==3.13
+    # via kubernetes
+requests==2.31.0
+    # via
+    #   kubernetes
+    #   requests-oauthlib
+requests-oauthlib==1.3.1
+    # via kubernetes
+rsa==4.9
+    # via google-auth
+six==1.16.0
+    # via
+    #   kubernetes
+    #   ml-metadata
+    #   python-dateutil
+urllib3==1.26.16
+    # via
+    #   google-auth
+    #   kubernetes
+    #   requests
+websocket-client==1.6.3
+    # via kubernetes
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools

backend/metadata_writer/update_requirements.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+# This image should be in sync with Dockerfile.
+IMAGE="python:3.8"
+../../hack/update-requirements.sh $IMAGE <requirements.in >requirements.txt

backend/requirements.txt

@@ -1,53 +1,55 @@
 #
-# This file is autogenerated by pip-compile with Python 3.11
+# This file is autogenerated by pip-compile with Python 3.7
 # by the following command:
 #
-#    pip-compile --no-emit-index-url requirements.in
+#    pip-compile --output-file=- -
 #
 cachetools==5.3.1
     # via google-auth
-certifi==2023.5.7
+certifi==2023.7.22
     # via
     #   kfp-server-api
     #   kubernetes
     #   requests
-charset-normalizer==3.1.0
+charset-normalizer==3.2.0
     # via requests
-click==8.1.3
+click==8.1.7
     # via kfp
 docstring-parser==0.15
     # via kfp
-google-api-core==2.11.0
+google-api-core==2.11.1
     # via
     #   google-cloud-core
     #   google-cloud-storage
     #   kfp
-google-auth==2.19.1
+google-auth==2.23.0
     # via
     #   google-api-core
     #   google-cloud-core
     #   google-cloud-storage
     #   kfp
     #   kubernetes
-google-cloud-core==2.3.2
+google-cloud-core==2.3.3
     # via google-cloud-storage
-google-cloud-storage==2.9.0
+google-cloud-storage==2.11.0
     # via kfp
 google-crc32c==1.5.0
     # via google-resumable-media
-google-resumable-media==2.5.0
+google-resumable-media==2.6.0
     # via google-cloud-storage
-googleapis-common-protos==1.59.0
+googleapis-common-protos==1.60.0
     # via google-api-core
 idna==3.4
     # via requests
-kfp==2.0.0rc1
-    # via -r requirements.in
+importlib-metadata==6.7.0
+    # via click
+kfp==2.0.1
+    # via -r -
 kfp-pipeline-spec==0.2.2
     # via kfp
-kfp-server-api==2.0.0-rc.1
+kfp-server-api==2.0.1
     # via kfp
-kubernetes==23.6.0
+kubernetes==26.1.0
     # via kfp
 oauthlib==3.2.2
     # via requests-oauthlib
@@ -67,7 +69,7 @@ python-dateutil==2.8.2
     # via
     #   kfp-server-api
     #   kubernetes
-pyyaml==6.0
+pyyaml==6.0.1
     # via
     #   kfp
     #   kubernetes
@@ -86,21 +88,26 @@ rsa==4.9
     # via google-auth
 six==1.16.0
     # via
-    #   google-auth
     #   kfp-server-api
     #   kubernetes
     #   python-dateutil
 tabulate==0.9.0
     # via kfp
+typing-extensions==4.7.1
+    # via
+    #   importlib-metadata
+    #   kfp
 urllib3==1.26.16
     # via
     #   google-auth
     #   kfp
     #   kfp-server-api
     #   kubernetes
     #   requests
-websocket-client==1.5.2
+websocket-client==1.6.1
     # via kubernetes
+zipp==3.15.0
+    # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools

backend/src/apiserver/visualization/requirements-test.txt

@@ -1 +1 @@
-snapshottest==0.5.1
+snapshottest==0.6.0

backend/src/apiserver/visualization/requirements.in

@@ -4,14 +4,16 @@ google-api-python-client==1.7.*
 itables==0.1.0
 ipykernel==5.1.1
 ipython==7.12.0
+jinja2==2.11.3
 jupyter_client==5.3.*
+markupsafe==2.0.1
 nbconvert==5.5.0
 nbformat==4.4.0
-scikit_learn==0.21.2
-tensorflow==2.5.1
-tensorflow-metadata==1.2.*
-tensorflow-model-analysis==0.33.*
-tensorflow-data-validation==1.2.*
-tensorflow-serving-api==2.5.1
+scikit-learn==0.24.2
+tensorflow==2.10.1
+tensorflow-metadata==1.9.*
+tensorflow-model-analysis==0.40.*
+tensorflow-data-validation==1.9.*
+tensorflow-serving-api==2.10.1
 tornado==6.*
 mistune<2.0.0