From b8d50b8a9461fc67cdd0afdce4d3487911767aeb Mon Sep 17 00:00:00 2001 From: mahdikhashan Date: Tue, 14 Jan 2025 14:04:56 +0100 Subject: [PATCH 01/16] init Signed-off-by: mahdikhashan --- .github/resources/scripts/test-requirements.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 .github/resources/scripts/test-requirements.sh diff --git a/.github/resources/scripts/test-requirements.sh b/.github/resources/scripts/test-requirements.sh new file mode 100644 index 00000000000..e69de29bb2d From d669f38367c6984547a53b06cde625d521c68b7e Mon Sep 17 00:00:00 2001 From: mahdikhashan Date: Tue, 14 Jan 2025 16:52:46 +0100 Subject: [PATCH 02/16] add test-requirements.sh and ci file Signed-off-by: mahdikhashan --- .../resources/scripts/test-requirements.sh | 40 +++++++++++++++++++ .github/workflows/requirements-test.yaml | 32 +++++++++++++++ 2 files changed, 72 insertions(+) create mode 100644 .github/workflows/requirements-test.yaml diff --git a/.github/resources/scripts/test-requirements.sh b/.github/resources/scripts/test-requirements.sh index e69de29bb2d..958423046a1 100644 --- a/.github/resources/scripts/test-requirements.sh +++ b/.github/resources/scripts/test-requirements.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +check_requirements() { + local req_in="$1" + local req_txt="$2" + + temp_file=$(mktemp) + + pip-compile --output-file=- "$req_in" > "$temp_file" + + # Compare the generated file with the existing requirements.txt + if diff -q "$temp_file" "$req_txt" >/dev/null; then + echo "Success: $req_txt matches the output of pip-compile for $req_in." + rm "$temp_file" + exit 0 + else + echo "Error: $req_txt does not match the output of pip-compile for $req_in." + echo "Differences:" + diff "$temp_file" "$req_txt" + rm "$temp_file" + exit 1 + fi +} + +for req_in_file in $(find . -name "requirements.in"); do + + echo $req_in_file + req_txt="${req_in_file/requirements.in/requirements.txt}" + + # Check if the corresponding requirements.txt exists + if [[ -f "$req_txt" ]]; then + echo "Checking $req_in_file against $req_txt..." + check_requirements "$req_in_file" "$req_txt" + else + echo "Error: $req_txt not found for $req_in_file." + exit 1 + fi +done + +echo "All checks completed successfully." diff --git a/.github/workflows/requirements-test.yaml b/.github/workflows/requirements-test.yaml new file mode 100644 index 00000000000..a2a1dd2ace2 --- /dev/null +++ b/.github/workflows/requirements-test.yaml @@ -0,0 +1,32 @@ +name: Requirements generated tests + +on: + push: + branches: [master] + + pull_request: + paths: + - 'test/**' + - 'backend/**' + - 'sdk/**' + +jobs: + requirements-generated-tests: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.9 + + - name: Install pip-tools + run: | + pip3 install pip-tools + + - name: Run Tests + run: | + ./.github/resources/scripts/test-requirements.sh From d23c23a972f936ca9eabc277f1fc7c013b6edf42 Mon Sep 17 00:00:00 2001 From: Mahdi Khashan <58775404+mahdikhashan@users.noreply.github.com> Date: Wed, 15 Jan 2025 13:53:55 +0100 Subject: [PATCH 03/16] update .github/workflows/requirements-test.yaml Co-authored-by: Helber Belmiro Signed-off-by: Mahdi Khashan <58775404+mahdikhashan@users.noreply.github.com> --- .github/workflows/requirements-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/requirements-test.yaml b/.github/workflows/requirements-test.yaml index a2a1dd2ace2..0e3378df954 100644 --- a/.github/workflows/requirements-test.yaml +++ b/.github/workflows/requirements-test.yaml @@ -1,4 +1,4 @@ -name: Requirements generated tests +name: requirements.txt automatically generated on: push: From a8bdb49625ebcb31528bd9e4c3ca714eaa3cade9 Mon Sep 17 00:00:00 2001 From: Mahdi Khashan <58775404+mahdikhashan@users.noreply.github.com> Date: Wed, 15 Jan 2025 13:56:00 +0100 Subject: [PATCH 04/16] update .github/workflows/requirements-test.yaml Co-authored-by: Helber Belmiro Signed-off-by: Mahdi Khashan <58775404+mahdikhashan@users.noreply.github.com> --- .github/workflows/requirements-test.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/requirements-test.yaml b/.github/workflows/requirements-test.yaml index 0e3378df954..8089a2d205b 100644 --- a/.github/workflows/requirements-test.yaml +++ b/.github/workflows/requirements-test.yaml @@ -6,9 +6,7 @@ on: pull_request: paths: - - 'test/**' - - 'backend/**' - - 'sdk/**' + - '.github/workflows/requirements-test.yaml' jobs: requirements-generated-tests: From 150a93c2e110b0130c3577128284d160f4e799db Mon Sep 17 00:00:00 2001 From: mahdikhashan Date: Wed, 15 Jan 2025 13:53:45 +0100 Subject: [PATCH 05/16] rename ci workflow file Signed-off-by: mahdikhashan --- ...ts-test.yaml => requirements-txt-automatically-generated.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{requirements-test.yaml => requirements-txt-automatically-generated.yaml} (100%) diff --git a/.github/workflows/requirements-test.yaml b/.github/workflows/requirements-txt-automatically-generated.yaml similarity index 100% rename from .github/workflows/requirements-test.yaml rename to .github/workflows/requirements-txt-automatically-generated.yaml From 74c7342e6bba6f9ab7a4f433e47898136d6e3f80 Mon Sep 17 00:00:00 2001 From: mahdikhashan Date: Wed, 15 Jan 2025 13:58:20 +0100 Subject: [PATCH 06/16] fix renaming issue Signed-off-by: mahdikhashan --- .github/workflows/requirements-txt-automatically-generated.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/requirements-txt-automatically-generated.yaml b/.github/workflows/requirements-txt-automatically-generated.yaml index 8089a2d205b..09a9d6b7892 100644 --- a/.github/workflows/requirements-txt-automatically-generated.yaml +++ b/.github/workflows/requirements-txt-automatically-generated.yaml @@ -6,7 +6,7 @@ on: pull_request: paths: - - '.github/workflows/requirements-test.yaml' + - '.github/workflows/requirements-txt-automatically-generated.yaml' jobs: requirements-generated-tests: From d9e935a01396dbfb0f0ad9a27cfeaff1087a9128 Mon Sep 17 00:00:00 2001 From: mahdikhashan Date: Wed, 15 Jan 2025 14:26:29 +0100 Subject: [PATCH 07/16] fix execute permission Signed-off-by: mahdikhashan --- .github/resources/scripts/test-requirements.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 .github/resources/scripts/test-requirements.sh diff --git a/.github/resources/scripts/test-requirements.sh b/.github/resources/scripts/test-requirements.sh old mode 100644 new mode 100755 From d0e52446143baabdf13194dff8bfb3b6700899bd Mon Sep 17 00:00:00 2001 From: mahdikhashan Date: Wed, 15 Jan 2025 19:56:32 +0100 Subject: [PATCH 08/16] update requirements in test/sample-test Signed-off-by: mahdikhashan --- test/sample-test/requirements.txt | 98 ++++++++++++++++++------------- 1 file changed, 58 insertions(+), 40 deletions(-) diff --git a/test/sample-test/requirements.txt b/test/sample-test/requirements.txt index 1fbb9b34077..e6ebc5fe176 100644 --- a/test/sample-test/requirements.txt +++ b/test/sample-test/requirements.txt @@ -1,34 +1,41 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # -# pip-compile +# pip-compile --resolver=backtracking # +aiohappyeyeballs==2.4.4 + # via aiohttp +aiohttp==3.11.11 + # via papermill +aiosignal==1.3.2 + # via aiohttp ansicolors==1.1.8 # via papermill argon2-cffi==23.1.0 # via minio argon2-cffi-bindings==21.2.0 # via argon2-cffi -attrs==24.2.0 +attrs==24.3.0 # via + # aiohttp # jsonschema # referencing -black==24.8.0 +black==24.10.0 # via -r requirements.in cachetools==5.5.0 # via google-auth -certifi==2024.7.4 +certifi==2024.12.14 # via # kfp-server-api # kubernetes # minio # requests -cffi==1.17.0 +cffi==1.17.1 # via argon2-cffi-bindings -charset-normalizer==3.3.2 +charset-normalizer==3.4.1 # via requests -click==8.1.7 +click==8.1.8 # via # black # kfp @@ -37,16 +44,20 @@ docstring-parser==0.16 # via kfp entrypoints==0.4 # via papermill -fastjsonschema==2.20.0 +fastjsonschema==2.21.1 # via nbformat -fire==0.6.0 +fire==0.7.0 # via -r requirements.in -google-api-core==2.19.1 +frozenlist==1.5.0 + # via + # aiohttp + # aiosignal +google-api-core==2.24.0 # via # google-cloud-core # google-cloud-storage # kfp -google-auth==2.34.0 +google-auth==2.37.0 # via # google-api-core # google-cloud-core @@ -55,25 +66,27 @@ google-auth==2.34.0 # kubernetes google-cloud-core==2.4.1 # via google-cloud-storage -google-cloud-storage==2.18.2 +google-cloud-storage==2.19.0 # via kfp -google-crc32c==1.5.0 +google-crc32c==1.6.0 # via # google-cloud-storage # google-resumable-media google-resumable-media==2.7.2 # via google-cloud-storage -googleapis-common-protos==1.64.0 +googleapis-common-protos==1.66.0 # via google-api-core -idna==3.8 - # via requests +idna==3.10 + # via + # requests + # yarl jsonschema==4.23.0 # via nbformat -jsonschema-specifications==2023.12.1 +jsonschema-specifications==2024.10.1 # via jsonschema junit-xml==1.9 # via -r requirements.in -jupyter-client==8.6.2 +jupyter-client==8.6.3 # via nbclient jupyter-core==5.7.2 # via @@ -90,11 +103,15 @@ kubernetes==26.1.0 # via # -r requirements.in # kfp -minio==7.2.8 +minio==7.2.14 # via -r requirements.in +multidict==6.1.0 + # via + # aiohttp + # yarl mypy-extensions==1.0.0 # via black -nbclient==0.10.0 +nbclient==0.10.2 # via papermill nbformat==5.10.4 # via @@ -102,34 +119,38 @@ nbformat==5.10.4 # papermill oauthlib==3.2.2 # via requests-oauthlib -packaging==24.1 +packaging==24.2 # via black papermill==2.6.0 # via -r requirements.in pathspec==0.12.1 # via black -platformdirs==4.2.2 +platformdirs==4.3.6 # via # black # jupyter-core -proto-plus==1.24.0 +propcache==0.2.1 + # via + # aiohttp + # yarl +proto-plus==1.25.0 # via google-api-core -protobuf==4.25.4 +protobuf==4.25.5 # via # google-api-core # googleapis-common-protos # kfp # kfp-pipeline-spec # proto-plus -pyasn1==0.6.0 +pyasn1==0.6.1 # via # pyasn1-modules # rsa -pyasn1-modules==0.4.0 +pyasn1-modules==0.4.1 # via google-auth pycparser==2.22 # via cffi -pycryptodome==3.20.0 +pycryptodome==3.21.0 # via minio python-dateutil==2.9.0.post0 # via @@ -160,15 +181,14 @@ requests-oauthlib==2.0.0 # via kubernetes requests-toolbelt==0.10.1 # via kfp -rpds-py==0.20.0 +rpds-py==0.22.3 # via # jsonschema # referencing rsa==4.9 # via google-auth -six==1.15.0 +six==1.17.0 # via - # fire # junit-xml # kfp-server-api # kubernetes @@ -177,13 +197,11 @@ tabulate==0.9.0 # via kfp tenacity==9.0.0 # via papermill -termcolor==2.4.0 +termcolor==2.5.0 # via fire -tomli==2.0.1 - # via black tornado==6.4.2 # via jupyter-client -tqdm==4.66.5 +tqdm==4.67.1 # via papermill traitlets==5.14.3 # via @@ -192,10 +210,8 @@ traitlets==5.14.3 # nbclient # nbformat typing-extensions==4.12.2 - # via - # black - # minio -urllib3==1.26.19 + # via minio +urllib3==1.26.20 # via # kfp # kfp-server-api @@ -204,8 +220,10 @@ urllib3==1.26.19 # requests websocket-client==1.8.0 # via kubernetes -yamale==5.2.1 +yamale==6.0.0 # via -r requirements.in +yarl==1.18.3 + # via aiohttp # The following packages are considered to be unsafe in a requirements file: # setuptools From 4b6d44f9963f8888595a05715dde0117835da4ab Mon Sep 17 00:00:00 2001 From: mahdikhashan Date: Wed, 15 Jan 2025 19:57:18 +0100 Subject: [PATCH 09/16] update requirements in test/kfp-functional-test Signed-off-by: mahdikhashan --- test/kfp-functional-test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/kfp-functional-test/requirements.txt b/test/kfp-functional-test/requirements.txt index 6c7eb5e6b68..b90f2ef4f26 100644 --- a/test/kfp-functional-test/requirements.txt +++ b/test/kfp-functional-test/requirements.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.12 # by the following command: # -# pip-compile requirements.in +# pip-compile --resolver=backtracking # cachetools==5.3.3 # via google-auth From 69f494032c8b37fb6bf2fd70c31a657e40d0ccc6 Mon Sep 17 00:00:00 2001 From: mahdikhashan Date: Thu, 16 Jan 2025 13:24:48 +0100 Subject: [PATCH 10/16] update requirements in backend Signed-off-by: mahdikhashan --- backend/requirements.txt | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/backend/requirements.txt b/backend/requirements.txt index 68cfc0f33f5..4b5467accd1 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1,8 +1,8 @@ # -# This file is autogenerated by pip-compile with Python 3.7 +# This file is autogenerated by pip-compile with Python 3.9 # by the following command: # -# pip-compile --output-file=- - +# pip-compile --resolver=backtracking # cachetools==5.3.1 # via google-auth @@ -41,10 +41,8 @@ googleapis-common-protos==1.60.0 # via google-api-core idna==3.4 # via requests -importlib-metadata==6.7.0 - # via click kfp==2.0.1 - # via -r - + # via -r requirements.in kfp-pipeline-spec==0.2.2 # via kfp kfp-server-api==2.0.1 @@ -93,10 +91,6 @@ six==1.16.0 # python-dateutil tabulate==0.9.0 # via kfp -typing-extensions==4.7.1 - # via - # importlib-metadata - # kfp urllib3==1.26.16 # via # google-auth @@ -106,8 +100,6 @@ urllib3==1.26.16 # requests websocket-client==1.6.1 # via kubernetes -zipp==3.15.0 - # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools From ee978e1ac54aeb2c7c0ace9a99d7d49971defe45 Mon Sep 17 00:00:00 2001 From: mahdikhashan Date: Thu, 16 Jan 2025 13:52:16 +0100 Subject: [PATCH 11/16] update requirements in sdk/python Signed-off-by: mahdikhashan --- sdk/python/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/python/requirements.txt b/sdk/python/requirements.txt index 7c532f43b75..cc6e059d3ef 100644 --- a/sdk/python/requirements.txt +++ b/sdk/python/requirements.txt @@ -1,8 +1,8 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.9 # by the following command: # -# pip-compile --no-emit-index-url requirements.in +# pip-compile --resolver=backtracking # cachetools==5.4.0 # via google-auth From 6fbfc08b55f8d38bfb5bced161019ba21e6bf55d Mon Sep 17 00:00:00 2001 From: mahdikhashan Date: Thu, 16 Jan 2025 17:14:11 +0100 Subject: [PATCH 12/16] update requirements in backend/src/apiserver/visualization Signed-off-by: mahdikhashan --- .../apiserver/visualization/requirements.txt | 102 ++++++++---------- 1 file changed, 42 insertions(+), 60 deletions(-) diff --git a/backend/src/apiserver/visualization/requirements.txt b/backend/src/apiserver/visualization/requirements.txt index fa15911aca8..5c1a396dc49 100644 --- a/backend/src/apiserver/visualization/requirements.txt +++ b/backend/src/apiserver/visualization/requirements.txt @@ -1,8 +1,8 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.9 # by the following command: # -# pip-compile --output-file=- --resolver=backtracking - +# pip-compile --output-file=- ./backend/src/apiserver/visualization/requirements.in # absl-py==1.4.0 # via @@ -12,13 +12,12 @@ absl-py==1.4.0 # tensorflow-metadata # tensorflow-model-analysis # tfx-bsl -anyio==4.5.2 +anyio==4.8.0 # via # httpx # jupyter-server apache-beam[gcp]==2.46.0 # via - # apache-beam # tensorflow-data-validation # tensorflow-model-analysis # tfx-bsl @@ -42,12 +41,10 @@ backcall==0.2.0 # via ipython beautifulsoup4==4.12.3 # via nbconvert -bleach[css]==6.1.0 - # via - # bleach - # nbconvert +bleach[css]==6.2.0 + # via nbconvert bokeh==1.2.0 - # via -r - + # via -r ./backend/src/apiserver/visualization/requirements.in cachetools==4.2.4 # via # apache-beam @@ -79,7 +76,7 @@ docopt==0.6.2 # via hdfs exceptiongroup==1.2.2 # via anyio -fastavro==1.9.7 +fastavro==1.10.0 # via apache-beam fasteners==0.19 # via @@ -94,10 +91,9 @@ fqdn==1.5.1 gast==0.4.0 # via tensorflow gcsfs==0.2.3 - # via -r - + # via -r ./backend/src/apiserver/visualization/requirements.in google-api-core[grpc]==2.24.0 # via - # google-api-core # google-cloud-bigquery # google-cloud-bigquery-storage # google-cloud-bigtable @@ -113,7 +109,7 @@ google-api-core[grpc]==2.24.0 # google-cloud-vision google-api-python-client==1.7.12 # via - # -r - + # -r ./backend/src/apiserver/visualization/requirements.in # tfx-bsl google-apitools==0.5.31 # via apache-beam @@ -139,7 +135,7 @@ google-auth-oauthlib==0.4.6 # via # gcsfs # tensorboard -google-cloud-bigquery==3.27.0 +google-cloud-bigquery==3.28.0 # via apache-beam google-cloud-bigquery-storage==2.16.2 # via apache-beam @@ -172,7 +168,7 @@ google-cloud-videointelligence==1.16.3 # via apache-beam google-cloud-vision==3.7.2 # via apache-beam -google-crc32c==1.5.0 +google-crc32c==1.6.0 # via google-resumable-media google-pasta==0.2.0 # via tensorflow @@ -211,7 +207,7 @@ grpcio-status==1.48.2 # google-cloud-pubsublite h11==0.14.0 # via httpcore -h5py==3.11.0 +h5py==3.12.1 # via tensorflow hdfs==2.7.3 # via apache-beam @@ -240,18 +236,13 @@ importlib-metadata==8.5.0 # jupyterlab-server # markdown # nbconvert -importlib-resources==6.4.5 - # via - # jsonschema - # jsonschema-specifications - # jupyterlab ipykernel==5.1.1 # via - # -r - + # -r ./backend/src/apiserver/visualization/requirements.in # jupyterlab ipython==7.12.0 # via - # -r - + # -r ./backend/src/apiserver/visualization/requirements.in # ipykernel # ipywidgets # tensorflow-model-analysis @@ -262,12 +253,12 @@ ipywidgets==7.8.5 isoduration==20.11.0 # via jsonschema itables==0.1.0 - # via -r - + # via -r ./backend/src/apiserver/visualization/requirements.in jedi==0.19.2 # via ipython jinja2==3.1.5 # via - # -r - + # -r ./backend/src/apiserver/visualization/requirements.in # bokeh # jupyter-server # jupyterlab @@ -286,11 +277,11 @@ jsonschema[format-nongpl]==4.23.0 # jupyter-events # jupyterlab-server # nbformat -jsonschema-specifications==2023.12.1 +jsonschema-specifications==2024.10.1 # via jsonschema jupyter-client==8.6.3 # via - # -r - + # -r ./backend/src/apiserver/visualization/requirements.in # ipykernel # jupyter-server # nbclient @@ -302,11 +293,11 @@ jupyter-core==5.7.2 # nbclient # nbconvert # nbformat -jupyter-events==0.10.0 +jupyter-events==0.11.0 # via jupyter-server jupyter-lsp==2.2.5 # via jupyterlab -jupyter-server==2.14.2 +jupyter-server==2.15.0 # via # jupyter-lsp # jupyterlab @@ -335,20 +326,20 @@ markdown==3.7 # via tensorboard markupsafe==2.0.1 # via - # -r - + # -r ./backend/src/apiserver/visualization/requirements.in # jinja2 # nbconvert mistune==3.1.0 # via nbconvert -nbclient==0.10.1 +nbclient==0.10.2 # via nbconvert nbconvert==7.16.5 # via - # -r - + # -r ./backend/src/apiserver/visualization/requirements.in # jupyter-server nbformat==5.10.4 # via - # -r - + # -r ./backend/src/apiserver/visualization/requirements.in # jupyter-server # nbclient # nbconvert @@ -410,10 +401,8 @@ pexpect==4.9.0 # via ipython pickleshare==0.7.5 # via ipython -pillow==10.4.0 +pillow==11.1.0 # via bokeh -pkgutil-resolve-name==1.3.10 - # via jsonschema platformdirs==4.3.6 # via jupyter-core prometheus-client==0.21.1 @@ -486,7 +475,7 @@ pygments==2.19.1 # nbconvert pymongo==3.13.0 # via apache-beam -pyparsing==3.1.4 +pyparsing==3.2.1 # via # httplib2 # pydot @@ -503,7 +492,6 @@ python-json-logger==3.2.1 pytz==2024.2 # via # apache-beam - # babel # pandas pyyaml==6.0.2 # via @@ -540,7 +528,7 @@ rfc3986-validator==0.1.1 # via # jsonschema # jupyter-events -rpds-py==0.20.1 +rpds-py==0.22.3 # via # jsonschema # referencing @@ -549,8 +537,8 @@ rsa==4.9 # google-auth # oauth2client scikit-learn==0.24.2 - # via -r - -scipy==1.10.1 + # via -r ./backend/src/apiserver/visualization/requirements.in +scipy==1.13.1 # via # scikit-learn # tensorflow-model-analysis @@ -559,7 +547,6 @@ send2trash==1.8.3 six==1.17.0 # via # astunparse - # bleach # bokeh # google-api-python-client # google-apitools @@ -586,30 +573,30 @@ tensorboard-plugin-wit==1.8.1 # via tensorboard tensorflow==2.10.1 # via - # -r - + # -r ./backend/src/apiserver/visualization/requirements.in # tensorflow-data-validation # tensorflow-model-analysis # tensorflow-serving-api # tfx-bsl tensorflow-data-validation==1.9.0 - # via -r - + # via -r ./backend/src/apiserver/visualization/requirements.in tensorflow-estimator==2.10.0 # via tensorflow -tensorflow-io-gcs-filesystem==0.34.0 +tensorflow-io-gcs-filesystem==0.37.1 # via tensorflow tensorflow-metadata==1.9.0 # via - # -r - + # -r ./backend/src/apiserver/visualization/requirements.in # tensorflow-data-validation # tensorflow-model-analysis # tfx-bsl tensorflow-model-analysis==0.40.0 - # via -r - + # via -r ./backend/src/apiserver/visualization/requirements.in tensorflow-serving-api==2.10.1 # via - # -r - + # -r ./backend/src/apiserver/visualization/requirements.in # tfx-bsl -termcolor==2.4.0 +termcolor==2.5.0 # via tensorflow terminado==0.18.1 # via @@ -621,13 +608,13 @@ tfx-bsl==1.9.0 # tensorflow-model-analysis threadpoolctl==3.5.0 # via scikit-learn -tinycss2==1.2.1 +tinycss2==1.4.0 # via bleach tomli==2.2.1 # via jupyterlab tornado==6.4.2 # via - # -r - + # -r ./backend/src/apiserver/visualization/requirements.in # bokeh # ipykernel # jupyter-client @@ -663,11 +650,11 @@ uri-template==1.3.0 # via jsonschema uritemplate==3.0.1 # via google-api-python-client -urllib3==2.2.3 +urllib3==2.3.0 # via requests wcwidth==0.2.13 # via prompt-toolkit -webcolors==24.8.0 +webcolors==24.11.1 # via jsonschema webencodings==0.5.1 # via @@ -685,12 +672,7 @@ widgetsnbextension==3.6.10 # via ipywidgets wrapt==1.17.2 # via tensorflow -zipp==3.20.2 - # via - # importlib-metadata - # importlib-resources +zipp==3.21.0 + # via importlib-metadata zstandard==0.23.0 # via apache-beam - -# The following packages are considered to be unsafe in a requirements file: -# setuptools From 019fca9fab9cb75e8c8d62f006aecce9abb34ea0 Mon Sep 17 00:00:00 2001 From: mahdikhashan Date: Fri, 17 Jan 2025 18:33:07 +0100 Subject: [PATCH 13/16] generate requirements.txt with '# via -r -' annotation for dependencies Signed-off-by: mahdikhashan --- .github/resources/scripts/test-requirements.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/resources/scripts/test-requirements.sh b/.github/resources/scripts/test-requirements.sh index 958423046a1..37f4acf7624 100755 --- a/.github/resources/scripts/test-requirements.sh +++ b/.github/resources/scripts/test-requirements.sh @@ -6,7 +6,7 @@ check_requirements() { temp_file=$(mktemp) - pip-compile --output-file=- "$req_in" > "$temp_file" + pip-compile - < "$req_in" --output-file="$temp_file" --annotate # Compare the generated file with the existing requirements.txt if diff -q "$temp_file" "$req_txt" >/dev/null; then From e6b4709c752a120f53c8c5649564907c84ef1e37 Mon Sep 17 00:00:00 2001 From: mahdikhashan Date: Fri, 17 Jan 2025 18:58:41 +0100 Subject: [PATCH 14/16] add backtracking as resolver Signed-off-by: mahdikhashan --- .github/resources/scripts/test-requirements.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/resources/scripts/test-requirements.sh b/.github/resources/scripts/test-requirements.sh index 37f4acf7624..bda8ba3f0de 100755 --- a/.github/resources/scripts/test-requirements.sh +++ b/.github/resources/scripts/test-requirements.sh @@ -6,7 +6,7 @@ check_requirements() { temp_file=$(mktemp) - pip-compile - < "$req_in" --output-file="$temp_file" --annotate + pip-compile - < "$req_in" --output-file="$temp_file" --annotate --resolver=backtracking # Compare the generated file with the existing requirements.txt if diff -q "$temp_file" "$req_txt" >/dev/null; then From abec4b696f0e63dc42903033074635dde3ba6efd Mon Sep 17 00:00:00 2001 From: mahdikhashan Date: Fri, 17 Jan 2025 18:59:23 +0100 Subject: [PATCH 15/16] update requirements in backend/src/apiserver/visulization Signed-off-by: mahdikhashan --- .../apiserver/visualization/requirements.txt | 44 ++++++++++--------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/backend/src/apiserver/visualization/requirements.txt b/backend/src/apiserver/visualization/requirements.txt index 5c1a396dc49..d678bccd20b 100644 --- a/backend/src/apiserver/visualization/requirements.txt +++ b/backend/src/apiserver/visualization/requirements.txt @@ -44,7 +44,7 @@ beautifulsoup4==4.12.3 bleach[css]==6.2.0 # via nbconvert bokeh==1.2.0 - # via -r ./backend/src/apiserver/visualization/requirements.in + # via -r - cachetools==4.2.4 # via # apache-beam @@ -91,7 +91,7 @@ fqdn==1.5.1 gast==0.4.0 # via tensorflow gcsfs==0.2.3 - # via -r ./backend/src/apiserver/visualization/requirements.in + # via -r - google-api-core[grpc]==2.24.0 # via # google-cloud-bigquery @@ -109,7 +109,7 @@ google-api-core[grpc]==2.24.0 # google-cloud-vision google-api-python-client==1.7.12 # via - # -r ./backend/src/apiserver/visualization/requirements.in + # -r - # tfx-bsl google-apitools==0.5.31 # via apache-beam @@ -135,7 +135,7 @@ google-auth-oauthlib==0.4.6 # via # gcsfs # tensorboard -google-cloud-bigquery==3.28.0 +google-cloud-bigquery==3.27.0 # via apache-beam google-cloud-bigquery-storage==2.16.2 # via apache-beam @@ -238,11 +238,11 @@ importlib-metadata==8.5.0 # nbconvert ipykernel==5.1.1 # via - # -r ./backend/src/apiserver/visualization/requirements.in + # -r - # jupyterlab ipython==7.12.0 # via - # -r ./backend/src/apiserver/visualization/requirements.in + # -r - # ipykernel # ipywidgets # tensorflow-model-analysis @@ -253,12 +253,12 @@ ipywidgets==7.8.5 isoduration==20.11.0 # via jsonschema itables==0.1.0 - # via -r ./backend/src/apiserver/visualization/requirements.in + # via -r - jedi==0.19.2 # via ipython jinja2==3.1.5 # via - # -r ./backend/src/apiserver/visualization/requirements.in + # -r - # bokeh # jupyter-server # jupyterlab @@ -281,7 +281,7 @@ jsonschema-specifications==2024.10.1 # via jsonschema jupyter-client==8.6.3 # via - # -r ./backend/src/apiserver/visualization/requirements.in + # -r - # ipykernel # jupyter-server # nbclient @@ -326,7 +326,7 @@ markdown==3.7 # via tensorboard markupsafe==2.0.1 # via - # -r ./backend/src/apiserver/visualization/requirements.in + # -r - # jinja2 # nbconvert mistune==3.1.0 @@ -335,11 +335,11 @@ nbclient==0.10.2 # via nbconvert nbconvert==7.16.5 # via - # -r ./backend/src/apiserver/visualization/requirements.in + # -r - # jupyter-server nbformat==5.10.4 # via - # -r ./backend/src/apiserver/visualization/requirements.in + # -r - # jupyter-server # nbclient # nbconvert @@ -501,7 +501,7 @@ pyzmq==26.2.0 # via # jupyter-client # jupyter-server -referencing==0.35.1 +referencing==0.36.1 # via # jsonschema # jsonschema-specifications @@ -537,7 +537,7 @@ rsa==4.9 # google-auth # oauth2client scikit-learn==0.24.2 - # via -r ./backend/src/apiserver/visualization/requirements.in + # via -r - scipy==1.13.1 # via # scikit-learn @@ -573,28 +573,28 @@ tensorboard-plugin-wit==1.8.1 # via tensorboard tensorflow==2.10.1 # via - # -r ./backend/src/apiserver/visualization/requirements.in + # -r - # tensorflow-data-validation # tensorflow-model-analysis # tensorflow-serving-api # tfx-bsl tensorflow-data-validation==1.9.0 - # via -r ./backend/src/apiserver/visualization/requirements.in + # via -r - tensorflow-estimator==2.10.0 # via tensorflow tensorflow-io-gcs-filesystem==0.37.1 # via tensorflow tensorflow-metadata==1.9.0 # via - # -r ./backend/src/apiserver/visualization/requirements.in + # -r - # tensorflow-data-validation # tensorflow-model-analysis # tfx-bsl tensorflow-model-analysis==0.40.0 - # via -r ./backend/src/apiserver/visualization/requirements.in + # via -r - tensorflow-serving-api==2.10.1 # via - # -r ./backend/src/apiserver/visualization/requirements.in + # -r - # tfx-bsl termcolor==2.5.0 # via tensorflow @@ -614,7 +614,7 @@ tomli==2.2.1 # via jupyterlab tornado==6.4.2 # via - # -r ./backend/src/apiserver/visualization/requirements.in + # -r - # bokeh # ipykernel # jupyter-client @@ -645,6 +645,7 @@ typing-extensions==4.12.2 # async-lru # mistune # python-json-logger + # referencing # tensorflow uri-template==1.3.0 # via jsonschema @@ -676,3 +677,6 @@ zipp==3.21.0 # via importlib-metadata zstandard==0.23.0 # via apache-beam + +# The following packages are considered to be unsafe in a requirements file: +# setuptools From ce7f54ca388ab4c1071837cb103bb89a1db5e8cb Mon Sep 17 00:00:00 2001 From: mahdikhashan Date: Mon, 20 Jan 2025 12:25:34 +0100 Subject: [PATCH 16/16] use make command to run tests in ci Signed-off-by: mahdikhashan --- .github/workflows/requirements-txt-automatically-generated.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/requirements-txt-automatically-generated.yaml b/.github/workflows/requirements-txt-automatically-generated.yaml index 09a9d6b7892..bc79b2076c2 100644 --- a/.github/workflows/requirements-txt-automatically-generated.yaml +++ b/.github/workflows/requirements-txt-automatically-generated.yaml @@ -27,4 +27,4 @@ jobs: - name: Run Tests run: | - ./.github/resources/scripts/test-requirements.sh + make test-requirements