From 55cf05aecf130b3b7dab207d9b2d70e7f9948ff3 Mon Sep 17 00:00:00 2001 From: Tristan Nixon Date: Thu, 2 Nov 2023 15:59:50 -0700 Subject: [PATCH] reorganizing tests to run a suite of tests for each LTS DBR version since Spark 3.0 --- docs/requirements.txt | 6 ++++++ python/requirements.txt | 19 ------------------- python/requirements/dbr104.txt | 10 ++++++++++ python/requirements/dbr113.txt | 9 +++++++++ python/requirements/dbr122.txt | 9 +++++++++ python/requirements/dbr133.txt | 9 +++++++++ python/requirements/dbr73.txt | 10 ++++++++++ python/requirements/dbr91.txt | 10 ++++++++++ python/tox.ini | 31 +++++++++++++++++++------------ 9 files changed, 82 insertions(+), 31 deletions(-) create mode 100644 docs/requirements.txt delete mode 100644 python/requirements.txt create mode 100644 python/requirements/dbr104.txt create mode 100644 python/requirements/dbr113.txt create mode 100644 python/requirements/dbr122.txt create mode 100644 python/requirements/dbr133.txt create mode 100644 python/requirements/dbr73.txt create mode 100644 python/requirements/dbr91.txt diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 00000000..7a76c34a --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,6 @@ +sphinx-autobuild==2021.3.14 +sphinx-copybutton==0.5.1 +Sphinx==4.5.0 +sphinx-design==0.2.0 +sphinx-panels==0.6.0 +furo==2022.9.29 \ No newline at end of file diff --git a/python/requirements.txt b/python/requirements.txt deleted file mode 100644 index 1a6844a9..00000000 --- a/python/requirements.txt +++ /dev/null @@ -1,19 +0,0 @@ -ipython==8.10.0 -numpy==1.24.3 -chispa==0.9.2 -pandas==1.5.2 -pyarrow==12.0.0 -python-dateutil==2.8.2 -pytz==2022.7.1 -scipy==1.10.1 -six==1.16.0 -wheel==0.38.4 -semver==2.13.0 -sphinx-autobuild==2021.3.14 -furo==2022.9.29 -sphinx-copybutton==0.5.1 -Sphinx==4.5.0 -sphinx-design==0.2.0 -sphinx-panels==0.6.0 -jsonref==1.1.0 -python-dateutil==2.8.2 diff --git a/python/requirements/dbr104.txt b/python/requirements/dbr104.txt new file mode 100644 index 00000000..f8b2ca74 --- /dev/null +++ b/python/requirements/dbr104.txt @@ -0,0 +1,10 @@ +ipython==7.22.0 +numpy==1.20.1 +pandas==1.2.4 +pyarrow==4.0.0 +python-dateutil==2.8.1 +pytz==2020.5 +scipy==1.6.2 +six==1.15.0 +wheel==0.36.2 + diff --git a/python/requirements/dbr113.txt b/python/requirements/dbr113.txt new file mode 100644 index 00000000..f593cb98 --- /dev/null +++ b/python/requirements/dbr113.txt @@ -0,0 +1,9 @@ +ipython==7.32.0 +numpy==1.20.3 +pandas==1.3.4 +pyarrow==7.0.0 +python-dateutil==2.8.2 +pytz==2021.3 +scipy==1.7.1 +six==1.16.0 +wheel==0.37.0 diff --git a/python/requirements/dbr122.txt b/python/requirements/dbr122.txt new file mode 100644 index 00000000..665831b5 --- /dev/null +++ b/python/requirements/dbr122.txt @@ -0,0 +1,9 @@ +ipython==8.5.0 +numpy==1.21.5 +pandas==1.4.2 +pyarrow==7.0.0 +python-dateutil==2.8.2 +pytz==2021.3 +scipy==1.7.3 +six==1.16.0 +wheel==0.37.0 diff --git a/python/requirements/dbr133.txt b/python/requirements/dbr133.txt new file mode 100644 index 00000000..d72d2cab --- /dev/null +++ b/python/requirements/dbr133.txt @@ -0,0 +1,9 @@ +ipython==8.10.0 +numpy==1.21.5 +pandas==1.4.4 +pyarrow==8.0.0 +python-dateutil==2.8.2 +pytz==2022.1 +scipy==1.9.1 +six==1.16.0 +wheel==0.37.1 diff --git a/python/requirements/dbr73.txt b/python/requirements/dbr73.txt new file mode 100644 index 00000000..cde477d4 --- /dev/null +++ b/python/requirements/dbr73.txt @@ -0,0 +1,10 @@ +ipython==7.12.0 +numpy==1.18.1 +pandas==1.0.1 +pyarrow==1.0.1 +python-dateutil==2.8.1 +pytz==2019.3 +scipy==1.4.1 +six==1.14.0 +wheel==0.34.2 + diff --git a/python/requirements/dbr91.txt b/python/requirements/dbr91.txt new file mode 100644 index 00000000..c293d28d --- /dev/null +++ b/python/requirements/dbr91.txt @@ -0,0 +1,10 @@ +ipython==7.22.0 +numpy==1.19.2 +pandas==1.2.4 +pyarrow==4.0.0 +python-dateutil==2.8.1 +pytz==2020.5 +scipy==1.6.2 +six==1.15.0 +wheel==0.36.2 + diff --git a/python/tox.ini b/python/tox.ini index d6af2f91..fc3cab45 100644 --- a/python/tox.ini +++ b/python/tox.ini @@ -2,7 +2,6 @@ requires = tox>4,<5 virtualenv>20,<21 - wheel>=0.38,<1 isolated_build = true envlist = format @@ -11,26 +10,34 @@ envlist = build-dist ; Mirror Supported LTS DBR versions here: https://docs.databricks.com/release-notes/runtime/ ; Use correct PySpark version based on Python version present in env name - py37-pyspark300, - py38-pyspark{312,321}, - py39-pyspark{330,332} + dbr{73,91,104,113,122,133} skip_missing_interpreters = true - [testenv] description = run the tests under {envname} package = wheel wheel_build_env = .pkg setenv = COVERAGE_FILE = .coverage.{envname} +basepython = + dbr133: py310 + dbr122: py39 + dbr113: py39 + dbr104: py38 + dbr91: py38 + dbr73: py37 deps = - pyspark300: pyspark==3.0.0 - pyspark312: pyspark==3.1.2 - pyspark321: pyspark==3.2.1 - pyspark330: pyspark==3.3.0 - pyspark332: pyspark==3.3.2 + dbr133: pyspark==3.4.1 + dbr122: pyspark==3.3.2 + dbr113: pyspark==3.3.0 + dbr104: pyspark==3.2.1 + dbr91: pyspark==3.1.2 + dbr73: pyspark==3.0.0 coverage>=7,<8 - -rrequirements.txt + chispa~=0.9.4 + semver>=3,<4 + jsonref>=1,<2 + -rrequirements/{envname}.txt commands = coverage --version coverage run -m unittest discover -s tests -p '*_tests.py' @@ -63,7 +70,7 @@ deps = mypy>=1,<2 pandas-stubs>=2,<3 types-pytz>=2023,<2024 - -rrequirements.txt + -rrequirements/dbr133.txt commands = mypy {toxinidir}/tempo