Skip to content

Commit

Permalink
chore(🦾): bump python et-xmlfile 1.1.0 -> 2.0.0 & remove pyhive[hive]…
Browse files Browse the repository at this point in the history
… from requirements/development.in (apache#31040)

Co-authored-by: GitHub Action <[email protected]>
Co-authored-by: Maxime Beauchemin <[email protected]>
  • Loading branch information
3 people authored Nov 24, 2024
1 parent e9e2c0b commit c5f6cc6
Show file tree
Hide file tree
Showing 9 changed files with 59 additions and 48 deletions.
2 changes: 1 addition & 1 deletion .github/actions/setup-backend/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ runs:
- name: Install dependencies
run: |
if [ "${{ inputs.install-superset }}" = "true" ]; then
sudo apt-get update && sudo apt-get -y install libldap2-dev libsasl2-dev
sudo apt-get update && sudo apt-get -y install libldap2-dev libsasl2-dev build-essential
pip install --upgrade pip setuptools wheel
if [ "${{ inputs.requirements-type }}" = "dev" ]; then
pip install -r requirements/development.txt
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/superset-python-presto-hive.yml
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,7 @@ jobs:
- name: Python unit tests (PostgreSQL)
if: steps.check.outputs.python
run: |
pip install -e .[hive]
./scripts/python_tests.sh -m 'chart_data_flow or sql_json_flow'
- name: Upload code coverage
uses: codecov/codecov-action@v4
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ dependencies = [
"nh3>=0.2.11, <0.3",
"numpy==1.23.5",
"packaging",
"pandas[performance]>=2.0.3, <2.1",
"pandas[excel,performance]>=2.0.3, <2.1",
"parsedatetime",
"paramiko>=3.4.0",
"pgsanity",
Expand Down
18 changes: 16 additions & 2 deletions requirements/base.txt
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,8 @@ cryptography==42.0.8
# apache-superset
# paramiko
# pyopenssl
defusedxml==0.7.1
# via odfpy
deprecated==1.2.14
# via limits
deprecation==2.1.0
Expand All @@ -93,6 +95,8 @@ dnspython==2.6.1
# via email-validator
email-validator==2.1.1
# via flask-appbuilder
et-xmlfile==2.0.0
# via openpyxl
exceptiongroup==1.2.2
# via cattrs
flask==2.3.3
Expand Down Expand Up @@ -231,6 +235,10 @@ numpy==1.23.5
# numexpr
# pandas
# pyarrow
odfpy==1.4.1
# via pandas
openpyxl==3.1.5
# via pandas
ordered-set==4.1.0
# via flask-limiter
packaging==23.2
Expand All @@ -243,7 +251,7 @@ packaging==23.2
# marshmallow
# marshmallow-sqlalchemy
# shillelagh
pandas[performance]==2.0.3
pandas[excel,performance]==2.0.3
# via apache-superset
paramiko==3.4.0
# via
Expand Down Expand Up @@ -306,6 +314,8 @@ pytz==2024.2
# croniter
# flask-babel
# pandas
pyxlsb==1.0.10
# via pandas
pyyaml==6.0.1
# via
# apache-superset
Expand Down Expand Up @@ -404,8 +414,12 @@ wtforms==3.2.1
# wtforms-json
wtforms-json==0.3.5
# via apache-superset
xlrd==2.0.1
# via pandas
xlsxwriter==3.0.9
# via apache-superset
# via
# apache-superset
# pandas
zipp==3.19.0
# via importlib-metadata
zstandard==0.22.0
Expand Down
2 changes: 1 addition & 1 deletion requirements/development.in
Original file line number Diff line number Diff line change
Expand Up @@ -17,4 +17,4 @@
# under the License.
#
-r base.in
-e .[development,bigquery,cors,druid,gevent,gsheets,hive,mysql,playwright,postgres,presto,prophet,trino,thumbnails]
-e .[development,bigquery,cors,druid,gevent,gsheets,mysql,playwright,postgres,presto,prophet,trino,thumbnails]
42 changes: 3 additions & 39 deletions requirements/development.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# SHA1:c186006a3f82c8775e1039f37c52309f6c858197
# SHA1:dc767a7288b56c785b0cd3c38e95e7b5e66be1ac
#
# This file is autogenerated by pip-compile-multi
# To update, run:
Expand All @@ -13,23 +13,17 @@
astroid==3.1.0
# via pylint
boto3==1.34.112
# via
# apache-superset
# dataflows-tabulator
# via apache-superset
botocore==1.34.112
# via
# boto3
# s3transfer
build==1.2.1
# via pip-tools
cached-property==1.5.2
# via tableschema
cfgv==3.3.1
# via pre-commit
chardet==5.1.0
# via
# dataflows-tabulator
# tox
# via tox
cmdstanpy==1.1.0
# via prophet
contourpy==1.0.7
Expand All @@ -38,8 +32,6 @@ coverage[toml]==7.2.5
# via pytest-cov
cycler==0.11.0
# via matplotlib
dataflows-tabulator==1.54.3
# via tableschema
db-dtypes==1.2.0
# via pandas-gbq
dill==0.3.8
Expand All @@ -48,8 +40,6 @@ distlib==0.3.8
# via virtualenv
docker==7.0.0
# via apache-superset
et-xmlfile==1.1.0
# via openpyxl
filelock==3.12.2
# via
# tox
Expand Down Expand Up @@ -103,8 +93,6 @@ grpcio-status==1.60.1
# via google-api-core
identify==2.5.36
# via pre-commit
ijson==3.2.3
# via dataflows-tabulator
iniconfig==2.0.0
# via pytest
isort==5.12.0
Expand All @@ -113,16 +101,12 @@ jmespath==1.0.1
# via
# boto3
# botocore
jsonlines==4.0.0
# via dataflows-tabulator
jsonschema-spec==0.1.6
# via openapi-spec-validator
kiwisolver==1.4.5
# via matplotlib
lazy-object-proxy==1.10.0
# via openapi-spec-validator
linear-tsv==1.1.0
# via dataflows-tabulator
matplotlib==3.9.0
# via prophet
mccabe==0.7.0
Expand All @@ -137,8 +121,6 @@ openapi-schema-validator==0.4.4
# via openapi-spec-validator
openapi-spec-validator==0.5.6
# via apache-superset
openpyxl==3.1.2
# via dataflows-tabulator
pandas-gbq==0.19.1
# via apache-superset
parameterized==0.9.0
Expand Down Expand Up @@ -178,8 +160,6 @@ psutil==6.0.0
# via apache-superset
psycopg2-binary==2.9.6
# via apache-superset
pure-sasl==0.6.2
# via thrift-sasl
pydata-google-auth==1.7.0
# via pandas-gbq
pydruid==0.6.9
Expand Down Expand Up @@ -215,8 +195,6 @@ requests-oauthlib==2.0.0
# via google-auth-oauthlib
rfc3339-validator==0.1.4
# via openapi-schema-validator
rfc3986==2.0.0
# via tableschema
ruff==0.4.5
# via apache-superset
s3transfer==0.10.1
Expand All @@ -227,14 +205,6 @@ sqloxide==0.1.43
# via apache-superset
statsd==4.0.1
# via apache-superset
tableschema==1.20.10
# via apache-superset
thrift==0.16.0
# via
# apache-superset
# thrift-sasl
thrift-sasl==0.4.3
# via apache-superset
tomli==2.0.1
# via
# build
Expand All @@ -259,18 +229,12 @@ trino==0.328.0
# via apache-superset
tzlocal==5.2
# via trino
unicodecsv==0.14.1
# via
# dataflows-tabulator
# tableschema
virtualenv==20.23.1
# via
# pre-commit
# tox
wheel==0.43.0
# via pip-tools
xlrd==2.0.1
# via dataflows-tabulator
zope-event==5.0
# via gevent
zope-interface==5.4.0
Expand Down
5 changes: 4 additions & 1 deletion tests/integration_tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,10 @@ def setup_sample_data() -> Any:
# relying on `tests.integration_tests.test_app.app` leveraging an `app` fixture
# which is purposely scoped to the function level to ensure tests remain idempotent.
with app.app_context():
setup_presto_if_needed()
try:
setup_presto_if_needed()
except Exception:
pass

from superset.examples.css_templates import load_css_templates

Expand Down
14 changes: 14 additions & 0 deletions tests/integration_tests/db_engine_specs/hive_tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@
# under the License.
# isort:skip_file
from unittest import mock
import unittest
from .base_tests import SupersetTestCase

import pytest
import pandas as pd
Expand Down Expand Up @@ -154,6 +156,9 @@ def test_df_to_sql_if_exists_fail(mock_g):


@mock.patch("superset.db_engine_specs.hive.g", spec={})
@unittest.skipUnless(
SupersetTestCase.is_module_installed("thrift"), "thrift not installed"
)
def test_df_to_sql_if_exists_fail_with_schema(mock_g):
mock_g.user = True
mock_database = mock.MagicMock()
Expand Down Expand Up @@ -290,6 +295,9 @@ def test_upload_to_s3_success(client):
app.config = config


@unittest.skipUnless(
SupersetTestCase.is_module_installed("thrift"), "thrift not installed"
)
def test_fetch_data_query_error():
from TCLIService import ttypes

Expand All @@ -301,6 +309,9 @@ def test_fetch_data_query_error():
HiveEngineSpec.fetch_data(cursor)


@unittest.skipUnless(
SupersetTestCase.is_module_installed("thrift"), "thrift not installed"
)
@mock.patch("superset.db_engine_specs.base.BaseEngineSpec.fetch_data")
def test_fetch_data_programming_error(fetch_data_mock):
from pyhive.exc import ProgrammingError
Expand All @@ -310,6 +321,9 @@ def test_fetch_data_programming_error(fetch_data_mock):
assert HiveEngineSpec.fetch_data(cursor) == []


@unittest.skipUnless(
SupersetTestCase.is_module_installed("thrift"), "thrift not installed"
)
@mock.patch("superset.db_engine_specs.base.BaseEngineSpec.fetch_data")
def test_fetch_data_success(fetch_data_mock):
return_value = ["a", "b"]
Expand Down
21 changes: 18 additions & 3 deletions tests/integration_tests/model_tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,9 @@ class TestDatabaseModel(SupersetTestCase):
@unittest.skipUnless(
SupersetTestCase.is_module_installed("requests"), "requests not installed"
)
@unittest.skipUnless(
SupersetTestCase.is_module_installed("pyhive"), "pyhive not installed"
)
def test_database_schema_presto(self):
sqlalchemy_uri = "presto://presto.airbnb.io:8080/hive/default"
model = Database(database_name="test_database", sqlalchemy_uri=sqlalchemy_uri)
Expand Down Expand Up @@ -108,7 +111,7 @@ def test_database_schema_hive(self):
assert "core_db" == db

@unittest.skipUnless(
SupersetTestCase.is_module_installed("MySQLdb"), "mysqlclient not installed"
SupersetTestCase.is_module_installed("mysqlclient"), "mysqlclient not installed"
)
def test_database_schema_mysql(self):
sqlalchemy_uri = "mysql://root@localhost/superset"
Expand All @@ -123,7 +126,7 @@ def test_database_schema_mysql(self):
assert "staging" == db

@unittest.skipUnless(
SupersetTestCase.is_module_installed("MySQLdb"), "mysqlclient not installed"
SupersetTestCase.is_module_installed("mysqlclient"), "mysqlclient not installed"
)
def test_database_impersonate_user(self):
uri = "mysql://root@localhost"
Expand All @@ -142,6 +145,9 @@ def test_database_impersonate_user(self):
assert example_user.username != username

@mock.patch("superset.models.core.create_engine")
@unittest.skipUnless(
SupersetTestCase.is_module_installed("pyhive"), "pyhive not installed"
)
def test_impersonate_user_presto(self, mocked_create_engine):
uri = "presto://localhost"
principal_user = security_manager.find_user(username="gamma")
Expand Down Expand Up @@ -190,7 +196,7 @@ def test_impersonate_user_presto(self, mocked_create_engine):
}

@unittest.skipUnless(
SupersetTestCase.is_module_installed("MySQLdb"), "mysqlclient not installed"
SupersetTestCase.is_module_installed("mysqlclient"), "mysqlclient not installed"
)
@mock.patch("superset.models.core.create_engine")
def test_adjust_engine_params_mysql(self, mocked_create_engine):
Expand Down Expand Up @@ -245,6 +251,12 @@ def test_impersonate_user_trino(self, mocked_create_engine):
assert call_args[1]["connect_args"]["user"] == "gamma"

@mock.patch("superset.models.core.create_engine")
@unittest.skipUnless(
SupersetTestCase.is_module_installed("pyhive"), "pyhive not installed"
)
@unittest.skipUnless(
SupersetTestCase.is_module_installed("thrift"), "thrift not installed"
)
def test_impersonate_user_hive(self, mocked_create_engine):
uri = "hive://localhost"
principal_user = security_manager.find_user(username="gamma")
Expand Down Expand Up @@ -293,6 +305,9 @@ def test_impersonate_user_hive(self, mocked_create_engine):
}

@pytest.mark.usefixtures("load_energy_table_with_slice")
@unittest.skipUnless(
SupersetTestCase.is_module_installed("pyhive"), "pyhive not installed"
)
def test_select_star(self):
db = get_example_database()
table_name = "energy_usage"
Expand Down

0 comments on commit c5f6cc6

Please sign in to comment.