diff --git a/superset/charts/post_processing.py b/superset/charts/client_processing.py similarity index 97% rename from superset/charts/post_processing.py rename to superset/charts/client_processing.py index 807760cb6b5ca..5a873ab8986df 100644 --- a/superset/charts/post_processing.py +++ b/superset/charts/client_processing.py @@ -15,15 +15,13 @@ # specific language governing permissions and limitations # under the License. """ -Functions to reproduce the post-processing of data on text charts. +Functions to reproduce the client post-processing of data on charts. -Some text-based charts (pivot tables and t-test table) perform -post-processing of the data in JavaScript. When sending the data -to users in reports we want to show the same data they would see -on Explore. +Some text-based charts (pivot tables and t-test table) perform post-processing of the +data in JavaScript. When sending the data to users in reports we want to show the same +data they would see on Explore. -In order to do that, we reproduce the post-processing in Python -for these chart types. +In order to do that, we reproduce the post-processing in Python for these chart types. """ from io import StringIO @@ -298,7 +296,7 @@ def table( @event_logger.log_this -def apply_post_process( # noqa: C901 +def apply_client_processing( # noqa: C901 result: dict[Any, Any], form_data: Optional[dict[str, Any]] = None, datasource: Optional[Union["BaseDatasource", "Query"]] = None, diff --git a/superset/charts/data/api.py b/superset/charts/data/api.py index a71171d29d837..e8080c646f2cb 100644 --- a/superset/charts/data/api.py +++ b/superset/charts/data/api.py @@ -28,8 +28,8 @@ from superset import is_feature_enabled, security_manager from superset.async_events.async_query_manager import AsyncQueryTokenException from superset.charts.api import ChartRestApi +from superset.charts.client_processing import apply_client_processing from superset.charts.data.query_context_cache_loader import QueryContextCacheLoader -from superset.charts.post_processing import apply_post_process from superset.charts.schemas import ChartDataQueryContextSchema from superset.commands.chart.data.create_async_job_command import ( CreateAsyncChartDataJobCommand, @@ -356,7 +356,7 @@ def _send_chart_response( # noqa: C901 # This is needed for sending reports based on text charts that do the # post-processing of data, eg, the pivot table. if result_type == ChartDataResultType.POST_PROCESSED: - result = apply_post_process(result, form_data, datasource) + result = apply_client_processing(result, form_data, datasource) if result_format in ChartDataResultFormat.table_like(): # Verify user has permission to export file diff --git a/superset/common/query_object.py b/superset/common/query_object.py index 32ab93448da93..a9956d6f73fdd 100644 --- a/superset/common/query_object.py +++ b/superset/common/query_object.py @@ -33,6 +33,7 @@ QueryClauseValidationException, QueryObjectValidationError, ) +from superset.extensions import event_logger from superset.sql_parse import sanitize_clause from superset.superset_typing import Column, Metric, OrderBy from superset.utils import json, pandas_postprocessing @@ -428,19 +429,20 @@ def exec_post_processing(self, df: DataFrame) -> DataFrame: is incorrect """ logger.debug("post_processing: \n %s", pformat(self.post_processing)) - for post_process in self.post_processing: - operation = post_process.get("operation") - if not operation: - raise InvalidPostProcessingError( - _("`operation` property of post processing object undefined") - ) - if not hasattr(pandas_postprocessing, operation): - raise InvalidPostProcessingError( - _( - "Unsupported post processing operation: %(operation)s", - type=operation, + with event_logger.log_context(f"{self.__class__.__name__}.post_processing"): + for post_process in self.post_processing: + operation = post_process.get("operation") + if not operation: + raise InvalidPostProcessingError( + _("`operation` property of post processing object undefined") ) - ) - options = post_process.get("options", {}) - df = getattr(pandas_postprocessing, operation)(df, **options) - return df + if not hasattr(pandas_postprocessing, operation): + raise InvalidPostProcessingError( + _( + "Unsupported post processing operation: %(operation)s", + type=operation, + ) + ) + options = post_process.get("options", {}) + df = getattr(pandas_postprocessing, operation)(df, **options) + return df diff --git a/tests/unit_tests/charts/test_post_processing.py b/tests/unit_tests/charts/test_client_processing.py similarity index 98% rename from tests/unit_tests/charts/test_post_processing.py rename to tests/unit_tests/charts/test_client_processing.py index f3eaf6fa7e953..e35229554b503 100644 --- a/tests/unit_tests/charts/test_post_processing.py +++ b/tests/unit_tests/charts/test_client_processing.py @@ -21,7 +21,7 @@ from flask_babel import lazy_gettext as _ from sqlalchemy.orm.session import Session -from superset.charts.post_processing import apply_post_process, pivot_df, table +from superset.charts.client_processing import apply_client_processing, pivot_df, table from superset.common.chart_data import ChartDataResultFormat from superset.utils.core import GenericDataType @@ -1841,16 +1841,16 @@ def test_table(): ) -def test_apply_post_process_no_form_invalid_viz_type(): +def test_apply_client_processing_no_form_invalid_viz_type(): """ Test with invalid viz type. It should just return the result """ result = {"foo": "bar"} form_data = {"viz_type": "baz"} - assert apply_post_process(result, form_data) == result + assert apply_client_processing(result, form_data) == result -def test_apply_post_process_without_result_format(): +def test_apply_client_processing_without_result_format(): """ A query without result_format should raise an exception """ @@ -1858,12 +1858,12 @@ def test_apply_post_process_without_result_format(): form_data = {"viz_type": "pivot_table_v2"} with pytest.raises(Exception) as ex: # noqa: PT011 - apply_post_process(result, form_data) + apply_client_processing(result, form_data) assert ex.match("Result format foo not supported") is True # noqa: E712 -def test_apply_post_process_json_format(): +def test_apply_client_processing_json_format(): """ It should be able to process json results """ @@ -1944,7 +1944,7 @@ def test_apply_post_process_json_format(): "result_type": "results", } - assert apply_post_process(result, form_data) == { + assert apply_client_processing(result, form_data) == { "queries": [ { "result_format": ChartDataResultFormat.JSON, @@ -1966,7 +1966,7 @@ def test_apply_post_process_json_format(): } -def test_apply_post_process_csv_format(): +def test_apply_client_processing_csv_format(): """ It should be able to process csv results """ @@ -2042,7 +2042,7 @@ def test_apply_post_process_csv_format(): "result_type": "results", } - assert apply_post_process(result, form_data) == { + assert apply_client_processing(result, form_data) == { "queries": [ { "result_format": ChartDataResultFormat.CSV, @@ -2056,7 +2056,7 @@ def test_apply_post_process_csv_format(): } -def test_apply_post_process_csv_format_empty_string(): +def test_apply_client_processing_csv_format_empty_string(): """ It should be able to process csv results with no data """ @@ -2122,13 +2122,13 @@ def test_apply_post_process_csv_format_empty_string(): "result_type": "results", } - assert apply_post_process(result, form_data) == { + assert apply_client_processing(result, form_data) == { "queries": [{"result_format": ChartDataResultFormat.CSV, "data": ""}] } @pytest.mark.parametrize("data", [None, "", "\n"]) -def test_apply_post_process_csv_format_no_data(data): +def test_apply_client_processing_csv_format_no_data(data): """ It should be able to process csv results with no data """ @@ -2194,12 +2194,12 @@ def test_apply_post_process_csv_format_no_data(data): "result_type": "results", } - assert apply_post_process(result, form_data) == { + assert apply_client_processing(result, form_data) == { "queries": [{"result_format": ChartDataResultFormat.CSV, "data": data}] } -def test_apply_post_process_csv_format_no_data_multiple_queries(): +def test_apply_client_processing_csv_format_no_data_multiple_queries(): """ It should be able to process csv results multiple queries if one query has no data """ @@ -2276,7 +2276,7 @@ def test_apply_post_process_csv_format_no_data_multiple_queries(): "result_type": "results", } - assert apply_post_process(result, form_data) == { + assert apply_client_processing(result, form_data) == { "queries": [ {"result_format": ChartDataResultFormat.CSV, "data": ""}, { @@ -2291,7 +2291,7 @@ def test_apply_post_process_csv_format_no_data_multiple_queries(): } -def test_apply_post_process_json_format_empty_string(): +def test_apply_client_processing_json_format_empty_string(): """ It should be able to process json results with no data """ @@ -2357,12 +2357,12 @@ def test_apply_post_process_json_format_empty_string(): "result_type": "results", } - assert apply_post_process(result, form_data) == { + assert apply_client_processing(result, form_data) == { "queries": [{"result_format": ChartDataResultFormat.JSON, "data": ""}] } -def test_apply_post_process_json_format_data_is_none(): +def test_apply_client_processing_json_format_data_is_none(): """ It should be able to process json results with no data """ @@ -2428,12 +2428,12 @@ def test_apply_post_process_json_format_data_is_none(): "result_type": "results", } - assert apply_post_process(result, form_data) == { + assert apply_client_processing(result, form_data) == { "queries": [{"result_format": ChartDataResultFormat.JSON, "data": None}] } -def test_apply_post_process_verbose_map(session: Session): +def test_apply_client_processing_verbose_map(session: Session): from superset import db from superset.connectors.sqla.models import SqlaTable, SqlMetric from superset.models.core import Database @@ -2487,7 +2487,7 @@ def test_apply_post_process_verbose_map(session: Session): "result_type": "results", } - assert apply_post_process(result, form_data, datasource=sqla_table) == { + assert apply_client_processing(result, form_data, datasource=sqla_table) == { "queries": [ { "result_format": ChartDataResultFormat.JSON,