From 9449c73aff8706de14eb1c9d51b6aa2194bf61d8 Mon Sep 17 00:00:00 2001 From: Antony Milne Date: Mon, 4 Nov 2024 10:52:03 +0000 Subject: [PATCH 01/64] Reduce to just one load function in filter.py --- ...30_170000_antony.milne_vvv_link_targets.md | 48 +++++++++ .../src/vizro/models/_controls/filter.py | 100 +++++++++--------- 2 files changed, 99 insertions(+), 49 deletions(-) create mode 100644 vizro-core/changelog.d/20241030_170000_antony.milne_vvv_link_targets.md diff --git a/vizro-core/changelog.d/20241030_170000_antony.milne_vvv_link_targets.md b/vizro-core/changelog.d/20241030_170000_antony.milne_vvv_link_targets.md new file mode 100644 index 000000000..7c0d58d4f --- /dev/null +++ b/vizro-core/changelog.d/20241030_170000_antony.milne_vvv_link_targets.md @@ -0,0 +1,48 @@ + + + + + + + + + diff --git a/vizro-core/src/vizro/models/_controls/filter.py b/vizro-core/src/vizro/models/_controls/filter.py index a66e8d62d..ac8f7c259 100644 --- a/vizro-core/src/vizro/models/_controls/filter.py +++ b/vizro-core/src/vizro/models/_controls/filter.py @@ -1,5 +1,6 @@ from __future__ import annotations +import numpy as np from typing import Literal, Union import pandas as pd @@ -86,6 +87,7 @@ class Filter(VizroBaseModel): ) selector: SelectorType = None _column_type: Literal["numerical", "categorical", "temporal"] = PrivateAttr() + _data_frames = PrivateAttr() @validator("targets", each_item=True) def check_target_present(cls, target): @@ -108,31 +110,57 @@ def build(self): return self.selector.build() def _set_targets(self): - if not self.targets: - for component_id in model_manager._get_page_model_ids_with_figure( - page_id=model_manager._get_model_page_id(model_id=ModelID(str(self.id))) - ): - # TODO: consider making a helper method in data_manager or elsewhere to reduce this operation being - # duplicated across Filter so much, and/or consider storing the result to avoid repeating it. - # Need to think about this in connection with how to update filters on the fly and duplicated calls - # issue outlined in https://github.com/mckinsey/vizro/pull/398#discussion_r1559120849. - data_source_name = model_manager[component_id]["data_frame"] - data_frame = data_manager[data_source_name].load() - if self.column in data_frame.columns: - self.targets.append(component_id) - if not self.targets: - raise ValueError(f"Selected column {self.column} not found in any dataframe on this page.") + # TODO: consider moving to data_manager, depending on Petar's work + # TODO: write tests + potential_targets = self.targets or model_manager._get_page_model_ids_with_figure( + page_id=model_manager._get_model_page_id(model_id=ModelID(str(self.id))) + ) + + potential_target_to_data_source_name = { + target: model_manager[target]["data_frame"] for target in potential_targets + } + # Using set() here ensures we only load each data source once rather than repeating the operation for each + # target. + data_source_name_to_data = { + data_source_name: data_manager[data_source_name].load() + for data_source_name in set(potential_target_to_data_source_name.values()) + } + target_to_series = dict() + + for target, data_source_name in potential_target_to_data_source_name.items(): + data_frame = data_source_name_to_data[data_source_name] + + if self.column in data_frame.columns: + target_to_series[target] = data_frame + elif self.targets: + # targets were manually specified so it's not ok the column isn't there. If targets were not specified + # then it's fine, we just skip this target, and error is not raised. + raise ValueError(f"Selected column {self.column} not found in dataframe for {target}.") + + if not target_to_series: + raise ValueError(f"Selected column {self.column} not found in any dataframe on this page.") + + self.targets = list(target_to_series) + # COMMEnt. will have repeats + self._targeted_data = pd.DataFrame.from_dict(target_to_series) def _set_column_type(self): - data_source_name = model_manager[self.targets[0]]["data_frame"] - data_frame = data_manager[data_source_name].load() + # TODO: check + is_numerical = self._targeted_data.apply(is_numeric_dtype) + is_temporal = self._targeted_data.apply(is_datetime64_any_dtype) + is_categorical = ~is_numerical & ~is_temporal - if is_numeric_dtype(data_frame[self.column]): + if is_numerical.all(): self._column_type = "numerical" - elif is_datetime64_any_dtype(data_frame[self.column]): + elif is_temporal.all(): self._column_type = "temporal" - else: + elif is_categorical.all(): self._column_type = "categorical" + else: + raise ValueError( + f"Inconsistent types detected in the shared data column {self.column}. This column must " + "have the same type for all targets." + ) def _set_selector(self): self.selector = self.selector or SELECTORS[self._column_type][0]() @@ -149,40 +177,14 @@ def _set_numerical_and_temporal_selectors_values(self): # If the selector is a numerical or temporal selector, and the min and max values are not set, then set them # N.B. All custom selectors inherit from numerical or temporal selector should also pass this check if isinstance(self.selector, SELECTORS["numerical"] + SELECTORS["temporal"]): - min_values = [] - max_values = [] - for target_id in self.targets: - data_source_name = model_manager[target_id]["data_frame"] - data_frame = data_manager[data_source_name].load() - min_values.append(data_frame[self.column].min()) - max_values.append(data_frame[self.column].max()) - - if not ( - (is_numeric_dtype(pd.Series(min_values)) and is_numeric_dtype(pd.Series(max_values))) - or (is_datetime64_any_dtype(pd.Series(min_values)) and is_datetime64_any_dtype(pd.Series(max_values))) - ): - raise ValueError( - f"Inconsistent types detected in the shared data column '{self.column}' for targeted charts " - f"{self.targets}. Please ensure that the data column contains the same data type across all " - f"targeted charts." - ) - - if self.selector.min is None: - self.selector.min = min(min_values) - if self.selector.max is None: - self.selector.max = max(max_values) + self.selector.min = self.selector.min or self._targeted_data.to_numpy().min() + self.selector.max = self.selector.max or self._targeted_data.to_numpy().max() def _set_categorical_selectors_options(self): # If the selector is a categorical selector, and the options are not set, then set them # N.B. All custom selectors inherit from categorical selector should also pass this check - if isinstance(self.selector, SELECTORS["categorical"]) and not self.selector.options: - options = set() - for target_id in self.targets: - data_source_name = model_manager[target_id]["data_frame"] - data_frame = data_manager[data_source_name].load() - options |= set(data_frame[self.column]) - - self.selector.options = sorted(options) + if isinstance(self.selector, SELECTORS["categorical"]): + self.selector.options = self.selector.options or sorted(np.unique(self._targeted_data.to_numpy())) def _set_actions(self): if not self.selector.actions: From b37f38f2c2994ca5823b7241bb96759db684dde6 Mon Sep 17 00:00:00 2001 From: Antony Milne Date: Mon, 4 Nov 2024 16:18:02 +0000 Subject: [PATCH 02/64] Refactor set methods and add in __call__ --- vizro-core/hatch.toml | 3 +- vizro-core/pyproject.toml | 2 +- .../src/vizro/models/_controls/filter.py | 188 ++++++++++-------- 3 files changed, 113 insertions(+), 80 deletions(-) diff --git a/vizro-core/hatch.toml b/vizro-core/hatch.toml index b532cb558..649fdd187 100644 --- a/vizro-core/hatch.toml +++ b/vizro-core/hatch.toml @@ -111,7 +111,8 @@ VIZRO_LOG_LEVEL = "DEBUG" extra-dependencies = [ "pydantic==1.10.16", "dash==2.17.1", - "plotly==5.12.0" + "plotly==5.12.0", + "pandas==2.0.0", ] features = ["kedro"] python = "3.9" diff --git a/vizro-core/pyproject.toml b/vizro-core/pyproject.toml index aa511adc6..a1992ddc8 100644 --- a/vizro-core/pyproject.toml +++ b/vizro-core/pyproject.toml @@ -18,7 +18,7 @@ dependencies = [ "dash>=2.17.1", # 2.17.1 needed for no_output fix in clientside_callback "dash_bootstrap_components", "dash-ag-grid>=31.0.0", - "pandas", + "pandas>=2", "plotly>=5.12.0", "pydantic>=1.10.16", # must be synced with pre-commit mypy hook manually "dash_mantine_components<0.13.0", # 0.13.0 is not compatible with 0.12, diff --git a/vizro-core/src/vizro/models/_controls/filter.py b/vizro-core/src/vizro/models/_controls/filter.py index ac8f7c259..4fbd105fd 100644 --- a/vizro-core/src/vizro/models/_controls/filter.py +++ b/vizro-core/src/vizro/models/_controls/filter.py @@ -87,7 +87,6 @@ class Filter(VizroBaseModel): ) selector: SelectorType = None _column_type: Literal["numerical", "categorical", "temporal"] = PrivateAttr() - _data_frames = PrivateAttr() @validator("targets", each_item=True) def check_target_present(cls, target): @@ -97,107 +96,140 @@ def check_target_present(cls, target): @_log_call def pre_build(self): - self._set_targets() - self._set_column_type() - self._set_selector() - self._validate_disallowed_selector() - self._set_numerical_and_temporal_selectors_values() - self._set_categorical_selectors_options() - self._set_actions() + if self.targets: + targeted_data = self._validate_targeted_data(targets=self.targets) + else: + # If targets aren't explicitly provided then try to target all figures on the page. In this case we don't + # want to raise an error if the column is not found in a figure's data_frame, it will just be ignored. + # Possibly in future this will change (which would be breaking change). + targeted_data = self._validate_targeted_data( + targets=model_manager._get_page_model_ids_with_figure( + page_id=model_manager._get_model_page_id(model_id=ModelID(str(self.id))) + ), + raise_column_not_found_error=False, + ) + if targeted_data.empty: + raise ValueError(f"Selected column {self.column} not found in any dataframe on this page.") + self.targets = targeted_data.columns + + # Set default selector according to column type. + self._column_type = self._validate_column_type(targeted_data) + self.selector = self.selector or SELECTORS[self._column_type][0]() + self.selector.title = self.selector.title or self.column.title() + + if isinstance(self.selector, DISALLOWED_SELECTORS.get(self._column_type, ())): + raise ValueError( + f"Chosen selector {type(self.selector)} is not compatible with ({self._column_type}) column '" + f"{self.column}'." + ) + + # Set appropriate properties for the selector. + if self._column_type == "categorical": + self.selector.options = self.selector.options or self._get_options(targeted_data) + elif self._column_type in ("temporal", "numerical"): + _min, _max = self._get_min_max(targeted_data) + self.selector.min = self.selector.min or _min + self.selector.max = self.selector.max or _max + + if not self.selector.actions: + if isinstance(self.selector, RangeSlider) or ( + isinstance(self.selector, DatePicker) and self.selector.range + ): + filter_function = _filter_between + else: + filter_function = _filter_isin + + self.selector.actions = [ + Action( + id=f"{FILTER_ACTION_PREFIX}_{self.id}", + function=_filter(filter_column=self.column, targets=self.targets, filter_function=filter_function), + ) + ] + + def __call__(self, **kwargs): + # Only relevant for a dynamic filter. + # TODO: this will need to pass parametrised data_frame arguments through to _validate_targeted_data. + # Although targets are fixed at build time, the validation logic is repeated during runtime, so if a column + # is missing then it will raise an error. We could change this if we wanted. + targeted_data = self._validate_targeted_data(targets=self.targets) + + if (column_type := self._validate_column_type(targeted_data)) != self._column_type: + raise ValueError( + f"{self.column} has changed type from {self._column_type} to {column_type}. A filtered column cannot " + "change type while the dashboard is running." + ) + + if self._column_type == "categorical": + options = self._get_options(targeted_data) + return options + # TODO: when implement dynamic, will need to do something with this e.g. pass to selector.__call__. + elif self._column_type in ("temporal", "numerical"): + _min, _max = self._get_min_max(targeted_data) + return _min, _max + # TODO: when implement dynamic, will need to do something with this e.g. pass to selector.__call__. @_log_call def build(self): return self.selector.build() - def _set_targets(self): - # TODO: consider moving to data_manager, depending on Petar's work - # TODO: write tests - potential_targets = self.targets or model_manager._get_page_model_ids_with_figure( - page_id=model_manager._get_model_page_id(model_id=ModelID(str(self.id))) - ) - - potential_target_to_data_source_name = { - target: model_manager[target]["data_frame"] for target in potential_targets - } - # Using set() here ensures we only load each data source once rather than repeating the operation for each - # target. + def _validate_targeted_data(self, targets: list[ModelID], raise_column_not_found_error=True) -> pd.DataFrame: + # TODO: consider moving some of this logic to data_manager when implement dynamic filter. Make sure + # get_modified_figures is as efficient as code here. + + # When loading data_frame there are possible keys: + # 1. target. In worst case scenario this is needed but can lead to unnecessary repeated data loading. + # 2. data_source_name. No repeated data loading but won't work when applying data_frame parameters at runtime. + # 3. target + data_frame parameters keyword-argument pairs. This is the correct key to use at runtime. + # For now we follow scheme 2 for data loading (due to set() below) and 1 for the returned targeted_data + # pd.DataFrame, i.e. a separate column for each target even if some data is repeated. + # TODO: when this works with data_frame parameters load() will need to take arguments and the structures here + # might change a bit. + target_to_data_source_name = {target: model_manager[target]["data_frame"] for target in targets} data_source_name_to_data = { data_source_name: data_manager[data_source_name].load() - for data_source_name in set(potential_target_to_data_source_name.values()) + for data_source_name in set(target_to_data_source_name.values()) } target_to_series = dict() - for target, data_source_name in potential_target_to_data_source_name.items(): + for target, data_source_name in target_to_data_source_name.items(): data_frame = data_source_name_to_data[data_source_name] if self.column in data_frame.columns: - target_to_series[target] = data_frame - elif self.targets: - # targets were manually specified so it's not ok the column isn't there. If targets were not specified - # then it's fine, we just skip this target, and error is not raised. + # reset_index so that when we make a DataFrame out of all these pd.Series pandas doesn't try to align + # the columns by index. + target_to_series[target] = data_frame[self.column].reset_index(drop=True) + elif raise_column_not_found_error: raise ValueError(f"Selected column {self.column} not found in dataframe for {target}.") - if not target_to_series: - raise ValueError(f"Selected column {self.column} not found in any dataframe on this page.") + return pd.DataFrame(target_to_series) - self.targets = list(target_to_series) - # COMMEnt. will have repeats - self._targeted_data = pd.DataFrame.from_dict(target_to_series) - - def _set_column_type(self): - # TODO: check - is_numerical = self._targeted_data.apply(is_numeric_dtype) - is_temporal = self._targeted_data.apply(is_datetime64_any_dtype) + def _validate_column_type(self, targeted_data: pd.DataFrame) -> Literal["numerical", "categorical", "temporal"]: + is_numerical = targeted_data.apply(is_numeric_dtype) + is_temporal = targeted_data.apply(is_datetime64_any_dtype) is_categorical = ~is_numerical & ~is_temporal if is_numerical.all(): - self._column_type = "numerical" + return "numerical" elif is_temporal.all(): - self._column_type = "temporal" + return "temporal" elif is_categorical.all(): - self._column_type = "categorical" + return "categorical" else: raise ValueError( f"Inconsistent types detected in the shared data column {self.column}. This column must " "have the same type for all targets." ) - def _set_selector(self): - self.selector = self.selector or SELECTORS[self._column_type][0]() - self.selector.title = self.selector.title or self.column.title() - - def _validate_disallowed_selector(self): - if isinstance(self.selector, DISALLOWED_SELECTORS.get(self._column_type, ())): - raise ValueError( - f"Chosen selector {self.selector.type} is not compatible " - f"with {self._column_type} column '{self.column}'. " - ) - - def _set_numerical_and_temporal_selectors_values(self): - # If the selector is a numerical or temporal selector, and the min and max values are not set, then set them - # N.B. All custom selectors inherit from numerical or temporal selector should also pass this check - if isinstance(self.selector, SELECTORS["numerical"] + SELECTORS["temporal"]): - self.selector.min = self.selector.min or self._targeted_data.to_numpy().min() - self.selector.max = self.selector.max or self._targeted_data.to_numpy().max() - - def _set_categorical_selectors_options(self): - # If the selector is a categorical selector, and the options are not set, then set them - # N.B. All custom selectors inherit from categorical selector should also pass this check - if isinstance(self.selector, SELECTORS["categorical"]): - self.selector.options = self.selector.options or sorted(np.unique(self._targeted_data.to_numpy())) - - def _set_actions(self): - if not self.selector.actions: - if isinstance(self.selector, RangeSlider) or ( - isinstance(self.selector, DatePicker) and self.selector.range - ): - filter_function = _filter_between - else: - filter_function = _filter_isin - - self.selector.actions = [ - Action( - id=f"{FILTER_ACTION_PREFIX}_{self.id}", - function=_filter(filter_column=self.column, targets=self.targets, filter_function=filter_function), - ) - ] + # TODO: write tests. Include N/A + # TODO: block all update of models during runtime + def _get_min_max(self, targeted_data: pd.DataFrame) -> tuple(float, float): + # Use item() to convert to convert scalar from numpy to Python type. This isn't needed during pre_build because + # pydantic will coerce the type, but it is necessary in __call__ where we don't update model field values + # and instead just pass straight to the Dash component. + return targeted_data.min(axis=None).item(), targeted_data.max(axis=None).item() + + def _get_options(self, targeted_data: pd.DataFrame) -> list: + # Use tolist() to convert to convert scalar from numpy to Python type. This isn't needed during pre_build + # because pydantic will coerce the type, but it is necessary in __call__ where we don't update model field values + # and instead just pass straight to the Dash component. + return np.unique(targeted_data.stack().dropna()).tolist() From 3d6f57008a8d68c4a988f3624a1d158cb40310a8 Mon Sep 17 00:00:00 2001 From: Antony Milne Date: Mon, 4 Nov 2024 17:00:41 +0000 Subject: [PATCH 03/64] Fix tests --- .../src/vizro/models/_controls/filter.py | 48 +++++++++++-------- .../unit/vizro/models/_action/test_action.py | 2 +- .../vizro/models/_controls/test_filter.py | 25 +++++----- 3 files changed, 40 insertions(+), 35 deletions(-) diff --git a/vizro-core/src/vizro/models/_controls/filter.py b/vizro-core/src/vizro/models/_controls/filter.py index 4fbd105fd..e55dedda0 100644 --- a/vizro-core/src/vizro/models/_controls/filter.py +++ b/vizro-core/src/vizro/models/_controls/filter.py @@ -106,11 +106,9 @@ def pre_build(self): targets=model_manager._get_page_model_ids_with_figure( page_id=model_manager._get_model_page_id(model_id=ModelID(str(self.id))) ), - raise_column_not_found_error=False, + eagerly_raise_column_not_found_error=False, ) - if targeted_data.empty: - raise ValueError(f"Selected column {self.column} not found in any dataframe on this page.") - self.targets = targeted_data.columns + self.targets = list(targeted_data.columns) # Set default selector according to column type. self._column_type = self._validate_column_type(targeted_data) @@ -119,17 +117,18 @@ def pre_build(self): if isinstance(self.selector, DISALLOWED_SELECTORS.get(self._column_type, ())): raise ValueError( - f"Chosen selector {type(self.selector)} is not compatible with ({self._column_type}) column '" - f"{self.column}'." + f"Chosen selector {type(self.selector).__name__} is not compatible with {self._column_type} column " + f"'{self.column}'." ) # Set appropriate properties for the selector. - if self._column_type == "categorical": - self.selector.options = self.selector.options or self._get_options(targeted_data) - elif self._column_type in ("temporal", "numerical"): + if isinstance(self.selector, SELECTORS["numerical"] + SELECTORS["temporal"]): _min, _max = self._get_min_max(targeted_data) self.selector.min = self.selector.min or _min self.selector.max = self.selector.max or _max + else: + # Categorical selector. + self.selector.options = self.selector.options or self._get_options(targeted_data) if not self.selector.actions: if isinstance(self.selector, RangeSlider) or ( @@ -159,20 +158,20 @@ def __call__(self, **kwargs): "change type while the dashboard is running." ) - if self._column_type == "categorical": - options = self._get_options(targeted_data) - return options - # TODO: when implement dynamic, will need to do something with this e.g. pass to selector.__call__. - elif self._column_type in ("temporal", "numerical"): - _min, _max = self._get_min_max(targeted_data) - return _min, _max - # TODO: when implement dynamic, will need to do something with this e.g. pass to selector.__call__. + # TODO: when implement dynamic, will need to do something with this e.g. pass to selector.__call__. + # if isinstance(self.selector, SELECTORS["numerical"] + SELECTORS["temporal"]): + # options = self._get_options(targeted_data) + # else: + # # Categorical selector. + # _min, _max = self._get_min_max(targeted_data) @_log_call def build(self): return self.selector.build() - def _validate_targeted_data(self, targets: list[ModelID], raise_column_not_found_error=True) -> pd.DataFrame: + def _validate_targeted_data( + self, targets: list[ModelID], eagerly_raise_column_not_found_error=True + ) -> pd.DataFrame: # TODO: consider moving some of this logic to data_manager when implement dynamic filter. Make sure # get_modified_figures is as efficient as code here. @@ -198,10 +197,17 @@ def _validate_targeted_data(self, targets: list[ModelID], raise_column_not_found # reset_index so that when we make a DataFrame out of all these pd.Series pandas doesn't try to align # the columns by index. target_to_series[target] = data_frame[self.column].reset_index(drop=True) - elif raise_column_not_found_error: + elif eagerly_raise_column_not_found_error: raise ValueError(f"Selected column {self.column} not found in dataframe for {target}.") - return pd.DataFrame(target_to_series) + targeted_data = pd.DataFrame(target_to_series) + if targeted_data.columns.empty: + # Still raised when eagerly_raise_column_not_found_error=False. + raise ValueError(f"Selected column {self.column} not found in any dataframe for {targets}.") + if targeted_data.empty: + raise ValueError(f"Selected column {self.column} does not contain any data.") + + return targeted_data def _validate_column_type(self, targeted_data: pd.DataFrame) -> Literal["numerical", "categorical", "temporal"]: is_numerical = targeted_data.apply(is_numeric_dtype) @@ -222,7 +228,7 @@ def _validate_column_type(self, targeted_data: pd.DataFrame) -> Literal["numeric # TODO: write tests. Include N/A # TODO: block all update of models during runtime - def _get_min_max(self, targeted_data: pd.DataFrame) -> tuple(float, float): + def _get_min_max(self, targeted_data: pd.DataFrame) -> tuple[float, float]: # Use item() to convert to convert scalar from numpy to Python type. This isn't needed during pre_build because # pydantic will coerce the type, but it is necessary in __call__ where we don't update model field values # and instead just pass straight to the Dash component. diff --git a/vizro-core/tests/unit/vizro/models/_action/test_action.py b/vizro-core/tests/unit/vizro/models/_action/test_action.py index 24b14f04b..48ed759d8 100644 --- a/vizro-core/tests/unit/vizro/models/_action/test_action.py +++ b/vizro-core/tests/unit/vizro/models/_action/test_action.py @@ -150,7 +150,7 @@ def managers_one_page_without_graphs_one_button(): vm.Page( id="test_page", title="Test page", - components=[vm.Graph(figure=px.scatter(data_frame=pd.DataFrame(columns=["A"]), x="A", y="A"))], + components=[vm.Graph(figure=px.scatter(data_frame=pd.DataFrame(data={"A": [1], "B": [2]}), x="A", y="B"))], controls=[vm.Filter(id="test_filter", column="A")], ) Vizro._pre_build() diff --git a/vizro-core/tests/unit/vizro/models/_controls/test_filter.py b/vizro-core/tests/unit/vizro/models/_controls/test_filter.py index 6a87e8c83..2ffe79603 100644 --- a/vizro-core/tests/unit/vizro/models/_controls/test_filter.py +++ b/vizro-core/tests/unit/vizro/models/_controls/test_filter.py @@ -274,22 +274,24 @@ def test_allowed_selectors_per_column_type(self, filtered_column, selector, mana assert isinstance(filter.selector, selector) @pytest.mark.parametrize( - "filtered_column, selector", + "filtered_column, selector, selector_name, column_type", [ - ("country", vm.Slider), - ("country", vm.RangeSlider), - ("country", vm.DatePicker), - ("lifeExp", vm.DatePicker), - ("year", vm.Slider), - ("year", vm.RangeSlider), + ("country", vm.Slider, "Slider", "categorical"), + ("country", vm.RangeSlider, "RangeSlider", "categorical"), + ("country", vm.DatePicker, "DatePicker", "categorical"), + ("lifeExp", vm.DatePicker, "DatePicker", "numerical"), + ("year", vm.Slider, "Slider", "temporal"), + ("year", vm.RangeSlider, "RangeSlider", "temporal"), ], ) - def test_disallowed_selectors_per_column_type(self, filtered_column, selector, managers_one_page_two_graphs): + def test_disallowed_selectors_per_column_type( + self, filtered_column, selector, selector_name, column_type, managers_one_page_two_graphs + ): filter = vm.Filter(column=filtered_column, selector=selector()) model_manager["test_page"].controls = [filter] with pytest.raises( ValueError, - match=f"Chosen selector {selector().type} is not compatible with .* column '{filtered_column}'. ", + match=f"Chosen selector {selector_name} is not compatible with {column_type} column '{filtered_column}'.", ): filter.pre_build() @@ -306,10 +308,7 @@ def test_set_slider_values_shared_column_inconsistent_dtype(self, targets, manag model_manager["graphs_with_shared_column"].controls = [filter] with pytest.raises( ValueError, - match=re.escape( - f"Inconsistent types detected in the shared data column 'shared_column' for targeted charts {targets}. " - f"Please ensure that the data column contains the same data type across all targeted charts." - ), + match="Inconsistent types detected in the shared data column shared_column.", ): filter.pre_build() From 68f6f610f1e587dc7c9daef592e1186ab37198dd Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 4 Nov 2024 17:03:00 +0000 Subject: [PATCH 04/64] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- vizro-core/hatch.toml | 2 +- vizro-core/src/vizro/models/_controls/filter.py | 2 +- vizro-core/tests/unit/vizro/models/_controls/test_filter.py | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/vizro-core/hatch.toml b/vizro-core/hatch.toml index 649fdd187..26e0fc53f 100644 --- a/vizro-core/hatch.toml +++ b/vizro-core/hatch.toml @@ -112,7 +112,7 @@ extra-dependencies = [ "pydantic==1.10.16", "dash==2.17.1", "plotly==5.12.0", - "pandas==2.0.0", + "pandas==2.0.0" ] features = ["kedro"] python = "3.9" diff --git a/vizro-core/src/vizro/models/_controls/filter.py b/vizro-core/src/vizro/models/_controls/filter.py index e55dedda0..f6213bbb7 100644 --- a/vizro-core/src/vizro/models/_controls/filter.py +++ b/vizro-core/src/vizro/models/_controls/filter.py @@ -1,8 +1,8 @@ from __future__ import annotations -import numpy as np from typing import Literal, Union +import numpy as np import pandas as pd from pandas.api.types import is_datetime64_any_dtype, is_numeric_dtype diff --git a/vizro-core/tests/unit/vizro/models/_controls/test_filter.py b/vizro-core/tests/unit/vizro/models/_controls/test_filter.py index 2ffe79603..8fb6851b9 100644 --- a/vizro-core/tests/unit/vizro/models/_controls/test_filter.py +++ b/vizro-core/tests/unit/vizro/models/_controls/test_filter.py @@ -1,4 +1,3 @@ -import re from datetime import date, datetime from typing import Literal From 9e6f62e014da8fde73975924e6f90da0d0c5858c Mon Sep 17 00:00:00 2001 From: Antony Milne Date: Mon, 4 Nov 2024 18:04:16 +0000 Subject: [PATCH 05/64] Add numpy lower bound --- vizro-core/hatch.toml | 3 ++- vizro-core/src/vizro/models/_controls/filter.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/vizro-core/hatch.toml b/vizro-core/hatch.toml index 26e0fc53f..3d045bb88 100644 --- a/vizro-core/hatch.toml +++ b/vizro-core/hatch.toml @@ -112,7 +112,8 @@ extra-dependencies = [ "pydantic==1.10.16", "dash==2.17.1", "plotly==5.12.0", - "pandas==2.0.0" + "pandas==2.0.0", + "numpy==1.23.0", ] features = ["kedro"] python = "3.9" diff --git a/vizro-core/src/vizro/models/_controls/filter.py b/vizro-core/src/vizro/models/_controls/filter.py index f6213bbb7..4e8b7e4ae 100644 --- a/vizro-core/src/vizro/models/_controls/filter.py +++ b/vizro-core/src/vizro/models/_controls/filter.py @@ -173,7 +173,7 @@ def _validate_targeted_data( self, targets: list[ModelID], eagerly_raise_column_not_found_error=True ) -> pd.DataFrame: # TODO: consider moving some of this logic to data_manager when implement dynamic filter. Make sure - # get_modified_figures is as efficient as code here. + # get_modified_figures and stuff in _actions_utils.py is as efficient as code here. # When loading data_frame there are possible keys: # 1. target. In worst case scenario this is needed but can lead to unnecessary repeated data loading. From 574169b7bda8b1d530258833d55d1ed794c3c331 Mon Sep 17 00:00:00 2001 From: Antony Milne Date: Tue, 5 Nov 2024 10:05:58 +0000 Subject: [PATCH 06/64] Fix tests --- vizro-core/src/vizro/models/_controls/filter.py | 2 +- vizro-core/tests/unit/vizro/models/_controls/test_filter.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/vizro-core/src/vizro/models/_controls/filter.py b/vizro-core/src/vizro/models/_controls/filter.py index 4e8b7e4ae..061c906b9 100644 --- a/vizro-core/src/vizro/models/_controls/filter.py +++ b/vizro-core/src/vizro/models/_controls/filter.py @@ -203,7 +203,7 @@ def _validate_targeted_data( targeted_data = pd.DataFrame(target_to_series) if targeted_data.columns.empty: # Still raised when eagerly_raise_column_not_found_error=False. - raise ValueError(f"Selected column {self.column} not found in any dataframe for {targets}.") + raise ValueError(f"Selected column {self.column} not found in any dataframe for {", ".join(targets)}.") if targeted_data.empty: raise ValueError(f"Selected column {self.column} does not contain any data.") diff --git a/vizro-core/tests/unit/vizro/models/_controls/test_filter.py b/vizro-core/tests/unit/vizro/models/_controls/test_filter.py index 8fb6851b9..c4084f570 100644 --- a/vizro-core/tests/unit/vizro/models/_controls/test_filter.py +++ b/vizro-core/tests/unit/vizro/models/_controls/test_filter.py @@ -217,7 +217,9 @@ def test_set_targets_invalid(self, managers_one_page_two_graphs): filter = vm.Filter(column="invalid_choice") model_manager["test_page"].controls = [filter] - with pytest.raises(ValueError, match="Selected column invalid_choice not found in any dataframe on this page."): + with pytest.raises( + ValueError, match="Selected column invalid_choice not found in any dataframe for scatter_chart, bar_chart." + ): filter.pre_build() @pytest.mark.parametrize( From 568dc58fb3f2d5a9bd580503b59e20b5262d5586 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 5 Nov 2024 10:06:22 +0000 Subject: [PATCH 07/64] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- vizro-core/hatch.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vizro-core/hatch.toml b/vizro-core/hatch.toml index 3d045bb88..9620b876e 100644 --- a/vizro-core/hatch.toml +++ b/vizro-core/hatch.toml @@ -113,7 +113,7 @@ extra-dependencies = [ "dash==2.17.1", "plotly==5.12.0", "pandas==2.0.0", - "numpy==1.23.0", + "numpy==1.23.0" ] features = ["kedro"] python = "3.9" From b13e5748652daecaad28644610385951e147c055 Mon Sep 17 00:00:00 2001 From: Antony Milne Date: Tue, 5 Nov 2024 10:23:31 +0000 Subject: [PATCH 08/64] Fix tests --- vizro-core/src/vizro/models/_controls/filter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vizro-core/src/vizro/models/_controls/filter.py b/vizro-core/src/vizro/models/_controls/filter.py index 061c906b9..207cf264f 100644 --- a/vizro-core/src/vizro/models/_controls/filter.py +++ b/vizro-core/src/vizro/models/_controls/filter.py @@ -203,7 +203,7 @@ def _validate_targeted_data( targeted_data = pd.DataFrame(target_to_series) if targeted_data.columns.empty: # Still raised when eagerly_raise_column_not_found_error=False. - raise ValueError(f"Selected column {self.column} not found in any dataframe for {", ".join(targets)}.") + raise ValueError(f"Selected column {self.column} not found in any dataframe for {', '.join(targets)}.") if targeted_data.empty: raise ValueError(f"Selected column {self.column} does not contain any data.") From 01307bdbd6bbfcae07f7fe8938fd2c7dbcc82ae1 Mon Sep 17 00:00:00 2001 From: Antony Milne Date: Tue, 5 Nov 2024 11:42:46 +0000 Subject: [PATCH 09/64] Add new tests and lint --- .pre-commit-config.yaml | 1 + .../src/vizro/models/_controls/filter.py | 28 ++-- .../unit/vizro/models/_controls/conftest.py | 36 ----- .../vizro/models/_controls/test_filter.py | 135 +++++++++++++++--- 4 files changed, 132 insertions(+), 68 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9061d1b59..acccb5e13 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -48,6 +48,7 @@ repos: hooks: - id: ruff args: [--fix] + exclude: "vizro-core/examples/scratch_dev/app.py" - id: ruff-format - repo: https://github.com/PyCQA/bandit diff --git a/vizro-core/src/vizro/models/_controls/filter.py b/vizro-core/src/vizro/models/_controls/filter.py index 207cf264f..da86cace5 100644 --- a/vizro-core/src/vizro/models/_controls/filter.py +++ b/vizro-core/src/vizro/models/_controls/filter.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Literal, Union +from typing import Any, Literal, Union import numpy as np import pandas as pd @@ -188,7 +188,7 @@ def _validate_targeted_data( data_source_name: data_manager[data_source_name].load() for data_source_name in set(target_to_data_source_name.values()) } - target_to_series = dict() + target_to_series = {} for target, data_source_name in target_to_data_source_name.items(): data_frame = data_source_name_to_data[data_source_name] @@ -205,7 +205,9 @@ def _validate_targeted_data( # Still raised when eagerly_raise_column_not_found_error=False. raise ValueError(f"Selected column {self.column} not found in any dataframe for {', '.join(targets)}.") if targeted_data.empty: - raise ValueError(f"Selected column {self.column} does not contain any data.") + raise ValueError( + f"Selected column {self.column} does not contain anything in any dataframe for {', '.join(targets)}." + ) return targeted_data @@ -222,20 +224,22 @@ def _validate_column_type(self, targeted_data: pd.DataFrame) -> Literal["numeric return "categorical" else: raise ValueError( - f"Inconsistent types detected in the shared data column {self.column}. This column must " - "have the same type for all targets." + f"Inconsistent types detected in column {self.column}. This column must have the same type for all " + "targets." ) - # TODO: write tests. Include N/A - # TODO: block all update of models during runtime - def _get_min_max(self, targeted_data: pd.DataFrame) -> tuple[float, float]: + @staticmethod + def _get_min_max(targeted_data: pd.DataFrame) -> tuple[float, float]: # Use item() to convert to convert scalar from numpy to Python type. This isn't needed during pre_build because # pydantic will coerce the type, but it is necessary in __call__ where we don't update model field values # and instead just pass straight to the Dash component. return targeted_data.min(axis=None).item(), targeted_data.max(axis=None).item() - def _get_options(self, targeted_data: pd.DataFrame) -> list: + @staticmethod + def _get_options(targeted_data: pd.DataFrame) -> list[Any]: # Use tolist() to convert to convert scalar from numpy to Python type. This isn't needed during pre_build - # because pydantic will coerce the type, but it is necessary in __call__ where we don't update model field values - # and instead just pass straight to the Dash component. - return np.unique(targeted_data.stack().dropna()).tolist() + # because pydantic will coerce the type, but it is necessary in __call__ where we don't update model field + # values and instead just pass straight to the Dash component. + # The dropna() isn't strictly required here but will be in future pandas versions when the behavior of stack + # changes. See https://pandas.pydata.org/docs/whatsnew/v2.1.0.html#whatsnew-210-enhancements-new-stack. + return np.unique(targeted_data.stack().dropna()).tolist() # noqa: PD013 diff --git a/vizro-core/tests/unit/vizro/models/_controls/conftest.py b/vizro-core/tests/unit/vizro/models/_controls/conftest.py index 1d376439e..8887876ff 100644 --- a/vizro-core/tests/unit/vizro/models/_controls/conftest.py +++ b/vizro-core/tests/unit/vizro/models/_controls/conftest.py @@ -1,8 +1,3 @@ -import datetime -import random - -import numpy as np -import pandas as pd import pytest import vizro.models as vm @@ -10,21 +5,6 @@ from vizro import Vizro -@pytest.fixture -def dfs_with_shared_column(): - df1 = pd.DataFrame() - df1["x"] = np.random.uniform(0, 10, 100) - df1["y"] = np.random.uniform(0, 10, 100) - df2 = df1.copy() - df3 = df1.copy() - - df1["shared_column"] = np.random.uniform(0, 10, 100) - df2["shared_column"] = [datetime.datetime(2024, 1, 1) + datetime.timedelta(days=i) for i in range(100)] - df3["shared_column"] = random.choices(["CATEGORY 1", "CATEGORY 2"], k=100) - - return df1, df2, df3 - - @pytest.fixture def managers_one_page_two_graphs(gapminder): """Instantiates a simple model_manager and data_manager with a page, and two graph models and gapminder data.""" @@ -37,19 +17,3 @@ def managers_one_page_two_graphs(gapminder): ], ) Vizro._pre_build() - - -@pytest.fixture -def managers_shared_column_different_dtype(dfs_with_shared_column): - """Instantiates the managers with a page and two graphs sharing the same column but of different data types.""" - df1, df2, df3 = dfs_with_shared_column - vm.Page( - id="graphs_with_shared_column", - title="Page Title", - components=[ - vm.Graph(id="id_shared_column_numerical", figure=px.scatter(df1, x="x", y="y", color="shared_column")), - vm.Graph(id="id_shared_column_temporal", figure=px.scatter(df2, x="x", y="y", color="shared_column")), - vm.Graph(id="id_shared_column_categorical", figure=px.scatter(df3, x="x", y="y", color="shared_column")), - ], - ) - Vizro._pre_build() diff --git a/vizro-core/tests/unit/vizro/models/_controls/test_filter.py b/vizro-core/tests/unit/vizro/models/_controls/test_filter.py index c4084f570..c3f9a0c13 100644 --- a/vizro-core/tests/unit/vizro/models/_controls/test_filter.py +++ b/vizro-core/tests/unit/vizro/models/_controls/test_filter.py @@ -6,12 +6,52 @@ from asserts import assert_component_equal import vizro.models as vm +import vizro.plotly.express as px +from vizro import Vizro from vizro.managers import model_manager from vizro.models._action._actions_chain import ActionsChain from vizro.models._controls.filter import Filter, _filter_between, _filter_isin from vizro.models.types import CapturedCallable +@pytest.fixture +def managers_column_different_type(): + """Instantiates the managers with a page and two graphs sharing the same column but of different data types.""" + df_numerical = pd.DataFrame({"shared_column": [1]}) + df_temporal = pd.DataFrame({"shared_column": [datetime(2024, 1, 1)]}) + df_categorical = pd.DataFrame({"shared_column": ["a"]}) + + vm.Page( + id="test_page", + title="Page Title", + components=[ + vm.Graph(id="column_numerical", figure=px.scatter(df_numerical)), + vm.Graph(id="column_temporal", figure=px.scatter(df_temporal)), + vm.Graph(id="column_categorical", figure=px.scatter(df_categorical)), + ], + ) + Vizro._pre_build() + + +@pytest.fixture +def managers_column_only_exists_in_some(): + """Dataframes with column_numerical and column_categorical, which can be different lengths.""" + vm.Page( + id="test_page", + title="Page Title", + components=[ + vm.Graph(id="column_numerical_exists_1", figure=px.scatter(pd.DataFrame({"column_numerical": [1]}))), + vm.Graph(id="column_numerical_exists_2", figure=px.scatter(pd.DataFrame({"column_numerical": [1, 2]}))), + vm.Graph(id="column_numerical_exists_empty", figure=px.scatter(pd.DataFrame({"column_numerical": []}))), + vm.Graph(id="column_categorical_exists_1", figure=px.scatter(pd.DataFrame({"column_categorical": ["a"]}))), + vm.Graph( + id="column_categorical_exists_2", figure=px.scatter(pd.DataFrame({"column_categorical": ["a", "b"]})) + ), + ], + ) + Vizro._pre_build() + + class TestFilterFunctions: @pytest.mark.parametrize( "data, value, expected", @@ -204,21 +244,63 @@ def test_check_target_present_invalid(self): Filter(column="foo", targets=["invalid_target"]) +""" +two figures, only one has target column - default should find right one, no error +two figures, only one has target column - specific invalid should error + +columns empty default and specific -> error +""" + + class TestPreBuildMethod: - def test_set_targets_valid(self, managers_one_page_two_graphs): + def test_targets_default_valid(self, managers_column_only_exists_in_some): # Core of tests is still interface level - filter = vm.Filter(column="country") + filter = vm.Filter(column="column_numerical") # Special case - need filter in the context of page in order to run filter.pre_build model_manager["test_page"].controls = [filter] filter.pre_build() - assert set(filter.targets) == {"scatter_chart", "bar_chart"} + assert filter.targets == [ + "column_numerical_exists_1", + "column_numerical_exists_2", + "column_numerical_exists_empty", + ] + + def test_targets_specific_valid(self, managers_column_only_exists_in_some): + filter = vm.Filter(column="column_numerical", targets=["column_numerical_exists_1"]) + model_manager["test_page"].controls = [filter] + filter.pre_build() + assert filter.targets == ["column_numerical_exists_1"] - def test_set_targets_invalid(self, managers_one_page_two_graphs): + def test_targets_default_invalid(self, managers_column_only_exists_in_some): filter = vm.Filter(column="invalid_choice") model_manager["test_page"].controls = [filter] with pytest.raises( - ValueError, match="Selected column invalid_choice not found in any dataframe for scatter_chart, bar_chart." + ValueError, + match="Selected column invalid_choice not found in any dataframe for column_numerical_exists_1, " + "column_numerical_exists_2, column_numerical_exists_empty, column_categorical_exists_1, " + "column_categorical_exists_2.", + ): + filter.pre_build() + + def test_targets_specific_invalid(self, managers_column_only_exists_in_some): + filter = vm.Filter(column="column_numerical", targets=["column_categorical_exists_1"]) + model_manager["test_page"].controls = [filter] + + with pytest.raises( + ValueError, + match="Selected column column_numerical not found in dataframe for column_categorical_exists_1.", + ): + filter.pre_build() + + def test_targets_empty(self, managers_column_only_exists_in_some): + filter = vm.Filter(column="column_numerical", targets=["column_numerical_exists_empty"]) + model_manager["test_page"].controls = [filter] + + with pytest.raises( + ValueError, + match="Selected column column_numerical does not contain anything in any dataframe for " + "column_numerical_exists_empty.", ): filter.pre_build() @@ -226,7 +308,7 @@ def test_set_targets_invalid(self, managers_one_page_two_graphs): "filtered_column, expected_column_type", [("country", "categorical"), ("year", "temporal"), ("lifeExp", "numerical")], ) - def test_set_column_type(self, filtered_column, expected_column_type, managers_one_page_two_graphs): + def test_column_type(self, filtered_column, expected_column_type, managers_one_page_two_graphs): filter = vm.Filter(column=filtered_column) model_manager["test_page"].controls = [filter] filter.pre_build() @@ -236,7 +318,7 @@ def test_set_column_type(self, filtered_column, expected_column_type, managers_o "filtered_column, expected_selector", [("country", vm.Dropdown), ("year", vm.DatePicker), ("lifeExp", vm.RangeSlider)], ) - def test_set_selector_default_selector(self, filtered_column, expected_selector, managers_one_page_two_graphs): + def test_selector_default_selector(self, filtered_column, expected_selector, managers_one_page_two_graphs): filter = vm.Filter(column=filtered_column) model_manager["test_page"].controls = [filter] filter.pre_build() @@ -244,7 +326,7 @@ def test_set_selector_default_selector(self, filtered_column, expected_selector, assert filter.selector.title == filtered_column.title() @pytest.mark.parametrize("filtered_column", ["country", "year", "lifeExp"]) - def test_set_selector_specific_selector(self, filtered_column, managers_one_page_two_graphs): + def test_selector_specific_selector(self, filtered_column, managers_one_page_two_graphs): filter = vm.Filter(column=filtered_column, selector=vm.RadioItems(title="Title")) model_manager["test_page"].controls = [filter] filter.pre_build() @@ -299,29 +381,36 @@ def test_disallowed_selectors_per_column_type( @pytest.mark.parametrize( "targets", [ - ["id_shared_column_numerical", "id_shared_column_temporal"], - ["id_shared_column_numerical", "id_shared_column_categorical"], - ["id_shared_column_temporal", "id_shared_column_categorical"], + ["column_numerical", "column_temporal"], + ["column_numerical", "column_categorical"], + ["column_temporal", "column_categorical"], ], ) - def test_set_slider_values_shared_column_inconsistent_dtype(self, targets, managers_shared_column_different_dtype): + def test_validate_column_type(self, targets, managers_column_different_type): filter = vm.Filter(column="shared_column", targets=targets) - model_manager["graphs_with_shared_column"].controls = [filter] + model_manager["test_page"].controls = [filter] with pytest.raises( ValueError, - match="Inconsistent types detected in the shared data column shared_column.", + match="Inconsistent types detected in column shared_column.", ): filter.pre_build() @pytest.mark.parametrize("selector", [vm.Slider, vm.RangeSlider]) - def test_set_numerical_selectors_values_min_max_default(self, selector, gapminder, managers_one_page_two_graphs): + def test_numerical_min_max_default(self, selector, gapminder, managers_one_page_two_graphs): filter = vm.Filter(column="lifeExp", selector=selector()) model_manager["test_page"].controls = [filter] filter.pre_build() assert filter.selector.min == gapminder.lifeExp.min() assert filter.selector.max == gapminder.lifeExp.max() - def test_set_temporal_selectors_values_min_max_default(self, gapminder, managers_one_page_two_graphs): + def test_numerical_min_max_different_column_lengths(self, gapminder, managers_column_only_exists_in_some): + filter = vm.Filter(column="column_numerical", selector=vm.Slider()) + model_manager["test_page"].controls = [filter] + filter.pre_build() + assert filter.selector.min == 1 + assert filter.selector.max == 2 + + def test_temporal_min_max_default(self, gapminder, managers_one_page_two_graphs): filter = vm.Filter(column="year", selector=vm.DatePicker()) model_manager["test_page"].controls = [filter] filter.pre_build() @@ -329,14 +418,14 @@ def test_set_temporal_selectors_values_min_max_default(self, gapminder, managers assert filter.selector.max == gapminder.year.max().to_pydatetime().date() @pytest.mark.parametrize("selector", [vm.Slider, vm.RangeSlider]) - def test_set_numerical_selectors_values_min_max_specific(self, selector, managers_one_page_two_graphs): + def test_numerical_min_max_specific(self, selector, managers_one_page_two_graphs): filter = vm.Filter(column="lifeExp", selector=selector(min=3, max=5)) model_manager["test_page"].controls = [filter] filter.pre_build() assert filter.selector.min == 3 assert filter.selector.max == 5 - def test_set_temporal_selectors_values_min_max_specific(self, managers_one_page_two_graphs): + def test_temporal_min_max_specific(self, managers_one_page_two_graphs): filter = vm.Filter(column="year", selector=vm.DatePicker(min="1952-01-01", max="2007-01-01")) model_manager["test_page"].controls = [filter] filter.pre_build() @@ -344,14 +433,20 @@ def test_set_temporal_selectors_values_min_max_specific(self, managers_one_page_ assert filter.selector.max == date(2007, 1, 1) @pytest.mark.parametrize("selector", [vm.Checklist, vm.Dropdown, vm.RadioItems]) - def test_set_categorical_selectors_options_default(self, selector, gapminder, managers_one_page_two_graphs): + def test_categorical_options_default(self, selector, gapminder, managers_one_page_two_graphs): filter = vm.Filter(column="continent", selector=selector()) model_manager["test_page"].controls = [filter] filter.pre_build() assert filter.selector.options == sorted(set(gapminder["continent"])) + def test_categorical_options_different_column_lengths(self, gapminder, managers_column_only_exists_in_some): + filter = vm.Filter(column="column_categorical", selector=vm.Checklist()) + model_manager["test_page"].controls = [filter] + filter.pre_build() + assert filter.selector.options == ["a", "b"] + @pytest.mark.parametrize("selector", [vm.Checklist, vm.Dropdown, vm.RadioItems]) - def test_set_categorical_selectors_options_specific(self, selector, managers_one_page_two_graphs): + def test_categorical_options_specific(self, selector, managers_one_page_two_graphs): filter = vm.Filter(column="continent", selector=selector(options=["Africa", "Europe"])) model_manager["test_page"].controls = [filter] filter.pre_build() From 2ac895b1595742816fafd1ef5e2844f1f9a51ae3 Mon Sep 17 00:00:00 2001 From: Antony Milne Date: Tue, 5 Nov 2024 11:46:16 +0000 Subject: [PATCH 10/64] Final tidy --- vizro-core/hatch.toml | 2 +- .../tests/unit/vizro/models/_controls/test_filter.py | 8 -------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/vizro-core/hatch.toml b/vizro-core/hatch.toml index 9620b876e..ca1ef48e8 100644 --- a/vizro-core/hatch.toml +++ b/vizro-core/hatch.toml @@ -113,7 +113,7 @@ extra-dependencies = [ "dash==2.17.1", "plotly==5.12.0", "pandas==2.0.0", - "numpy==1.23.0" + "numpy==1.23.0" # Need numpy<2 to work with pandas==2.0.0. See https://stackoverflow.com/questions/78634235/. ] features = ["kedro"] python = "3.9" diff --git a/vizro-core/tests/unit/vizro/models/_controls/test_filter.py b/vizro-core/tests/unit/vizro/models/_controls/test_filter.py index c3f9a0c13..a6654dc08 100644 --- a/vizro-core/tests/unit/vizro/models/_controls/test_filter.py +++ b/vizro-core/tests/unit/vizro/models/_controls/test_filter.py @@ -244,14 +244,6 @@ def test_check_target_present_invalid(self): Filter(column="foo", targets=["invalid_target"]) -""" -two figures, only one has target column - default should find right one, no error -two figures, only one has target column - specific invalid should error - -columns empty default and specific -> error -""" - - class TestPreBuildMethod: def test_targets_default_valid(self, managers_column_only_exists_in_some): # Core of tests is still interface level From 2e4e0e23f144b218dcdee533bbf296ed49e0c563 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 5 Nov 2024 11:46:39 +0000 Subject: [PATCH 11/64] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- vizro-core/hatch.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vizro-core/hatch.toml b/vizro-core/hatch.toml index ca1ef48e8..a8165f25e 100644 --- a/vizro-core/hatch.toml +++ b/vizro-core/hatch.toml @@ -113,7 +113,7 @@ extra-dependencies = [ "dash==2.17.1", "plotly==5.12.0", "pandas==2.0.0", - "numpy==1.23.0" # Need numpy<2 to work with pandas==2.0.0. See https://stackoverflow.com/questions/78634235/. + "numpy==1.23.0" # Need numpy<2 to work with pandas==2.0.0. See https://stackoverflow.com/questions/78634235/. ] features = ["kedro"] python = "3.9" From 983940b636fc53c60d093dde982b242f4438e3cf Mon Sep 17 00:00:00 2001 From: Antony Milne Date: Wed, 6 Nov 2024 10:45:16 +0000 Subject: [PATCH 12/64] Fix min/max=0 bug --- vizro-core/src/vizro/models/_controls/filter.py | 7 +++++-- .../tests/unit/vizro/models/_controls/test_filter.py | 9 +++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/vizro-core/src/vizro/models/_controls/filter.py b/vizro-core/src/vizro/models/_controls/filter.py index da86cace5..e70fad98c 100644 --- a/vizro-core/src/vizro/models/_controls/filter.py +++ b/vizro-core/src/vizro/models/_controls/filter.py @@ -124,8 +124,11 @@ def pre_build(self): # Set appropriate properties for the selector. if isinstance(self.selector, SELECTORS["numerical"] + SELECTORS["temporal"]): _min, _max = self._get_min_max(targeted_data) - self.selector.min = self.selector.min or _min - self.selector.max = self.selector.max or _max + # Note that manually set self.selector.min/max = 0 are Falsey but should not be overwritten. + if self.selector.min is None: + self.selector.min = _min + if self.selector.max is None: + self.selector.max = _max else: # Categorical selector. self.selector.options = self.selector.options or self._get_options(targeted_data) diff --git a/vizro-core/tests/unit/vizro/models/_controls/test_filter.py b/vizro-core/tests/unit/vizro/models/_controls/test_filter.py index a6654dc08..3f3d909e9 100644 --- a/vizro-core/tests/unit/vizro/models/_controls/test_filter.py +++ b/vizro-core/tests/unit/vizro/models/_controls/test_filter.py @@ -410,12 +410,13 @@ def test_temporal_min_max_default(self, gapminder, managers_one_page_two_graphs) assert filter.selector.max == gapminder.year.max().to_pydatetime().date() @pytest.mark.parametrize("selector", [vm.Slider, vm.RangeSlider]) - def test_numerical_min_max_specific(self, selector, managers_one_page_two_graphs): - filter = vm.Filter(column="lifeExp", selector=selector(min=3, max=5)) + @pytest.mark.parametrize("min, max", [(3, 5), (0, 5), (-5, 0)]) + def test_numerical_min_max_specific(self, selector, min, max, managers_one_page_two_graphs): + filter = vm.Filter(column="lifeExp", selector=selector(min=min, max=max)) model_manager["test_page"].controls = [filter] filter.pre_build() - assert filter.selector.min == 3 - assert filter.selector.max == 5 + assert filter.selector.min == min + assert filter.selector.max == max def test_temporal_min_max_specific(self, managers_one_page_two_graphs): filter = vm.Filter(column="year", selector=vm.DatePicker(min="1952-01-01", max="2007-01-01")) From 13f022e5fa3efffd6a4fc7fd477d6cad0d969e81 Mon Sep 17 00:00:00 2001 From: Antony Milne Date: Wed, 6 Nov 2024 11:36:30 +0000 Subject: [PATCH 13/64] Move _get_targets_data_and_config into _get_modified_page_figures with minimal loading - tests pass --- .../src/vizro/actions/_actions_utils.py | 50 ++++++++++++++++--- .../src/vizro/actions/_filter_action.py | 1 - .../src/vizro/models/_controls/filter.py | 3 +- 3 files changed, 44 insertions(+), 10 deletions(-) diff --git a/vizro-core/src/vizro/actions/_actions_utils.py b/vizro-core/src/vizro/actions/_actions_utils.py index a4d6a78c0..c2d9f4581 100644 --- a/vizro-core/src/vizro/actions/_actions_utils.py +++ b/vizro-core/src/vizro/actions/_actions_utils.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json from collections import defaultdict from copy import deepcopy from typing import TYPE_CHECKING, Any, Literal, Optional, TypedDict, Union @@ -23,7 +24,7 @@ class CallbackTriggerDict(TypedDict): """Represent dash.ctx.args_grouping item. Shortened as 'ctd' in the code. Args: - id: The component ID. If it`s a pattern matching ID, it will be a dict. + id: The component ID. If it's a pattern matching ID, it will be a dict. property: The component property used in the callback. value: The value of the component property at the time the callback was fired. str_id: For pattern matching IDs, it's the stringified dict ID without white spaces. @@ -210,15 +211,48 @@ def _get_modified_page_figures( ) -> dict[str, Any]: targets = targets or [] - filtered_data, parameterized_config = _get_targets_data_and_config( - ctds_filter=ctds_filter, - ctds_filter_interaction=ctds_filter_interaction, - ctds_parameters=ctds_parameters, - targets=targets, - ) + target_to_data_source_load_key = {} + + # TODO: Check doesn't give duplicates for static data + # TODO: separate out _get_parametrized_config for data_frame and general keys + for target in targets: + # parametrized_config includes a key "data_frame" that is used in the data loading function. + parameterized_config = _get_parametrized_config(target=target, ctd_parameters=ctds_parameters) + data_source_load_key = json.dumps( + { + "data_source_name": model_manager[target]["data_frame"], + "dynamic_data_load_params": parameterized_config["data_frame"], + }, + sort_keys=True, + ) + target_to_data_source_load_key[target] = data_source_load_key + + data_source_load_key_to_data = {} + + for data_source_load_key in set(target_to_data_source_load_key.values()): + data_source_load = json.loads(data_source_load_key) + data = data_manager[data_source_load["data_source_name"]].load(**data_source_load["dynamic_data_load_params"]) + data_source_load_key_to_data[data_source_load_key] = data outputs: dict[str, Any] = {} + + # TODO: deduplicate filtering operation for target in targets: - outputs[target] = model_manager[target](data_frame=filtered_data[target], **parameterized_config[target]) + data_frame = data_source_load_key_to_data[target_to_data_source_load_key[target]] + filtered_data = _apply_filters(data_frame=data_frame, ctds_filters=ctds_filter, target=target) + filtered_data = _apply_filter_interaction( + data_frame=filtered_data, ctds_filter_interaction=ctds_filter_interaction, target=target + ) + + outputs[target] = model_manager[target]( + data_frame=filtered_data, + **{ + key: value + for key, value in _get_parametrized_config(target=target, ctd_parameters=ctds_parameters).items() + if key != "data_frame" + }, + ) + + # TODO: think about where filter call goes return outputs diff --git a/vizro-core/src/vizro/actions/_filter_action.py b/vizro-core/src/vizro/actions/_filter_action.py index 409428e01..30784c708 100644 --- a/vizro-core/src/vizro/actions/_filter_action.py +++ b/vizro-core/src/vizro/actions/_filter_action.py @@ -28,7 +28,6 @@ def _filter( Returns: Dict mapping target component ids to modified charts/components e.g. {'my_scatter': Figure({})} - """ return _get_modified_page_figures( targets=targets, diff --git a/vizro-core/src/vizro/models/_controls/filter.py b/vizro-core/src/vizro/models/_controls/filter.py index e70fad98c..ea6390b72 100644 --- a/vizro-core/src/vizro/models/_controls/filter.py +++ b/vizro-core/src/vizro/models/_controls/filter.py @@ -181,7 +181,8 @@ def _validate_targeted_data( # When loading data_frame there are possible keys: # 1. target. In worst case scenario this is needed but can lead to unnecessary repeated data loading. # 2. data_source_name. No repeated data loading but won't work when applying data_frame parameters at runtime. - # 3. target + data_frame parameters keyword-argument pairs. This is the correct key to use at runtime. + # 3. data_source_name + data_frame parameters keyword-argument pairs. This is the correct key to use at + # runtime. # For now we follow scheme 2 for data loading (due to set() below) and 1 for the returned targeted_data # pd.DataFrame, i.e. a separate column for each target even if some data is repeated. # TODO: when this works with data_frame parameters load() will need to take arguments and the structures here From a07fda4e463cdadda062d61e60939bfeba7bc500 Mon Sep 17 00:00:00 2001 From: petar-qb Date: Wed, 6 Nov 2024 13:11:58 +0100 Subject: [PATCH 14/64] Dynamic before filter tidy changes --- .../scratch_dev/_poc_dynamic_controls.py | 397 ++++++++++++++++++ vizro-core/examples/scratch_dev/app.py | 196 ++++++++- vizro-core/examples/scratch_dev/data.yaml | 6 + vizro-core/hatch.toml | 2 +- .../_callback_mapping_utils.py | 16 + .../_get_action_callback_mapping.py | 3 +- .../src/vizro/actions/_filter_action.py | 1 + .../src/vizro/actions/_on_page_load_action.py | 35 +- .../models/_components/form/_form_utils.py | 5 +- .../models/_components/form/checklist.py | 20 +- .../vizro/models/_components/form/dropdown.py | 41 +- .../models/_components/form/radio_items.py | 20 +- .../models/_components/form/range_slider.py | 26 +- .../vizro/models/_components/form/slider.py | 59 ++- .../src/vizro/models/_controls/filter.py | 62 ++- .../src/vizro/static/js/models/dashboard.js | 2 +- .../vizro/static/js/models/range_slider.js | 19 +- .../src/vizro/static/js/models/slider.js | 13 +- 18 files changed, 879 insertions(+), 44 deletions(-) create mode 100644 vizro-core/examples/scratch_dev/_poc_dynamic_controls.py create mode 100644 vizro-core/examples/scratch_dev/data.yaml diff --git a/vizro-core/examples/scratch_dev/_poc_dynamic_controls.py b/vizro-core/examples/scratch_dev/_poc_dynamic_controls.py new file mode 100644 index 000000000..6cf07f73d --- /dev/null +++ b/vizro-core/examples/scratch_dev/_poc_dynamic_controls.py @@ -0,0 +1,397 @@ +import dash +import plotly.express as px + +from dash import Dash, html, dcc, Output, callback, clientside_callback, Input, State, set_props +import dash_mantine_components as dmc + + +CONTROL_SELECTOR = dcc.RangeSlider + +IS_DROPDOWN_MULTI = True + + +if CONTROL_SELECTOR in {dcc.Checklist, dcc.RangeSlider}: + MULTI = True +elif CONTROL_SELECTOR in {dcc.Slider, dcc.RadioItems}: + MULTI = False +elif CONTROL_SELECTOR == dcc.Dropdown: + if IS_DROPDOWN_MULTI not in [False, True]: + raise ValueError("IS_DROPDOWN_MULTI must be set to True or False for dcc.Dropdown selector.") + MULTI = IS_DROPDOWN_MULTI +else: + raise ValueError( + "Invalid CONTROL_SELECTOR. Must be one of: " + "dcc.Dropdown, dcc.Checklist, dcc.RadioItems, dcc.Slider, or dcc.RangeSlider." + ) + + +# Hardcoded global variable. +SELECTOR_TYPE = { + "categorical": [dcc.Dropdown, dcc.Checklist, dcc.RadioItems], + "numerical": [dcc.Slider, dcc.RangeSlider], +} + + +# like dynamic data +def slow_load(): + print("running slow_load") + time.sleep(0.1) + return px.data.iris().sample(6) + + +# Like pre-build - doesn't get run again when reload page +def categorical_filter_pre_build(): + df = slow_load() + options = sorted(df["species"].unique().tolist()) + return options, options if MULTI else options[0] + + +def numerical_filter_pre_build(): + df = slow_load() + _min = float(df["sepal_length"].min()) + _max = float(df["sepal_length"].max()) + return _min, _max, [_min, _max] if MULTI else _min + + +# TODO-TEST: You can hardcode these values for testing purposes. They represent initial options/min/max/value +# for the filter that are created in and sent from page.build() every time page is refreshed. +pre_build_options, pre_build_categorical_value = categorical_filter_pre_build() +pre_build_min, pre_build_max, pre_build_numerical_value = numerical_filter_pre_build() + + +# --- Pages --- +common = [ + html.H1(id="dashboard_title", children="Dashboard"), + html.Div(dcc.Link("Homepage", href="/")), + html.Div(dcc.Link("Another page", href="/another-page")), +] + + +def make_page(content): + page_build_obj = html.Div( + [ + *common, + html.P(datetime.datetime.now()), + *content, + ] + ) + return page_build_obj + + +# homepage build +def homepage(**kwargs): + return make_page([html.H2("Homepage")]) + + +# Like filter build - gets run every time page is loaded +def categorical_filter_build(options=None): + kwargs = {} + if CONTROL_SELECTOR == dcc.Dropdown: + kwargs["multi"] = MULTI + + return CONTROL_SELECTOR( + id=f'filter', + options=options or pre_build_options, + value=pre_build_categorical_value, + persistence=True, + persistence_type="session", + **kwargs + ) + + +def numerical_filter_build(min_value=None, max_value=None): + return CONTROL_SELECTOR( + id=f'filter', + min=min_value or pre_build_min, + max=max_value or pre_build_max, + value=pre_build_numerical_value, + step=0.1, + persistence=True, + persistence_type="session", + ) + + +# Like another-page build +def another_page(**kwargs): + def _get_initial_page_build_object(): + if CONTROL_SELECTOR == dcc.Dropdown: + # A hack explained in on_page_load To-do:"Limitations" section below in this page. + return dmc.DateRangePicker( + id='filter', + value=pre_build_categorical_value, + persistence=True, + persistence_type="session", + ) + elif CONTROL_SELECTOR in SELECTOR_TYPE["categorical"]: + return categorical_filter_build() + elif CONTROL_SELECTOR in SELECTOR_TYPE["numerical"]: + return numerical_filter_build() + else: + raise ValueError("Invalid CONTROL_SELECTOR.") + + return make_page( + [ + dcc.Store(id="on_page_load_trigger_another_page"), + html.H2("Another page"), + + # # This does NOT work because id="filter" doesn't exist but is used as OPL callback State. + # dcc.Loading(id="filter_container"), + + # # Possible solution is to alter filter.options from on_page_load. This would work, but it's not optimal. + # dcc.Dropdown(id="filter", options=options, value=options, multi=True, persistence=True), + + # # Outer container can be changed with dcc.Loading. + html.Div( + _get_initial_page_build_object(), + id="filter_container", + ), + + # # Does not work because OPL filter input is missing, but it's used for filtering figures data_frame. + # html.Div( + # html.Div(id="filter"), + # id="filter_container", + # ), + + html.Br(), + dcc.RadioItems(id="parameter", options=["sepal_width", "sepal_length"], value="sepal_width", persistence=True, persistence_type="session"), + dcc.Loading(dcc.Graph(id="graph1")), + dcc.Loading(dcc.Graph(id="graph2")), + ] + ) + + +def graph1_call(data_frame): + return px.bar(data_frame, x="species", color="species") + + +def graph2_call(data_frame, x): + return px.scatter(data_frame, x=x, y="petal_width", color="species") + + +# NOTE: +# You could do just update_filter to update options/value rather than replacing whole dcc.Dropdown object. Then would +# need to write it for rangeslider and dropdown etc. separately though. Probably easier to just replace whole object. +# This is consistent with how Graph, AgGrid etc. work. +# BUT controls are different from Graphs since you can set the pre-selected value that should be shown when +# user first visits page. Is this possible still with dynamic filter? -> YES + + +def get_data(species): + df = slow_load() + return df[df["species"].isin(species)] + + +# This mechanism is not actually necessary in this example since can just let OPL run without this trigger removing +# prevent_initial_call=True from it. +clientside_callback( + """ + function trigger_to_global_store(data) { + return data; + } + """, + Output("global_on_page_load_another_page_action_trigger", "data"), + Input("on_page_load_trigger_another_page", "data"), + prevent_initial_call=True, # doesn't do anything - callback still runs +) + + +# TODO: write something like get_modified_figures function to reduce repetition. + + +@callback( + output=[ + Output("graph1", "figure"), + Output("graph2", "figure"), + Output("filter_container", "children"), + ], + inputs=[ + Input("global_on_page_load_another_page_action_trigger", "data"), + + State("filter", "value"), + State("parameter", "value"), + ], + prevent_initial_call=True +) +def on_page_load(data, persisted_filter_value, x): + # Ideally, OPL flow should look like this: + # 1. Page.build() -> returns static layout (placeholder elements for dynamic components). + # 2. Persistence is applied. -> So, filter values are the same as the last time the page was visited. + # 3. OPL -> returns dynamic components based on the controls values (persisted) + # 3.1. Load DFs (include DFP values here) + # 3.2. Calculate new filter values: + # e.g. new_filter_values = [value for value in persisted_filter_value if value in new_filter_options] + # e.g. new_min = max(persisted_min, new_min); new_max = min(persisted_max, new_max) + # 3.3. Apply filters on DFs + # 3.4. Apply parameters on config + # 3.5. Return dynamic components (Figures and dynamic controls) + + # Why actions are better than dash.callback here? + # 1. They solve the circular dependency problem of the full graph. + # 2. They are explicit which means they can be configured in any way users want. There's no undesired behavior. + + # TODO: Last solution found -> hence put in highlighted TODO: + # 1. page.build() -> returns: + # 1.1. html.Div(html.Div(id="filter"), id="filter_container") + # * Does not work because we need persisted filter input value in OPL, so we can filter figures data_frame. * + # 1.2. html.Div(dcc.Dropdown(id="filter", ...), id="filter_container") + # * It works! :D * + # 2. OPL -> Manipulations with filter and options: + # 2.1. Recalculate options. + # 2.2. Recalculated value. (persisted_filter_value that exists in recalculated options) + # 2.3. Filter figures data_frame with recalculated value. + # 2.4. Create a new filter object with recalculated options and original value. + # Limitations: + # 1. do_filter is triggered automatically after OPL. + # This shouldn't be the issue since actions loop controls it. + # 2. Component persistence updating works slightly different for dcc.Dropdown than for other selector components. + # Persistence for Dropdown is set even when the Dropdown is returned as a new object from page_build or OPL. + # In persistence.js -> LN:309 "recordUiEdit" function is triggered when dropdown is returned from the server. + # It causes storage.SetItem() to be triggered which mess-ups the persistence for the Dropdown. + # This is probably dash bug because Dropdown is handled a lot with async which probably causes that returned + # Dropdown from the page_build or OPL triggers the "recordUiEdit" which should not trigger. + # ** Problem is solved by returning dmc.DateRangePicker instead of dcc.Dropdown from page.build. ** + # --- (A.M.): How to achieve all of these: --- + # * get correct selected value passed into graph calls -> Works with this solution. + # * populate filter with right values for user on first page load -> Works with this solution. + # * update filter options on page load -> Works with this solution. + # * persist filter values on page change -> Works with this solution. + + print("running on_page_load") + df = slow_load() + + # --- Calculate categorical filter --- + if CONTROL_SELECTOR in SELECTOR_TYPE["categorical"]: + categorical_filter_options = sorted(df["species"].unique().tolist()) + if MULTI: + categorical_filter_value = [value for value in persisted_filter_value if value in categorical_filter_options] + else: + categorical_filter_value = persisted_filter_value if persisted_filter_value in categorical_filter_options else None + new_filter_obj = categorical_filter_build(options=categorical_filter_options) + + # --- Filtering data: --- + if MULTI: + df = df[df["species"].isin(categorical_filter_value)] + else: + df = df[df["species"].isin([categorical_filter_value])] + + # --- set_props --- + # set_props(component_id="filter_container", props={"children": new_filter_obj}) + # More about set_props: + # -> https://dash.plotly.com/advanced-callbacks#setting-properties-directly + # -> https://community.plotly.com/t/dash-2-17-0-released-callback-updates-with-set-props-no-output-callbacks-layout-as-list-dcc-loading-trace-zorder/84343 + # Limitations: + # 1. Component properties updated using set_props won't appear in the callback graph for debugging. + # - This is not a problem because our graph debugging is already unreadable. :D + # 2. Component properties updated using set_props won't appear as loading when they are wrapped with a `dcc.Loading` component. + # - Potential solution. Set controls as dash.Output and then use set_props to update them + dash.no_update as a return value for them. + # 3. set_props doesn't validate the id or property names provided, so no error will be displayed if they contain typos. This can make apps that use set_props harder to debug. + # - That's okay since it's internal Vizro stuff and shouldn't affect user. + # 4. Using set_props with chained callbacks may lead to unexpected results. + # - It even behaves better because it doesn't trigger the "do_filter" callback. + # Open questions: + # 1. Is there any concern about different filter selectors? -> No. (I haven't tested the DatePicker it yet.) + # 2. Can we handle if filter selector changes dynamically? -> Potentially, (I haven't tested it yet.) + # 3. Is there a bug with set_props or with dash.Output?! + + # --- Calculate numerical filter --- + if CONTROL_SELECTOR in SELECTOR_TYPE["numerical"]: + numerical_filter_min = float(df["sepal_length"].min()) + numerical_filter_max = float(df["sepal_length"].max()) + if MULTI: + numerical_filter_value = [max(numerical_filter_min, persisted_filter_value[0]), min(numerical_filter_max, persisted_filter_value[1])] + else: + numerical_filter_value = persisted_filter_value if numerical_filter_min < persisted_filter_value < numerical_filter_max else numerical_filter_min + new_filter_obj = numerical_filter_build(min_value=numerical_filter_min, max_value=numerical_filter_max) + # set_props(component_id="numerical_filter_container", props={"children": new_filter_obj}) + + # --- Filtering data: --- + if MULTI: + df = df[(df["sepal_length"] >= numerical_filter_value[0]) & (df["sepal_length"] <= numerical_filter_value[1])] + else: + df = df[(df["sepal_length"] == numerical_filter_value)] + + print("") + return graph1_call(df), graph2_call(df, x), new_filter_obj + + +# @callback( +# Output("graph1", "figure", allow_duplicate=True), +# Output("graph2", "figure", allow_duplicate=True), +# Input("filter", "value"), +# State("parameter", "value"), +# prevent_initial_call=True, +# ) +# def do_filter(species, x): +# print("running do_filter") +# +# # This also works - filter is calculated on filter value select: +# # It also makes that filter/df1/df2 are calculated based on the same data. Should we enable that? +# # df = slow_load() +# # filter_options = df["species"].unique() +# # filter_value = [value for value in species if value in filter_options] +# # filter_obj = filter_call(filter_options, filter_value) +# # df1 = df2 = df[df["species"].isin(filter_value)] +# # set_props(component_id="filter_container", props={"children": filter_obj}) +# +# df1 = get_data(species) +# df2 = get_data(species) +# print("") +# return graph1_call(df1), graph2_call(df2, x) +# +# +# @callback( +# Output("graph2", "figure", allow_duplicate=True), +# Input("parameter", "value"), +# State("filter", "value"), +# prevent_initial_call=True, +# ) +# def do_parameter(x, species): +# print("running do_parameter") +# df1 = get_data(species) +# print("") +# return graph2_call(df1, x) + + +app = Dash(use_pages=True, pages_folder="", suppress_callback_exceptions=True) +dash.register_page("/", layout=homepage) +dash.register_page("another_page", layout=another_page) + +app.layout = html.Div([dcc.Store("global_on_page_load_another_page_action_trigger"), dash.page_container]) + + +##### NEXT STEPS FOR PETAR + +# How to update dynamic filter? +# Options: +# 1. on_page_load_controls and then on_page_load_components sequentially. Need to figure out how to get components +# into loading state to begin with - set as loading build and then change back in OPL callback? Means two callbacks. +# 2. on_page_load_controls and then on_page_load_components in parallel. NO, bad when caching +# 3. on_page_load_everything. THIS IS THE ONE WE PREFER. +# Can't have on_page_load_controls trigger regular "apply filter" etc. callbacks as could lead to many of them in +# parallel. + +# So need to make sure that either method 1 or 3 doesn't trigger regular callbacks. Not sure +# how to achieve this... +# Could put manual no_update in those regular callbacks but is not nice. +# Could actually just do on_page_load_controls and then use all regular callbacks in parallel - so long as caching +# turned on then on_page_load_controls will have warmed it up so then no problem with regular callbacks. +# But still not good because regular callbacks will override same output graph multiple times. + +# Maybe actually need on_page_load_controls to trigger regular filters in general? And just not have too many of them. + +# persistence still works +# changing page now does on_page_load which then triggers do_filter +# so effectively running do_filter twice +# How can we avoid this? + +# Consider actions loop and when one callback should trigger another etc. + +# How does persistence work? +# How does triggering callbacks work in vizro? +# How *should* triggering callbacks work in vizro? Can we align it more with Dash? +# How to handle filter options persistence and updating etc.? +# How to avoid the regular filters being triggered after on_page_load runs? +# IMPORTANT: also consider parametrised data case. + +if __name__ == "__main__": + app.run(debug=True, dev_tools_hot_reload=False) \ No newline at end of file diff --git a/vizro-core/examples/scratch_dev/app.py b/vizro-core/examples/scratch_dev/app.py index 303d62217..cd49caa9d 100644 --- a/vizro-core/examples/scratch_dev/app.py +++ b/vizro-core/examples/scratch_dev/app.py @@ -1,23 +1,203 @@ """Dev app to try things out.""" +import time +import yaml + +import dash +import pandas as pd +from flask_caching import Cache import vizro.models as vm import vizro.plotly.express as px from vizro import Vizro +from vizro.managers import data_manager + +print("INITIALIZING") + +FILTER_COLUMN = "species" +# FILTER_COLUMN = "sepal_length" + + +def slow_load(sample_size=3): + # time.sleep(2) + df = px.data.iris().sample(sample_size) + print(f'SLOW LOAD - {sorted(df["species"].unique().tolist())} - sample_size = {sample_size}') + return df + + +def load_from_file(): + with open('data.yaml', 'r') as file: + data = yaml.safe_load(file) + data = data or pd.DataFrame() -df = px.data.iris() + # Load the full iris dataset + df = px.data.iris() -page = vm.Page( - title="My first dashboard", + # Load the first N rows of each species. N per species is defined in the data.yaml file. + if FILTER_COLUMN == "species": + final_df = pd.concat(objs=[ + df[df[FILTER_COLUMN] == 'setosa'].head(data.get("setosa", 0)), + df[df[FILTER_COLUMN] == 'versicolor'].head(data.get("versicolor", 0)), + df[df[FILTER_COLUMN] == 'virginica'].head(data.get("virginica", 0)), + ], ignore_index=True) + elif FILTER_COLUMN == "sepal_length": + final_df = df[ + df[FILTER_COLUMN].between(data.get("min", 0), data.get("max", 0), inclusive="both") + ] + else: + raise ValueError("Invalid FILTER_COLUMN") + + return final_df + + +data_manager["dynamic_df"] = load_from_file + +# # TODO-DEV: Turn on/off caching to see how it affects the app. +# data_manager.cache = Cache(config={"CACHE_TYPE": "SimpleCache"}) +# data_manager["dynamic_df"] = slow_load +# data_manager["dynamic_df"].timeout = 5 + + +homepage = vm.Page( + title="Homepage", components=[ - vm.Graph(id="scatter_chart", figure=px.scatter(df, x="sepal_length", y="petal_width", color="species")), - vm.Graph(id="hist_chart", figure=px.histogram(df, x="sepal_width", color="species")), + vm.Card(text="This is the homepage."), + ], +) + +another_page = vm.Page( + title="Test update control options", + components=[ + vm.Graph( + id="graph_1_id", + figure=px.bar( + data_frame="dynamic_df", + x="species", + color="species", + color_discrete_map={"setosa": "#00b4ff", "versicolor": "#ff9222", "virginica": "#3949ab"}, + ) + ), + # vm.Graph( + # id="graph_2_id", + # figure=px.scatter( + # data_frame="dynamic_df", + # x="sepal_length", + # y="petal_width", + # color="species", + # ) + # ), + ], controls=[ - vm.Filter(column="species", selector=vm.Dropdown(value=["ALL"])), + vm.Filter( + id="filter_container_id", + column=FILTER_COLUMN, + + # selector=vm.Dropdown(id="filter_id"), + # selector=vm.Dropdown(id="filter_id", value=["setosa"]), + + selector=vm.Checklist(id="filter_id"), + # selector=vm.Checklist(id="filter_id", value=["setosa"]), + + # selector=vm.Dropdown(id="filter_id", multi=False), + # selector=vm.Dropdown(id="filter_id", multi=False, value="setosa"), + + # selector=vm.RadioItems(id="filter_id"), + # selector=vm.RadioItems(id="filter_id", value="setosa"), + + # selector=vm.Slider(id="filter_id"), + # selector=vm.Slider(id="filter_id", value=6), + + # selector=vm.RangeSlider(id="filter_id"), + # selector=vm.RangeSlider(id="filter_id", value=[5, 7]), + + # TEST CASES: + # no selector + # WORKS + # multi=True + # default + # WORKS + # selector=vm.Slider(id="filter_id", step=0.5), + # options: list + # WORKS - but options doesn't mean anything because options are dynamically overwritten. + # selector=vm.Dropdown(options=["setosa", "versicolor"]), + # options: empty list + # WORKS + # selector=vm.Dropdown(options=[]), + # options: dict + # WORKS - but "label" is always overwritten. + # selector=vm.Dropdown(options=[{"label": "Setosa", "value": "setosa"}, {"label": "Versicolor", "value": "versicolor"}]), + # options list; value + # WORKS + # selector=vm.Dropdown(options=["setosa", "versicolor"], value=["setosa"]), + # options list; empty value + # WORKS + # selector=vm.Dropdown(options=["setosa", "versicolor"], value=[]), + # strange options + # WORKS + # selector=vm.Dropdown(options=["A", "B", "C"]), + # strange options with strange value + # WORKS - works even for the dynamic False, and this is OK. + # selector=vm.Dropdown(options=["A", "B", "C"], value=["XYZ"]), + # + # + # multi=False -> TLDR: Doesn't work if value is cleared. Other combinations are same as multi=True. + # default + # DOES NOT WORK - Doesn't work if value is cleared. Then persistence storage become "null" and our + # placeholder component dmc.DateRangePicker can't process null value. It expects a value or a list + # of values. SOLUTION -> Create the "universal Vizro placeholder component" + # selector=vm.Dropdown(multi=False), + # options: list - because options are dynamically overwritten. + # WORKS - but options doesn't mean anything because options are dynamically overwritten. + # selector=vm.Dropdown(multi=False, options=["setosa", "versicolor"]), + # options: empty list + # WORKS + # selector=vm.Dropdown(multi=False, options=[]), + # options: dict + # selector=vm.Dropdown(multi=False, options=[{"label": "Setosa", "value": "setosa"}, {"label": "Versicolor", "value": "versicolor"}]), + # options list; value + # WORKS + # selector=vm.Dropdown(multi=False, options=["setosa", "versicolor"], value="setosa"), + # options list; None value + # WORKS + # selector=vm.Dropdown(multi=False, options=["setosa", "versicolor"], value=None), + # strange options + # WORKS + # selector=vm.Dropdown(multi=False, options=["A", "B", "C"]), + # strange options with strange value + # WORKS + # selector=vm.Dropdown(multi=False, options=["A", "B", "C"], value="XYZ"), + ), + vm.Parameter( + id="parameter_x", + targets=["graph_1_id.x",], + selector=vm.Dropdown( + options=["species", "sepal_width"], + value="species", + multi=False, + ) + ) + # vm.Parameter( + # id="parameter_container_id", + # targets=[ + # "graph_1_id.data_frame.sample_size", + # # "graph_2_id.data_frame.sample_size", + # ], + # selector=vm.Slider( + # id="parameter_id", + # min=1, + # max=10, + # step=1, + # value=10, + # ) + # ), ], ) -dashboard = vm.Dashboard(pages=[page]) +dashboard = vm.Dashboard(pages=[homepage, another_page]) if __name__ == "__main__": - Vizro().build(dashboard).run() + app = Vizro().build(dashboard) + + print("RUNNING\n") + + app.run(dev_tools_hot_reload=False) diff --git a/vizro-core/examples/scratch_dev/data.yaml b/vizro-core/examples/scratch_dev/data.yaml new file mode 100644 index 000000000..e50c2e8a9 --- /dev/null +++ b/vizro-core/examples/scratch_dev/data.yaml @@ -0,0 +1,6 @@ +setosa: 5 +versicolor: 10 +virginica: 15 + +min: 5 +max: 7 \ No newline at end of file diff --git a/vizro-core/hatch.toml b/vizro-core/hatch.toml index 037f28644..44ac498de 100644 --- a/vizro-core/hatch.toml +++ b/vizro-core/hatch.toml @@ -98,7 +98,7 @@ template = "examples" [envs.examples.env-vars] DASH_DEBUG = "true" -VIZRO_LOG_LEVEL = "DEBUG" +VIZRO_LOG_LEVEL = "WARNING" [envs.lower-bounds] extra-dependencies = [ diff --git a/vizro-core/src/vizro/actions/_callback_mapping/_callback_mapping_utils.py b/vizro-core/src/vizro/actions/_callback_mapping/_callback_mapping_utils.py index 214f04589..6bbba5653 100644 --- a/vizro-core/src/vizro/actions/_callback_mapping/_callback_mapping_utils.py +++ b/vizro-core/src/vizro/actions/_callback_mapping/_callback_mapping_utils.py @@ -103,6 +103,22 @@ def _get_action_callback_outputs(action_id: ModelID) -> Dict[str, Output]: } +def _get_on_page_load_callback_outputs(action_id: ModelID) -> Dict[str, Output]: + """Creates mapping of target names and their `Output`.""" + component_targets = _get_action_callback_outputs(action_id=action_id) + + # TODO-WIP: Add only dynamic filters from the current page + import vizro.models as vm + for filter_id, filter_obj in model_manager._items_with_type(vm.Filter): + if filter_obj._dynamic: + component_targets[filter_id] = Output( + component_id=filter_id, + component_property="children", + ) + + return component_targets + + def _get_export_data_callback_outputs(action_id: ModelID) -> Dict[str, Output]: """Gets mapping of relevant output target name and `Outputs` for `export_data` action.""" action = model_manager[action_id] diff --git a/vizro-core/src/vizro/actions/_callback_mapping/_get_action_callback_mapping.py b/vizro-core/src/vizro/actions/_callback_mapping/_get_action_callback_mapping.py index a147ec35d..4eb2dc7df 100644 --- a/vizro-core/src/vizro/actions/_callback_mapping/_get_action_callback_mapping.py +++ b/vizro-core/src/vizro/actions/_callback_mapping/_get_action_callback_mapping.py @@ -11,6 +11,7 @@ _get_action_callback_outputs, _get_export_data_callback_components, _get_export_data_callback_outputs, + _get_on_page_load_callback_outputs, ) from vizro.actions._filter_action import _filter from vizro.actions._on_page_load_action import _on_page_load @@ -45,7 +46,7 @@ def _get_action_callback_mapping( }, _on_page_load.__wrapped__: { "inputs": _get_action_callback_inputs, - "outputs": _get_action_callback_outputs, + "outputs": _get_on_page_load_callback_outputs, }, } action_call = action_callback_mapping.get(action_function, {}).get(argument) diff --git a/vizro-core/src/vizro/actions/_filter_action.py b/vizro-core/src/vizro/actions/_filter_action.py index 93c36870d..e27028427 100644 --- a/vizro-core/src/vizro/actions/_filter_action.py +++ b/vizro-core/src/vizro/actions/_filter_action.py @@ -30,6 +30,7 @@ def _filter( Dict mapping target component ids to modified charts/components e.g. {'my_scatter': Figure({})} """ + print("FILTER ACTION TRIGGERED!\n") return _get_modified_page_figures( targets=targets, ctds_filter=ctx.args_grouping["external"]["filters"], diff --git a/vizro-core/src/vizro/actions/_on_page_load_action.py b/vizro-core/src/vizro/actions/_on_page_load_action.py index 151fa77c0..3d37e0896 100644 --- a/vizro-core/src/vizro/actions/_on_page_load_action.py +++ b/vizro-core/src/vizro/actions/_on_page_load_action.py @@ -7,6 +7,7 @@ from vizro.actions._actions_utils import _get_modified_page_figures from vizro.managers._model_manager import ModelID from vizro.models.types import capture +from vizro.managers import model_manager, data_manager @capture("action") @@ -22,9 +23,41 @@ def _on_page_load(targets: List[ModelID], **inputs: Dict[str, Any]) -> Dict[str, Dict mapping target chart ids to modified figures e.g. {'my_scatter': Figure({})} """ - return _get_modified_page_figures( + print("\nON PAGE LOAD - START") + print(f'Filter value: {ctx.args_grouping["external"]["filters"]}') + return_obj = _get_modified_page_figures( targets=targets, ctds_filter=ctx.args_grouping["external"]["filters"], ctds_filter_interaction=ctx.args_grouping["external"]["filter_interaction"], ctds_parameters=ctx.args_grouping["external"]["parameters"], ) + + import vizro.models as vm + from time import sleep + sleep(1) + + # TODO-WIP: Add filters to OPL targets. Return only dynamic filter from the targets and not from the entire app. + for filter_id, filter_obj in model_manager._items_with_type(vm.Filter): + if filter_obj._dynamic: + current_value = [ + item for item in ctx.args_grouping["external"]["filters"] + if item["id"] == filter_obj.selector.id + ][0]["value"] + + if current_value in ["ALL", ["ALL"]]: + current_value = [] + + # TODO-CONSIDER: Does calculating options/min/max significantly slow down the app? + # TODO: Also propagate DFP values into the load() method + # 1. "new_options"/"min/max" DOES NOT include the "current_value" + # filter_obj._set_categorical_selectors_options(force=True, current_value=[]) + + # 2. "new_options" DOES include the "current_value" + filter_obj._set_categorical_selectors_options(force=True, current_value=current_value) + filter_obj._set_numerical_and_temporal_selectors_values(force=True, current_value=current_value) + + return_obj[filter_id] = filter_obj.selector(on_page_load_value=current_value) + + print("ON PAGE LOAD - END\n") + + return return_obj diff --git a/vizro-core/src/vizro/models/_components/form/_form_utils.py b/vizro-core/src/vizro/models/_components/form/_form_utils.py index 18e666882..618f60b01 100644 --- a/vizro-core/src/vizro/models/_components/form/_form_utils.py +++ b/vizro-core/src/vizro/models/_components/form/_form_utils.py @@ -45,6 +45,7 @@ def validate_options_dict(cls, values): return values +# TODO: Check this below again def validate_value(cls, value, values): """Reusable validator for the "value" argument of categorical selectors.""" if "options" not in values or not values["options"]: @@ -54,8 +55,8 @@ def validate_value(cls, value, values): [entry["value"] for entry in values["options"]] if isinstance(values["options"][0], dict) else values["options"] ) - if value and not is_value_contained(value, possible_values): - raise ValueError("Please provide a valid value from `options`.") + # if value and not is_value_contained(value, possible_values): + # raise ValueError("Please provide a valid value from `options`.") return value diff --git a/vizro-core/src/vizro/models/_components/form/checklist.py b/vizro-core/src/vizro/models/_components/form/checklist.py index 3279f940d..133d46d57 100644 --- a/vizro-core/src/vizro/models/_components/form/checklist.py +++ b/vizro-core/src/vizro/models/_components/form/checklist.py @@ -38,6 +38,8 @@ class Checklist(VizroBaseModel): title: str = Field("", description="Title to be displayed") actions: List[Action] = [] + _dynamic: bool = PrivateAttr(False) + # Component properties for actions and interactions _input_property: str = PrivateAttr("value") @@ -46,8 +48,10 @@ class Checklist(VizroBaseModel): _validate_options = root_validator(allow_reuse=True, pre=True)(validate_options_dict) _validate_value = validator("value", allow_reuse=True, always=True)(validate_value) - @_log_call - def build(self): + def __call__(self, **kwargs): + return self._build_static() + + def _build_static(self): full_options, default_value = get_options_and_default(options=self.options, multi=True) return html.Fieldset( @@ -62,3 +66,15 @@ def build(self): ), ] ) + + def _build_dynamic_placeholder(self): + if not self.value: + self.value = [get_options_and_default(self.options, multi=True)[1]] + + return self._build_static() + + @_log_call + def build(self): + # TODO: We don't have to implement _build_dynamic_placeholder, _build_static here. It's possible to: + # if dynamic and self.value is None -> set self.value + return standard build (static) + return self._build_dynamic_placeholder() if self._dynamic else self._build_static() diff --git a/vizro-core/src/vizro/models/_components/form/dropdown.py b/vizro-core/src/vizro/models/_components/form/dropdown.py index 60d7981f6..ddbc6986c 100755 --- a/vizro-core/src/vizro/models/_components/form/dropdown.py +++ b/vizro-core/src/vizro/models/_components/form/dropdown.py @@ -10,6 +10,7 @@ from pydantic import Field, PrivateAttr, StrictBool, root_validator, validator import dash_bootstrap_components as dbc +import dash_mantine_components as dmc from vizro.models import Action, VizroBaseModel from vizro.models._action._actions_chain import _action_validator_factory @@ -65,6 +66,11 @@ class Dropdown(VizroBaseModel): title: str = Field("", description="Title to be displayed") actions: List[Action] = [] + # A private property that allows dynamically updating components + # TODO: Consider making the _dynamic public later. The same property also could be used for all other components. + # For example: vm.Graph could have a dynamic that is by default set on True. + _dynamic: bool = PrivateAttr(False) + # Component properties for actions and interactions _input_property: str = PrivateAttr("value") @@ -82,8 +88,11 @@ def validate_multi(cls, multi, values): raise ValueError("Please set multi=True if providing a list of default values.") return multi - @_log_call - def build(self): + # Convenience wrapper/syntactic sugar. + def __call__(self, **kwargs): + return self._build_static() + + def _build_static(self): full_options, default_value = get_options_and_default(options=self.options, multi=self.multi) option_height = _calculate_option_height(full_options) @@ -95,9 +104,35 @@ def build(self): options=full_options, value=self.value if self.value is not None else default_value, multi=self.multi, - persistence=True, optionHeight=option_height, + persistence=True, persistence_type="session", ), ] ) + + def _build_dynamic_placeholder(self): + # Setting self.value is kind of Dropdown pre_build method. It sets self.value only the first time if it's None. + # We cannot create pre_build for the Dropdown because it has to be called after vm.Filter.pre_build, but + # nothing guarantees that. + if not self.value: + self.value = get_options_and_default(self.options, self.multi)[1] + + # return self._build_static() + + return html.Div( + children=[ + dbc.Label(self.title, html_for=self.id) if self.title else None, + dmc.DateRangePicker( + id=self.id, + value=self.value, + persistence=True, + persistence_type="session", + style={'opacity': 0} + ) + ] + ) + + @_log_call + def build(self): + return self._build_dynamic_placeholder() if self._dynamic else self._build_static() diff --git a/vizro-core/src/vizro/models/_components/form/radio_items.py b/vizro-core/src/vizro/models/_components/form/radio_items.py index e91d53f51..82d4c984c 100644 --- a/vizro-core/src/vizro/models/_components/form/radio_items.py +++ b/vizro-core/src/vizro/models/_components/form/radio_items.py @@ -39,6 +39,8 @@ class RadioItems(VizroBaseModel): title: str = Field("", description="Title to be displayed") actions: List[Action] = [] + _dynamic: bool = PrivateAttr(False) + # Component properties for actions and interactions _input_property: str = PrivateAttr("value") @@ -47,8 +49,10 @@ class RadioItems(VizroBaseModel): _validate_options = root_validator(allow_reuse=True, pre=True)(validate_options_dict) _validate_value = validator("value", allow_reuse=True, always=True)(validate_value) - @_log_call - def build(self): + def __call__(self, **kwargs): + return self._build_static() + + def _build_static(self): full_options, default_value = get_options_and_default(options=self.options, multi=False) return html.Fieldset( @@ -63,3 +67,15 @@ def build(self): ), ] ) + + def _build_dynamic_placeholder(self): + if not self.value: + self.value = get_options_and_default(self.options, multi=False)[1] + + return self._build_static() + + @_log_call + def build(self): + # TODO: We don't have to implement _build_dynamic_placeholder, _build_static here. It's possible to: + # if dynamic and self.value is None -> set self.value + return standard build (static) + return self._build_dynamic_placeholder() if self._dynamic else self._build_static() diff --git a/vizro-core/src/vizro/models/_components/form/range_slider.py b/vizro-core/src/vizro/models/_components/form/range_slider.py index 503ba87c8..003187d0f 100644 --- a/vizro-core/src/vizro/models/_components/form/range_slider.py +++ b/vizro-core/src/vizro/models/_components/form/range_slider.py @@ -50,6 +50,8 @@ class RangeSlider(VizroBaseModel): title: str = Field("", description="Title to be displayed.") actions: List[Action] = [] + _dynamic: bool = PrivateAttr(False) + # Component properties for actions and interactions _input_property: str = PrivateAttr("value") @@ -60,9 +62,12 @@ class RangeSlider(VizroBaseModel): _set_default_marks = validator("marks", allow_reuse=True, always=True)(set_default_marks) _set_actions = _action_validator_factory("value") + def __call__(self, on_page_load_value=None): + return self._build_static(on_page_load_value=on_page_load_value) + @_log_call - def build(self): - init_value = self.value or [self.min, self.max] # type: ignore[list-item] + def _build_static(self, is_dynamic_build=False, on_page_load_value=None): + init_value = on_page_load_value or self.value or [self.min, self.max] # type: ignore[list-item] output = [ Output(f"{self.id}_start_value", "value"), @@ -86,7 +91,7 @@ def build(self): return html.Div( children=[ - dcc.Store(f"{self.id}_callback_data", data={"id": self.id, "min": self.min, "max": self.max}), + dcc.Store(f"{self.id}_callback_data", data={"id": self.id, "min": self.min, "max": self.max, "is_dynamic_build": is_dynamic_build}), html.Div( children=[ dbc.Label(children=self.title, html_for=self.id) if self.title else None, @@ -117,7 +122,9 @@ def build(self): persistence_type="session", className="slider-text-input-field", ), - dcc.Store(id=f"{self.id}_input_store", storage_type="session", data=init_value), + dcc.Store(id=f"{self.id}_input_store", storage_type="session", data=self.value) + if is_dynamic_build + else dcc.Store(id=f"{self.id}_input_store", storage_type="session"), ], className="slider-text-input-container", ), @@ -137,3 +144,14 @@ def build(self): ), ] ) + + def _build_dynamic_placeholder(self): + if not self.value: + self.value = [self.min, self.max] + return self._build_static(is_dynamic_build=True) + + @_log_call + def build(self): + # TODO: We don't have to implement _build_dynamic_placeholder, _build_static here. It's possible to: + # if dynamic and self.value is None -> set self.value + return standard build (static) + return self._build_dynamic_placeholder() if self._dynamic else self._build_static() \ No newline at end of file diff --git a/vizro-core/src/vizro/models/_components/form/slider.py b/vizro-core/src/vizro/models/_components/form/slider.py index f3854e2b1..c0fe2ffe3 100644 --- a/vizro-core/src/vizro/models/_components/form/slider.py +++ b/vizro-core/src/vizro/models/_components/form/slider.py @@ -48,6 +48,8 @@ class Slider(VizroBaseModel): title: str = Field("", description="Title to be displayed.") actions: List[Action] = [] + _dynamic: bool = PrivateAttr(False) + # Component properties for actions and interactions _input_property: str = PrivateAttr("value") @@ -58,9 +60,11 @@ class Slider(VizroBaseModel): _set_default_marks = validator("marks", allow_reuse=True, always=True)(set_default_marks) _set_actions = _action_validator_factory("value") - @_log_call - def build(self): - init_value = self.value or self.min + def __call__(self, on_page_load_value=None): + return self._build_static(on_page_load_value=on_page_load_value) + + def _build_static(self, is_dynamic_build=False, on_page_load_value=None): + init_value = on_page_load_value or self.value or self.min output = [ Output(f"{self.id}_end_value", "value"), @@ -80,9 +84,40 @@ def build(self): inputs=inputs, ) + # TODO - TLDR: + # if static: -> assign init_value to the dcc.Store + # if dynamic: + # if dynamic_build: -> dcc.Store(id=f"{self.id}_input_store", storage_type="session", data=None) + # if static_build: -> dcc.Store(id=f"{self.id}_input_store", storage_type="session") + on_page_load_value + # to UI components like dcc.Slider and dcc.Input. on_page_load_value is propagated from the OPL() + # + changes on the slider.js so it returns value if is dynamic build and raises no_update of it's static_build. + + # How-it-works?: + # 1. If it's a static component: + # 0. Build method is only called once - in the page.build(), so before the OPL(). + # 1. Return always dcc.Store(data=init_value) -> the persistence will work here. + # 2. Make client_side callback that maps stored value to the one or many UI components. + # -> This callback is triggered before OPL and it ensures that the correct value is propagated to the OPL + # 3. Outcome: persistence storage keeps only dcc.store value. UI components are always correctly selected. + # 2. If it's a dynamic compoenent: + # Build method is called twice - from the page.build() and from the OPL() + # 1. page.build(): + # 1. page_build() is _build_dynamic and it returns dcc.Store(data=None) - "none" is immediatelly + # overwritten with the persited value if it exists. Otherwise it's overwritten with self.value or min. + # 2. Make client_side callback that maps stored value to the one or many UI components. + # -> This callback is triggered before OPL and ensures that the correct value is propagated to the OPL + # 2. OPL(): + # 1. OPL propagates currently selected value (e.g. slider value) to _build_static() + # 2. build static returns dcc.Store(). But it returns slider and other compoents with the slider_value + # propagated from the OPL. + # 3. clienside_callback is triggered again but as all input values are the same it raises + # dash_clienside.no_update and the process is done. Otherwise, filter_action would be triggered + + stop = 0 + return html.Div( children=[ - dcc.Store(f"{self.id}_callback_data", data={"id": self.id, "min": self.min, "max": self.max}), + dcc.Store(f"{self.id}_callback_data", data={"id": self.id, "min": self.min, "max": self.max, "is_dynamic_build": is_dynamic_build}), html.Div( children=[ dbc.Label(children=self.title, html_for=self.id) if self.title else None, @@ -100,7 +135,9 @@ def build(self): persistence_type="session", className="slider-text-input-field", ), - dcc.Store(id=f"{self.id}_input_store", storage_type="session", data=init_value), + dcc.Store(id=f"{self.id}_input_store", storage_type="session", data=init_value) + if is_dynamic_build + else dcc.Store(id=f"{self.id}_input_store", storage_type="session"), ], className="slider-text-input-container", ), @@ -121,3 +158,15 @@ def build(self): ), ] ) + + def _build_dynamic_placeholder(self): + if self.value is None: + self.value = self.min + + return self._build_static(is_dynamic_build=True) + + @_log_call + def build(self): + # TODO: We don't have to implement _build_dynamic_placeholder, _build_static here. It's possible to: + # if dynamic and self.value is None -> set self.value + return standard build (static) + return self._build_dynamic_placeholder() if self._dynamic else self._build_static() diff --git a/vizro-core/src/vizro/models/_controls/filter.py b/vizro-core/src/vizro/models/_controls/filter.py index 5ae2e0f13..87c95a4fd 100644 --- a/vizro-core/src/vizro/models/_controls/filter.py +++ b/vizro-core/src/vizro/models/_controls/filter.py @@ -1,5 +1,7 @@ from __future__ import annotations +from dash import dcc, html + from typing import List, Literal, Union import pandas as pd @@ -25,6 +27,8 @@ ) from vizro.models._models_utils import _log_call from vizro.models.types import MultiValueType, SelectorType +from vizro.models._components.form._form_utils import get_options_and_default +from vizro.managers._data_manager import _DynamicData # Ideally we might define these as NumericalSelectorType = Union[RangeSlider, Slider] etc., but that will not work # with isinstance checks. @@ -43,6 +47,10 @@ "categorical": SELECTORS["numerical"] + SELECTORS["temporal"], } +# TODO: Remove this check because all vizro selectors support dynamic mode +# Tuple of filter selectors that support dynamic mode +DYNAMIC_SELECTORS = (Dropdown, Checklist, RadioItems, Slider, RangeSlider) + def _filter_between(series: pd.Series, value: Union[List[float], List[str]]) -> pd.Series: if is_datetime64_any_dtype(series): @@ -85,6 +93,9 @@ class Filter(VizroBaseModel): "If none are given then target all components on the page that use `column`.", ) selector: SelectorType = None + + _dynamic: bool = PrivateAttr(None) + _column_type: Literal["numerical", "categorical", "temporal"] = PrivateAttr() @validator("targets", each_item=True) @@ -99,13 +110,15 @@ def pre_build(self): self._set_column_type() self._set_selector() self._validate_disallowed_selector() + self._set_dynamic() self._set_numerical_and_temporal_selectors_values() self._set_categorical_selectors_options() self._set_actions() @_log_call def build(self): - return self.selector.build() + selector_build_obj = self.selector.build() + return dcc.Loading(id=self.id, children=selector_build_obj) if self._dynamic else selector_build_obj def _set_targets(self): if not self.targets: @@ -145,12 +158,41 @@ def _validate_disallowed_selector(self): f"with {self._column_type} column '{self.column}'. " ) - def _set_numerical_and_temporal_selectors_values(self): + def _set_dynamic(self): + self._dynamic = False + + # Selector can't be dynamic if: + # Selector doesn't support dynamic mode + # Selector is categorical and "options" is defined + # Selector is numerical/Temporal and "min" and "max" are defined + if ( + not isinstance(self.selector, DYNAMIC_SELECTORS) + or getattr(self.selector, "options", False) + or any(getattr(self.selector, attr, False) for attr in ["min", "max"]) + ): + return + + for target_id in self.targets: + data_source_name = model_manager[target_id]["data_frame"] + if isinstance(data_manager[data_source_name], _DynamicData): + self._dynamic = True + self.selector._dynamic = True + return + + def _set_numerical_and_temporal_selectors_values(self, force=False, current_value=None): # If the selector is a numerical or temporal selector, and the min and max values are not set, then set them # N.B. All custom selectors inherit from numerical or temporal selector should also pass this check if isinstance(self.selector, SELECTORS["numerical"] + SELECTORS["temporal"]): - min_values = [] - max_values = [] + lvalue, hvalue = ( + (current_value[0], current_value[1]) + if isinstance(current_value, list) and len(current_value) == 2 + else (current_value[0], current_value[0]) + if isinstance(current_value, list) and len(current_value) == 1 + else (current_value, current_value) + ) + + min_values = [] if lvalue is None else [lvalue] + max_values = [] if hvalue is None else [hvalue] for target_id in self.targets: data_source_name = model_manager[target_id]["data_frame"] data_frame = data_manager[data_source_name].load() @@ -169,16 +211,18 @@ def _set_numerical_and_temporal_selectors_values(self): f"targeted charts." ) - if self.selector.min is None: + if self.selector.min is None or force: self.selector.min = min(min_values) - if self.selector.max is None: + if self.selector.max is None or force: self.selector.max = max(max_values) - def _set_categorical_selectors_options(self): + def _set_categorical_selectors_options(self, force=False, current_value=None): # If the selector is a categorical selector, and the options are not set, then set them # N.B. All custom selectors inherit from categorical selector should also pass this check - if isinstance(self.selector, SELECTORS["categorical"]) and not self.selector.options: - options = set() + if isinstance(self.selector, SELECTORS["categorical"]) and (not self.selector.options or force): + current_value = current_value or [] + current_value = current_value if isinstance(current_value, list) else [current_value] + options = set(current_value) for target_id in self.targets: data_source_name = model_manager[target_id]["data_frame"] data_frame = data_manager[data_source_name].load() diff --git a/vizro-core/src/vizro/static/js/models/dashboard.js b/vizro-core/src/vizro/static/js/models/dashboard.js index 0ac2482f4..680a1cd23 100644 --- a/vizro-core/src/vizro/static/js/models/dashboard.js +++ b/vizro-core/src/vizro/static/js/models/dashboard.js @@ -34,7 +34,7 @@ export function _update_graph_theme( export function _collapse_nav_panel(n_clicks, is_open) { if (!n_clicks) { /* Automatically collapses left-side if xs and s-devices are detected*/ - if (window.innerWidth < 576 || window.innerHeight < 576) { + if (window.innerWidth < 6 || window.innerHeight < 6) { return [ false, { diff --git a/vizro-core/src/vizro/static/js/models/range_slider.js b/vizro-core/src/vizro/static/js/models/range_slider.js index 94e877368..239cec3e5 100644 --- a/vizro-core/src/vizro/static/js/models/range_slider.js +++ b/vizro-core/src/vizro/static/js/models/range_slider.js @@ -10,13 +10,16 @@ export function _update_range_slider_values( slider_value, start_text_value, start_value, - trigger_id; + trigger_id, + is_on_page_load_triggered=false; trigger_id = dash_clientside.callback_context.triggered; if (trigger_id.length != 0) { trigger_id = dash_clientside.callback_context.triggered[0]["prop_id"].split(".")[0]; } + + // input form component is the trigger if ( trigger_id === `${self_data["id"]}_start_value` || trigger_id === `${self_data["id"]}_end_value` @@ -25,11 +28,15 @@ export function _update_range_slider_values( return dash_clientside.no_update; } [start_text_value, end_text_value] = [start, end]; + + // slider component is the trigger } else if (trigger_id === self_data["id"]) { [start_text_value, end_text_value] = [slider[0], slider[1]]; + + // on_page_load is the trigger } else { - [start_text_value, end_text_value] = - input_store !== null ? input_store : [slider[0], slider[1]]; + is_on_page_load_triggered = true; + [start_text_value, end_text_value] = input_store !== null ? input_store : [slider[0], slider[1]]; } start_value = Math.min(start_text_value, end_text_value); @@ -38,5 +45,9 @@ export function _update_range_slider_values( end_value = Math.min(self_data["max"], end_value); slider_value = [start_value, end_value]; - return [start_value, end_value, slider_value, [start_value, end_value]]; + if (is_on_page_load_triggered && !self_data["is_dynamic_build"]) { + return [dash_clientside.no_update, dash_clientside.no_update, dash_clientside.no_update, slider_value]; + } + + return [start_value, end_value, slider_value, slider_value]; } diff --git a/vizro-core/src/vizro/static/js/models/slider.js b/vizro-core/src/vizro/static/js/models/slider.js index 784a30a41..8930c9a37 100644 --- a/vizro-core/src/vizro/static/js/models/slider.js +++ b/vizro-core/src/vizro/static/js/models/slider.js @@ -1,23 +1,34 @@ export function _update_slider_values(start, slider, input_store, self_data) { - var end_value, trigger_id; + var end_value, trigger_id, is_on_page_load_triggered=false; trigger_id = dash_clientside.callback_context.triggered; if (trigger_id.length != 0) { trigger_id = dash_clientside.callback_context.triggered[0]["prop_id"].split(".")[0]; } + + // input form component is the trigger if (trigger_id === `${self_data["id"]}_end_value`) { if (isNaN(start)) { return dash_clientside.no_update; } end_value = start; + + // slider component is the trigger } else if (trigger_id === self_data["id"]) { end_value = slider; + + // on_page_load is the trigger } else { + is_on_page_load_triggered = true; end_value = input_store !== null ? input_store : self_data["min"]; } end_value = Math.min(Math.max(self_data["min"], end_value), self_data["max"]); + if (is_on_page_load_triggered && !self_data["is_dynamic_build"]) { + return [dash_clientside.no_update, dash_clientside.no_update, end_value]; + } return [end_value, end_value, end_value]; + } From f2bd3624c54de62081dff7475d0b8be1dd870ead Mon Sep 17 00:00:00 2001 From: Antony Milne Date: Wed, 6 Nov 2024 12:22:34 +0000 Subject: [PATCH 15/64] Turn _get_targets_data_and_config into _get_targets_data - tests pass --- .../src/vizro/actions/_actions_utils.py | 86 ++++++++----------- .../src/vizro/actions/export_data_action.py | 4 +- .../unit/vizro/actions/test_actions_utils.py | 18 ++-- 3 files changed, 48 insertions(+), 60 deletions(-) diff --git a/vizro-core/src/vizro/actions/_actions_utils.py b/vizro-core/src/vizro/actions/_actions_utils.py index c2d9f4581..68dfa36e6 100644 --- a/vizro-core/src/vizro/actions/_actions_utils.py +++ b/vizro-core/src/vizro/actions/_actions_utils.py @@ -116,17 +116,17 @@ def _create_target_arg_mapping(dot_separated_strings: list[str]) -> dict[str, li return results -def _update_nested_graph_properties( - graph_config: dict[str, Any], dot_separated_string: str, value: Any +def _update_nested_figure_properties( + figure_config: dict[str, Any], dot_separated_string: str, value: Any ) -> dict[str, Any]: keys = dot_separated_string.split(".") - current_property = graph_config + current_property = figure_config for key in keys[:-1]: current_property = current_property.setdefault(key, {}) current_property[keys[-1]] = value - return graph_config + return figure_config def _get_parametrized_config(target: ModelID, ctd_parameters: list[CallbackTriggerDict]) -> dict[str, Any]: @@ -165,59 +165,30 @@ def _get_parametrized_config(target: ModelID, ctd_parameters: list[CallbackTrigg continue for action_targets_arg in action_targets[target]: - config = _update_nested_graph_properties( - graph_config=config, dot_separated_string=action_targets_arg, value=selector_value + config = _update_nested_figure_properties( + figure_config=config, dot_separated_string=action_targets_arg, value=selector_value ) return config # Helper functions used in pre-defined actions ---- -def _get_targets_data_and_config( +def _get_targets_data( ctds_filter: list[CallbackTriggerDict], ctds_filter_interaction: list[dict[str, CallbackTriggerDict]], ctds_parameters: list[CallbackTriggerDict], targets: list[ModelID], ): - all_filtered_data = {} - all_parameterized_config = {} - - for target in targets: - # parametrized_config includes a key "data_frame" that is used in the data loading function. - parameterized_config = _get_parametrized_config(target=target, ctd_parameters=ctds_parameters) - data_source_name = model_manager[target]["data_frame"] - data_frame = data_manager[data_source_name].load(**parameterized_config["data_frame"]) - - filtered_data = _apply_filters(data_frame=data_frame, ctds_filters=ctds_filter, target=target) - filtered_data = _apply_filter_interaction( - data_frame=filtered_data, ctds_filter_interaction=ctds_filter_interaction, target=target - ) - - # Parameters affecting data_frame have already been used above in data loading and so are excluded from - # all_parameterized_config. - all_filtered_data[target] = filtered_data - all_parameterized_config[target] = { - key: value for key, value in parameterized_config.items() if key != "data_frame" - } - - return all_filtered_data, all_parameterized_config - - -def _get_modified_page_figures( - ctds_filter: list[CallbackTriggerDict], - ctds_filter_interaction: list[dict[str, CallbackTriggerDict]], - ctds_parameters: list[CallbackTriggerDict], - targets: Optional[list[ModelID]] = None, -) -> dict[str, Any]: - targets = targets or [] + target_to_parameterized_config = { + target: _get_parametrized_config(target=target, ctd_parameters=ctds_parameters) for target in targets + } target_to_data_source_load_key = {} # TODO: Check doesn't give duplicates for static data - # TODO: separate out _get_parametrized_config for data_frame and general keys - for target in targets: + + for target, parameterized_config in target_to_parameterized_config.items(): # parametrized_config includes a key "data_frame" that is used in the data loading function. - parameterized_config = _get_parametrized_config(target=target, ctd_parameters=ctds_parameters) data_source_load_key = json.dumps( { "data_source_name": model_manager[target]["data_frame"], @@ -227,6 +198,7 @@ def _get_modified_page_figures( ) target_to_data_source_load_key[target] = data_source_load_key + # TODO: don't duplicate data in memory. Now this isn't true so maybe change structure of dictionaries? data_source_load_key_to_data = {} for data_source_load_key in set(target_to_data_source_load_key.values()): @@ -234,7 +206,7 @@ def _get_modified_page_figures( data = data_manager[data_source_load["data_source_name"]].load(**data_source_load["dynamic_data_load_params"]) data_source_load_key_to_data[data_source_load_key] = data - outputs: dict[str, Any] = {} + target_to_filtered_data = {} # TODO: deduplicate filtering operation for target in targets: @@ -243,16 +215,32 @@ def _get_modified_page_figures( filtered_data = _apply_filter_interaction( data_frame=filtered_data, ctds_filter_interaction=ctds_filter_interaction, target=target ) + target_to_filtered_data[target] = filtered_data + + return target_to_filtered_data + +def _get_modified_page_figures( + ctds_filter: list[CallbackTriggerDict], + ctds_filter_interaction: list[dict[str, CallbackTriggerDict]], + ctds_parameters: list[CallbackTriggerDict], + targets: Optional[list[ModelID]] = None, +) -> dict[str, Any]: + targets = targets or [] + outputs: dict[str, Any] = {} + + all_filtered_data = _get_targets_data(ctds_filter, ctds_filter_interaction, ctds_parameters, targets) + target_to_parameterized_config = { + target: _get_parametrized_config(target=target, ctd_parameters=ctds_parameters) for target in targets + } + for target, parameterized_config in target_to_parameterized_config.items(): outputs[target] = model_manager[target]( - data_frame=filtered_data, - **{ - key: value - for key, value in _get_parametrized_config(target=target, ctd_parameters=ctds_parameters).items() - if key != "data_frame" - }, + data_frame=all_filtered_data[target], + **{key: value for key, value in parameterized_config.items() if key != "data_frame"}, ) - # TODO: think about where filter call goes + # here have new data_frame = True/False argument + + # TODO: think about where new dynamic filter call goes return outputs diff --git a/vizro-core/src/vizro/actions/export_data_action.py b/vizro-core/src/vizro/actions/export_data_action.py index fb87f419e..f65ac68d0 100644 --- a/vizro-core/src/vizro/actions/export_data_action.py +++ b/vizro-core/src/vizro/actions/export_data_action.py @@ -5,7 +5,7 @@ from dash import ctx, dcc from typing_extensions import Literal -from vizro.actions._actions_utils import _get_targets_data_and_config +from vizro.actions._actions_utils import _get_targets_data from vizro.managers import model_manager from vizro.managers._model_manager import ModelID from vizro.models.types import capture @@ -41,7 +41,7 @@ def export_data( if target not in model_manager: raise ValueError(f"Component '{target}' does not exist.") - data_frames, _ = _get_targets_data_and_config( + data_frames = _get_targets_data( targets=targets, ctds_filter=ctx.args_grouping["external"]["filters"], ctds_filter_interaction=ctx.args_grouping["external"]["filter_interaction"], diff --git a/vizro-core/tests/unit/vizro/actions/test_actions_utils.py b/vizro-core/tests/unit/vizro/actions/test_actions_utils.py index 56e09f899..53a57970a 100644 --- a/vizro-core/tests/unit/vizro/actions/test_actions_utils.py +++ b/vizro-core/tests/unit/vizro/actions/test_actions_utils.py @@ -1,12 +1,12 @@ import pytest -from vizro.actions._actions_utils import _create_target_arg_mapping, _update_nested_graph_properties +from vizro.actions._actions_utils import _create_target_arg_mapping, _update_nested_figure_properties class TestUpdateNestedGraphProperties: - def test_update_nested_graph_properties_single_level(self): + def test_update_nested_figure_properties_single_level(self): graph = {"color": "blue"} - result = _update_nested_graph_properties(graph, "color", "red") + result = _update_nested_figure_properties(graph, "color", "red") expected = {"color": "red"} assert result == expected @@ -22,8 +22,8 @@ def test_update_nested_graph_properties_single_level(self): ), ], ) - def test_update_nested_graph_properties_multiple_levels(self, graph, dot_separated_strings, value, expected): - result = _update_nested_graph_properties(graph, dot_separated_strings, value) + def test_update_nested_figure_properties_multiple_levels(self, graph, dot_separated_strings, value, expected): + result = _update_nested_figure_properties(graph, dot_separated_strings, value) assert result == expected @pytest.mark.parametrize( @@ -50,14 +50,14 @@ def test_update_nested_graph_properties_multiple_levels(self, graph, dot_separat ({}, "color", "red", {"color": "red"}), ], ) - def test_update_nested_graph_properties_add_or_overwrite_keys(self, graph, dot_separated_strings, value, expected): - result = _update_nested_graph_properties(graph, dot_separated_strings, value) + def test_update_nested_figure_properties_add_or_overwrite_keys(self, graph, dot_separated_strings, value, expected): + result = _update_nested_figure_properties(graph, dot_separated_strings, value) assert result == expected - def test_update_nested_graph_properties_invalid_type(self): + def test_update_nested_figure_properties_invalid_type(self): graph = {"color": "blue"} with pytest.raises(TypeError, match="'str' object does not support item assignment"): - _update_nested_graph_properties(graph, "color.value", "42") + _update_nested_figure_properties(graph, "color.value", "42") class TestCreateTargetArgMapping: From 1dbacb001208fcf8bfda77ed82764d5db528fe19 Mon Sep 17 00:00:00 2001 From: Antony Milne Date: Wed, 6 Nov 2024 15:53:01 +0000 Subject: [PATCH 16/64] Turn _create_target_arg_mapping into _filter_dot_separated_string - tests pass --- .../src/vizro/actions/_actions_utils.py | 89 +++++++++---------- .../src/vizro/actions/export_data_action.py | 6 +- .../unit/vizro/actions/test_actions_utils.py | 78 +++++++++------- 3 files changed, 90 insertions(+), 83 deletions(-) diff --git a/vizro-core/src/vizro/actions/_actions_utils.py b/vizro-core/src/vizro/actions/_actions_utils.py index 68dfa36e6..3a5478537 100644 --- a/vizro-core/src/vizro/actions/_actions_utils.py +++ b/vizro-core/src/vizro/actions/_actions_utils.py @@ -106,14 +106,17 @@ def _validate_selector_value_none(value: Union[SingleValueType, MultiValueType]) return value -def _create_target_arg_mapping(dot_separated_strings: list[str]) -> dict[str, list[str]]: - results = defaultdict(list) - for string in dot_separated_strings: - if "." not in string: - raise ValueError(f"Provided string {string} must contain a '.'") - component, arg = string.split(".", 1) - results[component].append(arg) - return results +def _filter_dot_separated_strings(dot_separated_strings: list[str], target: str, data_frame: bool) -> list[str]: + result = [] + + for dot_separated_string in dot_separated_strings: + if dot_separated_string.startswith(f"{target}."): + dot_separated_string = dot_separated_string.removeprefix(f"{target}.") + if (data_frame and dot_separated_string.startswith("data_frame.")) or ( + not data_frame and not dot_separated_string.startswith("data_frame.") + ): + result.append(dot_separated_string) + return result def _update_nested_figure_properties( @@ -129,14 +132,18 @@ def _update_nested_figure_properties( return figure_config -def _get_parametrized_config(target: ModelID, ctd_parameters: list[CallbackTriggerDict]) -> dict[str, Any]: - # TODO - avoid calling _captured_callable. Once we have done this we can remove _arguments from - # CapturedCallable entirely. - config = deepcopy(model_manager[target].figure._arguments) - - # It's not possible to address nested argument of data_frame like data_frame.x.y, just top-level ones like - # data_frame.x. - config["data_frame"] = {} +def _get_parametrized_config( + ctd_parameters: list[CallbackTriggerDict], target: ModelID, data_frame: bool +) -> dict[str, Any]: + if data_frame: + # It's not possible to address nested argument of data_frame like data_frame.x.y, just top-level ones like + # data_frame.x. + config = {"data_frame": {}} + else: + # TODO - avoid calling _captured_callable. Once we have done this we can remove _arguments from + # CapturedCallable entirely. This might mean not being able to address nested parameters. + config = deepcopy(model_manager[target].figure._arguments) + del config["data_frame"] for ctd in ctd_parameters: # TODO: needs to be refactored so that it is independent of implementation details @@ -153,52 +160,44 @@ def _get_parametrized_config(target: ModelID, ctd_parameters: list[CallbackTrigg selector_value = selector.options selector_value = _validate_selector_value_none(selector_value) - selector_actions = _get_component_actions(model_manager[ctd["id"]]) - for action in selector_actions: + for action in _get_component_actions(model_manager[ctd["id"]]): if action.function._function.__name__ != "_parameter": continue - action_targets = _create_target_arg_mapping(action.function["targets"]) - - if target not in action_targets: - continue - - for action_targets_arg in action_targets[target]: + for dot_separated_string in _filter_dot_separated_strings(action.function["targets"], target, data_frame): config = _update_nested_figure_properties( - figure_config=config, dot_separated_string=action_targets_arg, value=selector_value + figure_config=config, dot_separated_string=dot_separated_string, value=selector_value ) return config # Helper functions used in pre-defined actions ---- -def _get_targets_data( +def _get_target_to_filtered_data( ctds_filter: list[CallbackTriggerDict], ctds_filter_interaction: list[dict[str, CallbackTriggerDict]], ctds_parameters: list[CallbackTriggerDict], targets: list[ModelID], ): - target_to_parameterized_config = { - target: _get_parametrized_config(target=target, ctd_parameters=ctds_parameters) for target in targets - } - target_to_data_source_load_key = {} # TODO: Check doesn't give duplicates for static data - for target, parameterized_config in target_to_parameterized_config.items(): - # parametrized_config includes a key "data_frame" that is used in the data loading function. + for target in targets: + dynamic_data_load_params = _get_parametrized_config( + ctd_parameters=ctds_parameters, target=target, data_frame=True + ) data_source_load_key = json.dumps( { "data_source_name": model_manager[target]["data_frame"], - "dynamic_data_load_params": parameterized_config["data_frame"], + "dynamic_data_load_params": dynamic_data_load_params["data_frame"], }, sort_keys=True, ) target_to_data_source_load_key[target] = data_source_load_key - # TODO: don't duplicate data in memory. Now this isn't true so maybe change structure of dictionaries? + # TODO: comment data_source_load_key_to_data = {} for data_source_load_key in set(target_to_data_source_load_key.values()): @@ -208,7 +207,7 @@ def _get_targets_data( target_to_filtered_data = {} - # TODO: deduplicate filtering operation + # TODO: deduplicate filtering operation - save for future for target in targets: data_frame = data_source_load_key_to_data[target_to_data_source_load_key[target]] filtered_data = _apply_filters(data_frame=data_frame, ctds_filters=ctds_filter, target=target) @@ -227,19 +226,17 @@ def _get_modified_page_figures( targets: Optional[list[ModelID]] = None, ) -> dict[str, Any]: targets = targets or [] - outputs: dict[str, Any] = {} - all_filtered_data = _get_targets_data(ctds_filter, ctds_filter_interaction, ctds_parameters, targets) - target_to_parameterized_config = { - target: _get_parametrized_config(target=target, ctd_parameters=ctds_parameters) for target in targets - } - for target, parameterized_config in target_to_parameterized_config.items(): - outputs[target] = model_manager[target]( - data_frame=all_filtered_data[target], - **{key: value for key, value in parameterized_config.items() if key != "data_frame"}, + target_to_filtered_data = _get_target_to_filtered_data( + ctds_filter, ctds_filter_interaction, ctds_parameters, targets + ) + outputs = { + target: model_manager[target]( + data_frame=filtered_data, + **_get_parametrized_config(ctd_parameters=ctds_parameters, target=target, data_frame=False), ) - - # here have new data_frame = True/False argument + for target, filtered_data in target_to_filtered_data.items() + } # TODO: think about where new dynamic filter call goes diff --git a/vizro-core/src/vizro/actions/export_data_action.py b/vizro-core/src/vizro/actions/export_data_action.py index f65ac68d0..ab20fe619 100644 --- a/vizro-core/src/vizro/actions/export_data_action.py +++ b/vizro-core/src/vizro/actions/export_data_action.py @@ -5,7 +5,7 @@ from dash import ctx, dcc from typing_extensions import Literal -from vizro.actions._actions_utils import _get_targets_data +from vizro.actions._actions_utils import _get_target_to_filtered_data from vizro.managers import model_manager from vizro.managers._model_manager import ModelID from vizro.models.types import capture @@ -41,11 +41,11 @@ def export_data( if target not in model_manager: raise ValueError(f"Component '{target}' does not exist.") - data_frames = _get_targets_data( - targets=targets, + data_frames = _get_target_to_filtered_data( ctds_filter=ctx.args_grouping["external"]["filters"], ctds_filter_interaction=ctx.args_grouping["external"]["filter_interaction"], ctds_parameters=ctx.args_grouping["external"]["parameters"], + targets=targets, ) outputs = {} diff --git a/vizro-core/tests/unit/vizro/actions/test_actions_utils.py b/vizro-core/tests/unit/vizro/actions/test_actions_utils.py index 53a57970a..7e322130c 100644 --- a/vizro-core/tests/unit/vizro/actions/test_actions_utils.py +++ b/vizro-core/tests/unit/vizro/actions/test_actions_utils.py @@ -1,6 +1,6 @@ import pytest -from vizro.actions._actions_utils import _create_target_arg_mapping, _update_nested_figure_properties +from vizro.actions._actions_utils import _filter_dot_separated_strings, _update_nested_figure_properties class TestUpdateNestedGraphProperties: @@ -60,37 +60,47 @@ def test_update_nested_figure_properties_invalid_type(self): _update_nested_figure_properties(graph, "color.value", "42") -class TestCreateTargetArgMapping: - def test_single_string_one_component(self): - input_strings = ["component1.argument1"] - expected = {"component1": ["argument1"]} - result = _create_target_arg_mapping(input_strings) - assert result == expected - - def test_multiple_strings_different_components(self): - input_strings = ["component1.argument1", "component2.argument2", "component1.argument3"] - expected = {"component1": ["argument1", "argument3"], "component2": ["argument2"]} - result = _create_target_arg_mapping(input_strings) - assert result == expected - - def test_multiple_strings_same_component(self): - input_strings = ["component1.argument1", "component1.argument2", "component1.argument3"] - expected = {"component1": ["argument1", "argument2", "argument3"]} - result = _create_target_arg_mapping(input_strings) - assert result == expected - - def test_empty_input_list(self): - input_strings = [] - expected = {} - result = _create_target_arg_mapping(input_strings) - assert result == expected - - def test_strings_without_separator(self): - input_strings = ["component1_argument1", "component2_argument2"] - with pytest.raises(ValueError, match="must contain a '.'"): - _create_target_arg_mapping(input_strings) +class TestFilterDotSeparatedStrings: + @pytest.mark.parametrize( + "dot_separated_strings, expected", + [ + ([], []), + (["component1.argument1", "component1.data_frame.x"], ["data_frame.x"]), + ( + [ + "component1.argument1", + "component1.data_frame.x", + "component1.data_frame.y", + "component2.argument2", + "component2.data_frame.z", + "component1.argument3", + ], + ["data_frame.x", "data_frame.y"], + ), + (["component1.argument1.extra", "component1.data_frame.x"], ["data_frame.x"]), + ], + ) + def test_filter_data_frame_parameters(self, dot_separated_strings, expected): + assert _filter_dot_separated_strings(dot_separated_strings, "component1", data_frame=True) == expected - def test_strings_with_multiple_separators(self): - input_strings = ["component1.argument1.extra", "component2.argument2.extra"] - expected = {"component1": ["argument1.extra"], "component2": ["argument2.extra"]} - assert _create_target_arg_mapping(input_strings) == expected + @pytest.mark.parametrize( + "dot_separated_strings, expected", + [ + ([], []), + (["component1.argument1", "component1.data_frame.x"], ["argument1"]), + ( + [ + "component1.argument1", + "component1.data_frame.x", + "component1.data_frame.y", + "component2.argument2", + "component2.data_frame.z", + "component1.argument3", + ], + ["argument1", "argument3"], + ), + (["component1.argument1.extra", "component1.data_frame.x"], ["argument1.extra"]), + ], + ) + def test_filter_non_data_frame_parameters(self, dot_separated_strings, expected): + assert _filter_dot_separated_strings(dot_separated_strings, "component1", data_frame=False) == expected From 4d0fb0ed2f42942564e12cdbd5ed134db369efc6 Mon Sep 17 00:00:00 2001 From: Antony Milne Date: Thu, 7 Nov 2024 11:21:21 +0000 Subject: [PATCH 17/64] Split up filtered and unfiltered data, create _multi_load, rework Filter.pre_build - tests pass --- .../src/vizro/actions/_actions_utils.py | 79 +++++++++---------- .../src/vizro/actions/export_data_action.py | 22 +++--- .../src/vizro/managers/_data_manager.py | 38 ++++++++- .../src/vizro/models/_controls/filter.py | 72 ++++++++--------- 4 files changed, 116 insertions(+), 95 deletions(-) diff --git a/vizro-core/src/vizro/actions/_actions_utils.py b/vizro-core/src/vizro/actions/_actions_utils.py index 3a5478537..3fd45212e 100644 --- a/vizro-core/src/vizro/actions/_actions_utils.py +++ b/vizro-core/src/vizro/actions/_actions_utils.py @@ -48,7 +48,9 @@ def _get_component_actions(component) -> list[Action]: ) -def _apply_filters(data_frame: pd.DataFrame, ctds_filters: list[CallbackTriggerDict], target: str) -> pd.DataFrame: +def _apply_control_filters( + data_frame: pd.DataFrame, ctds_filters: list[CallbackTriggerDict], target: str +) -> pd.DataFrame: for ctd in ctds_filters: selector_value = ctd["value"] selector_value = selector_value if isinstance(selector_value, list) else [selector_value] @@ -174,49 +176,38 @@ def _get_parametrized_config( # Helper functions used in pre-defined actions ---- -def _get_target_to_filtered_data( +def _apply_filters( + data: pd.DataFrame, ctds_filter: list[CallbackTriggerDict], ctds_filter_interaction: list[dict[str, CallbackTriggerDict]], - ctds_parameters: list[CallbackTriggerDict], - targets: list[ModelID], + target: ModelID, ): - target_to_data_source_load_key = {} + # Takes in just one target, so dataframe is filtered repeatedly for every target that uses it. + # Potentially this could be de-duplicated but it's not so important since filtering is a relatively fast + # operation (compared to data loading). + filtered_data = _apply_control_filters(data_frame=data, ctds_filters=ctds_filter, target=target) + filtered_data = _apply_filter_interaction( + data_frame=filtered_data, ctds_filter_interaction=ctds_filter_interaction, target=target + ) + return filtered_data - # TODO: Check doesn't give duplicates for static data +def _get_unfiltered_data( + ctds_parameters: list[CallbackTriggerDict], targets: list[ModelID] +) -> dict[ModelID, pd.DataFrame]: + # Takes in multiple targets to ensure that data can be loaded efficiently using _multi_load and not repeated for + # every single target. + # Getting unfiltered data requires data frame parameters. We pass in all ctd_parameters and then find the + # data_frame ones by passing data_frame=True in the call to _get_paramaterized_config. + multi_data_source_name_load_kwargs = [] for target in targets: dynamic_data_load_params = _get_parametrized_config( ctd_parameters=ctds_parameters, target=target, data_frame=True ) - data_source_load_key = json.dumps( - { - "data_source_name": model_manager[target]["data_frame"], - "dynamic_data_load_params": dynamic_data_load_params["data_frame"], - }, - sort_keys=True, - ) - target_to_data_source_load_key[target] = data_source_load_key - - # TODO: comment - data_source_load_key_to_data = {} - - for data_source_load_key in set(target_to_data_source_load_key.values()): - data_source_load = json.loads(data_source_load_key) - data = data_manager[data_source_load["data_source_name"]].load(**data_source_load["dynamic_data_load_params"]) - data_source_load_key_to_data[data_source_load_key] = data - - target_to_filtered_data = {} - - # TODO: deduplicate filtering operation - save for future - for target in targets: - data_frame = data_source_load_key_to_data[target_to_data_source_load_key[target]] - filtered_data = _apply_filters(data_frame=data_frame, ctds_filters=ctds_filter, target=target) - filtered_data = _apply_filter_interaction( - data_frame=filtered_data, ctds_filter_interaction=ctds_filter_interaction, target=target - ) - target_to_filtered_data[target] = filtered_data + data_source_name = model_manager[target]["data_frame"] + multi_data_source_name_load_kwargs.append((data_source_name, dynamic_data_load_params["data_frame"])) - return target_to_filtered_data + return dict(zip(targets, data_manager._multi_load(multi_data_source_name_load_kwargs))) def _get_modified_page_figures( @@ -226,18 +217,22 @@ def _get_modified_page_figures( targets: Optional[list[ModelID]] = None, ) -> dict[str, Any]: targets = targets or [] + outputs = {} - target_to_filtered_data = _get_target_to_filtered_data( - ctds_filter, ctds_filter_interaction, ctds_parameters, targets - ) - outputs = { - target: model_manager[target]( + # TODO: the structure here would be nicer if we could get just the ctds for a single target at one time, + # so you could do apply_filters on a target a pass only the ctds relevant for that target. + # Consider restructuring ctds to a more convenient form to make this possible. + + for target, unfiltered_data in _get_unfiltered_data(ctds_parameters, targets).items(): + filtered_data = _apply_filters(unfiltered_data, ctds_filter, ctds_filter_interaction, target) + outputs[target] = model_manager[target]( data_frame=filtered_data, **_get_parametrized_config(ctd_parameters=ctds_parameters, target=target, data_frame=False), ) - for target, filtered_data in target_to_filtered_data.items() - } - # TODO: think about where new dynamic filter call goes + # TODO NEXT: will need to pass unfiltered_data into Filter.__call__. + # This dictionary is filtered for correct targets already selected in Filter.__call__ or that could be done here + # instead. + # {target: data_frame for target, data_frame in unfiltered_data.items() if target in self.targets} return outputs diff --git a/vizro-core/src/vizro/actions/export_data_action.py b/vizro-core/src/vizro/actions/export_data_action.py index ab20fe619..1853abe46 100644 --- a/vizro-core/src/vizro/actions/export_data_action.py +++ b/vizro-core/src/vizro/actions/export_data_action.py @@ -5,7 +5,7 @@ from dash import ctx, dcc from typing_extensions import Literal -from vizro.actions._actions_utils import _get_target_to_filtered_data +from vizro.actions._actions_utils import _get_unfiltered_data, _apply_filters from vizro.managers import model_manager from vizro.managers._model_manager import ModelID from vizro.models.types import capture @@ -41,23 +41,19 @@ def export_data( if target not in model_manager: raise ValueError(f"Component '{target}' does not exist.") - data_frames = _get_target_to_filtered_data( - ctds_filter=ctx.args_grouping["external"]["filters"], - ctds_filter_interaction=ctx.args_grouping["external"]["filter_interaction"], - ctds_parameters=ctx.args_grouping["external"]["parameters"], - targets=targets, - ) - + ctds = ctx.args_grouping["external"] outputs = {} - for target_id in targets: + + for target, unfiltered_data in _get_unfiltered_data(ctds["parameters"], targets).items(): + filtered_data = _apply_filters(unfiltered_data, ctds["filters"], ctds["filter_interaction"], target) if file_format == "csv": - writer = data_frames[target_id].to_csv + writer = filtered_data.to_csv elif file_format == "xlsx": - writer = data_frames[target_id].to_excel + writer = filtered_data.to # Invalid file_format should be caught by Action validation - outputs[f"download_dataframe_{target_id}"] = dcc.send_data_frame( - writer=writer, filename=f"{target_id}.{file_format}", index=False + outputs[f"download_dataframe_{target}"] = dcc.send_data_frame( + writer=writer, filename=f"{target}.{file_format}", index=False ) return outputs diff --git a/vizro-core/src/vizro/managers/_data_manager.py b/vizro-core/src/vizro/managers/_data_manager.py index 44e7972db..324b8d8fa 100644 --- a/vizro-core/src/vizro/managers/_data_manager.py +++ b/vizro-core/src/vizro/managers/_data_manager.py @@ -2,12 +2,14 @@ from __future__ import annotations +import json + import functools import logging import os import warnings from functools import partial -from typing import Callable, Optional, Union +from typing import Callable, Optional, Union, Any import pandas as pd import wrapt @@ -196,6 +198,40 @@ def __getitem__(self, name: DataSourceName) -> Union[_DynamicData, _StaticData]: except KeyError as exc: raise KeyError(f"Data source {name} does not exist.") from exc + def _multi_load(self, multi_name_load_kwargs: list[tuple[DataSourceName, dict[str, Any]]]) -> list[pd.DataFrame]: + """Loads multiple data sources as efficiently as possible. + + Deduplicates a list of (data source name, load keyword argument dictionary) tuples so that each one corresponds + to only a single load() call. In the worst case scenario where there are no repeated tuples then performance of + this function is identical to doing a load call for each tuple. + + Args: + multi_name_load_kwargs: List of (data source name, load keyword argument dictionary). + + Returns: + Loaded data in the same order as `multi_name_load_kwargs` was supplied. + """ + # TODO NOW: Check doesn't give duplicates for static data. Write tests with load call_count + + # Easiest way to make a key to de-duplicate each (data source name, load keyword argument dictionary) tuple. + def encode_load_key(name, load_kwargs): + return json.dumps([name, load_kwargs], sort_keys=True) + + def decode_load_key(key): + return json.loads(key) + + # dict.fromkeys does the de-duplication. + load_key_to_data = dict.fromkeys( + encode_load_key(name, load_kwargs) for name, load_kwargs in multi_name_load_kwargs + ) + + # Load each key only once. + for load_key in load_key_to_data.keys(): + name, load_kwargs = decode_load_key(load_key) + load_key_to_data[load_key] = self[name].load(**load_kwargs) + + return [load_key_to_data[encode_load_key(name, load_kwargs)] for name, load_kwargs in multi_name_load_kwargs] + def _clear(self): # We do not actually call self.cache.clear() because (a) it would only work when self._cache_has_app is True, # which is not the case when e.g. Vizro._reset is called, and (b) because we do not want to accidentally diff --git a/vizro-core/src/vizro/models/_controls/filter.py b/vizro-core/src/vizro/models/_controls/filter.py index ea6390b72..9f91081f2 100644 --- a/vizro-core/src/vizro/models/_controls/filter.py +++ b/vizro-core/src/vizro/models/_controls/filter.py @@ -96,19 +96,25 @@ def check_target_present(cls, target): @_log_call def pre_build(self): - if self.targets: - targeted_data = self._validate_targeted_data(targets=self.targets) - else: - # If targets aren't explicitly provided then try to target all figures on the page. In this case we don't - # want to raise an error if the column is not found in a figure's data_frame, it will just be ignored. - # Possibly in future this will change (which would be breaking change). - targeted_data = self._validate_targeted_data( - targets=model_manager._get_page_model_ids_with_figure( - page_id=model_manager._get_model_page_id(model_id=ModelID(str(self.id))) - ), - eagerly_raise_column_not_found_error=False, - ) - self.targets = list(targeted_data.columns) + # If targets aren't explicitly provided then try to target all figures on the page. In this case we don't + # want to raise an error if the column is not found in a figure's data_frame, it will just be ignored. + # This is the case when bool(self.targets) is False. + # Possibly in future this will change (which would be breaking change). + proposed_targets = self.targets or model_manager._get_page_model_ids_with_figure( + page_id=model_manager._get_model_page_id(model_id=ModelID(str(self.id))) + ) + # TODO NEXT: how to handle pre_build for dynamic filters? Do we still require default argument values in + # `load` to establish selector type etc.? Can we take selector values from model_manager to supply these? Or just don't + # do validation at pre_build time and wait until state is available during build time instead? What should the + # load kwargs be here? + # Note that currently _get_unfiltered_data is only suitable for use at runtime since it requires + # ctd_parameters. That could be changed to just reuse that function. + multi_data_source_name_load_kwargs = [(model_manager[target]["data_frame"], {}) for target in proposed_targets] + target_to_data_frame = dict(zip(proposed_targets, data_manager._multi_load(multi_data_source_name_load_kwargs))) + targeted_data = self._validate_targeted_data( + target_to_data_frame, eagerly_raise_column_not_found_error=bool(self.targets) + ) + self.targets = list(targeted_data.columns) # Set default selector according to column type. self._column_type = self._validate_column_type(targeted_data) @@ -148,12 +154,15 @@ def pre_build(self): ) ] - def __call__(self, **kwargs): + def __call__(self, target_to_data_frame: dict[ModelID, pd.DataFrame]): # Only relevant for a dynamic filter. - # TODO: this will need to pass parametrised data_frame arguments through to _validate_targeted_data. # Although targets are fixed at build time, the validation logic is repeated during runtime, so if a column # is missing then it will raise an error. We could change this if we wanted. - targeted_data = self._validate_targeted_data(targets=self.targets) + # Call this from actions_utils + targeted_data = self._validate_targeted_data( + {target: data_frame for target, data_frame in target_to_data_frame.items() if target in self.targets}, + eagerly_raise_column_not_found_error=True, + ) if (column_type := self._validate_column_type(targeted_data)) != self._column_type: raise ValueError( @@ -173,30 +182,11 @@ def build(self): return self.selector.build() def _validate_targeted_data( - self, targets: list[ModelID], eagerly_raise_column_not_found_error=True + self, target_to_data_frame: dict[ModelID, pd.DataFrame], eagerly_raise_column_not_found_error ) -> pd.DataFrame: - # TODO: consider moving some of this logic to data_manager when implement dynamic filter. Make sure - # get_modified_figures and stuff in _actions_utils.py is as efficient as code here. - - # When loading data_frame there are possible keys: - # 1. target. In worst case scenario this is needed but can lead to unnecessary repeated data loading. - # 2. data_source_name. No repeated data loading but won't work when applying data_frame parameters at runtime. - # 3. data_source_name + data_frame parameters keyword-argument pairs. This is the correct key to use at - # runtime. - # For now we follow scheme 2 for data loading (due to set() below) and 1 for the returned targeted_data - # pd.DataFrame, i.e. a separate column for each target even if some data is repeated. - # TODO: when this works with data_frame parameters load() will need to take arguments and the structures here - # might change a bit. - target_to_data_source_name = {target: model_manager[target]["data_frame"] for target in targets} - data_source_name_to_data = { - data_source_name: data_manager[data_source_name].load() - for data_source_name in set(target_to_data_source_name.values()) - } target_to_series = {} - for target, data_source_name in target_to_data_source_name.items(): - data_frame = data_source_name_to_data[data_source_name] - + for target, data_frame in target_to_data_frame.items(): if self.column in data_frame.columns: # reset_index so that when we make a DataFrame out of all these pd.Series pandas doesn't try to align # the columns by index. @@ -207,10 +197,14 @@ def _validate_targeted_data( targeted_data = pd.DataFrame(target_to_series) if targeted_data.columns.empty: # Still raised when eagerly_raise_column_not_found_error=False. - raise ValueError(f"Selected column {self.column} not found in any dataframe for {', '.join(targets)}.") + raise ValueError( + f"Selected column {self.column} not found in any dataframe for " + f"{', '.join(target_to_data_frame.keys())}." + ) if targeted_data.empty: raise ValueError( - f"Selected column {self.column} does not contain anything in any dataframe for {', '.join(targets)}." + f"Selected column {self.column} does not contain anything in any dataframe for " + f"{', '.join(target_to_data_frame.keys())}." ) return targeted_data From 1a859ff65223f818cf25aa7a0abdfb4fb9846cea Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 7 Nov 2024 11:24:17 +0000 Subject: [PATCH 18/64] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- vizro-core/src/vizro/actions/_actions_utils.py | 2 -- vizro-core/src/vizro/actions/export_data_action.py | 2 +- vizro-core/src/vizro/managers/_data_manager.py | 5 ++--- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/vizro-core/src/vizro/actions/_actions_utils.py b/vizro-core/src/vizro/actions/_actions_utils.py index 3fd45212e..51530edbd 100644 --- a/vizro-core/src/vizro/actions/_actions_utils.py +++ b/vizro-core/src/vizro/actions/_actions_utils.py @@ -2,8 +2,6 @@ from __future__ import annotations -import json -from collections import defaultdict from copy import deepcopy from typing import TYPE_CHECKING, Any, Literal, Optional, TypedDict, Union diff --git a/vizro-core/src/vizro/actions/export_data_action.py b/vizro-core/src/vizro/actions/export_data_action.py index 1853abe46..e1620d93e 100644 --- a/vizro-core/src/vizro/actions/export_data_action.py +++ b/vizro-core/src/vizro/actions/export_data_action.py @@ -5,7 +5,7 @@ from dash import ctx, dcc from typing_extensions import Literal -from vizro.actions._actions_utils import _get_unfiltered_data, _apply_filters +from vizro.actions._actions_utils import _apply_filters, _get_unfiltered_data from vizro.managers import model_manager from vizro.managers._model_manager import ModelID from vizro.models.types import capture diff --git a/vizro-core/src/vizro/managers/_data_manager.py b/vizro-core/src/vizro/managers/_data_manager.py index 324b8d8fa..b3b3243a6 100644 --- a/vizro-core/src/vizro/managers/_data_manager.py +++ b/vizro-core/src/vizro/managers/_data_manager.py @@ -2,14 +2,13 @@ from __future__ import annotations -import json - import functools +import json import logging import os import warnings from functools import partial -from typing import Callable, Optional, Union, Any +from typing import Any, Callable, Optional, Union import pandas as pd import wrapt From 2b3b0aeee9fb6a4eaf546c25d006d01a6bc2ebd3 Mon Sep 17 00:00:00 2001 From: petar-qb Date: Thu, 7 Nov 2024 14:41:55 +0100 Subject: [PATCH 19/64] Support for Slider and RangeSlider to work as RadioItems --- vizro-core/examples/scratch_dev/data.yaml | 2 +- .../models/_components/form/range_slider.py | 32 +++++++++--------- .../vizro/models/_components/form/slider.py | 13 +++----- .../vizro/static/js/models/range_slider.js | 33 ++++++++----------- .../src/vizro/static/js/models/slider.js | 26 ++++++++------- 5 files changed, 49 insertions(+), 57 deletions(-) diff --git a/vizro-core/examples/scratch_dev/data.yaml b/vizro-core/examples/scratch_dev/data.yaml index 8531be9a5..e50c2e8a9 100644 --- a/vizro-core/examples/scratch_dev/data.yaml +++ b/vizro-core/examples/scratch_dev/data.yaml @@ -2,5 +2,5 @@ setosa: 5 versicolor: 10 virginica: 15 -min: 6 +min: 5 max: 7 \ No newline at end of file diff --git a/vizro-core/src/vizro/models/_components/form/range_slider.py b/vizro-core/src/vizro/models/_components/form/range_slider.py index 9141c0ab2..bcc77405e 100644 --- a/vizro-core/src/vizro/models/_components/form/range_slider.py +++ b/vizro-core/src/vizro/models/_components/form/range_slider.py @@ -62,12 +62,14 @@ class RangeSlider(VizroBaseModel): _set_default_marks = validator("marks", allow_reuse=True, always=True)(set_default_marks) _set_actions = _action_validator_factory("value") - def __call__(self, on_page_load_value=None): - return self._build_static(on_page_load_value=on_page_load_value) + def __call__(self, current_value=None, new_min=None, new_max=None, **kwargs): + return self._build_static(current_value=current_value, new_min=new_min, new_max=new_max, **kwargs) @_log_call - def _build_static(self, is_dynamic_build=False, on_page_load_value=None): - init_value = on_page_load_value or self.value or [self.min, self.max] # type: ignore[list-item] + def _build_static(self, current_value=None, new_min=None, new_max=None, **kwargs): + _min = new_min if new_min else self.min + _max = new_max if new_max else self.max + init_value = current_value or self.value or [_min, _max] output = [ Output(f"{self.id}_start_value", "value"), @@ -89,9 +91,11 @@ def _build_static(self, is_dynamic_build=False, on_page_load_value=None): inputs=inputs, ) + stop = 0 + return html.Div( children=[ - dcc.Store(f"{self.id}_callback_data", data={"id": self.id, "min": self.min, "max": self.max, "is_dynamic_build": is_dynamic_build}), + dcc.Store(f"{self.id}_callback_data", data={"id": self.id, "min": _min, "max": _max}), html.Div( children=[ dbc.Label(children=self.title, html_for=self.id) if self.title else None, @@ -101,8 +105,8 @@ def _build_static(self, is_dynamic_build=False, on_page_load_value=None): id=f"{self.id}_start_value", type="number", placeholder="min", - min=self.min, - max=self.max, + min=_min, + max=_max, step=self.step, value=init_value[0], persistence=True, @@ -114,17 +118,15 @@ def _build_static(self, is_dynamic_build=False, on_page_load_value=None): id=f"{self.id}_end_value", type="number", placeholder="max", - min=self.min, - max=self.max, + min=_min, + max=_max, step=self.step, value=init_value[1], persistence=True, persistence_type="session", className="slider-text-input-field", ), - dcc.Store(id=f"{self.id}_input_store", storage_type="session", data=self.value) - if is_dynamic_build - else dcc.Store(id=f"{self.id}_input_store", storage_type="session"), + dcc.Store(id=f"{self.id}_input_store", storage_type="session") ], className="slider-text-input-container", ), @@ -133,8 +135,8 @@ def _build_static(self, is_dynamic_build=False, on_page_load_value=None): ), dcc.RangeSlider( id=self.id, - min=self.min, - max=self.max, + min=_min, + max=_max, step=self.step, marks=self.marks, value=init_value, @@ -146,8 +148,6 @@ def _build_static(self, is_dynamic_build=False, on_page_load_value=None): ) def _build_dynamic_placeholder(self): - if not self.value: - self.value = [self.min, self.max] return self._build_static(is_dynamic_build=True) @_log_call diff --git a/vizro-core/src/vizro/models/_components/form/slider.py b/vizro-core/src/vizro/models/_components/form/slider.py index 48122be9c..20bc6a1b2 100644 --- a/vizro-core/src/vizro/models/_components/form/slider.py +++ b/vizro-core/src/vizro/models/_components/form/slider.py @@ -63,7 +63,7 @@ class Slider(VizroBaseModel): def __call__(self, current_value=None, new_min=None, new_max=None, **kwargs): return self._build_static(current_value=current_value, new_min=new_min, new_max=new_max, **kwargs) - def _build_static(self, is_dynamic_build=False, current_value=None, new_min=None, new_max=None, **kwargs): + def _build_static(self, current_value=None, new_min=None, new_max=None, **kwargs): _min = new_min if new_min else self.min _max = new_max if new_max else self.max init_value = current_value or self.value or _min @@ -119,7 +119,7 @@ def _build_static(self, is_dynamic_build=False, current_value=None, new_min=None return html.Div( children=[ - dcc.Store(f"{self.id}_callback_data", data={"id": self.id, "min": _min, "max": _max, "is_dynamic_build": is_dynamic_build}), + dcc.Store(f"{self.id}_callback_data", data={"id": self.id, "min": _min, "max": _max}), html.Div( children=[ dbc.Label(children=self.title, html_for=self.id) if self.title else None, @@ -137,9 +137,7 @@ def _build_static(self, is_dynamic_build=False, current_value=None, new_min=None persistence_type="session", className="slider-text-input-field", ), - dcc.Store(id=f"{self.id}_input_store", storage_type="session", data=init_value) - if is_dynamic_build - else dcc.Store(id=f"{self.id}_input_store", storage_type="session"), + dcc.Store(id=f"{self.id}_input_store", storage_type="session") ], className="slider-text-input-container", ), @@ -162,10 +160,7 @@ def _build_static(self, is_dynamic_build=False, current_value=None, new_min=None ) def _build_dynamic_placeholder(self): - if self.value is None: - self.value = self.min - - return self._build_static(is_dynamic_build=True) + return self._build_static() @_log_call def build(self): diff --git a/vizro-core/src/vizro/static/js/models/range_slider.js b/vizro-core/src/vizro/static/js/models/range_slider.js index 5b1472754..e9c782324 100644 --- a/vizro-core/src/vizro/static/js/models/range_slider.js +++ b/vizro-core/src/vizro/static/js/models/range_slider.js @@ -10,8 +10,7 @@ function update_range_slider_values( slider_value, start_text_value, start_value, - trigger_id, - is_on_page_load_triggered=false; + trigger_id; trigger_id = dash_clientside.callback_context.triggered; if (trigger_id.length != 0) { @@ -19,7 +18,7 @@ function update_range_slider_values( dash_clientside.callback_context.triggered[0]["prop_id"].split(".")[0]; } - // input form component is the trigger + // text form component is the trigger if ( trigger_id === `${self_data["id"]}_start_value` || trigger_id === `${self_data["id"]}_end_value` @@ -27,29 +26,25 @@ function update_range_slider_values( if (isNaN(start) || isNaN(end)) { return dash_clientside.no_update; } - [start_text_value, end_text_value] = [start, end]; + return [start, end, [start, end], [start, end]]; // slider component is the trigger } else if (trigger_id === self_data["id"]) { - [start_text_value, end_text_value] = [slider[0], slider[1]]; + return [slider[0], slider[1], slider, slider]; // on_page_load is the trigger } else { - is_on_page_load_triggered = true; - [start_text_value, end_text_value] = input_store !== null ? input_store : [slider[0], slider[1]]; - } - - start_value = Math.min(start_text_value, end_text_value); - end_value = Math.max(start_text_value, end_text_value); - start_value = Math.max(self_data["min"], start_value); - end_value = Math.min(self_data["max"], end_value); - slider_value = [start_value, end_value]; - - if (is_on_page_load_triggered && !self_data["is_dynamic_build"]) { - return [dash_clientside.no_update, dash_clientside.no_update, dash_clientside.no_update, slider_value]; + if (input_store === null) { + return [dash_clientside.no_update, dash_clientside.no_update, dash_clientside.no_update, slider]; + } + else { + if (slider[0] === start && input_store[0] === start && slider[1] === end && input_store[1] === end){ + // To prevent filter_action to be triggered after on_page_load + return [dash_clientside.no_update, dash_clientside.no_update, dash_clientside.no_update, dash_clientside.no_update]; + } + return [input_store[0], input_store[1], input_store, input_store]; + } } - - return [start_value, end_value, slider_value, slider_value]; } window.dash_clientside = { diff --git a/vizro-core/src/vizro/static/js/models/slider.js b/vizro-core/src/vizro/static/js/models/slider.js index 1cac9d8ed..2155bf83f 100644 --- a/vizro-core/src/vizro/static/js/models/slider.js +++ b/vizro-core/src/vizro/static/js/models/slider.js @@ -7,31 +7,33 @@ function update_slider_values(start, slider, input_store, self_data) { dash_clientside.callback_context.triggered[0]["prop_id"].split(".")[0]; } - // input form component is the trigger + // text form component is the trigger if (trigger_id === `${self_data["id"]}_end_value`) { if (isNaN(start)) { return dash_clientside.no_update; } - end_value = start; + return [start, start, start]; // slider component is the trigger } else if (trigger_id === self_data["id"]) { - end_value = slider; + return [slider, slider, slider]; // on_page_load is the trigger } else { - is_on_page_load_triggered = true; - end_value = input_store !== null ? input_store : self_data["min"]; - } - - end_value = Math.min(Math.max(self_data["min"], end_value), self_data["max"]); - - if (is_on_page_load_triggered && !self_data["is_dynamic_build"]) { - return [dash_clientside.no_update, dash_clientside.no_update, end_value]; + if (input_store === null) { + return [dash_clientside.no_update, dash_clientside.no_update, slider]; + } + else { + if (slider === start && start === input_store) { + // To prevent filter_action to be triggered after on_page_load + return [dash_clientside.no_update, dash_clientside.no_update, dash_clientside.no_update]; + } + return [input_store, input_store, input_store]; + } } - return [end_value, end_value, end_value]; } + window.dash_clientside = { ...window.dash_clientside, slider: { update_slider_values: update_slider_values }, From d097733a683bb999a4aa4c5ae0f521467926dc2f Mon Sep 17 00:00:00 2001 From: Antony Milne Date: Thu, 7 Nov 2024 16:44:07 +0000 Subject: [PATCH 20/64] Lint and small fixes --- ...105_170003_antony.milne_new_interaction.md | 48 +++++++++++++++++++ ...06_104745_antony.milne_dynamic_filter_2.md | 47 ++++++++++++++++++ .../src/vizro/actions/_actions_utils.py | 15 +++--- .../src/vizro/actions/_filter_action.py | 4 +- .../src/vizro/actions/_on_page_load_action.py | 4 +- .../src/vizro/actions/_parameter_action.py | 4 +- .../src/vizro/actions/export_data_action.py | 2 +- .../actions/filter_interaction_action.py | 4 +- .../src/vizro/models/_controls/filter.py | 8 ++-- 9 files changed, 115 insertions(+), 21 deletions(-) create mode 100644 vizro-core/changelog.d/20241105_170003_antony.milne_new_interaction.md create mode 100644 vizro-core/changelog.d/20241106_104745_antony.milne_dynamic_filter_2.md diff --git a/vizro-core/changelog.d/20241105_170003_antony.milne_new_interaction.md b/vizro-core/changelog.d/20241105_170003_antony.milne_new_interaction.md new file mode 100644 index 000000000..7c0d58d4f --- /dev/null +++ b/vizro-core/changelog.d/20241105_170003_antony.milne_new_interaction.md @@ -0,0 +1,48 @@ + + + + + + + + + diff --git a/vizro-core/changelog.d/20241106_104745_antony.milne_dynamic_filter_2.md b/vizro-core/changelog.d/20241106_104745_antony.milne_dynamic_filter_2.md new file mode 100644 index 000000000..6108280fd --- /dev/null +++ b/vizro-core/changelog.d/20241106_104745_antony.milne_dynamic_filter_2.md @@ -0,0 +1,47 @@ + + + + + + +### Changed + +- Improve performance of data loading. ([#850](https://github.com/mckinsey/vizro/pull/850), [#857](https://github.com/mckinsey/vizro/pull/857)) + + + + diff --git a/vizro-core/src/vizro/actions/_actions_utils.py b/vizro-core/src/vizro/actions/_actions_utils.py index 51530edbd..4a1c25b2b 100644 --- a/vizro-core/src/vizro/actions/_actions_utils.py +++ b/vizro-core/src/vizro/actions/_actions_utils.py @@ -109,9 +109,9 @@ def _validate_selector_value_none(value: Union[SingleValueType, MultiValueType]) def _filter_dot_separated_strings(dot_separated_strings: list[str], target: str, data_frame: bool) -> list[str]: result = [] - for dot_separated_string in dot_separated_strings: - if dot_separated_string.startswith(f"{target}."): - dot_separated_string = dot_separated_string.removeprefix(f"{target}.") + for dot_separated_string_with_target in dot_separated_strings: + if dot_separated_string_with_target.startswith(f"{target}."): + dot_separated_string = dot_separated_string_with_target.removeprefix(f"{target}.") if (data_frame and dot_separated_string.startswith("data_frame.")) or ( not data_frame and not dot_separated_string.startswith("data_frame.") ): @@ -138,7 +138,7 @@ def _get_parametrized_config( if data_frame: # It's not possible to address nested argument of data_frame like data_frame.x.y, just top-level ones like # data_frame.x. - config = {"data_frame": {}} + config: dict[str, Any] = {"data_frame": {}} else: # TODO - avoid calling _captured_callable. Once we have done this we can remove _arguments from # CapturedCallable entirely. This might mean not being able to address nested parameters. @@ -212,10 +212,9 @@ def _get_modified_page_figures( ctds_filter: list[CallbackTriggerDict], ctds_filter_interaction: list[dict[str, CallbackTriggerDict]], ctds_parameters: list[CallbackTriggerDict], - targets: Optional[list[ModelID]] = None, -) -> dict[str, Any]: - targets = targets or [] - outputs = {} + targets: list[ModelID], +) -> dict[ModelID, Any]: + outputs: dict[ModelID, Any] = {} # TODO: the structure here would be nicer if we could get just the ctds for a single target at one time, # so you could do apply_filters on a target a pass only the ctds relevant for that target. diff --git a/vizro-core/src/vizro/actions/_filter_action.py b/vizro-core/src/vizro/actions/_filter_action.py index 30784c708..f3ec21b37 100644 --- a/vizro-core/src/vizro/actions/_filter_action.py +++ b/vizro-core/src/vizro/actions/_filter_action.py @@ -16,7 +16,7 @@ def _filter( targets: list[ModelID], filter_function: Callable[[pd.Series, Any], pd.Series], **inputs: dict[str, Any], -) -> dict[str, Any]: +) -> dict[ModelID, Any]: """Filters targeted charts/components on page by interaction with `Filter` control. Args: @@ -30,8 +30,8 @@ def _filter( Dict mapping target component ids to modified charts/components e.g. {'my_scatter': Figure({})} """ return _get_modified_page_figures( - targets=targets, ctds_filter=ctx.args_grouping["external"]["filters"], ctds_filter_interaction=ctx.args_grouping["external"]["filter_interaction"], ctds_parameters=ctx.args_grouping["external"]["parameters"], + targets=targets, ) diff --git a/vizro-core/src/vizro/actions/_on_page_load_action.py b/vizro-core/src/vizro/actions/_on_page_load_action.py index 5b2d97cdb..306ed9b5e 100644 --- a/vizro-core/src/vizro/actions/_on_page_load_action.py +++ b/vizro-core/src/vizro/actions/_on_page_load_action.py @@ -10,7 +10,7 @@ @capture("action") -def _on_page_load(targets: list[ModelID], **inputs: dict[str, Any]) -> dict[str, Any]: +def _on_page_load(targets: list[ModelID], **inputs: dict[str, Any]) -> dict[ModelID, Any]: """Applies controls to charts on page once the page is opened (or refreshed). Args: @@ -23,8 +23,8 @@ def _on_page_load(targets: list[ModelID], **inputs: dict[str, Any]) -> dict[str, """ return _get_modified_page_figures( - targets=targets, ctds_filter=ctx.args_grouping["external"]["filters"], ctds_filter_interaction=ctx.args_grouping["external"]["filter_interaction"], ctds_parameters=ctx.args_grouping["external"]["parameters"], + targets=targets, ) diff --git a/vizro-core/src/vizro/actions/_parameter_action.py b/vizro-core/src/vizro/actions/_parameter_action.py index e96b136fb..6284481ec 100644 --- a/vizro-core/src/vizro/actions/_parameter_action.py +++ b/vizro-core/src/vizro/actions/_parameter_action.py @@ -10,7 +10,7 @@ @capture("action") -def _parameter(targets: list[str], **inputs: dict[str, Any]) -> dict[str, Any]: +def _parameter(targets: list[str], **inputs: dict[str, Any]) -> dict[ModelID, Any]: """Modifies parameters of targeted charts/components on page. Args: @@ -25,8 +25,8 @@ def _parameter(targets: list[str], **inputs: dict[str, Any]) -> dict[str, Any]: target_ids: list[ModelID] = [target.split(".")[0] for target in targets] # type: ignore[misc] return _get_modified_page_figures( - targets=target_ids, ctds_filter=ctx.args_grouping["external"]["filters"], ctds_filter_interaction=ctx.args_grouping["external"]["filter_interaction"], ctds_parameters=ctx.args_grouping["external"]["parameters"], + targets=target_ids, ) diff --git a/vizro-core/src/vizro/actions/export_data_action.py b/vizro-core/src/vizro/actions/export_data_action.py index e1620d93e..923639998 100644 --- a/vizro-core/src/vizro/actions/export_data_action.py +++ b/vizro-core/src/vizro/actions/export_data_action.py @@ -49,7 +49,7 @@ def export_data( if file_format == "csv": writer = filtered_data.to_csv elif file_format == "xlsx": - writer = filtered_data.to + writer = filtered_data.to_excel # Invalid file_format should be caught by Action validation outputs[f"download_dataframe_{target}"] = dcc.send_data_frame( diff --git a/vizro-core/src/vizro/actions/filter_interaction_action.py b/vizro-core/src/vizro/actions/filter_interaction_action.py index 1f40f3171..bc6659ab9 100644 --- a/vizro-core/src/vizro/actions/filter_interaction_action.py +++ b/vizro-core/src/vizro/actions/filter_interaction_action.py @@ -10,7 +10,7 @@ @capture("action") -def filter_interaction(targets: Optional[list[ModelID]] = None, **inputs: dict[str, Any]) -> dict[str, Any]: +def filter_interaction(targets: Optional[list[ModelID]] = None, **inputs: dict[str, Any]) -> dict[ModelID, Any]: """Filters targeted charts/components on page by clicking on data points or table cells of the source chart. To set up filtering on specific columns of the target graph(s), include these columns in the 'custom_data' @@ -29,8 +29,8 @@ def filter_interaction(targets: Optional[list[ModelID]] = None, **inputs: dict[s """ return _get_modified_page_figures( - targets=targets, ctds_filter=ctx.args_grouping["external"]["filters"], ctds_filter_interaction=ctx.args_grouping["external"]["filter_interaction"], ctds_parameters=ctx.args_grouping["external"]["parameters"], + targets=targets or [], ) diff --git a/vizro-core/src/vizro/models/_controls/filter.py b/vizro-core/src/vizro/models/_controls/filter.py index 9f91081f2..5d44fa8f5 100644 --- a/vizro-core/src/vizro/models/_controls/filter.py +++ b/vizro-core/src/vizro/models/_controls/filter.py @@ -104,12 +104,12 @@ def pre_build(self): page_id=model_manager._get_model_page_id(model_id=ModelID(str(self.id))) ) # TODO NEXT: how to handle pre_build for dynamic filters? Do we still require default argument values in - # `load` to establish selector type etc.? Can we take selector values from model_manager to supply these? Or just don't - # do validation at pre_build time and wait until state is available during build time instead? What should the - # load kwargs be here? + # `load` to establish selector type etc.? Can we take selector values from model_manager to supply these? + # Or just don't do validation at pre_build time and wait until state is availableduring build time instead? + # What should the load kwargs be here? # Note that currently _get_unfiltered_data is only suitable for use at runtime since it requires # ctd_parameters. That could be changed to just reuse that function. - multi_data_source_name_load_kwargs = [(model_manager[target]["data_frame"], {}) for target in proposed_targets] + multi_data_source_name_load_kwargs = [(model_manager[target]["data_frame"], {}) for target in proposed_targets] # type: ignore[var-annotated] target_to_data_frame = dict(zip(proposed_targets, data_manager._multi_load(multi_data_source_name_load_kwargs))) targeted_data = self._validate_targeted_data( target_to_data_frame, eagerly_raise_column_not_found_error=bool(self.targets) From d3c677e0a426ec9acbd7945ef67763017c6add96 Mon Sep 17 00:00:00 2001 From: petar-qb Date: Tue, 12 Nov 2024 13:48:46 +0100 Subject: [PATCH 21/64] dynamic filters implemented as on_page_load targets --- vizro-core/examples/scratch_dev/app.py | 38 +++++++++++-- .../src/vizro/actions/_actions_utils.py | 57 ++++++++++++++----- .../_callback_mapping_utils.py | 15 ----- .../_get_action_callback_mapping.py | 3 +- .../src/vizro/actions/_filter_action.py | 2 +- .../src/vizro/actions/_on_page_load_action.py | 33 +++++------ .../src/vizro/actions/_parameter_action.py | 2 +- .../actions/filter_interaction_action.py | 2 +- .../src/vizro/models/_controls/filter.py | 40 ++++++------- vizro-core/src/vizro/models/_page.py | 19 ++++++- 10 files changed, 130 insertions(+), 81 deletions(-) diff --git a/vizro-core/examples/scratch_dev/app.py b/vizro-core/examples/scratch_dev/app.py index cc2029e9c..e5d6ded24 100644 --- a/vizro-core/examples/scratch_dev/app.py +++ b/vizro-core/examples/scratch_dev/app.py @@ -13,8 +13,8 @@ print("INITIALIZING") -# FILTER_COLUMN = "species" -FILTER_COLUMN = "sepal_length" +FILTER_COLUMN = "species" +# FILTER_COLUMN = "sepal_length" def slow_load(sample_size=3): @@ -49,7 +49,15 @@ def load_from_file(): return final_df +def load_parametrised_species(species=None): + df = px.data.iris() + if species: + df = df[df["species"].isin(species)] + return df + + data_manager["dynamic_df"] = load_from_file +# data_manager["dynamic_df"] = load_parametrised_species # # TODO-DEV: Turn on/off caching to see how it affects the app. # data_manager.cache = Cache(config={"CACHE_TYPE": "SimpleCache"}) @@ -81,6 +89,16 @@ def load_from_file(): # figure=px.scatter( # data_frame="dynamic_df", # x="sepal_length", + # y="petal_length", + # color="species", + # color_discrete_map={"setosa": "#00b4ff", "versicolor": "#ff9222", "virginica": "#3949ab"}, + # ), + # ), + # vm.Graph( + # id="graph_2_id", + # figure=px.scatter( + # data_frame="dynamic_df", + # x="sepal_length", # y="petal_width", # color="species", # ) @@ -91,11 +109,11 @@ def load_from_file(): vm.Filter( id="filter_container_id", column=FILTER_COLUMN, - + # targets=["graph_2_id"], # selector=vm.Dropdown(id="filter_id"), # selector=vm.Dropdown(id="filter_id", value=["setosa"]), - # selector=vm.Checklist(id="filter_id"), + selector=vm.Checklist(id="filter_id"), # selector=vm.Checklist(id="filter_id", value=["setosa"]), # selector=vm.Dropdown(id="filter_id", multi=False), @@ -104,7 +122,7 @@ def load_from_file(): # selector=vm.RadioItems(id="filter_id"), # selector=vm.RadioItems(id="filter_id", value="setosa"), - selector=vm.Slider(id="filter_id"), + # selector=vm.Slider(id="filter_id"), # selector=vm.Slider(id="filter_id", value=6), # selector=vm.RangeSlider(id="filter_id"), @@ -175,7 +193,15 @@ def load_from_file(): value="species", multi=False, ) - ) + ), + # vm.Parameter( + # id="parameter_species", + # targets=[ + # "graph_1_id.data_frame.species", + # "filter_container_id.", + # ], + # selector=vm.Dropdown(options=["setosa", "versicolor", "virginica"]) + # ), # vm.Parameter( # id="parameter_container_id", # targets=[ diff --git a/vizro-core/src/vizro/actions/_actions_utils.py b/vizro-core/src/vizro/actions/_actions_utils.py index 4a1c25b2b..a91f40480 100644 --- a/vizro-core/src/vizro/actions/_actions_utils.py +++ b/vizro-core/src/vizro/actions/_actions_utils.py @@ -13,7 +13,7 @@ from vizro.models.types import MultiValueType, SelectorType, SingleValueType if TYPE_CHECKING: - from vizro.models import Action, VizroBaseModel + from vizro.models import Action, VizroBaseModel, Filter ValidatedNoneValueType = Union[SingleValueType, MultiValueType, None, list[None]] @@ -191,7 +191,7 @@ def _apply_filters( def _get_unfiltered_data( - ctds_parameters: list[CallbackTriggerDict], targets: list[ModelID] + ctds_parameter: list[CallbackTriggerDict], targets: list[ModelID] ) -> dict[ModelID, pd.DataFrame]: # Takes in multiple targets to ensure that data can be loaded efficiently using _multi_load and not repeated for # every single target. @@ -200,8 +200,11 @@ def _get_unfiltered_data( multi_data_source_name_load_kwargs = [] for target in targets: dynamic_data_load_params = _get_parametrized_config( - ctd_parameters=ctds_parameters, target=target, data_frame=True + ctd_parameters=ctds_parameter, target=target, data_frame=True ) + # This works for the figure objects but not for the Filter objects. Ideally, we should or enable multiple + # data_frame-s from figure objects or limit Filter to use a single data_frame object. Filter with a single + # data_frame object sounds like a better idea (although it's a breaking change). data_source_name = model_manager[target]["data_frame"] multi_data_source_name_load_kwargs.append((data_source_name, dynamic_data_load_params["data_frame"])) @@ -211,7 +214,7 @@ def _get_unfiltered_data( def _get_modified_page_figures( ctds_filter: list[CallbackTriggerDict], ctds_filter_interaction: list[dict[str, CallbackTriggerDict]], - ctds_parameters: list[CallbackTriggerDict], + ctds_parameter: list[CallbackTriggerDict], targets: list[ModelID], ) -> dict[ModelID, Any]: outputs: dict[ModelID, Any] = {} @@ -220,16 +223,44 @@ def _get_modified_page_figures( # so you could do apply_filters on a target a pass only the ctds relevant for that target. # Consider restructuring ctds to a more convenient form to make this possible. - for target, unfiltered_data in _get_unfiltered_data(ctds_parameters, targets).items(): - filtered_data = _apply_filters(unfiltered_data, ctds_filter, ctds_filter_interaction, target) + from vizro.models import Filter + + control_targets = [] + control_targets_targets = [] + figure_targets = [] + + for target in targets: + target_obj = model_manager[target] + if isinstance(target_obj, Filter): + control_targets.append(target) + control_targets_targets.extend(target_obj.targets) + else: + figure_targets.append(target) + + # Retrieving only figure_targets data_frames from multi_load is not the best solution. We assume that Filter.targets + # are the subset of the action's targets. This works for the on_page_load, but will not work if explicitly set. + # Also, it was a good decision to return action output as key: value pairs for the predefined actions. + _get_unfiltered_data_targets = list(set(figure_targets + control_targets_targets)) + + figure_targets_unfiltered_data: dict[ModelID, pd.DataFrame] = _get_unfiltered_data(ctds_parameter, _get_unfiltered_data_targets) + + for target, unfiltered_data in figure_targets_unfiltered_data.items(): + if target in figure_targets: + filtered_data = _apply_filters(unfiltered_data, ctds_filter, ctds_filter_interaction, target) + outputs[target] = model_manager[target]( + data_frame=filtered_data, + **_get_parametrized_config(ctd_parameters=ctds_parameter, target=target, data_frame=False), + ) + + for target in control_targets: + current_value = [item for item in ctds_filter if item["id"] == model_manager[target].selector.id] + current_value = current_value if not current_value else current_value[0]["value"] + if "ALL" in current_value or ["ALL"] in current_value: + current_value = [] + outputs[target] = model_manager[target]( - data_frame=filtered_data, - **_get_parametrized_config(ctd_parameters=ctds_parameters, target=target, data_frame=False), + target_to_data_frame=figure_targets_unfiltered_data, + current_value=current_value ) - # TODO NEXT: will need to pass unfiltered_data into Filter.__call__. - # This dictionary is filtered for correct targets already selected in Filter.__call__ or that could be done here - # instead. - # {target: data_frame for target, data_frame in unfiltered_data.items() if target in self.targets} - return outputs diff --git a/vizro-core/src/vizro/actions/_callback_mapping/_callback_mapping_utils.py b/vizro-core/src/vizro/actions/_callback_mapping/_callback_mapping_utils.py index 2454718f8..4bf82991c 100644 --- a/vizro-core/src/vizro/actions/_callback_mapping/_callback_mapping_utils.py +++ b/vizro-core/src/vizro/actions/_callback_mapping/_callback_mapping_utils.py @@ -126,21 +126,6 @@ def _get_export_data_callback_outputs(action_id: ModelID) -> dict[str, Output]: } -def _get_on_page_load_callback_outputs(action_id: ModelID) -> dict[str, Output]: - """Creates mapping of target names and their `Output`.""" - component_targets = _get_action_callback_outputs(action_id=action_id) - - # TODO-WIP: Add only dynamic filters from the current page - import vizro.models as vm - for filter_id, filter_obj in model_manager._items_with_type(vm.Filter): - if filter_obj._dynamic: - component_targets[filter_id] = Output( - component_id=filter_id, - component_property="children", - ) - - return component_targets - # CALLBACK COMPONENTS -------------- def _get_export_data_callback_components(action_id: ModelID) -> list[dcc.Download]: """Creates dcc.Downloads for target components of the `export_data` action.""" diff --git a/vizro-core/src/vizro/actions/_callback_mapping/_get_action_callback_mapping.py b/vizro-core/src/vizro/actions/_callback_mapping/_get_action_callback_mapping.py index 69ac4295d..10841e18b 100644 --- a/vizro-core/src/vizro/actions/_callback_mapping/_get_action_callback_mapping.py +++ b/vizro-core/src/vizro/actions/_callback_mapping/_get_action_callback_mapping.py @@ -11,7 +11,6 @@ _get_action_callback_outputs, _get_export_data_callback_components, _get_export_data_callback_outputs, - _get_on_page_load_callback_outputs, ) from vizro.actions._filter_action import _filter from vizro.actions._on_page_load_action import _on_page_load @@ -46,7 +45,7 @@ def _get_action_callback_mapping( }, _on_page_load.__wrapped__: { "inputs": _get_action_callback_inputs, - "outputs": _get_on_page_load_callback_outputs, + "outputs": _get_action_callback_outputs, }, } action_call = action_callback_mapping.get(action_function, {}).get(argument) diff --git a/vizro-core/src/vizro/actions/_filter_action.py b/vizro-core/src/vizro/actions/_filter_action.py index 1c490b080..4e97faf68 100644 --- a/vizro-core/src/vizro/actions/_filter_action.py +++ b/vizro-core/src/vizro/actions/_filter_action.py @@ -33,6 +33,6 @@ def _filter( return _get_modified_page_figures( ctds_filter=ctx.args_grouping["external"]["filters"], ctds_filter_interaction=ctx.args_grouping["external"]["filter_interaction"], - ctds_parameters=ctx.args_grouping["external"]["parameters"], + ctds_parameter=ctx.args_grouping["external"]["parameters"], targets=targets, ) diff --git a/vizro-core/src/vizro/actions/_on_page_load_action.py b/vizro-core/src/vizro/actions/_on_page_load_action.py index ed8dc7628..dbd06b016 100644 --- a/vizro-core/src/vizro/actions/_on_page_load_action.py +++ b/vizro-core/src/vizro/actions/_on_page_load_action.py @@ -28,26 +28,24 @@ def _on_page_load(targets: list[ModelID], **inputs: dict[str, Any]) -> dict[Mode return_obj = _get_modified_page_figures( ctds_filter=ctx.args_grouping["external"]["filters"], ctds_filter_interaction=ctx.args_grouping["external"]["filter_interaction"], - ctds_parameters=ctx.args_grouping["external"]["parameters"], + ctds_parameter=ctx.args_grouping["external"]["parameters"], targets=targets, ) - import vizro.models as vm - from time import sleep - sleep(1) + # import vizro.models as vm + # from time import sleep + # sleep(1) + # + # for filter_id, filter_obj in model_manager._items_with_type(vm.Filter): + # if filter_obj._dynamic: + # current_value = [ + # item for item in ctx.args_grouping["external"]["filters"] + # if item["id"] == filter_obj.selector.id + # ][0]["value"] + # + # if current_value in ["ALL", ["ALL"]]: + # current_value = [] - # TODO-WIP: Add filters to OPL targets. Return only dynamic filter from the targets and not from the entire app. - for filter_id, filter_obj in model_manager._items_with_type(vm.Filter): - if filter_obj._dynamic: - current_value = [ - item for item in ctx.args_grouping["external"]["filters"] - if item["id"] == filter_obj.selector.id - ][0]["value"] - - if current_value in ["ALL", ["ALL"]]: - current_value = [] - - # TODO-CONSIDER: Does calculating options/min/max significantly slow down the app? # TODO: Also propagate DFP values into the load() method # 1. "new_options"/"min/max" DOES NOT include the "current_value" # filter_obj._set_categorical_selectors_options(force=True, current_value=[]) @@ -58,8 +56,7 @@ def _on_page_load(targets: list[ModelID], **inputs: dict[str, Any]) -> dict[Mode # return_obj[filter_id] = filter_obj.selector(on_page_load_value=current_value) - - return_obj[filter_id] = filter_obj(current_value=current_value) + # return_obj[filter_id] = filter_obj(current_value=current_value) print("ON PAGE LOAD - END\n") diff --git a/vizro-core/src/vizro/actions/_parameter_action.py b/vizro-core/src/vizro/actions/_parameter_action.py index 6284481ec..bfc58014f 100644 --- a/vizro-core/src/vizro/actions/_parameter_action.py +++ b/vizro-core/src/vizro/actions/_parameter_action.py @@ -27,6 +27,6 @@ def _parameter(targets: list[str], **inputs: dict[str, Any]) -> dict[ModelID, An return _get_modified_page_figures( ctds_filter=ctx.args_grouping["external"]["filters"], ctds_filter_interaction=ctx.args_grouping["external"]["filter_interaction"], - ctds_parameters=ctx.args_grouping["external"]["parameters"], + ctds_parameter=ctx.args_grouping["external"]["parameters"], targets=target_ids, ) diff --git a/vizro-core/src/vizro/actions/filter_interaction_action.py b/vizro-core/src/vizro/actions/filter_interaction_action.py index bc6659ab9..9618d265f 100644 --- a/vizro-core/src/vizro/actions/filter_interaction_action.py +++ b/vizro-core/src/vizro/actions/filter_interaction_action.py @@ -31,6 +31,6 @@ def filter_interaction(targets: Optional[list[ModelID]] = None, **inputs: dict[s return _get_modified_page_figures( ctds_filter=ctx.args_grouping["external"]["filters"], ctds_filter_interaction=ctx.args_grouping["external"]["filter_interaction"], - ctds_parameters=ctx.args_grouping["external"]["parameters"], + ctds_parameter=ctx.args_grouping["external"]["parameters"], targets=targets or [], ) diff --git a/vizro-core/src/vizro/models/_controls/filter.py b/vizro-core/src/vizro/models/_controls/filter.py index 9d0767bd1..b960eea24 100644 --- a/vizro-core/src/vizro/models/_controls/filter.py +++ b/vizro-core/src/vizro/models/_controls/filter.py @@ -96,6 +96,10 @@ class Filter(VizroBaseModel): selector: SelectorType = None _dynamic: bool = PrivateAttr(None) + _pre_build_finished: bool = PrivateAttr(False) + + # Component properties for actions and interactions + _output_component_property: str = PrivateAttr("children") _column_type: Literal["numerical", "categorical", "temporal"] = PrivateAttr() @@ -105,7 +109,7 @@ def check_target_present(cls, target): raise ValueError(f"Target {target} not found in model_manager.") return target - def __call__(self, current_value, **kwargs): + def __call__(self, target_to_data_frame: dict[ModelID, pd.DataFrame], current_value: Any, **kwargs): # Only relevant for a dynamic filter. # Although targets are fixed at build time, the validation logic is repeated during runtime, so if a column # is missing then it will raise an error. We could change this if we wanted. @@ -121,7 +125,6 @@ def __call__(self, current_value, **kwargs): "change type while the dashboard is running." ) - # TODO: when implement dynamic, will need to do something with this e.g. pass to selector.__call__. if isinstance(self.selector, SELECTORS["categorical"]): # Categorical selector. new_options = self._get_options(targeted_data, current_value) @@ -133,6 +136,9 @@ def __call__(self, current_value, **kwargs): @_log_call def pre_build(self): + if self._pre_build_finished: + return + self._pre_build_finished = True # If targets aren't explicitly provided then try to target all figures on the page. In this case we don't # want to raise an error if the column is not found in a figure's data_frame, it will just be ignored. # This is the case when bool(self.targets) is False. @@ -142,7 +148,7 @@ def pre_build(self): ) # TODO NEXT: how to handle pre_build for dynamic filters? Do we still require default argument values in # `load` to establish selector type etc.? Can we take selector values from model_manager to supply these? - # Or just don't do validation at pre_build time and wait until state is availableduring build time instead? + # Or just don't do validation at pre_build time and wait until state is available during build time instead? # What should the load kwargs be here? # Note that currently _get_unfiltered_data is only suitable for use at runtime since it requires # ctd_parameters. That could be changed to just reuse that function. @@ -216,29 +222,17 @@ def build(self): return dcc.Loading(id=self.id, children=selector_build_obj) if self._dynamic else selector_build_obj def _validate_targeted_data( - self, targets: list[ModelID], eagerly_raise_column_not_found_error=True + self, target_to_data_frame: dict[ModelID, pd.DataFrame], eagerly_raise_column_not_found_error=True ) -> pd.DataFrame: - # TODO: consider moving some of this logic to data_manager when implement dynamic filter. Make sure - # get_modified_figures and stuff in _actions_utils.py is as efficient as code here. - - # When loading data_frame there are possible keys: - # 1. target. In worst case scenario this is needed but can lead to unnecessary repeated data loading. - # 2. data_source_name. No repeated data loading but won't work when applying data_frame parameters at runtime. - # 3. target + data_frame parameters keyword-argument pairs. This is the correct key to use at runtime. - # For now we follow scheme 2 for data loading (due to set() below) and 1 for the returned targeted_data - # pd.DataFrame, i.e. a separate column for each target even if some data is repeated. - # TODO: when this works with data_frame parameters load() will need to take arguments and the structures here - # might change a bit. - target_to_data_source_name = {target: model_manager[target]["data_frame"] for target in targets} - data_source_name_to_data = { - data_source_name: data_manager[data_source_name].load() - for data_source_name in set(target_to_data_source_name.values()) - } - target_to_series = {} + # target_to_data_source_name = {target: model_manager[target]["data_frame"] for target in targets} + # data_source_name_to_data = { + # data_source_name: data_manager[data_source_name].load() + # for data_source_name in set(target_to_data_source_name.values()) + # } - for target, data_source_name in target_to_data_source_name.items(): - data_frame = data_source_name_to_data[data_source_name] + target_to_series = {} + for target, data_frame in target_to_data_frame.items(): if self.column in data_frame.columns: # reset_index so that when we make a DataFrame out of all these pd.Series pandas doesn't try to align # the columns by index. diff --git a/vizro-core/src/vizro/models/_page.py b/vizro-core/src/vizro/models/_page.py index 6b45a409c..c805b6ea5 100644 --- a/vizro-core/src/vizro/models/_page.py +++ b/vizro-core/src/vizro/models/_page.py @@ -14,7 +14,7 @@ from vizro.actions import _on_page_load from vizro.managers import model_manager from vizro.managers._model_manager import DuplicateIDError, ModelID -from vizro.models import Action, Layout, VizroBaseModel +from vizro.models import Action, Filter, Layout, VizroBaseModel from vizro.models._action._actions_chain import ActionsChain, Trigger from vizro.models._layout import set_layout from vizro.models._models_utils import _log_call, check_captured_callable, validate_min_length @@ -98,6 +98,23 @@ def __vizro_exclude_fields__(self) -> Optional[Union[set[str], Mapping[str, Any] def pre_build(self): # TODO: Remove default on page load action if possible targets = model_manager._get_page_model_ids_with_figure(page_id=ModelID(str(self.id))) + + # TODO: In the Vizro._pre_build(), the loop passes through the components in the random order, but the + # "pre_build" of one component can depend on another component's "pre_build". + # Does the "postorder DFS" traversal pass work for us? -> More about it: + # https://www.geeksforgeeks.org/tree-traversals-inorder-preorder-and-postorder/#postorder-traversal + # By introducing this traversal algorithm we should ensure that child pre_build will always be called before + # parent pre_build. Is this the case that solves all the pre_build interdependency future problems as well? + + # The hack below is just to ensure that the pre_build of the Filter components is called before the page + # pre_build calculates targets. + for control in self.controls: + if isinstance(control, Filter): + if not control._pre_build_finished: + control.pre_build() + if control._dynamic: + targets.append(control.id) + if targets: self.actions = [ ActionsChain( From 1d602ea3890e216032d379926135c899e3258563 Mon Sep 17 00:00:00 2001 From: petar-qb Date: Wed, 13 Nov 2024 16:32:57 +0100 Subject: [PATCH 22/64] =?UTF-8?q?Propagating=20data=5Fframe=20parameter=20?= =?UTF-8?q?values=20as=20load=E2=80=93kwargs=20from=20the=20model=5Fmanage?= =?UTF-8?q?r=20to=20the=20Filter.pre=5Fbuild=20data=5Fmanager=20=5Fmulti?= =?UTF-8?q?=5Fload?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- vizro-core/examples/scratch_dev/app.py | 375 +++++++++++------- .../src/vizro/actions/_actions_utils.py | 2 +- .../src/vizro/models/_controls/filter.py | 35 +- 3 files changed, 263 insertions(+), 149 deletions(-) diff --git a/vizro-core/examples/scratch_dev/app.py b/vizro-core/examples/scratch_dev/app.py index e5d6ded24..e52310aa7 100644 --- a/vizro-core/examples/scratch_dev/app.py +++ b/vizro-core/examples/scratch_dev/app.py @@ -10,9 +10,12 @@ import vizro.plotly.express as px from vizro import Vizro from vizro.managers import data_manager +from functools import partial print("INITIALIZING") +SPECIES_COLORS = {"setosa": "#00b4ff", "versicolor": "#ff9222", "virginica": "#3949ab"} + FILTER_COLUMN = "species" # FILTER_COLUMN = "sepal_length" @@ -24,7 +27,9 @@ def slow_load(sample_size=3): return df -def load_from_file(): +def load_from_file(filter_column=None): + filter_column = filter_column or FILTER_COLUMN + with open('data.yaml', 'r') as file: data = yaml.safe_load(file) data = data or pd.DataFrame() @@ -33,15 +38,15 @@ def load_from_file(): df = px.data.iris() # Load the first N rows of each species. N per species is defined in the data.yaml file. - if FILTER_COLUMN == "species": + if filter_column == "species": final_df = pd.concat(objs=[ - df[df[FILTER_COLUMN] == 'setosa'].head(data.get("setosa", 0)), - df[df[FILTER_COLUMN] == 'versicolor'].head(data.get("versicolor", 0)), - df[df[FILTER_COLUMN] == 'virginica'].head(data.get("virginica", 0)), + df[df[filter_column] == 'setosa'].head(data.get("setosa", 0)), + df[df[filter_column] == 'versicolor'].head(data.get("versicolor", 0)), + df[df[filter_column] == 'virginica'].head(data.get("virginica", 0)), ], ignore_index=True) - elif FILTER_COLUMN == "sepal_length": + elif filter_column == "sepal_length": final_df = df[ - df[FILTER_COLUMN].between(data.get("min", 0), data.get("max", 0), inclusive="both") + df[filter_column].between(data.get("min", 0), data.get("max", 0), inclusive="both") ] else: raise ValueError("Invalid FILTER_COLUMN") @@ -56,13 +61,16 @@ def load_parametrised_species(species=None): return df -data_manager["dynamic_df"] = load_from_file -# data_manager["dynamic_df"] = load_parametrised_species +data_manager["load_from_file"] = load_from_file +data_manager["load_from_file_species"] = partial(load_from_file, filter_column="species") +data_manager["load_from_file_sepal_length"] = partial(load_from_file, filter_column="sepal_length") + +# data_manager["load_parametrised_species"] = load_parametrised_species # # TODO-DEV: Turn on/off caching to see how it affects the app. # data_manager.cache = Cache(config={"CACHE_TYPE": "SimpleCache"}) -# data_manager["dynamic_df"] = slow_load -# data_manager["dynamic_df"].timeout = 5 +# data_manager["load_from_file"] = slow_load +# data_manager["load_from_file"].timeout = 5 homepage = vm.Page( @@ -72,154 +80,227 @@ def load_parametrised_species(species=None): ], ) -another_page = vm.Page( - title="Test update control options", +page_1 = vm.Page( + title="Dynamic vs Static filter", components=[ vm.Graph( - id="graph_1_id", + id="p1-G-1", figure=px.bar( - data_frame="dynamic_df", - x="species", - color="species", - color_discrete_map={"setosa": "#00b4ff", "versicolor": "#ff9222", "virginica": "#3949ab"}, - ) + data_frame="load_from_file_species", + x="species", color="species", color_discrete_map=SPECIES_COLORS, + ), ), - # vm.Graph( - # id="graph_2_id", - # figure=px.scatter( - # data_frame="dynamic_df", - # x="sepal_length", - # y="petal_length", - # color="species", - # color_discrete_map={"setosa": "#00b4ff", "versicolor": "#ff9222", "virginica": "#3949ab"}, - # ), - # ), - # vm.Graph( - # id="graph_2_id", - # figure=px.scatter( - # data_frame="dynamic_df", - # x="sepal_length", - # y="petal_width", - # color="species", - # ) - # ), - + vm.Graph( + id="p1-G-2", + figure=px.scatter( + data_frame=px.data.iris(), + x="sepal_length", y="petal_length", color="species", color_discrete_map=SPECIES_COLORS, + ), + ) ], controls=[ - vm.Filter( - id="filter_container_id", - column=FILTER_COLUMN, - # targets=["graph_2_id"], - # selector=vm.Dropdown(id="filter_id"), - # selector=vm.Dropdown(id="filter_id", value=["setosa"]), - - selector=vm.Checklist(id="filter_id"), - # selector=vm.Checklist(id="filter_id", value=["setosa"]), - - # selector=vm.Dropdown(id="filter_id", multi=False), - # selector=vm.Dropdown(id="filter_id", multi=False, value="setosa"), - - # selector=vm.RadioItems(id="filter_id"), - # selector=vm.RadioItems(id="filter_id", value="setosa"), - - # selector=vm.Slider(id="filter_id"), - # selector=vm.Slider(id="filter_id", value=6), - - # selector=vm.RangeSlider(id="filter_id"), - # selector=vm.RangeSlider(id="filter_id", value=[5, 7]), - - # TEST CASES: - # no selector - # WORKS - # multi=True - # default - # WORKS - # selector=vm.Slider(id="filter_id", step=0.5), - # options: list - # WORKS - but options doesn't mean anything because options are dynamically overwritten. - # selector=vm.Dropdown(options=["setosa", "versicolor"]), - # options: empty list - # WORKS - # selector=vm.Dropdown(options=[]), - # options: dict - # WORKS - but "label" is always overwritten. - # selector=vm.Dropdown(options=[{"label": "Setosa", "value": "setosa"}, {"label": "Versicolor", "value": "versicolor"}]), - # options list; value - # WORKS - # selector=vm.Dropdown(options=["setosa", "versicolor"], value=["setosa"]), - # options list; empty value - # WORKS - # selector=vm.Dropdown(options=["setosa", "versicolor"], value=[]), - # strange options - # WORKS - # selector=vm.Dropdown(options=["A", "B", "C"]), - # strange options with strange value - # WORKS - works even for the dynamic False, and this is OK. - # selector=vm.Dropdown(options=["A", "B", "C"], value=["XYZ"]), - # - # - # multi=False -> TLDR: Doesn't work if value is cleared. Other combinations are same as multi=True. - # default - # DOES NOT WORK - Doesn't work if value is cleared. Then persistence storage become "null" and our - # placeholder component dmc.DateRangePicker can't process null value. It expects a value or a list - # of values. SOLUTION -> Create the "universal Vizro placeholder component" - # selector=vm.Dropdown(multi=False), - # options: list - because options are dynamically overwritten. - # WORKS - but options doesn't mean anything because options are dynamically overwritten. - # selector=vm.Dropdown(multi=False, options=["setosa", "versicolor"]), - # options: empty list - # WORKS - # selector=vm.Dropdown(multi=False, options=[]), - # options: dict - # selector=vm.Dropdown(multi=False, options=[{"label": "Setosa", "value": "setosa"}, {"label": "Versicolor", "value": "versicolor"}]), - # options list; value - # WORKS - # selector=vm.Dropdown(multi=False, options=["setosa", "versicolor"], value="setosa"), - # options list; None value - # WORKS - # selector=vm.Dropdown(multi=False, options=["setosa", "versicolor"], value=None), - # strange options - # WORKS - # selector=vm.Dropdown(multi=False, options=["A", "B", "C"]), - # strange options with strange value - # WORKS - # selector=vm.Dropdown(multi=False, options=["A", "B", "C"], value="XYZ"), + vm.Filter(id="p1-F-1", column="species", targets=["p1-G-1"], selector=vm.Dropdown(title="Dynamic filter")), + vm.Filter(id="p1-F-2", column="species", targets=["p1-G-2"], selector=vm.Dropdown(title="Static filter")), + vm.Parameter( + targets=["p1-G-1.x", "p1-G-2.x"], + selector=vm.RadioItems(options=["species", "sepal_width"], value="species", title="Simple X-axis parameter") + ) + ] +) + + +page_2 = vm.Page( + title="Categorical dynamic selectors", + components=[ + vm.Graph( + id="p2-G-1", + figure=px.bar( + data_frame="load_from_file_species", + x="species", color="species", color_discrete_map=SPECIES_COLORS, + ), ), + ], + controls=[ + vm.Filter(id="p2-F-1", column="species", selector=vm.Dropdown()), + vm.Filter(id="p2-F-2", column="species", selector=vm.Dropdown(multi=False)), + vm.Filter(id="p2-F-3", column="species", selector=vm.Checklist()), + vm.Filter(id="p2-F-4", column="species", selector=vm.RadioItems()), vm.Parameter( - id="parameter_x", - targets=["graph_1_id.x",], - selector=vm.Dropdown( - options=["species", "sepal_width"], - value="species", - multi=False, - ) + targets=["p2-G-1.x"], + selector=vm.RadioItems(options=["species", "sepal_width"], value="species", title="Simple X-axis parameter") + ) + ] +) + + +page_3 = vm.Page( + title="Numerical dynamic selectors", + components=[ + vm.Graph( + id="p3-G-1", + figure=px.bar( + data_frame="load_from_file_sepal_length", + x="species", color="species", color_discrete_map=SPECIES_COLORS, + ), ), - # vm.Parameter( - # id="parameter_species", - # targets=[ - # "graph_1_id.data_frame.species", - # "filter_container_id.", - # ], - # selector=vm.Dropdown(options=["setosa", "versicolor", "virginica"]) - # ), - # vm.Parameter( - # id="parameter_container_id", - # targets=[ - # "graph_1_id.data_frame.sample_size", - # # "graph_2_id.data_frame.sample_size", - # ], - # selector=vm.Slider( - # id="parameter_id", - # min=1, - # max=10, - # step=1, - # value=10, - # ) - # ), ], + controls=[ + vm.Filter(id="p3-F-1", column="sepal_length", selector=vm.Slider()), + vm.Filter(id="p3-F-2", column="sepal_length", selector=vm.RangeSlider()), + vm.Parameter( + targets=["p3-G-1.x"], + selector=vm.RadioItems(options=["species", "sepal_width"], value="species", title="Simple X-axis parameter") + ) + ] ) +# another_page = vm.Page( +# title="Test update control options", +# components=[ +# vm.Graph( +# id="graph_1_id", +# figure=px.bar( +# data_frame="dynamic_df", +# x="species", +# color="species", +# color_discrete_map={"setosa": "#00b4ff", "versicolor": "#ff9222", "virginica": "#3949ab"}, +# ) +# ), +# # vm.Graph( +# # id="graph_2_id", +# # figure=px.scatter( +# # data_frame="dynamic_df", +# # x="sepal_length", +# # y="petal_length", +# # color="species", +# # color_discrete_map={"setosa": "#00b4ff", "versicolor": "#ff9222", "virginica": "#3949ab"}, +# # ), +# # ), +# # vm.Graph( +# # id="graph_2_id", +# # figure=px.scatter( +# # data_frame="dynamic_df", +# # x="sepal_length", +# # y="petal_width", +# # color="species", +# # ) +# # ), +# +# ], +# controls=[ +# vm.Filter( +# id="filter_container_id", +# column=FILTER_COLUMN, +# # targets=["graph_2_id"], +# # selector=vm.Dropdown(id="filter_id"), +# # selector=vm.Dropdown(id="filter_id", value=["setosa"]), +# +# selector=vm.Checklist(id="filter_id"), +# # selector=vm.Checklist(id="filter_id", value=["setosa"]), +# +# # selector=vm.Dropdown(id="filter_id", multi=False), +# # selector=vm.Dropdown(id="filter_id", multi=False, value="setosa"), +# +# # selector=vm.RadioItems(id="filter_id"), +# # selector=vm.RadioItems(id="filter_id", value="setosa"), +# +# # selector=vm.Slider(id="filter_id"), +# # selector=vm.Slider(id="filter_id", value=6), +# +# # selector=vm.RangeSlider(id="filter_id"), +# # selector=vm.RangeSlider(id="filter_id", value=[5, 7]), +# +# # TEST CASES: +# # no selector +# # WORKS +# # multi=True +# # default +# # WORKS +# # selector=vm.Slider(id="filter_id", step=0.5), +# # options: list +# # WORKS - but options doesn't mean anything because options are dynamically overwritten. +# # selector=vm.Dropdown(options=["setosa", "versicolor"]), +# # options: empty list +# # WORKS +# # selector=vm.Dropdown(options=[]), +# # options: dict +# # WORKS - but "label" is always overwritten. +# # selector=vm.Dropdown(options=[{"label": "Setosa", "value": "setosa"}, {"label": "Versicolor", "value": "versicolor"}]), +# # options list; value +# # WORKS +# # selector=vm.Dropdown(options=["setosa", "versicolor"], value=["setosa"]), +# # options list; empty value +# # WORKS +# # selector=vm.Dropdown(options=["setosa", "versicolor"], value=[]), +# # strange options +# # WORKS +# # selector=vm.Dropdown(options=["A", "B", "C"]), +# # strange options with strange value +# # WORKS - works even for the dynamic False, and this is OK. +# # selector=vm.Dropdown(options=["A", "B", "C"], value=["XYZ"]), +# # +# # +# # multi=False -> TLDR: Doesn't work if value is cleared. Other combinations are same as multi=True. +# # default +# # DOES NOT WORK - Doesn't work if value is cleared. Then persistence storage become "null" and our +# # placeholder component dmc.DateRangePicker can't process null value. It expects a value or a list +# # of values. SOLUTION -> Create the "universal Vizro placeholder component" +# # selector=vm.Dropdown(multi=False), +# # options: list - because options are dynamically overwritten. +# # WORKS - but options doesn't mean anything because options are dynamically overwritten. +# # selector=vm.Dropdown(multi=False, options=["setosa", "versicolor"]), +# # options: empty list +# # WORKS +# # selector=vm.Dropdown(multi=False, options=[]), +# # options: dict +# # selector=vm.Dropdown(multi=False, options=[{"label": "Setosa", "value": "setosa"}, {"label": "Versicolor", "value": "versicolor"}]), +# # options list; value +# # WORKS +# # selector=vm.Dropdown(multi=False, options=["setosa", "versicolor"], value="setosa"), +# # options list; None value +# # WORKS +# # selector=vm.Dropdown(multi=False, options=["setosa", "versicolor"], value=None), +# # strange options +# # WORKS +# # selector=vm.Dropdown(multi=False, options=["A", "B", "C"]), +# # strange options with strange value +# # WORKS +# # selector=vm.Dropdown(multi=False, options=["A", "B", "C"], value="XYZ"), +# ), +# vm.Parameter( +# id="parameter_x", +# targets=["graph_1_id.x",], +# selector=vm.Dropdown( +# options=["species", "sepal_width"], +# value="species", +# multi=False, +# ) +# ), +# # vm.Parameter( +# # id="parameter_species", +# # targets=[ +# # "graph_1_id.data_frame.species", +# # "filter_container_id.", +# # ], +# # selector=vm.Dropdown(options=["setosa", "versicolor", "virginica"]) +# # ), +# # vm.Parameter( +# # id="parameter_container_id", +# # targets=[ +# # "graph_1_id.data_frame.sample_size", +# # # "graph_2_id.data_frame.sample_size", +# # ], +# # selector=vm.Slider( +# # id="parameter_id", +# # min=1, +# # max=10, +# # step=1, +# # value=10, +# # ) +# # ), +# ], +# ) -dashboard = vm.Dashboard(pages=[homepage, another_page]) +dashboard = vm.Dashboard(pages=[homepage, page_1, page_2, page_3]) if __name__ == "__main__": app = Vizro().build(dashboard) diff --git a/vizro-core/src/vizro/actions/_actions_utils.py b/vizro-core/src/vizro/actions/_actions_utils.py index a91f40480..d50312c23 100644 --- a/vizro-core/src/vizro/actions/_actions_utils.py +++ b/vizro-core/src/vizro/actions/_actions_utils.py @@ -255,7 +255,7 @@ def _get_modified_page_figures( for target in control_targets: current_value = [item for item in ctds_filter if item["id"] == model_manager[target].selector.id] current_value = current_value if not current_value else current_value[0]["value"] - if "ALL" in current_value or ["ALL"] in current_value: + if hasattr(current_value, "__iter__") and ALL_OPTION in current_value: current_value = [] outputs[target] = model_manager[target]( diff --git a/vizro-core/src/vizro/models/_controls/filter.py b/vizro-core/src/vizro/models/_controls/filter.py index b960eea24..1fc357739 100644 --- a/vizro-core/src/vizro/models/_controls/filter.py +++ b/vizro-core/src/vizro/models/_controls/filter.py @@ -152,7 +152,39 @@ def pre_build(self): # What should the load kwargs be here? # Note that currently _get_unfiltered_data is only suitable for use at runtime since it requires # ctd_parameters. That could be changed to just reuse that function. - multi_data_source_name_load_kwargs = [(model_manager[target]["data_frame"], {}) for target in proposed_targets] # type: ignore[var-annotated] + from vizro.models._controls import Parameter + + load_kwargs = {} + page_obj = model_manager[model_manager._get_model_page_id(model_id=ModelID(str(self.id)))] + for target in proposed_targets: + data_source_name = model_manager[target]["data_frame"] + load_kwargs[data_source_name] = {} + + for page_parameter in page_obj.controls: + if isinstance(page_parameter, Parameter): + for parameter_targets in page_parameter.targets: + if parameter_targets.startswith(f'{target}.data_frame'): + argument = parameter_targets.split('.')[2] + # argument is explicitly defined + if parameter_value := getattr(page_parameter.selector, value, None): + load_kwargs[data_source_name].append((argument, parameter_value)) + # find default value + else: + parameter_selector = page_parameter.selector + if parameter_selector == Dropdown: + default_parameter_value = get_options_and_default(parameter_selector.options, parameter_selector.multi) + elif parameter_selector == Checklist: + default_parameter_value = get_options_and_default(parameter_selector.options, True) + elif parameter_selector == RadioItems: + default_parameter_value = get_options_and_default(parameter_selector.options, False) + elif parameter_selector == Slider: + default_parameter_value = parameter_selector.min + elif parameter_selector == RangeSlider: + default_parameter_value = [parameter_selector.min, parameter_selector.max] + load_kwargs[data_source_name].append((argument, default_parameter_value)) + + # multi_data_source_name_load_kwargs = [(model_manager[target]["data_frame"], {}) for target in proposed_targets] # type: ignore[var-annotated] + multi_data_source_name_load_kwargs = [(a, s) for a, s in load_kwargs.items()] target_to_data_frame = dict(zip(proposed_targets, data_manager._multi_load(multi_data_source_name_load_kwargs))) targeted_data = self._validate_targeted_data( target_to_data_frame, eagerly_raise_column_not_found_error=bool(self.targets) @@ -244,6 +276,7 @@ def _validate_targeted_data( if targeted_data.columns.empty: # Still raised when eagerly_raise_column_not_found_error=False. raise ValueError(f"Selected column {self.column} not found in any dataframe for {', '.join(targets)}.") + # TODO: Enable empty data_frame handling if targeted_data.empty: raise ValueError( f"Selected column {self.column} does not contain anything in any dataframe for {', '.join(targets)}." From e066fdbfdd72146cdab44bf91fec911efdda9ece Mon Sep 17 00:00:00 2001 From: petar-qb Date: Fri, 15 Nov 2024 11:24:13 +0100 Subject: [PATCH 23/64] More improvements --- vizro-core/examples/scratch_dev/app.py | 374 +++++++----------- vizro-core/examples/scratch_dev/data.yaml | 9 +- .../src/vizro/models/_controls/filter.py | 36 +- 3 files changed, 163 insertions(+), 256 deletions(-) diff --git a/vizro-core/examples/scratch_dev/app.py b/vizro-core/examples/scratch_dev/app.py index e52310aa7..96befdc57 100644 --- a/vizro-core/examples/scratch_dev/app.py +++ b/vizro-core/examples/scratch_dev/app.py @@ -15,29 +15,28 @@ print("INITIALIZING") SPECIES_COLORS = {"setosa": "#00b4ff", "versicolor": "#ff9222", "virginica": "#3949ab"} +BAR_CHART_CONF = dict(x="species", color="species", color_discrete_map=SPECIES_COLORS) +SCATTER_CHART_CONF = dict(x="sepal_length", y="petal_length", color="species", color_discrete_map=SPECIES_COLORS) +# Relevant for the "page_6" only FILTER_COLUMN = "species" # FILTER_COLUMN = "sepal_length" +# FILTER_COLUMN = "date_column" -def slow_load(sample_size=3): - # time.sleep(2) - df = px.data.iris().sample(sample_size) - print(f'SLOW LOAD - {sorted(df["species"].unique().tolist())} - sample_size = {sample_size}') - return df - +def load_from_file(filter_column=None, parametrized_species=None): + # Load the full iris dataset + df = px.data.iris() + df['date_column'] = pd.date_range(start=pd.to_datetime("2024-01-01"), periods=len(df), freq='D') -def load_from_file(filter_column=None): - filter_column = filter_column or FILTER_COLUMN + if parametrized_species: + return df[df["species"].isin(parametrized_species)] with open('data.yaml', 'r') as file: data = yaml.safe_load(file) - data = data or pd.DataFrame() - - # Load the full iris dataset - df = px.data.iris() + data = data or {} - # Load the first N rows of each species. N per species is defined in the data.yaml file. + filter_column = filter_column or FILTER_COLUMN if filter_column == "species": final_df = pd.concat(objs=[ df[df[filter_column] == 'setosa'].head(data.get("setosa", 0)), @@ -45,32 +44,25 @@ def load_from_file(filter_column=None): df[df[filter_column] == 'virginica'].head(data.get("virginica", 0)), ], ignore_index=True) elif filter_column == "sepal_length": - final_df = df[ - df[filter_column].between(data.get("min", 0), data.get("max", 0), inclusive="both") - ] + final_df = df[df[filter_column].between(data.get("min"), data.get("max",), inclusive="both")] + elif filter_column == "date_column": + date_min = pd.to_datetime(data.get("date_min")) + date_max = pd.to_datetime(data.get("date_max")) + final_df = df[df[filter_column].between(date_min, date_max, inclusive="both")] else: raise ValueError("Invalid FILTER_COLUMN") return final_df -def load_parametrised_species(species=None): - df = px.data.iris() - if species: - df = df[df["species"].isin(species)] - return df - - data_manager["load_from_file"] = load_from_file data_manager["load_from_file_species"] = partial(load_from_file, filter_column="species") data_manager["load_from_file_sepal_length"] = partial(load_from_file, filter_column="sepal_length") +data_manager["load_from_file_date_column"] = partial(load_from_file, filter_column="date_column") -# data_manager["load_parametrised_species"] = load_parametrised_species # # TODO-DEV: Turn on/off caching to see how it affects the app. -# data_manager.cache = Cache(config={"CACHE_TYPE": "SimpleCache"}) -# data_manager["load_from_file"] = slow_load -# data_manager["load_from_file"].timeout = 5 +# data_manager.cache = Cache(config={"CACHE_TYPE": "SimpleCache", "CACHE_DEFAULT_TIMEOUT": 5}) homepage = vm.Page( @@ -85,17 +77,11 @@ def load_parametrised_species(species=None): components=[ vm.Graph( id="p1-G-1", - figure=px.bar( - data_frame="load_from_file_species", - x="species", color="species", color_discrete_map=SPECIES_COLORS, - ), + figure=px.bar(data_frame="load_from_file_species", **BAR_CHART_CONF), ), vm.Graph( id="p1-G-2", - figure=px.scatter( - data_frame=px.data.iris(), - x="sepal_length", y="petal_length", color="species", color_discrete_map=SPECIES_COLORS, - ), + figure=px.scatter(data_frame=px.data.iris(), **SCATTER_CHART_CONF), ) ], controls=[ @@ -114,10 +100,7 @@ def load_parametrised_species(species=None): components=[ vm.Graph( id="p2-G-1", - figure=px.bar( - data_frame="load_from_file_species", - x="species", color="species", color_discrete_map=SPECIES_COLORS, - ), + figure=px.bar(data_frame="load_from_file_species", **BAR_CHART_CONF), ), ], controls=[ @@ -138,10 +121,7 @@ def load_parametrised_species(species=None): components=[ vm.Graph( id="p3-G-1", - figure=px.bar( - data_frame="load_from_file_sepal_length", - x="species", color="species", color_discrete_map=SPECIES_COLORS, - ), + figure=px.bar(data_frame="load_from_file_sepal_length", **BAR_CHART_CONF), ), ], controls=[ @@ -153,154 +133,115 @@ def load_parametrised_species(species=None): ) ] ) -# another_page = vm.Page( -# title="Test update control options", -# components=[ -# vm.Graph( -# id="graph_1_id", -# figure=px.bar( -# data_frame="dynamic_df", -# x="species", -# color="species", -# color_discrete_map={"setosa": "#00b4ff", "versicolor": "#ff9222", "virginica": "#3949ab"}, -# ) -# ), -# # vm.Graph( -# # id="graph_2_id", -# # figure=px.scatter( -# # data_frame="dynamic_df", -# # x="sepal_length", -# # y="petal_length", -# # color="species", -# # color_discrete_map={"setosa": "#00b4ff", "versicolor": "#ff9222", "virginica": "#3949ab"}, -# # ), -# # ), -# # vm.Graph( -# # id="graph_2_id", -# # figure=px.scatter( -# # data_frame="dynamic_df", -# # x="sepal_length", -# # y="petal_width", -# # color="species", -# # ) -# # ), -# -# ], -# controls=[ -# vm.Filter( -# id="filter_container_id", -# column=FILTER_COLUMN, -# # targets=["graph_2_id"], -# # selector=vm.Dropdown(id="filter_id"), -# # selector=vm.Dropdown(id="filter_id", value=["setosa"]), -# -# selector=vm.Checklist(id="filter_id"), -# # selector=vm.Checklist(id="filter_id", value=["setosa"]), -# -# # selector=vm.Dropdown(id="filter_id", multi=False), -# # selector=vm.Dropdown(id="filter_id", multi=False, value="setosa"), -# -# # selector=vm.RadioItems(id="filter_id"), -# # selector=vm.RadioItems(id="filter_id", value="setosa"), -# -# # selector=vm.Slider(id="filter_id"), -# # selector=vm.Slider(id="filter_id", value=6), -# -# # selector=vm.RangeSlider(id="filter_id"), -# # selector=vm.RangeSlider(id="filter_id", value=[5, 7]), -# -# # TEST CASES: -# # no selector -# # WORKS -# # multi=True -# # default -# # WORKS -# # selector=vm.Slider(id="filter_id", step=0.5), -# # options: list -# # WORKS - but options doesn't mean anything because options are dynamically overwritten. -# # selector=vm.Dropdown(options=["setosa", "versicolor"]), -# # options: empty list -# # WORKS -# # selector=vm.Dropdown(options=[]), -# # options: dict -# # WORKS - but "label" is always overwritten. -# # selector=vm.Dropdown(options=[{"label": "Setosa", "value": "setosa"}, {"label": "Versicolor", "value": "versicolor"}]), -# # options list; value -# # WORKS -# # selector=vm.Dropdown(options=["setosa", "versicolor"], value=["setosa"]), -# # options list; empty value -# # WORKS -# # selector=vm.Dropdown(options=["setosa", "versicolor"], value=[]), -# # strange options -# # WORKS -# # selector=vm.Dropdown(options=["A", "B", "C"]), -# # strange options with strange value -# # WORKS - works even for the dynamic False, and this is OK. -# # selector=vm.Dropdown(options=["A", "B", "C"], value=["XYZ"]), -# # -# # -# # multi=False -> TLDR: Doesn't work if value is cleared. Other combinations are same as multi=True. -# # default -# # DOES NOT WORK - Doesn't work if value is cleared. Then persistence storage become "null" and our -# # placeholder component dmc.DateRangePicker can't process null value. It expects a value or a list -# # of values. SOLUTION -> Create the "universal Vizro placeholder component" -# # selector=vm.Dropdown(multi=False), -# # options: list - because options are dynamically overwritten. -# # WORKS - but options doesn't mean anything because options are dynamically overwritten. -# # selector=vm.Dropdown(multi=False, options=["setosa", "versicolor"]), -# # options: empty list -# # WORKS -# # selector=vm.Dropdown(multi=False, options=[]), -# # options: dict -# # selector=vm.Dropdown(multi=False, options=[{"label": "Setosa", "value": "setosa"}, {"label": "Versicolor", "value": "versicolor"}]), -# # options list; value -# # WORKS -# # selector=vm.Dropdown(multi=False, options=["setosa", "versicolor"], value="setosa"), -# # options list; None value -# # WORKS -# # selector=vm.Dropdown(multi=False, options=["setosa", "versicolor"], value=None), -# # strange options -# # WORKS -# # selector=vm.Dropdown(multi=False, options=["A", "B", "C"]), -# # strange options with strange value -# # WORKS -# # selector=vm.Dropdown(multi=False, options=["A", "B", "C"], value="XYZ"), -# ), -# vm.Parameter( -# id="parameter_x", -# targets=["graph_1_id.x",], -# selector=vm.Dropdown( -# options=["species", "sepal_width"], -# value="species", -# multi=False, -# ) -# ), -# # vm.Parameter( -# # id="parameter_species", -# # targets=[ -# # "graph_1_id.data_frame.species", -# # "filter_container_id.", -# # ], -# # selector=vm.Dropdown(options=["setosa", "versicolor", "virginica"]) -# # ), -# # vm.Parameter( -# # id="parameter_container_id", -# # targets=[ -# # "graph_1_id.data_frame.sample_size", -# # # "graph_2_id.data_frame.sample_size", -# # ], -# # selector=vm.Slider( -# # id="parameter_id", -# # min=1, -# # max=10, -# # step=1, -# # value=10, -# # ) -# # ), -# ], -# ) - -dashboard = vm.Dashboard(pages=[homepage, page_1, page_2, page_3]) + +page_4 = vm.Page( + title="Temporal dynamic selectors", + components=[ + vm.Graph( + id="p4-G-1", + figure=px.bar(data_frame="load_from_file_date_column", **BAR_CHART_CONF), + ), + ], + controls=[ + vm.Filter(id="p4-F-1", column="date_column", selector=vm.DatePicker(range=False)), + vm.Filter(id="p4-F-2", column="date_column", selector=vm.DatePicker()), + vm.Parameter( + targets=["p4-G-1.x"], + selector=vm.RadioItems(options=["species", "sepal_width"], value="species", title="Simple X-axis parameter") + ) + ] +) + +page_5 = vm.Page( + title="Parametrised dynamic selectors", + components=[ + vm.Graph( + id="p5-G-1", + figure=px.bar(data_frame="load_from_file_species", **BAR_CHART_CONF), + ), + ], + controls=[ + vm.Filter(id="p5-F-1", column="species", targets=["p5-G-1"], selector=vm.Checklist()), + vm.Parameter( + targets=[ + "p5-G-1.data_frame.parametrized_species", + # TODO: Uncomment the following target and see the magic :D + # Is this the indicator that parameter.targets prop has to support 'target' definition without the '.'? + # "p5-F-1.", + ], + selector=vm.Dropdown( + options=["setosa", "versicolor", "virginica"], + multi=True, + title="Parametrized species" + ) + ), + vm.Parameter( + targets=[ + "p5-G-1.x", + # TODO: Uncomment the following target and see the magic :D + # "p5-F-1.", + ], + selector=vm.RadioItems(options=["species", "sepal_width"], value="species", title="Simple X-axis parameter") + ), + ] +) + + +page_6 = vm.Page( + title="Page to test things out", + components=[ + vm.Graph( + id="graph_dynamic", + figure=px.bar(data_frame="load_from_file", **BAR_CHART_CONF) + ), + vm.Graph( + id="graph_static", + figure=px.scatter(data_frame=px.data.iris(), **SCATTER_CHART_CONF), + ) + ], + controls=[ + vm.Filter( + id="filter_container_id", + column=FILTER_COLUMN, + targets=["graph_dynamic"], + # targets=["graph_static"], + + # selector=vm.Dropdown(id="filter_id"), + # selector=vm.Dropdown(id="filter_id", value=["setosa"]), + + # selector=vm.Checklist(id="filter_id"), + # selector=vm.Checklist(id="filter_id", value=["setosa"]), + + # TODO-BUG: vm.Dropdown(multi=False) Doesn't work if value is cleared. The persistence storage become + # "null" and our placeholder component dmc.DateRangePicker can't process null value. It expects a value or + # a list of values. + # SOLUTION -> Create the "Universal Vizro placeholder component". + # TEMPORARY SOLUTION -> set clearable=False for the dynamic Dropdown(multi=False) + # selector=vm.Dropdown(id="filter_id", multi=False), -> + # selector=vm.Dropdown(id="filter_id", multi=False, value="setosa"), + + # selector=vm.RadioItems(id="filter_id"), + # selector=vm.RadioItems(id="filter_id", value="setosa"), + + # selector=vm.Slider(id="filter_id"), + # selector=vm.Slider(id="filter_id", value=5), + + # selector=vm.RangeSlider(id="filter_id"), + # selector=vm.RangeSlider(id="filter_id", value=[5, 7]), + ), + vm.Parameter( + id="parameter_x", + targets=["graph_dynamic.x",], + selector=vm.Dropdown( + options=["species", "sepal_width"], + value="species", + multi=False, + ) + ), + ], +) + +dashboard = vm.Dashboard(pages=[homepage, page_1, page_2, page_3, page_4, page_5, page_6]) if __name__ == "__main__": app = Vizro().build(dashboard) @@ -308,54 +249,3 @@ def load_parametrised_species(species=None): print("RUNNING\n") app.run(dev_tools_hot_reload=False) - - -# """Dev app to try things out.""" -# -# import pandas as pd -# import vizro.models as vm -# import vizro.plotly.express as px -# from vizro import Vizro -# -# from vizro.managers import data_manager -# -# -# def load_1(sample=1): -# print("load_1") -# return px.data.iris().sample(sample) -# -# -# def load_2(sample=2): -# print("load_2") -# return px.data.iris().sample(sample) -# -# -# data_manager["load_1"] = load_1 -# data_manager["load_2"] = load_2 -# -# page = vm.Page( -# title="Charts UI", -# components=[ -# vm.Graph(id="graph_1", figure=px.scatter("load_1", x="sepal_width", y="sepal_length")), -# vm.Graph(id="graph_2", figure=px.scatter("load_1", x="sepal_width", y="sepal_length")), -# vm.Graph(id="graph_3", figure=px.scatter("load_2", x="sepal_width", y="sepal_length")), -# vm.Graph(id="graph_4", figure=px.scatter("load_2", x="sepal_width", y="sepal_length")), -# ], -# controls=[ -# vm.Filter(column="species"), -# vm.Parameter( -# targets=['graph_1.data_frame.sample', 'graph_3.data_frame.sample'], -# selector=vm.Slider(min=1, max=10, step=1, value=1), -# ), -# vm.Parameter( -# targets=['graph_2.data_frame.sample', 'graph_4.data_frame.sample'], -# selector=vm.Slider(min=1, max=10, step=1, value=1), -# ) -# ], -# -# ) -# -# dashboard = vm.Dashboard(pages=[page]) -# -# if __name__ == "__main__": -# Vizro().build(dashboard).run() \ No newline at end of file diff --git a/vizro-core/examples/scratch_dev/data.yaml b/vizro-core/examples/scratch_dev/data.yaml index e50c2e8a9..dc352a757 100644 --- a/vizro-core/examples/scratch_dev/data.yaml +++ b/vizro-core/examples/scratch_dev/data.yaml @@ -1,6 +1,13 @@ +# Choose from 0-50 setosa: 5 versicolor: 10 virginica: 15 +# Choose from: 4.8 to 7.4 min: 5 -max: 7 \ No newline at end of file +max: 7 + +# Choose from: +# 2020-01-01 to 2020-05-29 +date_min: 2024-01-01 +date_max: 2024-05-29 \ No newline at end of file diff --git a/vizro-core/src/vizro/models/_controls/filter.py b/vizro-core/src/vizro/models/_controls/filter.py index af5aa877b..9ae3b01d2 100644 --- a/vizro-core/src/vizro/models/_controls/filter.py +++ b/vizro-core/src/vizro/models/_controls/filter.py @@ -50,7 +50,7 @@ "categorical": SELECTORS["numerical"] + SELECTORS["temporal"], } -# TODO: Remove this check because all vizro selectors support dynamic mode +# TODO: Remove this check when support dynamic mode for DatePicker selector. # Tuple of filter selectors that support dynamic mode DYNAMIC_SELECTORS = (Dropdown, Checklist, RadioItems, Slider, RangeSlider) @@ -156,11 +156,20 @@ def pre_build(self): # ctd_parameters. That could be changed to just reuse that function. from vizro.models._controls import Parameter - load_kwargs = {} + multi_data_source_name_load_kwargs: list[tuple[DataSourceName, dict[str, Any]]] = [] + + # One tuple per filter.target + # [ + # ('data_1', {'arg_1': 1, 'arg_2': 2,}), + # ('data_2', {'X': "ASD"}), + # ('data_2', {'X': "qwe"}), + # ] + + # TODO-NEXT: The code below is just a PoC and could be improved a lot. page_obj = model_manager[model_manager._get_model_page_id(model_id=ModelID(str(self.id)))] for target in proposed_targets: data_source_name = model_manager[target]["data_frame"] - load_kwargs[data_source_name] = {} + load_kwargs = {} for page_parameter in page_obj.controls: if isinstance(page_parameter, Parameter): @@ -168,25 +177,26 @@ def pre_build(self): if parameter_targets.startswith(f'{target}.data_frame'): argument = parameter_targets.split('.')[2] # argument is explicitly defined - if parameter_value := getattr(page_parameter.selector, value, None): - load_kwargs[data_source_name].append((argument, parameter_value)) + if parameter_value := getattr(page_parameter.selector, 'value', None): + load_kwargs[argument]=parameter_value # find default value else: parameter_selector = page_parameter.selector - if parameter_selector == Dropdown: + default_parameter_value = None + if isinstance(parameter_selector, Dropdown): default_parameter_value = get_options_and_default(parameter_selector.options, parameter_selector.multi) - elif parameter_selector == Checklist: + elif isinstance(parameter_selector, Checklist): default_parameter_value = get_options_and_default(parameter_selector.options, True) - elif parameter_selector == RadioItems: + elif isinstance(parameter_selector, RadioItems): default_parameter_value = get_options_and_default(parameter_selector.options, False) - elif parameter_selector == Slider: + elif isinstance(parameter_selector, Slider): default_parameter_value = parameter_selector.min - elif parameter_selector == RangeSlider: + elif isinstance(parameter_selector, RangeSlider): default_parameter_value = [parameter_selector.min, parameter_selector.max] - load_kwargs[data_source_name].append((argument, default_parameter_value)) + load_kwargs[argument] = default_parameter_value[1] if default_parameter_value[1] != "ALL" else parameter_selector.options + + multi_data_source_name_load_kwargs.append((data_source_name, load_kwargs)) - # multi_data_source_name_load_kwargs = [(model_manager[target]["data_frame"], {}) for target in proposed_targets] # type: ignore[var-annotated] - multi_data_source_name_load_kwargs = [(a, s) for a, s in load_kwargs.items()] target_to_data_frame = dict(zip(proposed_targets, data_manager._multi_load(multi_data_source_name_load_kwargs))) targeted_data = self._validate_targeted_data( target_to_data_frame, eagerly_raise_column_not_found_error=bool(self.targets) From 9e5a239e160fc2c8837ed3242ba425e217db7b3f Mon Sep 17 00:00:00 2001 From: petar-qb Date: Fri, 15 Nov 2024 11:28:30 +0100 Subject: [PATCH 24/64] Reverting: Sending DFP values from the MM to Filter.pre_build DM._multi_load --- .../src/vizro/models/_controls/filter.py | 47 ++----------------- 1 file changed, 4 insertions(+), 43 deletions(-) diff --git a/vizro-core/src/vizro/models/_controls/filter.py b/vizro-core/src/vizro/models/_controls/filter.py index 9ae3b01d2..0d1270381 100644 --- a/vizro-core/src/vizro/models/_controls/filter.py +++ b/vizro-core/src/vizro/models/_controls/filter.py @@ -151,51 +151,12 @@ def pre_build(self): # TODO NEXT: how to handle pre_build for dynamic filters? Do we still require default argument values in # `load` to establish selector type etc.? Can we take selector values from model_manager to supply these? # Or just don't do validation at pre_build time and wait until state is available during build time instead? - # What should the load kwargs be here? + # What should the load kwargs be here? Remember they need to be {} for static data. # Note that currently _get_unfiltered_data is only suitable for use at runtime since it requires # ctd_parameters. That could be changed to just reuse that function. - from vizro.models._controls import Parameter - - multi_data_source_name_load_kwargs: list[tuple[DataSourceName, dict[str, Any]]] = [] - - # One tuple per filter.target - # [ - # ('data_1', {'arg_1': 1, 'arg_2': 2,}), - # ('data_2', {'X': "ASD"}), - # ('data_2', {'X': "qwe"}), - # ] - - # TODO-NEXT: The code below is just a PoC and could be improved a lot. - page_obj = model_manager[model_manager._get_model_page_id(model_id=ModelID(str(self.id)))] - for target in proposed_targets: - data_source_name = model_manager[target]["data_frame"] - load_kwargs = {} - - for page_parameter in page_obj.controls: - if isinstance(page_parameter, Parameter): - for parameter_targets in page_parameter.targets: - if parameter_targets.startswith(f'{target}.data_frame'): - argument = parameter_targets.split('.')[2] - # argument is explicitly defined - if parameter_value := getattr(page_parameter.selector, 'value', None): - load_kwargs[argument]=parameter_value - # find default value - else: - parameter_selector = page_parameter.selector - default_parameter_value = None - if isinstance(parameter_selector, Dropdown): - default_parameter_value = get_options_and_default(parameter_selector.options, parameter_selector.multi) - elif isinstance(parameter_selector, Checklist): - default_parameter_value = get_options_and_default(parameter_selector.options, True) - elif isinstance(parameter_selector, RadioItems): - default_parameter_value = get_options_and_default(parameter_selector.options, False) - elif isinstance(parameter_selector, Slider): - default_parameter_value = parameter_selector.min - elif isinstance(parameter_selector, RangeSlider): - default_parameter_value = [parameter_selector.min, parameter_selector.max] - load_kwargs[argument] = default_parameter_value[1] if default_parameter_value[1] != "ALL" else parameter_selector.options - - multi_data_source_name_load_kwargs.append((data_source_name, load_kwargs)) + multi_data_source_name_load_kwargs: list[tuple[DataSourceName, dict[str, Any]]] = [ + (model_manager[target]["data_frame"], {}) for target in proposed_targets + ] target_to_data_frame = dict(zip(proposed_targets, data_manager._multi_load(multi_data_source_name_load_kwargs))) targeted_data = self._validate_targeted_data( From cfdf4fcebd0b8f4b64b9763a7fe9a1c54499aa1e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 15 Nov 2024 10:31:55 +0000 Subject: [PATCH 25/64] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../scratch_dev/_poc_dynamic_controls.py | 57 ++++++----- vizro-core/examples/scratch_dev/app.py | 95 +++++++++++-------- vizro-core/examples/scratch_dev/data.yaml | 2 +- .../src/vizro/actions/_actions_utils.py | 10 +- .../src/vizro/actions/_on_page_load_action.py | 17 ++-- .../vizro/models/_components/form/dropdown.py | 8 +- .../models/_components/form/range_slider.py | 4 +- .../vizro/models/_components/form/slider.py | 2 +- .../src/vizro/models/_controls/filter.py | 16 ++-- .../vizro/static/js/models/range_slider.js | 28 ++++-- .../src/vizro/static/js/models/slider.js | 14 +-- 11 files changed, 143 insertions(+), 110 deletions(-) diff --git a/vizro-core/examples/scratch_dev/_poc_dynamic_controls.py b/vizro-core/examples/scratch_dev/_poc_dynamic_controls.py index 6cf07f73d..77bab2540 100644 --- a/vizro-core/examples/scratch_dev/_poc_dynamic_controls.py +++ b/vizro-core/examples/scratch_dev/_poc_dynamic_controls.py @@ -1,9 +1,7 @@ import dash -import plotly.express as px - -from dash import Dash, html, dcc, Output, callback, clientside_callback, Input, State, set_props import dash_mantine_components as dmc - +import plotly.express as px +from dash import Dash, Input, Output, State, callback, clientside_callback, dcc, html CONTROL_SELECTOR = dcc.RangeSlider @@ -90,18 +88,18 @@ def categorical_filter_build(options=None): kwargs["multi"] = MULTI return CONTROL_SELECTOR( - id=f'filter', + id="filter", options=options or pre_build_options, value=pre_build_categorical_value, persistence=True, persistence_type="session", - **kwargs + **kwargs, ) def numerical_filter_build(min_value=None, max_value=None): return CONTROL_SELECTOR( - id=f'filter', + id="filter", min=min_value or pre_build_min, max=max_value or pre_build_max, value=pre_build_numerical_value, @@ -117,7 +115,7 @@ def _get_initial_page_build_object(): if CONTROL_SELECTOR == dcc.Dropdown: # A hack explained in on_page_load To-do:"Limitations" section below in this page. return dmc.DateRangePicker( - id='filter', + id="filter", value=pre_build_categorical_value, persistence=True, persistence_type="session", @@ -133,27 +131,28 @@ def _get_initial_page_build_object(): [ dcc.Store(id="on_page_load_trigger_another_page"), html.H2("Another page"), - # # This does NOT work because id="filter" doesn't exist but is used as OPL callback State. # dcc.Loading(id="filter_container"), - # # Possible solution is to alter filter.options from on_page_load. This would work, but it's not optimal. # dcc.Dropdown(id="filter", options=options, value=options, multi=True, persistence=True), - # # Outer container can be changed with dcc.Loading. html.Div( _get_initial_page_build_object(), id="filter_container", ), - # # Does not work because OPL filter input is missing, but it's used for filtering figures data_frame. # html.Div( # html.Div(id="filter"), # id="filter_container", # ), - html.Br(), - dcc.RadioItems(id="parameter", options=["sepal_width", "sepal_length"], value="sepal_width", persistence=True, persistence_type="session"), + dcc.RadioItems( + id="parameter", + options=["sepal_width", "sepal_length"], + value="sepal_width", + persistence=True, + persistence_type="session", + ), dcc.Loading(dcc.Graph(id="graph1")), dcc.Loading(dcc.Graph(id="graph2")), ] @@ -206,11 +205,10 @@ def get_data(species): ], inputs=[ Input("global_on_page_load_another_page_action_trigger", "data"), - State("filter", "value"), State("parameter", "value"), ], - prevent_initial_call=True + prevent_initial_call=True, ) def on_page_load(data, persisted_filter_value, x): # Ideally, OPL flow should look like this: @@ -263,9 +261,13 @@ def on_page_load(data, persisted_filter_value, x): if CONTROL_SELECTOR in SELECTOR_TYPE["categorical"]: categorical_filter_options = sorted(df["species"].unique().tolist()) if MULTI: - categorical_filter_value = [value for value in persisted_filter_value if value in categorical_filter_options] + categorical_filter_value = [ + value for value in persisted_filter_value if value in categorical_filter_options + ] else: - categorical_filter_value = persisted_filter_value if persisted_filter_value in categorical_filter_options else None + categorical_filter_value = ( + persisted_filter_value if persisted_filter_value in categorical_filter_options else None + ) new_filter_obj = categorical_filter_build(options=categorical_filter_options) # --- Filtering data: --- @@ -298,19 +300,28 @@ def on_page_load(data, persisted_filter_value, x): numerical_filter_min = float(df["sepal_length"].min()) numerical_filter_max = float(df["sepal_length"].max()) if MULTI: - numerical_filter_value = [max(numerical_filter_min, persisted_filter_value[0]), min(numerical_filter_max, persisted_filter_value[1])] + numerical_filter_value = [ + max(numerical_filter_min, persisted_filter_value[0]), + min(numerical_filter_max, persisted_filter_value[1]), + ] else: - numerical_filter_value = persisted_filter_value if numerical_filter_min < persisted_filter_value < numerical_filter_max else numerical_filter_min + numerical_filter_value = ( + persisted_filter_value + if numerical_filter_min < persisted_filter_value < numerical_filter_max + else numerical_filter_min + ) new_filter_obj = numerical_filter_build(min_value=numerical_filter_min, max_value=numerical_filter_max) # set_props(component_id="numerical_filter_container", props={"children": new_filter_obj}) # --- Filtering data: --- if MULTI: - df = df[(df["sepal_length"] >= numerical_filter_value[0]) & (df["sepal_length"] <= numerical_filter_value[1])] + df = df[ + (df["sepal_length"] >= numerical_filter_value[0]) & (df["sepal_length"] <= numerical_filter_value[1]) + ] else: df = df[(df["sepal_length"] == numerical_filter_value)] - print("") + print() return graph1_call(df), graph2_call(df, x), new_filter_obj @@ -394,4 +405,4 @@ def on_page_load(data, persisted_filter_value, x): # IMPORTANT: also consider parametrised data case. if __name__ == "__main__": - app.run(debug=True, dev_tools_hot_reload=False) \ No newline at end of file + app.run(debug=True, dev_tools_hot_reload=False) diff --git a/vizro-core/examples/scratch_dev/app.py b/vizro-core/examples/scratch_dev/app.py index 96befdc57..401f17a28 100644 --- a/vizro-core/examples/scratch_dev/app.py +++ b/vizro-core/examples/scratch_dev/app.py @@ -1,4 +1,5 @@ """Dev app to try things out.""" + import time import yaml @@ -27,24 +28,35 @@ def load_from_file(filter_column=None, parametrized_species=None): # Load the full iris dataset df = px.data.iris() - df['date_column'] = pd.date_range(start=pd.to_datetime("2024-01-01"), periods=len(df), freq='D') + df["date_column"] = pd.date_range(start=pd.to_datetime("2024-01-01"), periods=len(df), freq="D") if parametrized_species: return df[df["species"].isin(parametrized_species)] - with open('data.yaml', 'r') as file: + with open("data.yaml", "r") as file: data = yaml.safe_load(file) data = data or {} filter_column = filter_column or FILTER_COLUMN if filter_column == "species": - final_df = pd.concat(objs=[ - df[df[filter_column] == 'setosa'].head(data.get("setosa", 0)), - df[df[filter_column] == 'versicolor'].head(data.get("versicolor", 0)), - df[df[filter_column] == 'virginica'].head(data.get("virginica", 0)), - ], ignore_index=True) + final_df = pd.concat( + objs=[ + df[df[filter_column] == "setosa"].head(data.get("setosa", 0)), + df[df[filter_column] == "versicolor"].head(data.get("versicolor", 0)), + df[df[filter_column] == "virginica"].head(data.get("virginica", 0)), + ], + ignore_index=True, + ) elif filter_column == "sepal_length": - final_df = df[df[filter_column].between(data.get("min"), data.get("max",), inclusive="both")] + final_df = df[ + df[filter_column].between( + data.get("min"), + data.get( + "max", + ), + inclusive="both", + ) + ] elif filter_column == "date_column": date_min = pd.to_datetime(data.get("date_min")) date_max = pd.to_datetime(data.get("date_max")) @@ -82,16 +94,18 @@ def load_from_file(filter_column=None, parametrized_species=None): vm.Graph( id="p1-G-2", figure=px.scatter(data_frame=px.data.iris(), **SCATTER_CHART_CONF), - ) + ), ], controls=[ vm.Filter(id="p1-F-1", column="species", targets=["p1-G-1"], selector=vm.Dropdown(title="Dynamic filter")), vm.Filter(id="p1-F-2", column="species", targets=["p1-G-2"], selector=vm.Dropdown(title="Static filter")), vm.Parameter( targets=["p1-G-1.x", "p1-G-2.x"], - selector=vm.RadioItems(options=["species", "sepal_width"], value="species", title="Simple X-axis parameter") - ) - ] + selector=vm.RadioItems( + options=["species", "sepal_width"], value="species", title="Simple X-axis parameter" + ), + ), + ], ) @@ -110,9 +124,11 @@ def load_from_file(filter_column=None, parametrized_species=None): vm.Filter(id="p2-F-4", column="species", selector=vm.RadioItems()), vm.Parameter( targets=["p2-G-1.x"], - selector=vm.RadioItems(options=["species", "sepal_width"], value="species", title="Simple X-axis parameter") - ) - ] + selector=vm.RadioItems( + options=["species", "sepal_width"], value="species", title="Simple X-axis parameter" + ), + ), + ], ) @@ -129,9 +145,11 @@ def load_from_file(filter_column=None, parametrized_species=None): vm.Filter(id="p3-F-2", column="sepal_length", selector=vm.RangeSlider()), vm.Parameter( targets=["p3-G-1.x"], - selector=vm.RadioItems(options=["species", "sepal_width"], value="species", title="Simple X-axis parameter") - ) - ] + selector=vm.RadioItems( + options=["species", "sepal_width"], value="species", title="Simple X-axis parameter" + ), + ), + ], ) page_4 = vm.Page( @@ -147,9 +165,11 @@ def load_from_file(filter_column=None, parametrized_species=None): vm.Filter(id="p4-F-2", column="date_column", selector=vm.DatePicker()), vm.Parameter( targets=["p4-G-1.x"], - selector=vm.RadioItems(options=["species", "sepal_width"], value="species", title="Simple X-axis parameter") - ) - ] + selector=vm.RadioItems( + options=["species", "sepal_width"], value="species", title="Simple X-axis parameter" + ), + ), + ], ) page_5 = vm.Page( @@ -170,10 +190,8 @@ def load_from_file(filter_column=None, parametrized_species=None): # "p5-F-1.", ], selector=vm.Dropdown( - options=["setosa", "versicolor", "virginica"], - multi=True, - title="Parametrized species" - ) + options=["setosa", "versicolor", "virginica"], multi=True, title="Parametrized species" + ), ), vm.Parameter( targets=[ @@ -181,23 +199,22 @@ def load_from_file(filter_column=None, parametrized_species=None): # TODO: Uncomment the following target and see the magic :D # "p5-F-1.", ], - selector=vm.RadioItems(options=["species", "sepal_width"], value="species", title="Simple X-axis parameter") + selector=vm.RadioItems( + options=["species", "sepal_width"], value="species", title="Simple X-axis parameter" + ), ), - ] + ], ) page_6 = vm.Page( title="Page to test things out", components=[ - vm.Graph( - id="graph_dynamic", - figure=px.bar(data_frame="load_from_file", **BAR_CHART_CONF) - ), + vm.Graph(id="graph_dynamic", figure=px.bar(data_frame="load_from_file", **BAR_CHART_CONF)), vm.Graph( id="graph_static", figure=px.scatter(data_frame=px.data.iris(), **SCATTER_CHART_CONF), - ) + ), ], controls=[ vm.Filter( @@ -205,13 +222,10 @@ def load_from_file(filter_column=None, parametrized_species=None): column=FILTER_COLUMN, targets=["graph_dynamic"], # targets=["graph_static"], - # selector=vm.Dropdown(id="filter_id"), # selector=vm.Dropdown(id="filter_id", value=["setosa"]), - # selector=vm.Checklist(id="filter_id"), # selector=vm.Checklist(id="filter_id", value=["setosa"]), - # TODO-BUG: vm.Dropdown(multi=False) Doesn't work if value is cleared. The persistence storage become # "null" and our placeholder component dmc.DateRangePicker can't process null value. It expects a value or # a list of values. @@ -219,24 +233,23 @@ def load_from_file(filter_column=None, parametrized_species=None): # TEMPORARY SOLUTION -> set clearable=False for the dynamic Dropdown(multi=False) # selector=vm.Dropdown(id="filter_id", multi=False), -> # selector=vm.Dropdown(id="filter_id", multi=False, value="setosa"), - # selector=vm.RadioItems(id="filter_id"), # selector=vm.RadioItems(id="filter_id", value="setosa"), - # selector=vm.Slider(id="filter_id"), # selector=vm.Slider(id="filter_id", value=5), - # selector=vm.RangeSlider(id="filter_id"), # selector=vm.RangeSlider(id="filter_id", value=[5, 7]), - ), + ), vm.Parameter( id="parameter_x", - targets=["graph_dynamic.x",], + targets=[ + "graph_dynamic.x", + ], selector=vm.Dropdown( options=["species", "sepal_width"], value="species", multi=False, - ) + ), ), ], ) diff --git a/vizro-core/examples/scratch_dev/data.yaml b/vizro-core/examples/scratch_dev/data.yaml index dc352a757..04854efbd 100644 --- a/vizro-core/examples/scratch_dev/data.yaml +++ b/vizro-core/examples/scratch_dev/data.yaml @@ -10,4 +10,4 @@ max: 7 # Choose from: # 2020-01-01 to 2020-05-29 date_min: 2024-01-01 -date_max: 2024-05-29 \ No newline at end of file +date_max: 2024-05-29 diff --git a/vizro-core/src/vizro/actions/_actions_utils.py b/vizro-core/src/vizro/actions/_actions_utils.py index 5f2855ed6..69d929a94 100644 --- a/vizro-core/src/vizro/actions/_actions_utils.py +++ b/vizro-core/src/vizro/actions/_actions_utils.py @@ -14,7 +14,7 @@ from vizro.models.types import MultiValueType, SelectorType, SingleValueType if TYPE_CHECKING: - from vizro.models import Action, VizroBaseModel, Filter + from vizro.models import Action, VizroBaseModel ValidatedNoneValueType = Union[SingleValueType, MultiValueType, None, list[None]] @@ -162,6 +162,7 @@ def _update_nested_figure_properties( current_property[keys[-1]] = value return figure_config + def _get_parametrized_config( ctd_parameter: list[CallbackTriggerDict], target: ModelID, data_frame: bool ) -> dict[str, Any]: @@ -275,7 +276,9 @@ def _get_modified_page_figures( # Also, it was a good decision to return action output as key: value pairs for the predefined actions. _get_unfiltered_data_targets = list(set(figure_targets + control_targets_targets)) - figure_targets_unfiltered_data: dict[ModelID, pd.DataFrame] = _get_unfiltered_data(ctds_parameter, _get_unfiltered_data_targets) + figure_targets_unfiltered_data: dict[ModelID, pd.DataFrame] = _get_unfiltered_data( + ctds_parameter, _get_unfiltered_data_targets + ) for target, unfiltered_data in figure_targets_unfiltered_data.items(): if target in figure_targets: @@ -292,8 +295,7 @@ def _get_modified_page_figures( current_value = [] outputs[target] = model_manager[target]( - target_to_data_frame=figure_targets_unfiltered_data, - current_value=current_value + target_to_data_frame=figure_targets_unfiltered_data, current_value=current_value ) return outputs diff --git a/vizro-core/src/vizro/actions/_on_page_load_action.py b/vizro-core/src/vizro/actions/_on_page_load_action.py index dbd06b016..c4cea3e05 100644 --- a/vizro-core/src/vizro/actions/_on_page_load_action.py +++ b/vizro-core/src/vizro/actions/_on_page_load_action.py @@ -7,7 +7,6 @@ from vizro.actions._actions_utils import _get_modified_page_figures from vizro.managers._model_manager import ModelID from vizro.models.types import capture -from vizro.managers import model_manager, data_manager @capture("action") @@ -46,17 +45,17 @@ def _on_page_load(targets: list[ModelID], **inputs: dict[str, Any]) -> dict[Mode # if current_value in ["ALL", ["ALL"]]: # current_value = [] - # TODO: Also propagate DFP values into the load() method - # 1. "new_options"/"min/max" DOES NOT include the "current_value" - # filter_obj._set_categorical_selectors_options(force=True, current_value=[]) + # TODO: Also propagate DFP values into the load() method + # 1. "new_options"/"min/max" DOES NOT include the "current_value" + # filter_obj._set_categorical_selectors_options(force=True, current_value=[]) - # 2. "new_options" DOES include the "current_value" - # filter_obj._set_categorical_selectors_options(force=True, current_value=current_value) - # filter_obj._set_numerical_and_temporal_selectors_values(force=True, current_value=current_value) + # 2. "new_options" DOES include the "current_value" + # filter_obj._set_categorical_selectors_options(force=True, current_value=current_value) + # filter_obj._set_numerical_and_temporal_selectors_values(force=True, current_value=current_value) - # return_obj[filter_id] = filter_obj.selector(on_page_load_value=current_value) + # return_obj[filter_id] = filter_obj.selector(on_page_load_value=current_value) - # return_obj[filter_id] = filter_obj(current_value=current_value) + # return_obj[filter_id] = filter_obj(current_value=current_value) print("ON PAGE LOAD - END\n") diff --git a/vizro-core/src/vizro/models/_components/form/dropdown.py b/vizro-core/src/vizro/models/_components/form/dropdown.py index f6ef1fbe9..32bdd01be 100755 --- a/vizro-core/src/vizro/models/_components/form/dropdown.py +++ b/vizro-core/src/vizro/models/_components/form/dropdown.py @@ -125,12 +125,8 @@ def _build_dynamic_placeholder(self): children=[ dbc.Label(self.title, html_for=self.id) if self.title else None, dmc.DateRangePicker( - id=self.id, - value=self.value, - persistence=True, - persistence_type="session", - style={'opacity': 0} - ) + id=self.id, value=self.value, persistence=True, persistence_type="session", style={"opacity": 0} + ), ] ) diff --git a/vizro-core/src/vizro/models/_components/form/range_slider.py b/vizro-core/src/vizro/models/_components/form/range_slider.py index bcc77405e..cd2857afa 100644 --- a/vizro-core/src/vizro/models/_components/form/range_slider.py +++ b/vizro-core/src/vizro/models/_components/form/range_slider.py @@ -126,7 +126,7 @@ def _build_static(self, current_value=None, new_min=None, new_max=None, **kwargs persistence_type="session", className="slider-text-input-field", ), - dcc.Store(id=f"{self.id}_input_store", storage_type="session") + dcc.Store(id=f"{self.id}_input_store", storage_type="session"), ], className="slider-text-input-container", ), @@ -154,4 +154,4 @@ def _build_dynamic_placeholder(self): def build(self): # TODO: We don't have to implement _build_dynamic_placeholder, _build_static here. It's possible to: # if dynamic and self.value is None -> set self.value + return standard build (static) - return self._build_dynamic_placeholder() if self._dynamic else self._build_static() \ No newline at end of file + return self._build_dynamic_placeholder() if self._dynamic else self._build_static() diff --git a/vizro-core/src/vizro/models/_components/form/slider.py b/vizro-core/src/vizro/models/_components/form/slider.py index 20bc6a1b2..7e65ac644 100644 --- a/vizro-core/src/vizro/models/_components/form/slider.py +++ b/vizro-core/src/vizro/models/_components/form/slider.py @@ -137,7 +137,7 @@ def _build_static(self, current_value=None, new_min=None, new_max=None, **kwargs persistence_type="session", className="slider-text-input-field", ), - dcc.Store(id=f"{self.id}_input_store", storage_type="session") + dcc.Store(id=f"{self.id}_input_store", storage_type="session"), ], className="slider-text-input-container", ), diff --git a/vizro-core/src/vizro/models/_controls/filter.py b/vizro-core/src/vizro/models/_controls/filter.py index 0d1270381..90dfe55e9 100644 --- a/vizro-core/src/vizro/models/_controls/filter.py +++ b/vizro-core/src/vizro/models/_controls/filter.py @@ -1,11 +1,10 @@ from __future__ import annotations -from dash import dcc, html - from typing import Any, Literal, Union import numpy as np import pandas as pd +from dash import dcc from pandas.api.types import is_datetime64_any_dtype, is_numeric_dtype from vizro.managers._data_manager import DataSourceName @@ -18,6 +17,7 @@ from vizro._constants import FILTER_ACTION_PREFIX from vizro.actions import _filter from vizro.managers import data_manager, model_manager +from vizro.managers._data_manager import _DynamicData from vizro.managers._model_manager import ModelID from vizro.models import Action, VizroBaseModel from vizro.models._components.form import ( @@ -30,8 +30,6 @@ ) from vizro.models._models_utils import _log_call from vizro.models.types import MultiValueType, SelectorType -from vizro.models._components.form._form_utils import get_options_and_default -from vizro.managers._data_manager import _DynamicData # Ideally we might define these as NumericalSelectorType = Union[RangeSlider, Slider] etc., but that will not work # with isinstance checks. @@ -179,12 +177,10 @@ def pre_build(self): # Selector doesn't support dynamic mode # Selector is categorical and "options" is defined # Selector is numerical/Temporal and "min" and "max" are defined - if ( - isinstance(self.selector, DYNAMIC_SELECTORS) and - ( - hasattr(self.selector, "options") and not getattr(self.selector, "options") or - all(hasattr(self.selector, attr) and getattr(self.selector, attr) is None for attr in ["min", "max"]) - ) + if isinstance(self.selector, DYNAMIC_SELECTORS) and ( + hasattr(self.selector, "options") + and not getattr(self.selector, "options") + or all(hasattr(self.selector, attr) and getattr(self.selector, attr) is None for attr in ["min", "max"]) ): for target_id in self.targets: data_source_name = model_manager[target_id]["data_frame"] diff --git a/vizro-core/src/vizro/static/js/models/range_slider.js b/vizro-core/src/vizro/static/js/models/range_slider.js index e9c782324..7b3989ecf 100644 --- a/vizro-core/src/vizro/static/js/models/range_slider.js +++ b/vizro-core/src/vizro/static/js/models/range_slider.js @@ -28,19 +28,33 @@ function update_range_slider_values( } return [start, end, [start, end], [start, end]]; - // slider component is the trigger + // slider component is the trigger } else if (trigger_id === self_data["id"]) { return [slider[0], slider[1], slider, slider]; - // on_page_load is the trigger + // on_page_load is the trigger } else { if (input_store === null) { - return [dash_clientside.no_update, dash_clientside.no_update, dash_clientside.no_update, slider]; - } - else { - if (slider[0] === start && input_store[0] === start && slider[1] === end && input_store[1] === end){ + return [ + dash_clientside.no_update, + dash_clientside.no_update, + dash_clientside.no_update, + slider, + ]; + } else { + if ( + slider[0] === start && + input_store[0] === start && + slider[1] === end && + input_store[1] === end + ) { // To prevent filter_action to be triggered after on_page_load - return [dash_clientside.no_update, dash_clientside.no_update, dash_clientside.no_update, dash_clientside.no_update]; + return [ + dash_clientside.no_update, + dash_clientside.no_update, + dash_clientside.no_update, + dash_clientside.no_update, + ]; } return [input_store[0], input_store[1], input_store, input_store]; } diff --git a/vizro-core/src/vizro/static/js/models/slider.js b/vizro-core/src/vizro/static/js/models/slider.js index 2155bf83f..75264eecd 100644 --- a/vizro-core/src/vizro/static/js/models/slider.js +++ b/vizro-core/src/vizro/static/js/models/slider.js @@ -14,26 +14,28 @@ function update_slider_values(start, slider, input_store, self_data) { } return [start, start, start]; - // slider component is the trigger + // slider component is the trigger } else if (trigger_id === self_data["id"]) { return [slider, slider, slider]; - // on_page_load is the trigger + // on_page_load is the trigger } else { if (input_store === null) { return [dash_clientside.no_update, dash_clientside.no_update, slider]; - } - else { + } else { if (slider === start && start === input_store) { // To prevent filter_action to be triggered after on_page_load - return [dash_clientside.no_update, dash_clientside.no_update, dash_clientside.no_update]; + return [ + dash_clientside.no_update, + dash_clientside.no_update, + dash_clientside.no_update, + ]; } return [input_store, input_store, input_store]; } } } - window.dash_clientside = { ...window.dash_clientside, slider: { update_slider_values: update_slider_values }, From 8ca538239207839fb966bd24f913e17b1699b6ee Mon Sep 17 00:00:00 2001 From: petar-qb Date: Fri, 15 Nov 2024 14:41:15 +0100 Subject: [PATCH 26/64] Minor code cleaning --- .../scratch_dev/_poc_dynamic_controls.py | 397 ------------------ vizro-core/examples/scratch_dev/app.py | 4 +- vizro-core/examples/scratch_dev/data.yaml | 2 +- vizro-core/hatch.toml | 2 +- .../src/vizro/actions/_actions_utils.py | 39 +- .../src/vizro/actions/_on_page_load_action.py | 26 -- .../models/_components/form/_form_utils.py | 5 +- .../models/_components/form/checklist.py | 6 +- .../vizro/models/_components/form/dropdown.py | 14 +- .../models/_components/form/radio_items.py | 6 +- .../models/_components/form/range_slider.py | 8 +- .../vizro/models/_components/form/slider.py | 35 +- .../src/vizro/models/_controls/filter.py | 18 +- vizro-core/src/vizro/models/_page.py | 5 +- .../src/vizro/static/js/models/dashboard.js | 2 +- 15 files changed, 55 insertions(+), 514 deletions(-) delete mode 100644 vizro-core/examples/scratch_dev/_poc_dynamic_controls.py diff --git a/vizro-core/examples/scratch_dev/_poc_dynamic_controls.py b/vizro-core/examples/scratch_dev/_poc_dynamic_controls.py deleted file mode 100644 index 6cf07f73d..000000000 --- a/vizro-core/examples/scratch_dev/_poc_dynamic_controls.py +++ /dev/null @@ -1,397 +0,0 @@ -import dash -import plotly.express as px - -from dash import Dash, html, dcc, Output, callback, clientside_callback, Input, State, set_props -import dash_mantine_components as dmc - - -CONTROL_SELECTOR = dcc.RangeSlider - -IS_DROPDOWN_MULTI = True - - -if CONTROL_SELECTOR in {dcc.Checklist, dcc.RangeSlider}: - MULTI = True -elif CONTROL_SELECTOR in {dcc.Slider, dcc.RadioItems}: - MULTI = False -elif CONTROL_SELECTOR == dcc.Dropdown: - if IS_DROPDOWN_MULTI not in [False, True]: - raise ValueError("IS_DROPDOWN_MULTI must be set to True or False for dcc.Dropdown selector.") - MULTI = IS_DROPDOWN_MULTI -else: - raise ValueError( - "Invalid CONTROL_SELECTOR. Must be one of: " - "dcc.Dropdown, dcc.Checklist, dcc.RadioItems, dcc.Slider, or dcc.RangeSlider." - ) - - -# Hardcoded global variable. -SELECTOR_TYPE = { - "categorical": [dcc.Dropdown, dcc.Checklist, dcc.RadioItems], - "numerical": [dcc.Slider, dcc.RangeSlider], -} - - -# like dynamic data -def slow_load(): - print("running slow_load") - time.sleep(0.1) - return px.data.iris().sample(6) - - -# Like pre-build - doesn't get run again when reload page -def categorical_filter_pre_build(): - df = slow_load() - options = sorted(df["species"].unique().tolist()) - return options, options if MULTI else options[0] - - -def numerical_filter_pre_build(): - df = slow_load() - _min = float(df["sepal_length"].min()) - _max = float(df["sepal_length"].max()) - return _min, _max, [_min, _max] if MULTI else _min - - -# TODO-TEST: You can hardcode these values for testing purposes. They represent initial options/min/max/value -# for the filter that are created in and sent from page.build() every time page is refreshed. -pre_build_options, pre_build_categorical_value = categorical_filter_pre_build() -pre_build_min, pre_build_max, pre_build_numerical_value = numerical_filter_pre_build() - - -# --- Pages --- -common = [ - html.H1(id="dashboard_title", children="Dashboard"), - html.Div(dcc.Link("Homepage", href="/")), - html.Div(dcc.Link("Another page", href="/another-page")), -] - - -def make_page(content): - page_build_obj = html.Div( - [ - *common, - html.P(datetime.datetime.now()), - *content, - ] - ) - return page_build_obj - - -# homepage build -def homepage(**kwargs): - return make_page([html.H2("Homepage")]) - - -# Like filter build - gets run every time page is loaded -def categorical_filter_build(options=None): - kwargs = {} - if CONTROL_SELECTOR == dcc.Dropdown: - kwargs["multi"] = MULTI - - return CONTROL_SELECTOR( - id=f'filter', - options=options or pre_build_options, - value=pre_build_categorical_value, - persistence=True, - persistence_type="session", - **kwargs - ) - - -def numerical_filter_build(min_value=None, max_value=None): - return CONTROL_SELECTOR( - id=f'filter', - min=min_value or pre_build_min, - max=max_value or pre_build_max, - value=pre_build_numerical_value, - step=0.1, - persistence=True, - persistence_type="session", - ) - - -# Like another-page build -def another_page(**kwargs): - def _get_initial_page_build_object(): - if CONTROL_SELECTOR == dcc.Dropdown: - # A hack explained in on_page_load To-do:"Limitations" section below in this page. - return dmc.DateRangePicker( - id='filter', - value=pre_build_categorical_value, - persistence=True, - persistence_type="session", - ) - elif CONTROL_SELECTOR in SELECTOR_TYPE["categorical"]: - return categorical_filter_build() - elif CONTROL_SELECTOR in SELECTOR_TYPE["numerical"]: - return numerical_filter_build() - else: - raise ValueError("Invalid CONTROL_SELECTOR.") - - return make_page( - [ - dcc.Store(id="on_page_load_trigger_another_page"), - html.H2("Another page"), - - # # This does NOT work because id="filter" doesn't exist but is used as OPL callback State. - # dcc.Loading(id="filter_container"), - - # # Possible solution is to alter filter.options from on_page_load. This would work, but it's not optimal. - # dcc.Dropdown(id="filter", options=options, value=options, multi=True, persistence=True), - - # # Outer container can be changed with dcc.Loading. - html.Div( - _get_initial_page_build_object(), - id="filter_container", - ), - - # # Does not work because OPL filter input is missing, but it's used for filtering figures data_frame. - # html.Div( - # html.Div(id="filter"), - # id="filter_container", - # ), - - html.Br(), - dcc.RadioItems(id="parameter", options=["sepal_width", "sepal_length"], value="sepal_width", persistence=True, persistence_type="session"), - dcc.Loading(dcc.Graph(id="graph1")), - dcc.Loading(dcc.Graph(id="graph2")), - ] - ) - - -def graph1_call(data_frame): - return px.bar(data_frame, x="species", color="species") - - -def graph2_call(data_frame, x): - return px.scatter(data_frame, x=x, y="petal_width", color="species") - - -# NOTE: -# You could do just update_filter to update options/value rather than replacing whole dcc.Dropdown object. Then would -# need to write it for rangeslider and dropdown etc. separately though. Probably easier to just replace whole object. -# This is consistent with how Graph, AgGrid etc. work. -# BUT controls are different from Graphs since you can set the pre-selected value that should be shown when -# user first visits page. Is this possible still with dynamic filter? -> YES - - -def get_data(species): - df = slow_load() - return df[df["species"].isin(species)] - - -# This mechanism is not actually necessary in this example since can just let OPL run without this trigger removing -# prevent_initial_call=True from it. -clientside_callback( - """ - function trigger_to_global_store(data) { - return data; - } - """, - Output("global_on_page_load_another_page_action_trigger", "data"), - Input("on_page_load_trigger_another_page", "data"), - prevent_initial_call=True, # doesn't do anything - callback still runs -) - - -# TODO: write something like get_modified_figures function to reduce repetition. - - -@callback( - output=[ - Output("graph1", "figure"), - Output("graph2", "figure"), - Output("filter_container", "children"), - ], - inputs=[ - Input("global_on_page_load_another_page_action_trigger", "data"), - - State("filter", "value"), - State("parameter", "value"), - ], - prevent_initial_call=True -) -def on_page_load(data, persisted_filter_value, x): - # Ideally, OPL flow should look like this: - # 1. Page.build() -> returns static layout (placeholder elements for dynamic components). - # 2. Persistence is applied. -> So, filter values are the same as the last time the page was visited. - # 3. OPL -> returns dynamic components based on the controls values (persisted) - # 3.1. Load DFs (include DFP values here) - # 3.2. Calculate new filter values: - # e.g. new_filter_values = [value for value in persisted_filter_value if value in new_filter_options] - # e.g. new_min = max(persisted_min, new_min); new_max = min(persisted_max, new_max) - # 3.3. Apply filters on DFs - # 3.4. Apply parameters on config - # 3.5. Return dynamic components (Figures and dynamic controls) - - # Why actions are better than dash.callback here? - # 1. They solve the circular dependency problem of the full graph. - # 2. They are explicit which means they can be configured in any way users want. There's no undesired behavior. - - # TODO: Last solution found -> hence put in highlighted TODO: - # 1. page.build() -> returns: - # 1.1. html.Div(html.Div(id="filter"), id="filter_container") - # * Does not work because we need persisted filter input value in OPL, so we can filter figures data_frame. * - # 1.2. html.Div(dcc.Dropdown(id="filter", ...), id="filter_container") - # * It works! :D * - # 2. OPL -> Manipulations with filter and options: - # 2.1. Recalculate options. - # 2.2. Recalculated value. (persisted_filter_value that exists in recalculated options) - # 2.3. Filter figures data_frame with recalculated value. - # 2.4. Create a new filter object with recalculated options and original value. - # Limitations: - # 1. do_filter is triggered automatically after OPL. - # This shouldn't be the issue since actions loop controls it. - # 2. Component persistence updating works slightly different for dcc.Dropdown than for other selector components. - # Persistence for Dropdown is set even when the Dropdown is returned as a new object from page_build or OPL. - # In persistence.js -> LN:309 "recordUiEdit" function is triggered when dropdown is returned from the server. - # It causes storage.SetItem() to be triggered which mess-ups the persistence for the Dropdown. - # This is probably dash bug because Dropdown is handled a lot with async which probably causes that returned - # Dropdown from the page_build or OPL triggers the "recordUiEdit" which should not trigger. - # ** Problem is solved by returning dmc.DateRangePicker instead of dcc.Dropdown from page.build. ** - # --- (A.M.): How to achieve all of these: --- - # * get correct selected value passed into graph calls -> Works with this solution. - # * populate filter with right values for user on first page load -> Works with this solution. - # * update filter options on page load -> Works with this solution. - # * persist filter values on page change -> Works with this solution. - - print("running on_page_load") - df = slow_load() - - # --- Calculate categorical filter --- - if CONTROL_SELECTOR in SELECTOR_TYPE["categorical"]: - categorical_filter_options = sorted(df["species"].unique().tolist()) - if MULTI: - categorical_filter_value = [value for value in persisted_filter_value if value in categorical_filter_options] - else: - categorical_filter_value = persisted_filter_value if persisted_filter_value in categorical_filter_options else None - new_filter_obj = categorical_filter_build(options=categorical_filter_options) - - # --- Filtering data: --- - if MULTI: - df = df[df["species"].isin(categorical_filter_value)] - else: - df = df[df["species"].isin([categorical_filter_value])] - - # --- set_props --- - # set_props(component_id="filter_container", props={"children": new_filter_obj}) - # More about set_props: - # -> https://dash.plotly.com/advanced-callbacks#setting-properties-directly - # -> https://community.plotly.com/t/dash-2-17-0-released-callback-updates-with-set-props-no-output-callbacks-layout-as-list-dcc-loading-trace-zorder/84343 - # Limitations: - # 1. Component properties updated using set_props won't appear in the callback graph for debugging. - # - This is not a problem because our graph debugging is already unreadable. :D - # 2. Component properties updated using set_props won't appear as loading when they are wrapped with a `dcc.Loading` component. - # - Potential solution. Set controls as dash.Output and then use set_props to update them + dash.no_update as a return value for them. - # 3. set_props doesn't validate the id or property names provided, so no error will be displayed if they contain typos. This can make apps that use set_props harder to debug. - # - That's okay since it's internal Vizro stuff and shouldn't affect user. - # 4. Using set_props with chained callbacks may lead to unexpected results. - # - It even behaves better because it doesn't trigger the "do_filter" callback. - # Open questions: - # 1. Is there any concern about different filter selectors? -> No. (I haven't tested the DatePicker it yet.) - # 2. Can we handle if filter selector changes dynamically? -> Potentially, (I haven't tested it yet.) - # 3. Is there a bug with set_props or with dash.Output?! - - # --- Calculate numerical filter --- - if CONTROL_SELECTOR in SELECTOR_TYPE["numerical"]: - numerical_filter_min = float(df["sepal_length"].min()) - numerical_filter_max = float(df["sepal_length"].max()) - if MULTI: - numerical_filter_value = [max(numerical_filter_min, persisted_filter_value[0]), min(numerical_filter_max, persisted_filter_value[1])] - else: - numerical_filter_value = persisted_filter_value if numerical_filter_min < persisted_filter_value < numerical_filter_max else numerical_filter_min - new_filter_obj = numerical_filter_build(min_value=numerical_filter_min, max_value=numerical_filter_max) - # set_props(component_id="numerical_filter_container", props={"children": new_filter_obj}) - - # --- Filtering data: --- - if MULTI: - df = df[(df["sepal_length"] >= numerical_filter_value[0]) & (df["sepal_length"] <= numerical_filter_value[1])] - else: - df = df[(df["sepal_length"] == numerical_filter_value)] - - print("") - return graph1_call(df), graph2_call(df, x), new_filter_obj - - -# @callback( -# Output("graph1", "figure", allow_duplicate=True), -# Output("graph2", "figure", allow_duplicate=True), -# Input("filter", "value"), -# State("parameter", "value"), -# prevent_initial_call=True, -# ) -# def do_filter(species, x): -# print("running do_filter") -# -# # This also works - filter is calculated on filter value select: -# # It also makes that filter/df1/df2 are calculated based on the same data. Should we enable that? -# # df = slow_load() -# # filter_options = df["species"].unique() -# # filter_value = [value for value in species if value in filter_options] -# # filter_obj = filter_call(filter_options, filter_value) -# # df1 = df2 = df[df["species"].isin(filter_value)] -# # set_props(component_id="filter_container", props={"children": filter_obj}) -# -# df1 = get_data(species) -# df2 = get_data(species) -# print("") -# return graph1_call(df1), graph2_call(df2, x) -# -# -# @callback( -# Output("graph2", "figure", allow_duplicate=True), -# Input("parameter", "value"), -# State("filter", "value"), -# prevent_initial_call=True, -# ) -# def do_parameter(x, species): -# print("running do_parameter") -# df1 = get_data(species) -# print("") -# return graph2_call(df1, x) - - -app = Dash(use_pages=True, pages_folder="", suppress_callback_exceptions=True) -dash.register_page("/", layout=homepage) -dash.register_page("another_page", layout=another_page) - -app.layout = html.Div([dcc.Store("global_on_page_load_another_page_action_trigger"), dash.page_container]) - - -##### NEXT STEPS FOR PETAR - -# How to update dynamic filter? -# Options: -# 1. on_page_load_controls and then on_page_load_components sequentially. Need to figure out how to get components -# into loading state to begin with - set as loading build and then change back in OPL callback? Means two callbacks. -# 2. on_page_load_controls and then on_page_load_components in parallel. NO, bad when caching -# 3. on_page_load_everything. THIS IS THE ONE WE PREFER. -# Can't have on_page_load_controls trigger regular "apply filter" etc. callbacks as could lead to many of them in -# parallel. - -# So need to make sure that either method 1 or 3 doesn't trigger regular callbacks. Not sure -# how to achieve this... -# Could put manual no_update in those regular callbacks but is not nice. -# Could actually just do on_page_load_controls and then use all regular callbacks in parallel - so long as caching -# turned on then on_page_load_controls will have warmed it up so then no problem with regular callbacks. -# But still not good because regular callbacks will override same output graph multiple times. - -# Maybe actually need on_page_load_controls to trigger regular filters in general? And just not have too many of them. - -# persistence still works -# changing page now does on_page_load which then triggers do_filter -# so effectively running do_filter twice -# How can we avoid this? - -# Consider actions loop and when one callback should trigger another etc. - -# How does persistence work? -# How does triggering callbacks work in vizro? -# How *should* triggering callbacks work in vizro? Can we align it more with Dash? -# How to handle filter options persistence and updating etc.? -# How to avoid the regular filters being triggered after on_page_load runs? -# IMPORTANT: also consider parametrised data case. - -if __name__ == "__main__": - app.run(debug=True, dev_tools_hot_reload=False) \ No newline at end of file diff --git a/vizro-core/examples/scratch_dev/app.py b/vizro-core/examples/scratch_dev/app.py index 96befdc57..55a8fcb28 100644 --- a/vizro-core/examples/scratch_dev/app.py +++ b/vizro-core/examples/scratch_dev/app.py @@ -61,7 +61,7 @@ def load_from_file(filter_column=None, parametrized_species=None): data_manager["load_from_file_date_column"] = partial(load_from_file, filter_column="date_column") -# # TODO-DEV: Turn on/off caching to see how it affects the app. +# TODO-DEV: Turn on/off caching to see how it affects the app. # data_manager.cache = Cache(config={"CACHE_TYPE": "SimpleCache", "CACHE_DEFAULT_TIMEOUT": 5}) @@ -135,7 +135,7 @@ def load_from_file(filter_column=None, parametrized_species=None): ) page_4 = vm.Page( - title="Temporal dynamic selectors", + title="[TO BE DONE IN THE FOLLOW UP PR] Temporal dynamic selectors", components=[ vm.Graph( id="p4-G-1", diff --git a/vizro-core/examples/scratch_dev/data.yaml b/vizro-core/examples/scratch_dev/data.yaml index dc352a757..04854efbd 100644 --- a/vizro-core/examples/scratch_dev/data.yaml +++ b/vizro-core/examples/scratch_dev/data.yaml @@ -10,4 +10,4 @@ max: 7 # Choose from: # 2020-01-01 to 2020-05-29 date_min: 2024-01-01 -date_max: 2024-05-29 \ No newline at end of file +date_max: 2024-05-29 diff --git a/vizro-core/hatch.toml b/vizro-core/hatch.toml index 1a34adcba..a8165f25e 100644 --- a/vizro-core/hatch.toml +++ b/vizro-core/hatch.toml @@ -105,7 +105,7 @@ template = "examples" [envs.examples.env-vars] DASH_DEBUG = "true" -VIZRO_LOG_LEVEL = "WARNING" +VIZRO_LOG_LEVEL = "DEBUG" [envs.lower-bounds] extra-dependencies = [ diff --git a/vizro-core/src/vizro/actions/_actions_utils.py b/vizro-core/src/vizro/actions/_actions_utils.py index 5f2855ed6..422d10834 100644 --- a/vizro-core/src/vizro/actions/_actions_utils.py +++ b/vizro-core/src/vizro/actions/_actions_utils.py @@ -48,18 +48,18 @@ def _get_component_actions(component) -> list[Action]: def _apply_filter_controls( - data_frame: pd.DataFrame, ctds_filters: list[CallbackTriggerDict], target: ModelID + data_frame: pd.DataFrame, ctds_filter: list[CallbackTriggerDict], target: ModelID ) -> pd.DataFrame: """Applies filters from a vm.Filter model in the controls. Args: data_frame: unfiltered DataFrame. - ctds_filters: list of CallbackTriggerDict for filters. + ctds_filter: list of CallbackTriggerDict for filters. target: id of targeted Figure. Returns: filtered DataFrame. """ - for ctd in ctds_filters: + for ctd in ctds_filter: selector_value = ctd["value"] selector_value = selector_value if isinstance(selector_value, list) else [selector_value] selector_actions = _get_component_actions(model_manager[ctd["id"]]) @@ -163,12 +163,12 @@ def _update_nested_figure_properties( return figure_config def _get_parametrized_config( - ctd_parameter: list[CallbackTriggerDict], target: ModelID, data_frame: bool + ctds_parameter: list[CallbackTriggerDict], target: ModelID, data_frame: bool ) -> dict[str, Any]: """Convert parameters into a keyword-argument dictionary. Args: - ctd_parameter: list of CallbackTriggerDicts for vm.Parameter. + ctds_parameter: list of CallbackTriggerDicts for vm.Parameter. target: id of targeted figure. data_frame: whether to return only DataFrame parameters starting "data_frame." or only non-DataFrame parameters. @@ -186,7 +186,7 @@ def _get_parametrized_config( config = deepcopy(model_manager[target].figure._arguments) del config["data_frame"] - for ctd in ctd_parameter: + for ctd in ctds_parameter: # TODO: needs to be refactored so that it is independent of implementation details parameter_value = ctd["value"] @@ -222,7 +222,7 @@ def _apply_filters( # Takes in just one target, so dataframe is filtered repeatedly for every target that uses it. # Potentially this could be de-duplicated but it's not so important since filtering is a relatively fast # operation (compared to data loading). - filtered_data = _apply_filter_controls(data_frame=data, ctds_filters=ctds_filter, target=target) + filtered_data = _apply_filter_controls(data_frame=data, ctds_filter=ctds_filter, target=target) filtered_data = _apply_filter_interaction( data_frame=filtered_data, ctds_filter_interaction=ctds_filter_interaction, target=target ) @@ -234,13 +234,13 @@ def _get_unfiltered_data( ) -> dict[ModelID, pd.DataFrame]: # Takes in multiple targets to ensure that data can be loaded efficiently using _multi_load and not repeated for # every single target. - # Getting unfiltered data requires data frame parameters. We pass in all ctd_parameter and then find the + # Getting unfiltered data requires data frame parameters. We pass in all ctds_parameter and then find the # data_frame ones by passing data_frame=True in the call to _get_paramaterized_config. Static data is also # handled here and will just have empty dictionary for its kwargs. multi_data_source_name_load_kwargs: list[tuple[DataSourceName, dict[str, Any]]] = [] for target in targets: dynamic_data_load_params = _get_parametrized_config( - ctd_parameter=ctds_parameter, target=target, data_frame=True + ctds_parameter=ctds_parameter, target=target, data_frame=True ) data_source_name = model_manager[target]["data_frame"] multi_data_source_name_load_kwargs.append((data_source_name, dynamic_data_load_params["data_frame"])) @@ -254,14 +254,13 @@ def _get_modified_page_figures( ctds_parameter: list[CallbackTriggerDict], targets: list[ModelID], ) -> dict[ModelID, Any]: - outputs: dict[ModelID, Any] = {} - from vizro.models import Filter + outputs: dict[ModelID, Any] = {} + control_targets = [] control_targets_targets = [] figure_targets = [] - for target in targets: target_obj = model_manager[target] if isinstance(target_obj, Filter): @@ -270,19 +269,25 @@ def _get_modified_page_figures( else: figure_targets.append(target) - # Retrieving only figure_targets data_frames from multi_load is not the best solution. We assume that Filter.targets - # are the subset of the action's targets. This works for the on_page_load, but will not work if explicitly set. - # Also, it was a good decision to return action output as key: value pairs for the predefined actions. + # Retrieving only figure_targets data_frames from _multi_load is not the best solution. + # In that way, we assume that Filter.targets are the subset of the action's targets. This works for the + # on_page_load, but will not work if targets are explicitly set. + # For example, in future, if Parameter is targeting only a single Filter. _get_unfiltered_data_targets = list(set(figure_targets + control_targets_targets)) - figure_targets_unfiltered_data: dict[ModelID, pd.DataFrame] = _get_unfiltered_data(ctds_parameter, _get_unfiltered_data_targets) + figure_targets_unfiltered_data: dict[ModelID, pd.DataFrame] = _get_unfiltered_data( + ctds_parameter=ctds_parameter, targets=_get_unfiltered_data_targets + ) + # TODO: the structure here would be nicer if we could get just the ctds for a single target at one time, + # so you could do apply_filters on a target a pass only the ctds relevant for that target. + # Consider restructuring ctds to a more convenient form to make this possible. for target, unfiltered_data in figure_targets_unfiltered_data.items(): if target in figure_targets: filtered_data = _apply_filters(unfiltered_data, ctds_filter, ctds_filter_interaction, target) outputs[target] = model_manager[target]( data_frame=filtered_data, - **_get_parametrized_config(ctd_parameter=ctds_parameter, target=target, data_frame=False), + **_get_parametrized_config(ctds_parameter=ctds_parameter, target=target, data_frame=False), ) for target in control_targets: diff --git a/vizro-core/src/vizro/actions/_on_page_load_action.py b/vizro-core/src/vizro/actions/_on_page_load_action.py index dbd06b016..464a905bb 100644 --- a/vizro-core/src/vizro/actions/_on_page_load_action.py +++ b/vizro-core/src/vizro/actions/_on_page_load_action.py @@ -32,32 +32,6 @@ def _on_page_load(targets: list[ModelID], **inputs: dict[str, Any]) -> dict[Mode targets=targets, ) - # import vizro.models as vm - # from time import sleep - # sleep(1) - # - # for filter_id, filter_obj in model_manager._items_with_type(vm.Filter): - # if filter_obj._dynamic: - # current_value = [ - # item for item in ctx.args_grouping["external"]["filters"] - # if item["id"] == filter_obj.selector.id - # ][0]["value"] - # - # if current_value in ["ALL", ["ALL"]]: - # current_value = [] - - # TODO: Also propagate DFP values into the load() method - # 1. "new_options"/"min/max" DOES NOT include the "current_value" - # filter_obj._set_categorical_selectors_options(force=True, current_value=[]) - - # 2. "new_options" DOES include the "current_value" - # filter_obj._set_categorical_selectors_options(force=True, current_value=current_value) - # filter_obj._set_numerical_and_temporal_selectors_values(force=True, current_value=current_value) - - # return_obj[filter_id] = filter_obj.selector(on_page_load_value=current_value) - - # return_obj[filter_id] = filter_obj(current_value=current_value) - print("ON PAGE LOAD - END\n") return return_obj diff --git a/vizro-core/src/vizro/models/_components/form/_form_utils.py b/vizro-core/src/vizro/models/_components/form/_form_utils.py index 618f60b01..d1c9b80e9 100644 --- a/vizro-core/src/vizro/models/_components/form/_form_utils.py +++ b/vizro-core/src/vizro/models/_components/form/_form_utils.py @@ -45,7 +45,6 @@ def validate_options_dict(cls, values): return values -# TODO: Check this below again def validate_value(cls, value, values): """Reusable validator for the "value" argument of categorical selectors.""" if "options" not in values or not values["options"]: @@ -55,8 +54,8 @@ def validate_value(cls, value, values): [entry["value"] for entry in values["options"]] if isinstance(values["options"][0], dict) else values["options"] ) - # if value and not is_value_contained(value, possible_values): - # raise ValueError("Please provide a valid value from `options`.") + if value and ALL_OPTION not in value and not is_value_contained(value, possible_values): + raise ValueError("Please provide a valid value from `options`.") return value diff --git a/vizro-core/src/vizro/models/_components/form/checklist.py b/vizro-core/src/vizro/models/_components/form/checklist.py index e2ae77902..617b6c527 100644 --- a/vizro-core/src/vizro/models/_components/form/checklist.py +++ b/vizro-core/src/vizro/models/_components/form/checklist.py @@ -69,13 +69,13 @@ def _build_static(self, new_options=None, **kwargs): ) def _build_dynamic_placeholder(self): - if not self.value: + if self.value is None: self.value = [get_options_and_default(self.options, multi=True)[1]] return self._build_static() @_log_call def build(self): - # TODO: We don't have to implement _build_dynamic_placeholder, _build_static here. It's possible to: - # if dynamic and self.value is None -> set self.value + return standard build (static) + # We don't have to implement _build_dynamic_placeholder, _build_static here. It's possible to: if dynamic and + # self.value is None -> set self.value + return standard build (static), but let's align it with the Dropdown. return self._build_dynamic_placeholder() if self._dynamic else self._build_static() diff --git a/vizro-core/src/vizro/models/_components/form/dropdown.py b/vizro-core/src/vizro/models/_components/form/dropdown.py index f6ef1fbe9..56cffb878 100755 --- a/vizro-core/src/vizro/models/_components/form/dropdown.py +++ b/vizro-core/src/vizro/models/_components/form/dropdown.py @@ -67,8 +67,8 @@ class Dropdown(VizroBaseModel): actions: list[Action] = [] # A private property that allows dynamically updating components - # TODO: Consider making the _dynamic public later. The same property also could be used for all other components. - # For example: vm.Graph could have a dynamic that is by default set on True. + # Consider making the _dynamic public later. The same property could also be used for all other components. + # For example: vm.Graph could have a dynamic that is by default set on True. _dynamic: bool = PrivateAttr(False) # Component properties for actions and interactions @@ -88,7 +88,6 @@ def validate_multi(cls, multi, values): raise ValueError("Please set multi=True if providing a list of default values.") return multi - # Convenience wrapper/syntactic sugar. def __call__(self, new_options=None, **kwargs): return self._build_static(new_options=new_options, **kwargs) @@ -114,13 +113,12 @@ def _build_static(self, new_options=None, **kwargs): def _build_dynamic_placeholder(self): # Setting self.value is kind of Dropdown pre_build method. It sets self.value only the first time if it's None. - # We cannot create pre_build for the Dropdown because it has to be called after vm.Filter.pre_build, but - # nothing guarantees that. - if not self.value: + # We cannot create pre_build for the Dropdown because it has to be called after vm.Filter.pre_build, but nothing + # guarantees that. We can call Filter.selector.pre_build() from the Filter.pre_build() method it we decide that. + if self.value is None: self.value = get_options_and_default(self.options, self.multi)[1] - # return self._build_static() - + # Replace this with the Universal Vizro Placeholder component. return html.Div( children=[ dbc.Label(self.title, html_for=self.id) if self.title else None, diff --git a/vizro-core/src/vizro/models/_components/form/radio_items.py b/vizro-core/src/vizro/models/_components/form/radio_items.py index 99555606c..59004110e 100644 --- a/vizro-core/src/vizro/models/_components/form/radio_items.py +++ b/vizro-core/src/vizro/models/_components/form/radio_items.py @@ -70,13 +70,13 @@ def _build_static(self, new_options=None, **kwargs): ) def _build_dynamic_placeholder(self): - if not self.value: + if self.value is None: self.value = get_options_and_default(self.options, multi=False)[1] return self._build_static() @_log_call def build(self): - # TODO: We don't have to implement _build_dynamic_placeholder, _build_static here. It's possible to: - # if dynamic and self.value is None -> set self.value + return standard build (static) + # We don't have to implement _build_dynamic_placeholder, _build_static here. It's possible to: if dynamic and + # self.value is None -> set self.value + return standard build (static), but let's align it with the Dropdown. return self._build_dynamic_placeholder() if self._dynamic else self._build_static() diff --git a/vizro-core/src/vizro/models/_components/form/range_slider.py b/vizro-core/src/vizro/models/_components/form/range_slider.py index bcc77405e..4fcd175b6 100644 --- a/vizro-core/src/vizro/models/_components/form/range_slider.py +++ b/vizro-core/src/vizro/models/_components/form/range_slider.py @@ -91,8 +91,6 @@ def _build_static(self, current_value=None, new_min=None, new_max=None, **kwargs inputs=inputs, ) - stop = 0 - return html.Div( children=[ dcc.Store(f"{self.id}_callback_data", data={"id": self.id, "min": _min, "max": _max}), @@ -148,10 +146,10 @@ def _build_static(self, current_value=None, new_min=None, new_max=None, **kwargs ) def _build_dynamic_placeholder(self): - return self._build_static(is_dynamic_build=True) + return self._build_static() @_log_call def build(self): - # TODO: We don't have to implement _build_dynamic_placeholder, _build_static here. It's possible to: - # if dynamic and self.value is None -> set self.value + return standard build (static) + # We don't have to implement _build_dynamic_placeholder, _build_static here. It's possible to: if dynamic and + # self.value is None -> set self.value + return standard build (static), but let's align it with the Dropdown. return self._build_dynamic_placeholder() if self._dynamic else self._build_static() \ No newline at end of file diff --git a/vizro-core/src/vizro/models/_components/form/slider.py b/vizro-core/src/vizro/models/_components/form/slider.py index 20bc6a1b2..fc765f3cc 100644 --- a/vizro-core/src/vizro/models/_components/form/slider.py +++ b/vizro-core/src/vizro/models/_components/form/slider.py @@ -86,37 +86,6 @@ def _build_static(self, current_value=None, new_min=None, new_max=None, **kwargs inputs=inputs, ) - # TODO - TLDR: - # if static: -> assign init_value to the dcc.Store - # if dynamic: - # if dynamic_build: -> dcc.Store(id=f"{self.id}_input_store", storage_type="session", data=None) - # if static_build: -> dcc.Store(id=f"{self.id}_input_store", storage_type="session") + on_page_load_value - # to UI components like dcc.Slider and dcc.Input. on_page_load_value is propagated from the OPL() - # + changes on the slider.js so it returns value if is dynamic build and raises no_update of it's static_build. - - # How-it-works?: - # 1. If it's a static component: - # 0. Build method is only called once - in the page.build(), so before the OPL(). - # 1. Return always dcc.Store(data=init_value) -> the persistence will work here. - # 2. Make client_side callback that maps stored value to the one or many UI components. - # -> This callback is triggered before OPL and it ensures that the correct value is propagated to the OPL - # 3. Outcome: persistence storage keeps only dcc.store value. UI components are always correctly selected. - # 2. If it's a dynamic compoenent: - # Build method is called twice - from the page.build() and from the OPL() - # 1. page.build(): - # 1. page_build() is _build_dynamic and it returns dcc.Store(data=None) - "none" is immediatelly - # overwritten with the persited value if it exists. Otherwise it's overwritten with self.value or min. - # 2. Make client_side callback that maps stored value to the one or many UI components. - # -> This callback is triggered before OPL and ensures that the correct value is propagated to the OPL - # 2. OPL(): - # 1. OPL propagates currently selected value (e.g. slider value) to _build_static() - # 2. build static returns dcc.Store(). But it returns slider and other compoents with the slider_value - # propagated from the OPL. - # 3. clienside_callback is triggered again but as all input values are the same it raises - # dash_clienside.no_update and the process is done. Otherwise, filter_action would be triggered - - stop = 0 - return html.Div( children=[ dcc.Store(f"{self.id}_callback_data", data={"id": self.id, "min": _min, "max": _max}), @@ -164,6 +133,6 @@ def _build_dynamic_placeholder(self): @_log_call def build(self): - # TODO: We don't have to implement _build_dynamic_placeholder, _build_static here. It's possible to: - # if dynamic and self.value is None -> set self.value + return standard build (static) + # We don't have to implement _build_dynamic_placeholder, _build_static here. It's possible to: if dynamic and + # self.value is None -> set self.value + return standard build (static), but let's align it with the Dropdown. return self._build_dynamic_placeholder() if self._dynamic else self._build_static() diff --git a/vizro-core/src/vizro/models/_controls/filter.py b/vizro-core/src/vizro/models/_controls/filter.py index 0d1270381..e92787fbe 100644 --- a/vizro-core/src/vizro/models/_controls/filter.py +++ b/vizro-core/src/vizro/models/_controls/filter.py @@ -153,7 +153,7 @@ def pre_build(self): # Or just don't do validation at pre_build time and wait until state is available during build time instead? # What should the load kwargs be here? Remember they need to be {} for static data. # Note that currently _get_unfiltered_data is only suitable for use at runtime since it requires - # ctd_parameters. That could be changed to just reuse that function. + # ctds_parameter. That could be changed to just reuse that function. multi_data_source_name_load_kwargs: list[tuple[DataSourceName, dict[str, Any]]] = [ (model_manager[target]["data_frame"], {}) for target in proposed_targets ] @@ -176,9 +176,9 @@ def pre_build(self): ) # Selector can't be dynamic if: - # Selector doesn't support dynamic mode - # Selector is categorical and "options" is defined - # Selector is numerical/Temporal and "min" and "max" are defined + # 1. Selector doesn't support the dynamic mode + # 2. Selector is categorical and "options" prop is defined + # 3. Selector is numerical and "min" and "max" props are defined if ( isinstance(self.selector, DYNAMIC_SELECTORS) and ( @@ -222,19 +222,13 @@ def pre_build(self): @_log_call def build(self): - # TODO: Align inner and outer id to be the same as for other figure components. + # TODO: Align inner and outer ids to be handled in the same way as for other figure components. selector_build_obj = self.selector.build() return dcc.Loading(id=self.id, children=selector_build_obj) if self._dynamic else selector_build_obj def _validate_targeted_data( self, target_to_data_frame: dict[ModelID, pd.DataFrame], eagerly_raise_column_not_found_error ) -> pd.DataFrame: - # target_to_data_source_name = {target: model_manager[target]["data_frame"] for target in targets} - # data_source_name_to_data = { - # data_source_name: data_manager[data_source_name].load() - # for data_source_name in set(target_to_data_source_name.values()) - # } - target_to_series = {} for target, data_frame in target_to_data_frame.items(): @@ -287,7 +281,7 @@ def _get_min_max(targeted_data: pd.DataFrame, current_value=None) -> tuple[float _max = targeted_data.max(axis=None).item() if current_value: - if isinstance(current_value, list) and len(current_value) == 2: + if isinstance(current_value, list): _min = min(_min, current_value[0]) _max = max(_max, current_value[1]) else: diff --git a/vizro-core/src/vizro/models/_page.py b/vizro-core/src/vizro/models/_page.py index c805b6ea5..90e11b32b 100644 --- a/vizro-core/src/vizro/models/_page.py +++ b/vizro-core/src/vizro/models/_page.py @@ -104,10 +104,11 @@ def pre_build(self): # Does the "postorder DFS" traversal pass work for us? -> More about it: # https://www.geeksforgeeks.org/tree-traversals-inorder-preorder-and-postorder/#postorder-traversal # By introducing this traversal algorithm we should ensure that child pre_build will always be called before - # parent pre_build. Is this the case that solves all the pre_build interdependency future problems as well? + # parent pre_build. Does this case solve all the pre_build interdependency future problems for us? + # Should we also ensure that targeted components are always pre_build before the source components? # The hack below is just to ensure that the pre_build of the Filter components is called before the page - # pre_build calculates targets. + # pre_build calculates on_page_load targets. We need this to check is the Filter _dynamic or not. for control in self.controls: if isinstance(control, Filter): if not control._pre_build_finished: diff --git a/vizro-core/src/vizro/static/js/models/dashboard.js b/vizro-core/src/vizro/static/js/models/dashboard.js index 8b14a7934..e04084788 100644 --- a/vizro-core/src/vizro/static/js/models/dashboard.js +++ b/vizro-core/src/vizro/static/js/models/dashboard.js @@ -30,7 +30,7 @@ function update_graph_theme(figure, theme_selector_checked, vizro_themes) { function collapse_nav_panel(n_clicks, is_open) { if (!n_clicks) { /* Automatically collapses left-side if xs and s-devices are detected*/ - if (window.innerWidth < 6 || window.innerHeight < 6) { + if (window.innerWidth < 576 || window.innerHeight < 576) { return [ false, { From 1e3ba7fd71e3635a6d5e665acb18b3bf75442c70 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 15 Nov 2024 13:46:44 +0000 Subject: [PATCH 27/64] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- vizro-core/src/vizro/actions/_actions_utils.py | 3 +-- vizro-core/src/vizro/models/_controls/filter.py | 10 ++++------ 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/vizro-core/src/vizro/actions/_actions_utils.py b/vizro-core/src/vizro/actions/_actions_utils.py index 6f8851090..2a43d8384 100644 --- a/vizro-core/src/vizro/actions/_actions_utils.py +++ b/vizro-core/src/vizro/actions/_actions_utils.py @@ -298,8 +298,7 @@ def _get_modified_page_figures( current_value = [] outputs[target] = model_manager[target]( - target_to_data_frame=figure_targets_unfiltered_data, - current_value=current_value + target_to_data_frame=figure_targets_unfiltered_data, current_value=current_value ) return outputs diff --git a/vizro-core/src/vizro/models/_controls/filter.py b/vizro-core/src/vizro/models/_controls/filter.py index b67575964..e0f06704d 100644 --- a/vizro-core/src/vizro/models/_controls/filter.py +++ b/vizro-core/src/vizro/models/_controls/filter.py @@ -177,12 +177,10 @@ def pre_build(self): # 1. Selector doesn't support the dynamic mode # 2. Selector is categorical and "options" prop is defined # 3. Selector is numerical and "min" and "max" props are defined - if ( - isinstance(self.selector, DYNAMIC_SELECTORS) and - ( - hasattr(self.selector, "options") and not getattr(self.selector, "options") or - all(hasattr(self.selector, attr) and getattr(self.selector, attr) is None for attr in ["min", "max"]) - ) + if isinstance(self.selector, DYNAMIC_SELECTORS) and ( + hasattr(self.selector, "options") + and not getattr(self.selector, "options") + or all(hasattr(self.selector, attr) and getattr(self.selector, attr) is None for attr in ["min", "max"]) ): for target_id in self.targets: data_source_name = model_manager[target_id]["data_frame"] From 41adc9e9a9f6e22ff058ba914418083727709a9e Mon Sep 17 00:00:00 2001 From: petar-qb Date: Fri, 15 Nov 2024 14:51:05 +0100 Subject: [PATCH 28/64] Minor refactoring --- vizro-core/examples/scratch_dev/app.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/vizro-core/examples/scratch_dev/app.py b/vizro-core/examples/scratch_dev/app.py index 23bbfef3c..d88fcbcad 100644 --- a/vizro-core/examples/scratch_dev/app.py +++ b/vizro-core/examples/scratch_dev/app.py @@ -102,7 +102,7 @@ def load_from_file(filter_column=None, parametrized_species=None): vm.Parameter( targets=["p1-G-1.x", "p1-G-2.x"], selector=vm.RadioItems( - options=["species", "sepal_width"], value="species", title="Simple X-axis parameter" + options=["species", "sepal_width"], title="Simple X-axis parameter" ), ), ], @@ -241,15 +241,8 @@ def load_from_file(filter_column=None, parametrized_species=None): # selector=vm.RangeSlider(id="filter_id", value=[5, 7]), ), vm.Parameter( - id="parameter_x", - targets=[ - "graph_dynamic.x", - ], - selector=vm.Dropdown( - options=["species", "sepal_width"], - value="species", - multi=False, - ), + targets=["graph_dynamic.x"], + selector=vm.RadioItems(options=["species", "sepal_width"], title="Simple X-axis parameter"), ), ], ) From b912bb0db31ebd22f752ba9274bd5332983bb3d0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 15 Nov 2024 13:52:28 +0000 Subject: [PATCH 29/64] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- vizro-core/examples/scratch_dev/app.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/vizro-core/examples/scratch_dev/app.py b/vizro-core/examples/scratch_dev/app.py index d88fcbcad..9cbd9cad3 100644 --- a/vizro-core/examples/scratch_dev/app.py +++ b/vizro-core/examples/scratch_dev/app.py @@ -101,9 +101,7 @@ def load_from_file(filter_column=None, parametrized_species=None): vm.Filter(id="p1-F-2", column="species", targets=["p1-G-2"], selector=vm.Dropdown(title="Static filter")), vm.Parameter( targets=["p1-G-1.x", "p1-G-2.x"], - selector=vm.RadioItems( - options=["species", "sepal_width"], title="Simple X-axis parameter" - ), + selector=vm.RadioItems(options=["species", "sepal_width"], title="Simple X-axis parameter"), ), ], ) From 1d2a9c5d6532dd316f74126fcea2ee5ef1e42ed3 Mon Sep 17 00:00:00 2001 From: petar-qb Date: Fri, 15 Nov 2024 16:48:18 +0100 Subject: [PATCH 30/64] Minor refactoring --- vizro-core/examples/scratch_dev/app.py | 16 +++------------- vizro-core/src/vizro/actions/_actions_utils.py | 5 +++-- vizro-core/src/vizro/actions/_filter_action.py | 1 - .../src/vizro/actions/_on_page_load_action.py | 4 ---- vizro-core/src/vizro/models/_controls/filter.py | 2 +- 5 files changed, 7 insertions(+), 21 deletions(-) diff --git a/vizro-core/examples/scratch_dev/app.py b/vizro-core/examples/scratch_dev/app.py index d88fcbcad..cbfde94b1 100644 --- a/vizro-core/examples/scratch_dev/app.py +++ b/vizro-core/examples/scratch_dev/app.py @@ -48,15 +48,7 @@ def load_from_file(filter_column=None, parametrized_species=None): ignore_index=True, ) elif filter_column == "sepal_length": - final_df = df[ - df[filter_column].between( - data.get("min"), - data.get( - "max", - ), - inclusive="both", - ) - ] + final_df = df[df[filter_column].between(data.get("min"), data.get("max"), inclusive="both")] elif filter_column == "date_column": date_min = pd.to_datetime(data.get("date_min")) date_max = pd.to_datetime(data.get("date_max")) @@ -101,9 +93,7 @@ def load_from_file(filter_column=None, parametrized_species=None): vm.Filter(id="p1-F-2", column="species", targets=["p1-G-2"], selector=vm.Dropdown(title="Static filter")), vm.Parameter( targets=["p1-G-1.x", "p1-G-2.x"], - selector=vm.RadioItems( - options=["species", "sepal_width"], title="Simple X-axis parameter" - ), + selector=vm.RadioItems(options=["species", "sepal_width"], title="Simple X-axis parameter"), ), ], ) @@ -231,7 +221,7 @@ def load_from_file(filter_column=None, parametrized_species=None): # a list of values. # SOLUTION -> Create the "Universal Vizro placeholder component". # TEMPORARY SOLUTION -> set clearable=False for the dynamic Dropdown(multi=False) - # selector=vm.Dropdown(id="filter_id", multi=False), -> + # selector=vm.Dropdown(id="filter_id", multi=False), # selector=vm.Dropdown(id="filter_id", multi=False, value="setosa"), # selector=vm.RadioItems(id="filter_id"), # selector=vm.RadioItems(id="filter_id", value="setosa"), diff --git a/vizro-core/src/vizro/actions/_actions_utils.py b/vizro-core/src/vizro/actions/_actions_utils.py index 2a43d8384..c23b2554e 100644 --- a/vizro-core/src/vizro/actions/_actions_utils.py +++ b/vizro-core/src/vizro/actions/_actions_utils.py @@ -292,8 +292,9 @@ def _get_modified_page_figures( ) for target in control_targets: - current_value = [item for item in ctds_filter if item["id"] == model_manager[target].selector.id] - current_value = current_value if not current_value else current_value[0]["value"] + current_value: Any = [item for item in ctds_filter if item["id"] == model_manager[target].selector.id] + if current_value: + current_value = current_value[0]["value"] if hasattr(current_value, "__iter__") and ALL_OPTION in current_value: current_value = [] diff --git a/vizro-core/src/vizro/actions/_filter_action.py b/vizro-core/src/vizro/actions/_filter_action.py index 4e97faf68..d50f0125c 100644 --- a/vizro-core/src/vizro/actions/_filter_action.py +++ b/vizro-core/src/vizro/actions/_filter_action.py @@ -29,7 +29,6 @@ def _filter( Returns: Dict mapping target component ids to modified charts/components e.g. {'my_scatter': Figure({})} """ - print("FILTER ACTION TRIGGERED!\n") return _get_modified_page_figures( ctds_filter=ctx.args_grouping["external"]["filters"], ctds_filter_interaction=ctx.args_grouping["external"]["filter_interaction"], diff --git a/vizro-core/src/vizro/actions/_on_page_load_action.py b/vizro-core/src/vizro/actions/_on_page_load_action.py index 5b180c7df..c159485a0 100644 --- a/vizro-core/src/vizro/actions/_on_page_load_action.py +++ b/vizro-core/src/vizro/actions/_on_page_load_action.py @@ -22,8 +22,6 @@ def _on_page_load(targets: list[ModelID], **inputs: dict[str, Any]) -> dict[Mode Dict mapping target chart ids to modified figures e.g. {'my_scatter': Figure({})} """ - print("\nON PAGE LOAD - START") - print(f'Filter value: {ctx.args_grouping["external"]["filters"]}') return_obj = _get_modified_page_figures( ctds_filter=ctx.args_grouping["external"]["filters"], ctds_filter_interaction=ctx.args_grouping["external"]["filter_interaction"], @@ -31,6 +29,4 @@ def _on_page_load(targets: list[ModelID], **inputs: dict[str, Any]) -> dict[Mode targets=targets, ) - print("ON PAGE LOAD - END\n") - return return_obj diff --git a/vizro-core/src/vizro/models/_controls/filter.py b/vizro-core/src/vizro/models/_controls/filter.py index e0f06704d..c62eb23b0 100644 --- a/vizro-core/src/vizro/models/_controls/filter.py +++ b/vizro-core/src/vizro/models/_controls/filter.py @@ -294,4 +294,4 @@ def _get_options(targeted_data: pd.DataFrame, current_value=None) -> list[Any]: # The dropna() isn't strictly required here but will be in future pandas versions when the behavior of stack # changes. See https://pandas.pydata.org/docs/whatsnew/v2.1.0.html#whatsnew-210-enhancements-new-stack. current_value = current_value or [] - return np.unique(pd.concat([targeted_data.stack().dropna(), pd.Series(current_value)])).tolist() + return np.unique(pd.concat([targeted_data.stack().dropna(), pd.Series(current_value)])).tolist() # noqa: PD013 From a89b8bfacebec93f4a376c70e48f5ed55741ff96 Mon Sep 17 00:00:00 2001 From: petar-qb Date: Mon, 18 Nov 2024 10:31:45 +0100 Subject: [PATCH 31/64] Minor comment change --- vizro-core/src/vizro/actions/_actions_utils.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/vizro-core/src/vizro/actions/_actions_utils.py b/vizro-core/src/vizro/actions/_actions_utils.py index c23b2554e..c5582e33c 100644 --- a/vizro-core/src/vizro/actions/_actions_utils.py +++ b/vizro-core/src/vizro/actions/_actions_utils.py @@ -270,10 +270,7 @@ def _get_modified_page_figures( else: figure_targets.append(target) - # Retrieving only figure_targets data_frames from _multi_load is not the best solution. - # In that way, we assume that Filter.targets are the subset of the action's targets. This works for the - # on_page_load, but will not work if targets are explicitly set. - # For example, in future, if Parameter is targeting only a single Filter. + # Get data_frames for all figure_targets and components that are targets of controls _get_unfiltered_data_targets = list(set(figure_targets + control_targets_targets)) figure_targets_unfiltered_data: dict[ModelID, pd.DataFrame] = _get_unfiltered_data( From 48e6716ed4b63a954f63af95450e0477e0bd8c9d Mon Sep 17 00:00:00 2001 From: petar-qb Date: Mon, 18 Nov 2024 11:42:05 +0100 Subject: [PATCH 32/64] More refactoring --- .../src/vizro/actions/_on_page_load_action.py | 4 +--- .../vizro/models/_components/form/dropdown.py | 4 ++-- .../src/vizro/models/_controls/filter.py | 24 +++++++------------ 3 files changed, 12 insertions(+), 20 deletions(-) diff --git a/vizro-core/src/vizro/actions/_on_page_load_action.py b/vizro-core/src/vizro/actions/_on_page_load_action.py index c159485a0..c6611fbd5 100644 --- a/vizro-core/src/vizro/actions/_on_page_load_action.py +++ b/vizro-core/src/vizro/actions/_on_page_load_action.py @@ -22,11 +22,9 @@ def _on_page_load(targets: list[ModelID], **inputs: dict[str, Any]) -> dict[Mode Dict mapping target chart ids to modified figures e.g. {'my_scatter': Figure({})} """ - return_obj = _get_modified_page_figures( + return _get_modified_page_figures( ctds_filter=ctx.args_grouping["external"]["filters"], ctds_filter_interaction=ctx.args_grouping["external"]["filter_interaction"], ctds_parameter=ctx.args_grouping["external"]["parameters"], targets=targets, ) - - return return_obj diff --git a/vizro-core/src/vizro/models/_components/form/dropdown.py b/vizro-core/src/vizro/models/_components/form/dropdown.py index 5ad4a33b5..1f01eadc9 100755 --- a/vizro-core/src/vizro/models/_components/form/dropdown.py +++ b/vizro-core/src/vizro/models/_components/form/dropdown.py @@ -114,11 +114,11 @@ def _build_static(self, new_options=None, **kwargs): def _build_dynamic_placeholder(self): # Setting self.value is kind of Dropdown pre_build method. It sets self.value only the first time if it's None. # We cannot create pre_build for the Dropdown because it has to be called after vm.Filter.pre_build, but nothing - # guarantees that. We can call Filter.selector.pre_build() from the Filter.pre_build() method it we decide that. + # guarantees that. We can call Filter.selector.pre_build() from the Filter.pre_build() method if we decide that. if self.value is None: self.value = get_options_and_default(self.options, self.multi)[1] - # Replace this with the Universal Vizro Placeholder component. + # TODO-NEXT: Replace this with the "universal Vizro placeholder" component. return html.Div( children=[ dbc.Label(self.title, html_for=self.id) if self.title else None, diff --git a/vizro-core/src/vizro/models/_controls/filter.py b/vizro-core/src/vizro/models/_controls/filter.py index c62eb23b0..b48e3ddbb 100644 --- a/vizro-core/src/vizro/models/_controls/filter.py +++ b/vizro-core/src/vizro/models/_controls/filter.py @@ -48,7 +48,7 @@ "categorical": SELECTORS["numerical"] + SELECTORS["temporal"], } -# TODO: Remove this check when support dynamic mode for DatePicker selector. +# TODO: Remove DYNAMIC_SELECTORS along with its validation check when support dynamic mode for the DatePicker selector. # Tuple of filter selectors that support dynamic mode DYNAMIC_SELECTORS = (Dropdown, Checklist, RadioItems, Slider, RangeSlider) @@ -173,14 +173,11 @@ def pre_build(self): f"'{self.column}'." ) - # Selector can't be dynamic if: - # 1. Selector doesn't support the dynamic mode - # 2. Selector is categorical and "options" prop is defined - # 3. Selector is numerical and "min" and "max" props are defined + # Selector can be dynamic if selector support the dynamic mode and "options", "min" and "max" are not provided. if isinstance(self.selector, DYNAMIC_SELECTORS) and ( - hasattr(self.selector, "options") - and not getattr(self.selector, "options") - or all(hasattr(self.selector, attr) and getattr(self.selector, attr) is None for attr in ["min", "max"]) + not getattr(self.selector, "options", None) + and getattr(self.selector, "min", None) is None + and getattr(self.selector, "max", None) is None ): for target_id in self.targets: data_source_name = model_manager[target_id]["data_frame"] @@ -218,7 +215,6 @@ def pre_build(self): @_log_call def build(self): - # TODO: Align inner and outer ids to be handled in the same way as for other figure components. selector_build_obj = self.selector.build() return dcc.Loading(id=self.id, children=selector_build_obj) if self._dynamic else selector_build_obj @@ -277,12 +273,10 @@ def _get_min_max(targeted_data: pd.DataFrame, current_value=None) -> tuple[float _max = targeted_data.max(axis=None).item() if current_value: - if isinstance(current_value, list): - _min = min(_min, current_value[0]) - _max = max(_max, current_value[1]) - else: - _min = min(_min, current_value) - _max = max(_max, current_value) + # The current_value is a list of two elements when a range selector is used. Otherwise it is a single value. + _is_range_selector = isinstance(current_value, list) + _min = min(_min, current_value[0] if _is_range_selector else current_value) + _max = max(_max, current_value[1] if _is_range_selector else current_value) return _min, _max From 161a1e9e34c6c6032f9ff3c3081ec35ffdb262a1 Mon Sep 17 00:00:00 2001 From: petar-qb Date: Tue, 19 Nov 2024 08:18:07 +0100 Subject: [PATCH 33/64] Remove fetching unfiltered data for filter.targets --- .../src/vizro/actions/_actions_utils.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/vizro-core/src/vizro/actions/_actions_utils.py b/vizro-core/src/vizro/actions/_actions_utils.py index c5582e33c..2f6ccae8e 100644 --- a/vizro-core/src/vizro/actions/_actions_utils.py +++ b/vizro-core/src/vizro/actions/_actions_utils.py @@ -260,27 +260,22 @@ def _get_modified_page_figures( outputs: dict[ModelID, Any] = {} control_targets = [] - control_targets_targets = [] figure_targets = [] for target in targets: - target_obj = model_manager[target] - if isinstance(target_obj, Filter): + if isinstance(model_manager[target], Filter): control_targets.append(target) - control_targets_targets.extend(target_obj.targets) else: figure_targets.append(target) - # Get data_frames for all figure_targets and components that are targets of controls - _get_unfiltered_data_targets = list(set(figure_targets + control_targets_targets)) - - figure_targets_unfiltered_data: dict[ModelID, pd.DataFrame] = _get_unfiltered_data( - ctds_parameter=ctds_parameter, targets=_get_unfiltered_data_targets - ) + # TODO-NEXT: Add fetching unfiltered data for the Filter.targets as well, once dynamic filters become "targetable" + # from other actions too. For example, in future, if Parameter is targeting only a single Filter. + # Currently, it only works for the on_page_load because Filter.targets are indeed the part of the actions' targets. + target_to_data_frame = _get_unfiltered_data(ctds_parameter=ctds_parameter, targets=figure_targets) # TODO: the structure here would be nicer if we could get just the ctds for a single target at one time, # so you could do apply_filters on a target a pass only the ctds relevant for that target. # Consider restructuring ctds to a more convenient form to make this possible. - for target, unfiltered_data in figure_targets_unfiltered_data.items(): + for target, unfiltered_data in target_to_data_frame.items(): if target in figure_targets: filtered_data = _apply_filters(unfiltered_data, ctds_filter, ctds_filter_interaction, target) outputs[target] = model_manager[target]( @@ -296,7 +291,7 @@ def _get_modified_page_figures( current_value = [] outputs[target] = model_manager[target]( - target_to_data_frame=figure_targets_unfiltered_data, current_value=current_value + target_to_data_frame=target_to_data_frame, current_value=current_value ) return outputs From e1db7ecf82f86e524f86e292f04be15910534598 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 19 Nov 2024 07:18:29 +0000 Subject: [PATCH 34/64] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- vizro-core/src/vizro/actions/_actions_utils.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/vizro-core/src/vizro/actions/_actions_utils.py b/vizro-core/src/vizro/actions/_actions_utils.py index 2f6ccae8e..23d532f80 100644 --- a/vizro-core/src/vizro/actions/_actions_utils.py +++ b/vizro-core/src/vizro/actions/_actions_utils.py @@ -290,8 +290,6 @@ def _get_modified_page_figures( if hasattr(current_value, "__iter__") and ALL_OPTION in current_value: current_value = [] - outputs[target] = model_manager[target]( - target_to_data_frame=target_to_data_frame, current_value=current_value - ) + outputs[target] = model_manager[target](target_to_data_frame=target_to_data_frame, current_value=current_value) return outputs From b450404776382a124011b728f681f3cda08f66da Mon Sep 17 00:00:00 2001 From: petar-qb Date: Thu, 21 Nov 2024 12:37:32 +0100 Subject: [PATCH 35/64] Addressing PR comments --- vizro-core/examples/scratch_dev/app.py | 37 +++++++------ vizro-core/examples/scratch_dev/data.yaml | 15 +++--- vizro-core/src/vizro/_vizro.py | 10 +++- .../src/vizro/actions/_actions_utils.py | 20 ++++--- .../models/_components/form/checklist.py | 11 ++-- .../vizro/models/_components/form/dropdown.py | 9 ++-- .../models/_components/form/radio_items.py | 11 ++-- .../models/_components/form/range_slider.py | 34 ++++++------ .../vizro/models/_components/form/slider.py | 28 +++++----- .../src/vizro/models/_controls/filter.py | 53 ++++++++++--------- vizro-core/src/vizro/models/_page.py | 19 +------ .../vizro/static/js/models/range_slider.js | 53 +++++++++---------- .../src/vizro/static/js/models/slider.js | 31 +++++------ 13 files changed, 153 insertions(+), 178 deletions(-) diff --git a/vizro-core/examples/scratch_dev/app.py b/vizro-core/examples/scratch_dev/app.py index cbfde94b1..8b4d5fbeb 100644 --- a/vizro-core/examples/scratch_dev/app.py +++ b/vizro-core/examples/scratch_dev/app.py @@ -25,38 +25,41 @@ # FILTER_COLUMN = "date_column" -def load_from_file(filter_column=None, parametrized_species=None): +def load_from_file(filter_column=FILTER_COLUMN, parametrized_species=None): # Load the full iris dataset df = px.data.iris() df["date_column"] = pd.date_range(start=pd.to_datetime("2024-01-01"), periods=len(df), freq="D") - if parametrized_species: - return df[df["species"].isin(parametrized_species)] - with open("data.yaml", "r") as file: - data = yaml.safe_load(file) - data = data or {} + data = { + "setosa": 0, "versicolor": 0, "virginica": 0, + "min": 0, "max": 10, + "date_min": "2024-01-01", "date_max": "2024-05-29", + } + data.update(yaml.safe_load(file) or {}) - filter_column = filter_column or FILTER_COLUMN if filter_column == "species": - final_df = pd.concat( + df = pd.concat( objs=[ - df[df[filter_column] == "setosa"].head(data.get("setosa", 0)), - df[df[filter_column] == "versicolor"].head(data.get("versicolor", 0)), - df[df[filter_column] == "virginica"].head(data.get("virginica", 0)), + df[df[filter_column] == "setosa"].head(data["setosa"]), + df[df[filter_column] == "versicolor"].head(data["versicolor"]), + df[df[filter_column] == "virginica"].head(data["virginica"]), ], ignore_index=True, ) elif filter_column == "sepal_length": - final_df = df[df[filter_column].between(data.get("min"), data.get("max"), inclusive="both")] + df = df[df[filter_column].between(data["min"], data["max"], inclusive="both")] elif filter_column == "date_column": - date_min = pd.to_datetime(data.get("date_min")) - date_max = pd.to_datetime(data.get("date_max")) - final_df = df[df[filter_column].between(date_min, date_max, inclusive="both")] + date_min = pd.to_datetime(data["date_min"]) + date_max = pd.to_datetime(data["date_max"]) + df = df[df[filter_column].between(date_min, date_max, inclusive="both")] else: raise ValueError("Invalid FILTER_COLUMN") - return final_df + if parametrized_species: + df = df[df["species"].isin(parametrized_species)] + + return df data_manager["load_from_file"] = load_from_file @@ -66,7 +69,7 @@ def load_from_file(filter_column=None, parametrized_species=None): # TODO-DEV: Turn on/off caching to see how it affects the app. -# data_manager.cache = Cache(config={"CACHE_TYPE": "SimpleCache", "CACHE_DEFAULT_TIMEOUT": 5}) +# data_manager.cache = Cache(config={"CACHE_TYPE": "SimpleCache", "CACHE_DEFAULT_TIMEOUT": 10}) homepage = vm.Page( diff --git a/vizro-core/examples/scratch_dev/data.yaml b/vizro-core/examples/scratch_dev/data.yaml index 04854efbd..e6211f0e7 100644 --- a/vizro-core/examples/scratch_dev/data.yaml +++ b/vizro-core/examples/scratch_dev/data.yaml @@ -1,13 +1,12 @@ -# Choose from 0-50 -setosa: 5 -versicolor: 10 +# Choose between 0-50 +#setosa: 5 +#versicolor: 10 virginica: 15 -# Choose from: 4.8 to 7.4 -min: 5 -max: 7 +# Choose between: 4.3 to 7.4 +min: 7.1 +#max: 7 -# Choose from: -# 2020-01-01 to 2020-05-29 +# Choose between: 2020-01-01 to 2020-05-29 date_min: 2024-01-01 date_max: 2024-05-29 diff --git a/vizro-core/src/vizro/_vizro.py b/vizro-core/src/vizro/_vizro.py index 55d30a5fe..09f8b9f49 100644 --- a/vizro-core/src/vizro/_vizro.py +++ b/vizro-core/src/vizro/_vizro.py @@ -15,7 +15,7 @@ import vizro from vizro._constants import VIZRO_ASSETS_PATH from vizro.managers import data_manager, model_manager -from vizro.models import Dashboard +from vizro.models import Dashboard, Filter logger = logging.getLogger(__name__) @@ -144,9 +144,15 @@ def _pre_build(): # changes size. # Any models that are created during the pre-build process *will not* themselves have pre_build run on them. # In future may add a second pre_build loop after the first one. + + # model_manager results is wrapped into a list to avoid RuntimeError: dictionary changed size during iteration + for _, filter_obj in list(model_manager._items_with_type(Filter)): + # Run pre_build on all filters first, then on all other models. This handles dependency between Filter + # and Page pre_build and ensures that filters are pre-built before the Page objects that use them. + filter_obj.pre_build() for model_id in set(model_manager): model = model_manager[model_id] - if hasattr(model, "pre_build"): + if hasattr(model, "pre_build") and not isinstance(model, Filter): model.pre_build() def __call__(self, environ: WSGIEnvironment, start_response: StartResponse) -> Iterable[bytes]: diff --git a/vizro-core/src/vizro/actions/_actions_utils.py b/vizro-core/src/vizro/actions/_actions_utils.py index 23d532f80..fb2dbf184 100644 --- a/vizro-core/src/vizro/actions/_actions_utils.py +++ b/vizro-core/src/vizro/actions/_actions_utils.py @@ -276,19 +276,17 @@ def _get_modified_page_figures( # so you could do apply_filters on a target a pass only the ctds relevant for that target. # Consider restructuring ctds to a more convenient form to make this possible. for target, unfiltered_data in target_to_data_frame.items(): - if target in figure_targets: - filtered_data = _apply_filters(unfiltered_data, ctds_filter, ctds_filter_interaction, target) - outputs[target] = model_manager[target]( - data_frame=filtered_data, - **_get_parametrized_config(ctds_parameter=ctds_parameter, target=target, data_frame=False), - ) + filtered_data = _apply_filters(unfiltered_data, ctds_filter, ctds_filter_interaction, target) + outputs[target] = model_manager[target]( + data_frame=filtered_data, + **_get_parametrized_config(ctds_parameter=ctds_parameter, target=target, data_frame=False), + ) for target in control_targets: - current_value: Any = [item for item in ctds_filter if item["id"] == model_manager[target].selector.id] - if current_value: - current_value = current_value[0]["value"] - if hasattr(current_value, "__iter__") and ALL_OPTION in current_value: - current_value = [] + ctd_filter = [item for item in ctds_filter if item["id"] == model_manager[target].selector.id] + + # This only covers the case of cross-page actions when Filter in an output, but is not an input of the action. + current_value = ctd_filter[0]["value"] if ctd_filter else None outputs[target] = model_manager[target](target_to_data_frame=target_to_data_frame, current_value=current_value) diff --git a/vizro-core/src/vizro/models/_components/form/checklist.py b/vizro-core/src/vizro/models/_components/form/checklist.py index 617b6c527..43927629f 100644 --- a/vizro-core/src/vizro/models/_components/form/checklist.py +++ b/vizro-core/src/vizro/models/_components/form/checklist.py @@ -48,11 +48,10 @@ class Checklist(VizroBaseModel): _validate_options = root_validator(allow_reuse=True, pre=True)(validate_options_dict) _validate_value = validator("value", allow_reuse=True, always=True)(validate_value) - def __call__(self, new_options=None, **kwargs): - return self._build_static(new_options=new_options, **kwargs) + def __call__(self, options): + return self._build_static(options) - def _build_static(self, new_options=None, **kwargs): - options = new_options if new_options else self.options + def _build_static(self, options): full_options, default_value = get_options_and_default(options=options, multi=True) return html.Fieldset( @@ -72,10 +71,10 @@ def _build_dynamic_placeholder(self): if self.value is None: self.value = [get_options_and_default(self.options, multi=True)[1]] - return self._build_static() + return self._build_static(self.options) @_log_call def build(self): # We don't have to implement _build_dynamic_placeholder, _build_static here. It's possible to: if dynamic and # self.value is None -> set self.value + return standard build (static), but let's align it with the Dropdown. - return self._build_dynamic_placeholder() if self._dynamic else self._build_static() + return self._build_dynamic_placeholder() if self._dynamic else self._build_static(self.options) diff --git a/vizro-core/src/vizro/models/_components/form/dropdown.py b/vizro-core/src/vizro/models/_components/form/dropdown.py index 1f01eadc9..af22b71f4 100755 --- a/vizro-core/src/vizro/models/_components/form/dropdown.py +++ b/vizro-core/src/vizro/models/_components/form/dropdown.py @@ -88,11 +88,10 @@ def validate_multi(cls, multi, values): raise ValueError("Please set multi=True if providing a list of default values.") return multi - def __call__(self, new_options=None, **kwargs): - return self._build_static(new_options=new_options, **kwargs) + def __call__(self, options): + return self._build_static(options) - def _build_static(self, new_options=None, **kwargs): - options = new_options if new_options else self.options + def _build_static(self, options): full_options, default_value = get_options_and_default(options=options, multi=self.multi) option_height = _calculate_option_height(full_options) @@ -130,4 +129,4 @@ def _build_dynamic_placeholder(self): @_log_call def build(self): - return self._build_dynamic_placeholder() if self._dynamic else self._build_static() + return self._build_dynamic_placeholder() if self._dynamic else self._build_static(self.options) diff --git a/vizro-core/src/vizro/models/_components/form/radio_items.py b/vizro-core/src/vizro/models/_components/form/radio_items.py index 59004110e..6949d4992 100644 --- a/vizro-core/src/vizro/models/_components/form/radio_items.py +++ b/vizro-core/src/vizro/models/_components/form/radio_items.py @@ -49,11 +49,10 @@ class RadioItems(VizroBaseModel): _validate_options = root_validator(allow_reuse=True, pre=True)(validate_options_dict) _validate_value = validator("value", allow_reuse=True, always=True)(validate_value) - def __call__(self, new_options=None, **kwargs): - return self._build_static(new_options=new_options, **kwargs) + def __call__(self, options): + return self._build_static(options) - def _build_static(self, new_options=None, **kwargs): - options = new_options if new_options else self.options + def _build_static(self, options): full_options, default_value = get_options_and_default(options=options, multi=False) return html.Fieldset( @@ -73,10 +72,10 @@ def _build_dynamic_placeholder(self): if self.value is None: self.value = get_options_and_default(self.options, multi=False)[1] - return self._build_static() + return self._build_static(self.options) @_log_call def build(self): # We don't have to implement _build_dynamic_placeholder, _build_static here. It's possible to: if dynamic and # self.value is None -> set self.value + return standard build (static), but let's align it with the Dropdown. - return self._build_dynamic_placeholder() if self._dynamic else self._build_static() + return self._build_dynamic_placeholder() if self._dynamic else self._build_static(self.options) diff --git a/vizro-core/src/vizro/models/_components/form/range_slider.py b/vizro-core/src/vizro/models/_components/form/range_slider.py index 7110a2851..3441440ea 100644 --- a/vizro-core/src/vizro/models/_components/form/range_slider.py +++ b/vizro-core/src/vizro/models/_components/form/range_slider.py @@ -62,15 +62,11 @@ class RangeSlider(VizroBaseModel): _set_default_marks = validator("marks", allow_reuse=True, always=True)(set_default_marks) _set_actions = _action_validator_factory("value") - def __call__(self, current_value=None, new_min=None, new_max=None, **kwargs): - return self._build_static(current_value=current_value, new_min=new_min, new_max=new_max, **kwargs) + def __call__(self, min, max, current_value): + return self._build_static(min, max, current_value) @_log_call - def _build_static(self, current_value=None, new_min=None, new_max=None, **kwargs): - _min = new_min if new_min else self.min - _max = new_max if new_max else self.max - init_value = current_value or self.value or [_min, _max] - + def _build_static(self, min, max, current_value): output = [ Output(f"{self.id}_start_value", "value"), Output(f"{self.id}_end_value", "value"), @@ -93,7 +89,7 @@ def _build_static(self, current_value=None, new_min=None, new_max=None, **kwargs return html.Div( children=[ - dcc.Store(f"{self.id}_callback_data", data={"id": self.id, "min": _min, "max": _max}), + dcc.Store(f"{self.id}_callback_data", data={"id": self.id, "min": min, "max": max}), html.Div( children=[ dbc.Label(children=self.title, html_for=self.id) if self.title else None, @@ -103,10 +99,10 @@ def _build_static(self, current_value=None, new_min=None, new_max=None, **kwargs id=f"{self.id}_start_value", type="number", placeholder="min", - min=_min, - max=_max, + min=min, + max=max, step=self.step, - value=init_value[0], + value=current_value[0], persistence=True, persistence_type="session", className="slider-text-input-field", @@ -116,10 +112,10 @@ def _build_static(self, current_value=None, new_min=None, new_max=None, **kwargs id=f"{self.id}_end_value", type="number", placeholder="max", - min=_min, - max=_max, + min=min, + max=max, step=self.step, - value=init_value[1], + value=current_value[1], persistence=True, persistence_type="session", className="slider-text-input-field", @@ -133,11 +129,11 @@ def _build_static(self, current_value=None, new_min=None, new_max=None, **kwargs ), dcc.RangeSlider( id=self.id, - min=_min, - max=_max, + min=min, + max=max, step=self.step, marks=self.marks, - value=init_value, + value=current_value, persistence=True, persistence_type="session", className="slider-track-without-marks" if self.marks is None else "slider-track-with-marks", @@ -146,10 +142,10 @@ def _build_static(self, current_value=None, new_min=None, new_max=None, **kwargs ) def _build_dynamic_placeholder(self): - return self._build_static() + return self._build_static(self.min, self.max, self.value or [self.min, self.max]) @_log_call def build(self): # We don't have to implement _build_dynamic_placeholder, _build_static here. It's possible to: if dynamic and # self.value is None -> set self.value + return standard build (static), but let's align it with the Dropdown. - return self._build_dynamic_placeholder() if self._dynamic else self._build_static() + return self._build_dynamic_placeholder() if self._dynamic else self._build_static(self.min, self.max, self.value or [self.min, self.max]) diff --git a/vizro-core/src/vizro/models/_components/form/slider.py b/vizro-core/src/vizro/models/_components/form/slider.py index 0781015a1..67edf984c 100644 --- a/vizro-core/src/vizro/models/_components/form/slider.py +++ b/vizro-core/src/vizro/models/_components/form/slider.py @@ -60,14 +60,10 @@ class Slider(VizroBaseModel): _set_default_marks = validator("marks", allow_reuse=True, always=True)(set_default_marks) _set_actions = _action_validator_factory("value") - def __call__(self, current_value=None, new_min=None, new_max=None, **kwargs): - return self._build_static(current_value=current_value, new_min=new_min, new_max=new_max, **kwargs) - - def _build_static(self, current_value=None, new_min=None, new_max=None, **kwargs): - _min = new_min if new_min else self.min - _max = new_max if new_max else self.max - init_value = current_value or self.value or _min + def __call__(self, min, max, current_value): + return self._build_static(min, max, current_value) + def _build_static(self, min, max, current_value): output = [ Output(f"{self.id}_end_value", "value"), Output(self.id, "value"), @@ -88,7 +84,7 @@ def _build_static(self, current_value=None, new_min=None, new_max=None, **kwargs return html.Div( children=[ - dcc.Store(f"{self.id}_callback_data", data={"id": self.id, "min": _min, "max": _max}), + dcc.Store(f"{self.id}_callback_data", data={"id": self.id, "min": min, "max": max}), html.Div( children=[ dbc.Label(children=self.title, html_for=self.id) if self.title else None, @@ -98,10 +94,10 @@ def _build_static(self, current_value=None, new_min=None, new_max=None, **kwargs id=f"{self.id}_end_value", type="number", placeholder="max", - min=_min, - max=_max, + min=min, + max=max, step=self.step, - value=init_value, + value=current_value, persistence=True, persistence_type="session", className="slider-text-input-field", @@ -115,11 +111,11 @@ def _build_static(self, current_value=None, new_min=None, new_max=None, **kwargs ), dcc.Slider( id=self.id, - min=_min, - max=_max, + min=min, + max=max, step=self.step, marks=self.marks, - value=init_value, + value=current_value, included=False, persistence=True, persistence_type="session", @@ -129,10 +125,10 @@ def _build_static(self, current_value=None, new_min=None, new_max=None, **kwargs ) def _build_dynamic_placeholder(self): - return self._build_static() + return self._build_static(self.min, self.max, self.value or self.min) @_log_call def build(self): # We don't have to implement _build_dynamic_placeholder, _build_static here. It's possible to: if dynamic and # self.value is None -> set self.value + return standard build (static), but let's align it with the Dropdown. - return self._build_dynamic_placeholder() if self._dynamic else self._build_static() + return self._build_dynamic_placeholder() if self._dynamic else self._build_static(self.min, self.max, self.value or self.min) diff --git a/vizro-core/src/vizro/models/_controls/filter.py b/vizro-core/src/vizro/models/_controls/filter.py index b48e3ddbb..4b8c4fa6e 100644 --- a/vizro-core/src/vizro/models/_controls/filter.py +++ b/vizro-core/src/vizro/models/_controls/filter.py @@ -14,7 +14,7 @@ except ImportError: # pragma: no cov from pydantic import Field, PrivateAttr, validator -from vizro._constants import FILTER_ACTION_PREFIX +from vizro._constants import ALL_OPTION, FILTER_ACTION_PREFIX from vizro.actions import _filter from vizro.managers import data_manager, model_manager from vizro.managers._data_manager import _DynamicData @@ -96,7 +96,6 @@ class Filter(VizroBaseModel): selector: SelectorType = None _dynamic: bool = PrivateAttr(None) - _pre_build_finished: bool = PrivateAttr(False) # Component properties for actions and interactions _output_component_property: str = PrivateAttr("children") @@ -109,11 +108,10 @@ def check_target_present(cls, target): raise ValueError(f"Target {target} not found in model_manager.") return target - def __call__(self, target_to_data_frame: dict[ModelID, pd.DataFrame], current_value: Any, **kwargs): + def __call__(self, target_to_data_frame: dict[ModelID, pd.DataFrame], current_value: Any): # Only relevant for a dynamic filter. # Although targets are fixed at build time, the validation logic is repeated during runtime, so if a column # is missing then it will raise an error. We could change this if we wanted. - # Call this from actions_utils targeted_data = self._validate_targeted_data( {target: data_frame for target, data_frame in target_to_data_frame.items() if target in self.targets}, eagerly_raise_column_not_found_error=True, @@ -126,19 +124,15 @@ def __call__(self, target_to_data_frame: dict[ModelID, pd.DataFrame], current_va ) if isinstance(self.selector, SELECTORS["categorical"]): - # Categorical selector. - new_options = self._get_options(targeted_data, current_value) - return self.selector(current_value=current_value, new_options=new_options, **kwargs) + return self.selector(options=self._get_options(targeted_data, current_value)) else: - # Numerical or temporal selector. _min, _max = self._get_min_max(targeted_data, current_value) - return self.selector(current_value=current_value, new_min=_min, new_max=_max, **kwargs) + # "current_value" is propagated only to support dcc.Input and dcc.Store components in numerical selectors + # to work with a dynamic selector. This can be removed when dash persistence bug is fixed. + return self.selector(min=_min, max=_max, current_value=current_value) @_log_call def pre_build(self): - if self._pre_build_finished: - return - self._pre_build_finished = True # If targets aren't explicitly provided then try to target all figures on the page. In this case we don't # want to raise an error if the column is not found in a figure's data_frame, it will just be ignored. # This is the case when bool(self.targets) is False. @@ -146,12 +140,14 @@ def pre_build(self): proposed_targets = self.targets or model_manager._get_page_model_ids_with_figure( page_id=model_manager._get_model_page_id(model_id=ModelID(str(self.id))) ) - # TODO NEXT: how to handle pre_build for dynamic filters? Do we still require default argument values in - # `load` to establish selector type etc.? Can we take selector values from model_manager to supply these? - # Or just don't do validation at pre_build time and wait until state is available during build time instead? - # What should the load kwargs be here? Remember they need to be {} for static data. - # Note that currently _get_unfiltered_data is only suitable for use at runtime since it requires - # ctds_parameter. That could be changed to just reuse that function. + + # TODO: Currently dynamic data functions require a default value for every argument. Even when there is a + # dataframe parameter, the default value is used when pre-build the filter e.g. to find the targets, + # column type (and hence selector) and initial values. There are three ways to handle this: + # 1. (Current approach) - Propagate {} and use only default arguments value in the dynamic data function. + # 2. Propagate values from the model_manager and relax the limitation of requireing argument default values. + # 3. Skip the pre-build and do everything in the build method (if possible). + # Find more about the mentioned limitation at: https://github.com/mckinsey/vizro/pull/879/files#r1846609956 multi_data_source_name_load_kwargs: list[tuple[DataSourceName, dict[str, Any]]] = [ (model_manager[target]["data_frame"], {}) for target in proposed_targets ] @@ -173,7 +169,10 @@ def pre_build(self): f"'{self.column}'." ) - # Selector can be dynamic if selector support the dynamic mode and "options", "min" and "max" are not provided. + # Check if the filter is dynamic. Dynamic filter means that the filter is updated when the page is refreshed + # which causes that "options" for categorical or "min" and "max" for numerical/temporal selectors are updated. + # The filter is dynamic if mentioned attributes ("options"/"min"/"max") are not explicitly provided and if the + # filter targets at least one figure that uses the dynamic data source. if isinstance(self.selector, DYNAMIC_SELECTORS) and ( not getattr(self.selector, "options", None) and getattr(self.selector, "min", None) is None @@ -272,11 +271,10 @@ def _get_min_max(targeted_data: pd.DataFrame, current_value=None) -> tuple[float _min = targeted_data.min(axis=None).item() _max = targeted_data.max(axis=None).item() - if current_value: - # The current_value is a list of two elements when a range selector is used. Otherwise it is a single value. - _is_range_selector = isinstance(current_value, list) - _min = min(_min, current_value[0] if _is_range_selector else current_value) - _max = max(_max, current_value[1] if _is_range_selector else current_value) + if current_value is not None: + current_value = current_value if isinstance(current_value, list) else [current_value] + _min = min(_min, *current_value) + _max = max(_max, *current_value) return _min, _max @@ -287,5 +285,10 @@ def _get_options(targeted_data: pd.DataFrame, current_value=None) -> list[Any]: # values and instead just pass straight to the Dash component. # The dropna() isn't strictly required here but will be in future pandas versions when the behavior of stack # changes. See https://pandas.pydata.org/docs/whatsnew/v2.1.0.html#whatsnew-210-enhancements-new-stack. - current_value = current_value or [] + + # Remove ALL_OPTION from the string or list of currently selected value for the categorical filters. + current_value = [] if current_value in (None, ALL_OPTION) else current_value + if isinstance(current_value, list) and ALL_OPTION in current_value: + current_value.remove(ALL_OPTION) + return np.unique(pd.concat([targeted_data.stack().dropna(), pd.Series(current_value)])).tolist() # noqa: PD013 diff --git a/vizro-core/src/vizro/models/_page.py b/vizro-core/src/vizro/models/_page.py index 90e11b32b..4a11c74e7 100644 --- a/vizro-core/src/vizro/models/_page.py +++ b/vizro-core/src/vizro/models/_page.py @@ -96,25 +96,8 @@ def __vizro_exclude_fields__(self) -> Optional[Union[set[str], Mapping[str, Any] @_log_call def pre_build(self): - # TODO: Remove default on page load action if possible targets = model_manager._get_page_model_ids_with_figure(page_id=ModelID(str(self.id))) - - # TODO: In the Vizro._pre_build(), the loop passes through the components in the random order, but the - # "pre_build" of one component can depend on another component's "pre_build". - # Does the "postorder DFS" traversal pass work for us? -> More about it: - # https://www.geeksforgeeks.org/tree-traversals-inorder-preorder-and-postorder/#postorder-traversal - # By introducing this traversal algorithm we should ensure that child pre_build will always be called before - # parent pre_build. Does this case solve all the pre_build interdependency future problems for us? - # Should we also ensure that targeted components are always pre_build before the source components? - - # The hack below is just to ensure that the pre_build of the Filter components is called before the page - # pre_build calculates on_page_load targets. We need this to check is the Filter _dynamic or not. - for control in self.controls: - if isinstance(control, Filter): - if not control._pre_build_finished: - control.pre_build() - if control._dynamic: - targets.append(control.id) + targets.extend(control.id for control in self.controls if getattr(control, "_dynamic", False)) if targets: self.actions = [ diff --git a/vizro-core/src/vizro/static/js/models/range_slider.js b/vizro-core/src/vizro/static/js/models/range_slider.js index 7b3989ecf..69055dbcf 100644 --- a/vizro-core/src/vizro/static/js/models/range_slider.js +++ b/vizro-core/src/vizro/static/js/models/range_slider.js @@ -28,37 +28,34 @@ function update_range_slider_values( } return [start, end, [start, end], [start, end]]; - // slider component is the trigger + // slider component is the trigger } else if (trigger_id === self_data["id"]) { return [slider[0], slider[1], slider, slider]; - - // on_page_load is the trigger - } else { - if (input_store === null) { - return [ - dash_clientside.no_update, - dash_clientside.no_update, - dash_clientside.no_update, - slider, - ]; - } else { - if ( - slider[0] === start && - input_store[0] === start && - slider[1] === end && - input_store[1] === end - ) { - // To prevent filter_action to be triggered after on_page_load - return [ - dash_clientside.no_update, - dash_clientside.no_update, - dash_clientside.no_update, - dash_clientside.no_update, - ]; - } - return [input_store[0], input_store[1], input_store, input_store]; - } } + // on_page_load is the trigger + if (input_store === null) { + return [ + dash_clientside.no_update, + dash_clientside.no_update, + dash_clientside.no_update, + slider, + ]; + } + if ( + slider[0] === start && + input_store[0] === start && + slider[1] === end && + input_store[1] === end + ) { + // To prevent filter_action to be triggered after on_page_load + return [ + dash_clientside.no_update, + dash_clientside.no_update, + dash_clientside.no_update, + dash_clientside.no_update, + ]; + } + return [input_store[0], input_store[1], input_store, input_store]; } window.dash_clientside = { diff --git a/vizro-core/src/vizro/static/js/models/slider.js b/vizro-core/src/vizro/static/js/models/slider.js index 75264eecd..051cac3d4 100644 --- a/vizro-core/src/vizro/static/js/models/slider.js +++ b/vizro-core/src/vizro/static/js/models/slider.js @@ -14,26 +14,23 @@ function update_slider_values(start, slider, input_store, self_data) { } return [start, start, start]; - // slider component is the trigger + // slider component is the trigger } else if (trigger_id === self_data["id"]) { return [slider, slider, slider]; - - // on_page_load is the trigger - } else { - if (input_store === null) { - return [dash_clientside.no_update, dash_clientside.no_update, slider]; - } else { - if (slider === start && start === input_store) { - // To prevent filter_action to be triggered after on_page_load - return [ - dash_clientside.no_update, - dash_clientside.no_update, - dash_clientside.no_update, - ]; - } - return [input_store, input_store, input_store]; - } } + // on_page_load is the trigger + if (input_store === null) { + return [dash_clientside.no_update, dash_clientside.no_update, slider]; + } + if (slider === start && start === input_store) { + // To prevent filter_action to be triggered after on_page_load + return [ + dash_clientside.no_update, + dash_clientside.no_update, + dash_clientside.no_update, + ]; + } + return [input_store, input_store, input_store]; } window.dash_clientside = { From 82838f1f11eea26dc9af4908a40de2e11690b335 Mon Sep 17 00:00:00 2001 From: petar-qb Date: Thu, 21 Nov 2024 12:45:33 +0100 Subject: [PATCH 36/64] Lint --- vizro-core/examples/scratch_dev/app.py | 10 +++++++--- vizro-core/examples/scratch_dev/data.yaml | 8 ++++---- .../src/vizro/models/_components/form/range_slider.py | 11 ++++++++--- .../src/vizro/models/_components/form/slider.py | 11 ++++++++--- vizro-core/src/vizro/models/_controls/filter.py | 2 +- vizro-core/src/vizro/models/_page.py | 2 +- vizro-core/src/vizro/static/js/models/range_slider.js | 2 +- vizro-core/src/vizro/static/js/models/slider.js | 2 +- 8 files changed, 31 insertions(+), 17 deletions(-) diff --git a/vizro-core/examples/scratch_dev/app.py b/vizro-core/examples/scratch_dev/app.py index 8b4d5fbeb..380dc1968 100644 --- a/vizro-core/examples/scratch_dev/app.py +++ b/vizro-core/examples/scratch_dev/app.py @@ -32,9 +32,13 @@ def load_from_file(filter_column=FILTER_COLUMN, parametrized_species=None): with open("data.yaml", "r") as file: data = { - "setosa": 0, "versicolor": 0, "virginica": 0, - "min": 0, "max": 10, - "date_min": "2024-01-01", "date_max": "2024-05-29", + "setosa": 0, + "versicolor": 0, + "virginica": 0, + "min": 0, + "max": 10, + "date_min": "2024-01-01", + "date_max": "2024-05-29", } data.update(yaml.safe_load(file) or {}) diff --git a/vizro-core/examples/scratch_dev/data.yaml b/vizro-core/examples/scratch_dev/data.yaml index e6211f0e7..d8b0aea90 100644 --- a/vizro-core/examples/scratch_dev/data.yaml +++ b/vizro-core/examples/scratch_dev/data.yaml @@ -1,11 +1,11 @@ # Choose between 0-50 -#setosa: 5 -#versicolor: 10 +setosa: 5 +versicolor: 10 virginica: 15 # Choose between: 4.3 to 7.4 -min: 7.1 -#max: 7 +min: 5 +max: 7 # Choose between: 2020-01-01 to 2020-05-29 date_min: 2024-01-01 diff --git a/vizro-core/src/vizro/models/_components/form/range_slider.py b/vizro-core/src/vizro/models/_components/form/range_slider.py index 3441440ea..0800e6e75 100644 --- a/vizro-core/src/vizro/models/_components/form/range_slider.py +++ b/vizro-core/src/vizro/models/_components/form/range_slider.py @@ -141,11 +141,16 @@ def _build_static(self, min, max, current_value): ] ) - def _build_dynamic_placeholder(self): - return self._build_static(self.min, self.max, self.value or [self.min, self.max]) + def _build_dynamic_placeholder(self, current_value): + return self._build_static(self.min, self.max, current_value) @_log_call def build(self): # We don't have to implement _build_dynamic_placeholder, _build_static here. It's possible to: if dynamic and # self.value is None -> set self.value + return standard build (static), but let's align it with the Dropdown. - return self._build_dynamic_placeholder() if self._dynamic else self._build_static(self.min, self.max, self.value or [self.min, self.max]) + current_value = self.value or [self.min, self.max] # type: ignore[list-item] + return ( + self._build_dynamic_placeholder(current_value) + if self._dynamic + else self._build_static(self.min, self.max, current_value) + ) diff --git a/vizro-core/src/vizro/models/_components/form/slider.py b/vizro-core/src/vizro/models/_components/form/slider.py index 67edf984c..8f4107b5b 100644 --- a/vizro-core/src/vizro/models/_components/form/slider.py +++ b/vizro-core/src/vizro/models/_components/form/slider.py @@ -124,11 +124,16 @@ def _build_static(self, min, max, current_value): ] ) - def _build_dynamic_placeholder(self): - return self._build_static(self.min, self.max, self.value or self.min) + def _build_dynamic_placeholder(self, current_value): + return self._build_static(self.min, self.max, current_value) @_log_call def build(self): # We don't have to implement _build_dynamic_placeholder, _build_static here. It's possible to: if dynamic and # self.value is None -> set self.value + return standard build (static), but let's align it with the Dropdown. - return self._build_dynamic_placeholder() if self._dynamic else self._build_static(self.min, self.max, self.value or self.min) + current_value = self.value if self.value is not None else self.min + return ( + self._build_dynamic_placeholder(current_value) + if self._dynamic + else self._build_static(self.min, self.max, current_value) + ) diff --git a/vizro-core/src/vizro/models/_controls/filter.py b/vizro-core/src/vizro/models/_controls/filter.py index 4b8c4fa6e..b638d03d0 100644 --- a/vizro-core/src/vizro/models/_controls/filter.py +++ b/vizro-core/src/vizro/models/_controls/filter.py @@ -145,7 +145,7 @@ def pre_build(self): # dataframe parameter, the default value is used when pre-build the filter e.g. to find the targets, # column type (and hence selector) and initial values. There are three ways to handle this: # 1. (Current approach) - Propagate {} and use only default arguments value in the dynamic data function. - # 2. Propagate values from the model_manager and relax the limitation of requireing argument default values. + # 2. Propagate values from the model_manager and relax the limitation of requiring argument default values. # 3. Skip the pre-build and do everything in the build method (if possible). # Find more about the mentioned limitation at: https://github.com/mckinsey/vizro/pull/879/files#r1846609956 multi_data_source_name_load_kwargs: list[tuple[DataSourceName, dict[str, Any]]] = [ diff --git a/vizro-core/src/vizro/models/_page.py b/vizro-core/src/vizro/models/_page.py index 4a11c74e7..b64e6e250 100644 --- a/vizro-core/src/vizro/models/_page.py +++ b/vizro-core/src/vizro/models/_page.py @@ -14,7 +14,7 @@ from vizro.actions import _on_page_load from vizro.managers import model_manager from vizro.managers._model_manager import DuplicateIDError, ModelID -from vizro.models import Action, Filter, Layout, VizroBaseModel +from vizro.models import Action, Layout, VizroBaseModel from vizro.models._action._actions_chain import ActionsChain, Trigger from vizro.models._layout import set_layout from vizro.models._models_utils import _log_call, check_captured_callable, validate_min_length diff --git a/vizro-core/src/vizro/static/js/models/range_slider.js b/vizro-core/src/vizro/static/js/models/range_slider.js index 69055dbcf..8aafdde7f 100644 --- a/vizro-core/src/vizro/static/js/models/range_slider.js +++ b/vizro-core/src/vizro/static/js/models/range_slider.js @@ -28,7 +28,7 @@ function update_range_slider_values( } return [start, end, [start, end], [start, end]]; - // slider component is the trigger + // slider component is the trigger } else if (trigger_id === self_data["id"]) { return [slider[0], slider[1], slider, slider]; } diff --git a/vizro-core/src/vizro/static/js/models/slider.js b/vizro-core/src/vizro/static/js/models/slider.js index 051cac3d4..1b15d78ae 100644 --- a/vizro-core/src/vizro/static/js/models/slider.js +++ b/vizro-core/src/vizro/static/js/models/slider.js @@ -14,7 +14,7 @@ function update_slider_values(start, slider, input_store, self_data) { } return [start, start, start]; - // slider component is the trigger + // slider component is the trigger } else if (trigger_id === self_data["id"]) { return [slider, slider, slider]; } From db017a2acc606a6691419f86f300578777cd6d5a Mon Sep 17 00:00:00 2001 From: petar-qb Date: Mon, 25 Nov 2024 08:34:51 +0100 Subject: [PATCH 37/64] Unit tests --- .../models/_components/form/_form_utils.py | 5 +- .../src/vizro/models/_controls/filter.py | 25 +- .../src/vizro/models/_controls/parameter.py | 1 + .../tests/unit/vizro/actions/conftest.py | 46 --- vizro-core/tests/unit/vizro/conftest.py | 47 ++++ .../_components/form/test_range_slider.py | 4 +- .../models/_components/form/test_slider.py | 2 +- .../vizro/models/_controls/test_filter.py | 263 +++++++++++++++++- 8 files changed, 335 insertions(+), 58 deletions(-) diff --git a/vizro-core/src/vizro/models/_components/form/_form_utils.py b/vizro-core/src/vizro/models/_components/form/_form_utils.py index d1c9b80e9..14a20a169 100644 --- a/vizro-core/src/vizro/models/_components/form/_form_utils.py +++ b/vizro-core/src/vizro/models/_components/form/_form_utils.py @@ -54,7 +54,10 @@ def validate_value(cls, value, values): [entry["value"] for entry in values["options"]] if isinstance(values["options"][0], dict) else values["options"] ) - if value and ALL_OPTION not in value and not is_value_contained(value, possible_values): + if hasattr(value, "__iter__") and ALL_OPTION in value: + return value + + if value and not is_value_contained(value, possible_values): raise ValueError("Please provide a valid value from `options`.") return value diff --git a/vizro-core/src/vizro/models/_controls/filter.py b/vizro-core/src/vizro/models/_controls/filter.py index b638d03d0..c70e7032b 100644 --- a/vizro-core/src/vizro/models/_controls/filter.py +++ b/vizro-core/src/vizro/models/_controls/filter.py @@ -5,6 +5,7 @@ import numpy as np import pandas as pd from dash import dcc +from contextlib import suppress from pandas.api.types import is_datetime64_any_dtype, is_numeric_dtype from vizro.managers._data_manager import DataSourceName @@ -95,7 +96,7 @@ class Filter(VizroBaseModel): ) selector: SelectorType = None - _dynamic: bool = PrivateAttr(None) + _dynamic: bool = PrivateAttr(False) # Component properties for actions and interactions _output_component_property: str = PrivateAttr("children") @@ -265,11 +266,22 @@ def _validate_column_type(self, targeted_data: pd.DataFrame) -> Literal["numeric @staticmethod def _get_min_max(targeted_data: pd.DataFrame, current_value=None) -> tuple[float, float]: + _min = targeted_data.min(axis=None) + _max = targeted_data.max(axis=None) + + # Convert to datetime if the column is datetime64 + if targeted_data.apply(is_datetime64_any_dtype).all(): + _min = pd.to_datetime(_min) + _max = pd.to_datetime(_max) + current_value = pd.to_datetime(current_value) + # Convert DatetimeIndex to list of Timestamp objects so that we can use min and max functions below. + with suppress(AttributeError): current_value = current_value.tolist() + # Use item() to convert to convert scalar from numpy to Python type. This isn't needed during pre_build because # pydantic will coerce the type, but it is necessary in __call__ where we don't update model field values # and instead just pass straight to the Dash component. - _min = targeted_data.min(axis=None).item() - _max = targeted_data.max(axis=None).item() + with suppress(AttributeError): _min = _min.item() + with suppress(AttributeError): _max = _max.item() if current_value is not None: current_value = current_value if isinstance(current_value, list) else [current_value] @@ -285,10 +297,15 @@ def _get_options(targeted_data: pd.DataFrame, current_value=None) -> list[Any]: # values and instead just pass straight to the Dash component. # The dropna() isn't strictly required here but will be in future pandas versions when the behavior of stack # changes. See https://pandas.pydata.org/docs/whatsnew/v2.1.0.html#whatsnew-210-enhancements-new-stack. + # Also setting the dtype for the current_value_series to the dtype of the targeted_data_series to ensure it + # works when it's empty. See: https://pandas.pydata.org/docs/whatsnew/v2.1.0.html#other-deprecations # Remove ALL_OPTION from the string or list of currently selected value for the categorical filters. current_value = [] if current_value in (None, ALL_OPTION) else current_value if isinstance(current_value, list) and ALL_OPTION in current_value: current_value.remove(ALL_OPTION) - return np.unique(pd.concat([targeted_data.stack().dropna(), pd.Series(current_value)])).tolist() # noqa: PD013 + targeted_data_series = targeted_data.stack().dropna() + current_value_series = pd.Series(current_value).astype(targeted_data_series.dtypes) + + return sorted(list(pd.concat([targeted_data_series, current_value_series]).unique())) diff --git a/vizro-core/src/vizro/models/_controls/parameter.py b/vizro-core/src/vizro/models/_controls/parameter.py index e7d6d537c..9663c3f14 100644 --- a/vizro-core/src/vizro/models/_controls/parameter.py +++ b/vizro-core/src/vizro/models/_controls/parameter.py @@ -55,6 +55,7 @@ def check_data_frame_as_target_argument(cls, target): f"Invalid target {target}. 'data_frame' target must be supplied in the form " ".data_frame." ) + # TODO: Add validation: Make sure the target data_frame is _DynamicData. return target @validator("targets") diff --git a/vizro-core/tests/unit/vizro/actions/conftest.py b/vizro-core/tests/unit/vizro/actions/conftest.py index 902ca042f..6833fa0d9 100644 --- a/vizro-core/tests/unit/vizro/actions/conftest.py +++ b/vizro-core/tests/unit/vizro/actions/conftest.py @@ -25,37 +25,11 @@ def iris(): return px.data.iris() -@pytest.fixture -def gapminder_dynamic_first_n_last_n_function(gapminder): - return lambda first_n=None, last_n=None: ( - pd.concat([gapminder[:first_n], gapminder[-last_n:]]) - if last_n - else gapminder[:first_n] - if first_n - else gapminder - ) - - -@pytest.fixture -def box_params(): - return {"x": "continent", "y": "lifeExp", "custom_data": ["continent"]} - - @pytest.fixture def box_chart(gapminder_2007, box_params): return px.box(gapminder_2007, **box_params) -@pytest.fixture -def box_chart_dynamic_data_frame(box_params): - return px.box("gapminder_dynamic_first_n_last_n", **box_params) - - -@pytest.fixture -def scatter_params(): - return {"x": "gdpPercap", "y": "lifeExp"} - - @pytest.fixture def scatter_chart(gapminder_2007, scatter_params): return px.scatter(gapminder_2007, **scatter_params) @@ -71,11 +45,6 @@ def scatter_matrix_chart(iris, scatter_matrix_params): return px.scatter_matrix(iris, **scatter_matrix_params) -@pytest.fixture -def scatter_chart_dynamic_data_frame(scatter_params): - return px.scatter("gapminder_dynamic_first_n_last_n", **scatter_params) - - @pytest.fixture def target_scatter_filtered_continent(request, gapminder_2007, scatter_params): continent = request.param @@ -105,21 +74,6 @@ def managers_one_page_two_graphs_one_button(box_chart, scatter_chart): Vizro._pre_build() -@pytest.fixture -def managers_one_page_two_graphs_with_dynamic_data(box_chart_dynamic_data_frame, scatter_chart_dynamic_data_frame): - """Instantiates a simple model_manager and data_manager with a page, two graph models and the button component.""" - vm.Page( - id="test_page", - title="My first dashboard", - components=[ - vm.Graph(id="box_chart", figure=box_chart_dynamic_data_frame), - vm.Graph(id="scatter_chart", figure=scatter_chart_dynamic_data_frame), - vm.Button(id="button"), - ], - ) - Vizro._pre_build() - - @pytest.fixture def managers_one_page_two_graphs_one_table_one_aggrid_one_button( box_chart, scatter_chart, dash_data_table_with_id, ag_grid_with_id diff --git a/vizro-core/tests/unit/vizro/conftest.py b/vizro-core/tests/unit/vizro/conftest.py index cba2bccc7..dd15c9fa4 100644 --- a/vizro-core/tests/unit/vizro/conftest.py +++ b/vizro-core/tests/unit/vizro/conftest.py @@ -1,5 +1,6 @@ """Fixtures to be shared across several tests.""" +import pandas as pd import plotly.graph_objects as go import pytest @@ -20,6 +21,17 @@ def stocks(): return px.data.stocks() +@pytest.fixture +def gapminder_dynamic_first_n_last_n_function(gapminder): + return lambda first_n=None, last_n=None: ( + pd.concat([gapminder[:first_n], gapminder[-last_n:]]) + if last_n + else gapminder[:first_n] + if first_n + else gapminder + ) + + @pytest.fixture def standard_px_chart(gapminder): return px.scatter( @@ -33,6 +45,26 @@ def standard_px_chart(gapminder): ) +@pytest.fixture +def scatter_params(): + return {"x": "gdpPercap", "y": "lifeExp"} + + +@pytest.fixture +def scatter_chart_dynamic_data_frame(scatter_params): + return px.scatter("gapminder_dynamic_first_n_last_n", **scatter_params) + + +@pytest.fixture +def box_params(): + return {"x": "continent", "y": "lifeExp", "custom_data": ["continent"]} + + +@pytest.fixture +def box_chart_dynamic_data_frame(box_params): + return px.box("gapminder_dynamic_first_n_last_n", **box_params) + + @pytest.fixture def standard_ag_grid(gapminder): return dash_ag_grid(data_frame=gapminder) @@ -88,6 +120,21 @@ def page_2(): return vm.Page(title="Page 2", components=[vm.Button()]) +@pytest.fixture +def managers_one_page_two_graphs_with_dynamic_data(box_chart_dynamic_data_frame, scatter_chart_dynamic_data_frame): + """Instantiates a simple model_manager and data_manager with a page, two graph models and the button component.""" + vm.Page( + id="test_page", + title="My first dashboard", + components=[ + vm.Graph(id="box_chart", figure=box_chart_dynamic_data_frame), + vm.Graph(id="scatter_chart", figure=scatter_chart_dynamic_data_frame), + vm.Button(id="button"), + ], + ) + Vizro._pre_build() + + @pytest.fixture() def vizro_app(): """Fixture to instantiate Vizro/Dash app. diff --git a/vizro-core/tests/unit/vizro/models/_components/form/test_range_slider.py b/vizro-core/tests/unit/vizro/models/_components/form/test_range_slider.py index e0c9a9f13..4879aafc3 100644 --- a/vizro-core/tests/unit/vizro/models/_components/form/test_range_slider.py +++ b/vizro-core/tests/unit/vizro/models/_components/form/test_range_slider.py @@ -48,7 +48,7 @@ def expected_range_slider_default(): persistence_type="session", className="slider-text-input-field", ), - dcc.Store(id="range_slider_input_store", storage_type="session", data=[None, None]), + dcc.Store(id="range_slider_input_store", storage_type="session"), ], className="slider-text-input-container", ), @@ -105,7 +105,7 @@ def expected_range_slider_with_optional(): persistence_type="session", className="slider-text-input-field", ), - dcc.Store(id="range_slider_input_store", storage_type="session", data=[0, 10]), + dcc.Store(id="range_slider_input_store", storage_type="session"), ], className="slider-text-input-container", ), diff --git a/vizro-core/tests/unit/vizro/models/_components/form/test_slider.py b/vizro-core/tests/unit/vizro/models/_components/form/test_slider.py index 56d2c5f24..92b34429e 100755 --- a/vizro-core/tests/unit/vizro/models/_components/form/test_slider.py +++ b/vizro-core/tests/unit/vizro/models/_components/form/test_slider.py @@ -35,7 +35,7 @@ def expected_slider(): persistence_type="session", className="slider-text-input-field", ), - dcc.Store(id="slider_id_input_store", storage_type="session", data=5.0), + dcc.Store(id="slider_id_input_store", storage_type="session"), ], className="slider-text-input-container", ), diff --git a/vizro-core/tests/unit/vizro/models/_controls/test_filter.py b/vizro-core/tests/unit/vizro/models/_controls/test_filter.py index 3f3d909e9..04d0004c0 100644 --- a/vizro-core/tests/unit/vizro/models/_controls/test_filter.py +++ b/vizro-core/tests/unit/vizro/models/_controls/test_filter.py @@ -1,6 +1,7 @@ from datetime import date, datetime from typing import Literal +from dash import dcc import pandas as pd import pytest from asserts import assert_component_equal @@ -8,7 +9,8 @@ import vizro.models as vm import vizro.plotly.express as px from vizro import Vizro -from vizro.managers import model_manager +from vizro.managers import data_manager, model_manager +from vizro.models import DatePicker from vizro.models._action._actions_chain import ActionsChain from vizro.models._controls.filter import Filter, _filter_between, _filter_isin from vizro.models.types import CapturedCallable @@ -219,6 +221,147 @@ def test_filter_isin_date(self, data, value, expected): pd.testing.assert_series_equal(result, expected) +class TestFilterStaticMethods: + """Tests static methods of the Filter class.""" + + @pytest.mark.parametrize( + "data_columns, expected", + [ + ([[]], []), + ([["A", "B", "A"]], ["A", "B"]), + ([[1, 2, 1]], [1, 2]), + ([[1.1, 2.2, 1.1]], [1.1, 2.2]), + ([[ + datetime(2024, 1, 1), + datetime(2024, 1, 2), + datetime(2024, 1, 1), + ]], + [ + datetime(2024, 1, 1), + datetime(2024, 1, 2), + ]), + ([["A", "B"], ["B", "C"]], ["A", "B", "C"]), + ([["A", "B"], ["C"]], ["A", "B", "C"]), + ([["A" ], []], ["A" ]), + ], + ) + def test_get_options(self, data_columns, expected): + targeted_data = pd.DataFrame({f"target_{i}": pd.Series(data) for i, data in enumerate(data_columns)}) + result = Filter._get_options(targeted_data) + assert result == expected + + @pytest.mark.parametrize( + "data_columns, current_value, expected", + [ + ([[]], None, []), + ([[]], "ALL", []), + ([[]], ["ALL", "A"], ["A"]), + ([[]], "A", ["A"]), + ([[]], ["A", "B"], ["A", "B"]), + ([["A", "B"]], "C", ["A", "B", "C"]), + ([["A", "B"]], ["C", "D"], ["A", "B", "C", "D"]), + ([[1, 2]], 3, [1, 2, 3]), + ([[1, 2]], [3, 4], [1, 2, 3, 4]), + ([[1.1, 2.2]], 3.3, [1.1, 2.2, 3.3]), + ([[1.1, 2.2]], [3.3, 4.4], [1.1, 2.2, 3.3, 4.4]), + ([[ + datetime(2024, 1, 1), + datetime(2024, 1, 2), + ]], + datetime(2024, 1, 3), + [ + datetime(2024, 1, 1), + datetime(2024, 1, 2), + datetime(2024, 1, 3), + ]), + ([[ + datetime(2024, 1, 1), + datetime(2024, 1, 2), + ]], + [ + datetime(2024, 1, 3), + datetime(2024, 1, 4), + ], + [ + datetime(2024, 1, 1), + datetime(2024, 1, 2), + datetime(2024, 1, 3), + datetime(2024, 1, 4), + ]), + ], + ) + def test_get_options_with_current_value(self, data_columns, current_value, expected): + targeted_data = pd.DataFrame({f"target_{i}": pd.Series(data) for i, data in enumerate(data_columns)}) + result = Filter._get_options(targeted_data, current_value) + assert result == expected + + @pytest.mark.parametrize( + "data_columns, expected", + [ + ([[1, 2, 1]], (1, 2)), + ([[1.1, 2.2, 1.1]], (1.1, 2.2)), + ([[ + datetime(2024, 1, 1), + datetime(2024, 1, 2), + datetime(2024, 1, 1), + ]], + ( + datetime(2024, 1, 1), + datetime(2024, 1, 2), + )), + ([[1, 2], [2, 3]], (1, 3)), + ([[1, 2], [3]], (1, 3)), + ([[1, 2], []], (1, 2)), + ], + ) + def test_get_min_max(self, data_columns, expected): + targeted_data = pd.DataFrame({f"target_{i}": pd.Series(data) for i, data in enumerate(data_columns)}) + result = Filter._get_min_max(targeted_data) + assert result == expected + + @pytest.mark.parametrize( + "data_columns, current_value, expected", + [ + ([[1, 2]], 3, (1, 3)), + ([[1, 2]], [3, 4], (1, 4)), + ([[1.1, 2.2]], 3.3, (1.1, 3.3)), + ([[1.1, 2.2]], [3.3, 4.4], (1.1, 4.4)), + ([[ + datetime(2024, 1, 1), + datetime(2024, 1, 2), + datetime(2024, 1, 1), + ]], + datetime(2024, 1, 3), + ( + datetime(2024, 1, 1), + datetime(2024, 1, 3), + )), + ([[ + datetime(2024, 1, 1), + datetime(2024, 1, 2), + datetime(2024, 1, 1), + ]], + [ + datetime(2024, 1, 3), + datetime(2024, 1, 4), + ], + ( + datetime(2024, 1, 1), + datetime(2024, 1, 4), + ) + ), + ([[1, 2], [2, 3]], 4, (1, 4)), + ([[1, 2], [2, 3]], [4, 5], (1, 5)), + ([[1, 2], []], 3, (1, 3)), + ([[1, 2], []], [3, 4], (1, 4)), + ], + ) + def test_get_min_max_with_current_value(self, data_columns, current_value, expected): + targeted_data = pd.DataFrame({f"target_{i}": pd.Series(data) for i, data in enumerate(data_columns)}) + result = Filter._get_min_max(targeted_data, current_value) + assert result == expected + + @pytest.mark.usefixtures("managers_one_page_two_graphs") class TestFilterInstantiation: """Tests model instantiation and the validators run at that time.""" @@ -387,6 +530,78 @@ def test_validate_column_type(self, targets, managers_column_different_type): ): filter.pre_build() + @pytest.mark.usefixtures("managers_one_page_two_graphs") + def test_filter_is_not_dynamic(self): + filter = vm.Filter(column="continent") + model_manager["test_page"].controls = [filter] + filter.pre_build() + # Filter is not dynamic because it does not target a figure that uses dynamic data + assert filter._dynamic is False + assert filter.selector._dynamic is False + + @pytest.mark.usefixtures("managers_one_page_two_graphs_with_dynamic_data") + @pytest.mark.parametrize( + "test_column ,test_selector", + [ + ("continent", vm.Checklist()), + ("continent", vm.Dropdown()), + ("continent", vm.Dropdown(multi=False)), + ("continent", vm.RadioItems()), + ("pop", vm.Slider()), + ("pop", vm.RangeSlider()), + ], + ) + def test_filter_is_dynamic_with_dynamic_selectors( + self, + test_column, + test_selector, + gapminder_dynamic_first_n_last_n_function + ): + data_manager["gapminder_dynamic_first_n_last_n"] = gapminder_dynamic_first_n_last_n_function + filter = vm.Filter(column=test_column, selector=test_selector) + model_manager["test_page"].controls = [filter] + filter.pre_build() + # Filter is dynamic because it targets a figure that uses dynamic data + assert filter._dynamic is True + assert filter.selector._dynamic is True + + @pytest.mark.usefixtures("managers_one_page_two_graphs_with_dynamic_data") + def test_filter_is_not_dynamic_with_non_dynamic_selectors(self, gapminder_dynamic_first_n_last_n_function): + data_manager["gapminder_dynamic_first_n_last_n"] = gapminder_dynamic_first_n_last_n_function + filter = vm.Filter(column="year", selector=vm.DatePicker()) + model_manager["test_page"].controls = [filter] + filter.pre_build() + assert filter._dynamic is False + + @pytest.mark.usefixtures("managers_one_page_two_graphs_with_dynamic_data") + @pytest.mark.parametrize( + "test_column ,test_selector", + [ + ("continent", vm.Checklist(options=["Africa", "Europe"])), + ("continent", vm.Dropdown(options=["Africa", "Europe"])), + ("continent", vm.Dropdown(multi=False, options=["Africa", "Europe"])), + ("continent", vm.RadioItems(options=["Africa", "Europe"])), + ("pop", vm.Slider(min=2002)), + ("pop", vm.Slider(max=2007)), + ("pop", vm.Slider(min=2002, max=2007)), + ("pop", vm.RangeSlider(min=2002)), + ("pop", vm.RangeSlider(max=2007)), + ("pop", vm.RangeSlider(min=2002, max=2007)), + ], + ) + def test_filter_is_not_dynamic_with_options_min_max_specified( + self, + test_column, + test_selector, + gapminder_dynamic_first_n_last_n_function + ): + data_manager["gapminder_dynamic_first_n_last_n"] = gapminder_dynamic_first_n_last_n_function + filter = vm.Filter(column=test_column, selector=test_selector) + model_manager["test_page"].controls = [filter] + filter.pre_build() + assert filter._dynamic is False + assert filter.selector._dynamic is False + @pytest.mark.parametrize("selector", [vm.Slider, vm.RangeSlider]) def test_numerical_min_max_default(self, selector, gapminder, managers_one_page_two_graphs): filter = vm.Filter(column="lifeExp", selector=selector()) @@ -500,18 +715,19 @@ def build(self): assert default_action.actions[0].id == f"filter_action_{filter.id}" -@pytest.mark.usefixtures("managers_one_page_two_graphs") class TestFilterBuild: """Tests filter build method.""" + @pytest.mark.usefixtures("managers_one_page_two_graphs") @pytest.mark.parametrize( - "test_column,test_selector", + "test_column ,test_selector", [ ("continent", vm.Checklist()), ("continent", vm.Dropdown()), + ("continent", vm.Dropdown(multi=False)), ("continent", vm.RadioItems()), - ("pop", vm.RangeSlider()), ("pop", vm.Slider()), + ("pop", vm.RangeSlider()), ("year", vm.DatePicker()), ("year", vm.DatePicker(range=False)), ], @@ -524,3 +740,42 @@ def test_filter_build(self, test_column, test_selector): expected = test_selector.build() assert_component_equal(result, expected) + + @pytest.mark.usefixtures("managers_one_page_two_graphs_with_dynamic_data") + @pytest.mark.parametrize( + "test_column, test_selector", + [ + ("continent", vm.Checklist()), + ("continent", vm.Dropdown()), + ("continent", vm.Dropdown(multi=False)), + ("continent", vm.RadioItems()), + ("pop", vm.Slider()), + ("pop", vm.RangeSlider()), + ], + ) + def test_dynamic_filter_build(self, test_column, test_selector, gapminder_dynamic_first_n_last_n_function): + # Adding dynamic data_frame to data_manager + data_manager["gapminder_dynamic_first_n_last_n"] = gapminder_dynamic_first_n_last_n_function + filter = vm.Filter(id="filter_id", column=test_column, selector=test_selector) + model_manager["test_page"].controls = [filter] + filter.pre_build() + + result = filter.build() + expected = dcc.Loading(id="filter_id", children=test_selector.build()) + + assert_component_equal(result, expected) + + @pytest.mark.usefixtures("managers_one_page_two_graphs_with_dynamic_data") + def test_dynamic_filter_build_with_non_dynamic_selectors(self, gapminder_dynamic_first_n_last_n_function): + # Adding dynamic data_frame to data_manager + data_manager["gapminder_dynamic_first_n_last_n"] = gapminder_dynamic_first_n_last_n_function + + test_selector = vm.DatePicker() + filter = vm.Filter(column="year", selector=test_selector) + model_manager["test_page"].controls = [filter] + filter.pre_build() + + result = filter.build() + expected = test_selector.build() + + assert_component_equal(result, expected) From e6208faa19aa1b2fad6ae6d83f73d0b800effad0 Mon Sep 17 00:00:00 2001 From: petar-qb Date: Mon, 25 Nov 2024 08:44:33 +0100 Subject: [PATCH 38/64] Lint --- vizro-core/examples/scratch_dev/data.yaml | 2 +- .../src/vizro/models/_controls/filter.py | 16 +- .../tests/unit/vizro/actions/conftest.py | 1 - .../vizro/models/_controls/test_filter.py | 164 ++++++++++-------- 4 files changed, 100 insertions(+), 83 deletions(-) diff --git a/vizro-core/examples/scratch_dev/data.yaml b/vizro-core/examples/scratch_dev/data.yaml index d8b0aea90..63e64b40d 100644 --- a/vizro-core/examples/scratch_dev/data.yaml +++ b/vizro-core/examples/scratch_dev/data.yaml @@ -1,6 +1,6 @@ # Choose between 0-50 setosa: 5 -versicolor: 10 +#versicolor: 10 virginica: 15 # Choose between: 4.3 to 7.4 diff --git a/vizro-core/src/vizro/models/_controls/filter.py b/vizro-core/src/vizro/models/_controls/filter.py index c70e7032b..8920dce34 100644 --- a/vizro-core/src/vizro/models/_controls/filter.py +++ b/vizro-core/src/vizro/models/_controls/filter.py @@ -1,11 +1,10 @@ from __future__ import annotations +from contextlib import suppress from typing import Any, Literal, Union -import numpy as np import pandas as pd from dash import dcc -from contextlib import suppress from pandas.api.types import is_datetime64_any_dtype, is_numeric_dtype from vizro.managers._data_manager import DataSourceName @@ -275,13 +274,16 @@ def _get_min_max(targeted_data: pd.DataFrame, current_value=None) -> tuple[float _max = pd.to_datetime(_max) current_value = pd.to_datetime(current_value) # Convert DatetimeIndex to list of Timestamp objects so that we can use min and max functions below. - with suppress(AttributeError): current_value = current_value.tolist() + with suppress(AttributeError): + current_value = current_value.tolist() # Use item() to convert to convert scalar from numpy to Python type. This isn't needed during pre_build because # pydantic will coerce the type, but it is necessary in __call__ where we don't update model field values # and instead just pass straight to the Dash component. - with suppress(AttributeError): _min = _min.item() - with suppress(AttributeError): _max = _max.item() + with suppress(AttributeError): + _min = _min.item() + with suppress(AttributeError): + _max = _max.item() if current_value is not None: current_value = current_value if isinstance(current_value, list) else [current_value] @@ -305,7 +307,7 @@ def _get_options(targeted_data: pd.DataFrame, current_value=None) -> list[Any]: if isinstance(current_value, list) and ALL_OPTION in current_value: current_value.remove(ALL_OPTION) - targeted_data_series = targeted_data.stack().dropna() + targeted_data_series = targeted_data.stack().dropna() # noqa: PD013 current_value_series = pd.Series(current_value).astype(targeted_data_series.dtypes) - return sorted(list(pd.concat([targeted_data_series, current_value_series]).unique())) + return sorted(pd.concat([targeted_data_series, current_value_series]).unique()) diff --git a/vizro-core/tests/unit/vizro/actions/conftest.py b/vizro-core/tests/unit/vizro/actions/conftest.py index 6833fa0d9..3b7bc5337 100644 --- a/vizro-core/tests/unit/vizro/actions/conftest.py +++ b/vizro-core/tests/unit/vizro/actions/conftest.py @@ -1,4 +1,3 @@ -import pandas as pd import pytest import vizro.models as vm diff --git a/vizro-core/tests/unit/vizro/models/_controls/test_filter.py b/vizro-core/tests/unit/vizro/models/_controls/test_filter.py index 04d0004c0..6f4f5550d 100644 --- a/vizro-core/tests/unit/vizro/models/_controls/test_filter.py +++ b/vizro-core/tests/unit/vizro/models/_controls/test_filter.py @@ -1,16 +1,15 @@ from datetime import date, datetime from typing import Literal -from dash import dcc import pandas as pd import pytest from asserts import assert_component_equal +from dash import dcc import vizro.models as vm import vizro.plotly.express as px from vizro import Vizro from vizro.managers import data_manager, model_manager -from vizro.models import DatePicker from vizro.models._action._actions_chain import ActionsChain from vizro.models._controls.filter import Filter, _filter_between, _filter_isin from vizro.models.types import CapturedCallable @@ -231,18 +230,22 @@ class TestFilterStaticMethods: ([["A", "B", "A"]], ["A", "B"]), ([[1, 2, 1]], [1, 2]), ([[1.1, 2.2, 1.1]], [1.1, 2.2]), - ([[ - datetime(2024, 1, 1), - datetime(2024, 1, 2), - datetime(2024, 1, 1), - ]], - [ - datetime(2024, 1, 1), - datetime(2024, 1, 2), - ]), + ( + [ + [ + datetime(2024, 1, 1), + datetime(2024, 1, 2), + datetime(2024, 1, 1), + ] + ], + [ + datetime(2024, 1, 1), + datetime(2024, 1, 2), + ], + ), ([["A", "B"], ["B", "C"]], ["A", "B", "C"]), ([["A", "B"], ["C"]], ["A", "B", "C"]), - ([["A" ], []], ["A" ]), + ([["A"], []], ["A"]), ], ) def test_get_options(self, data_columns, expected): @@ -264,30 +267,38 @@ def test_get_options(self, data_columns, expected): ([[1, 2]], [3, 4], [1, 2, 3, 4]), ([[1.1, 2.2]], 3.3, [1.1, 2.2, 3.3]), ([[1.1, 2.2]], [3.3, 4.4], [1.1, 2.2, 3.3, 4.4]), - ([[ - datetime(2024, 1, 1), - datetime(2024, 1, 2), - ]], - datetime(2024, 1, 3), - [ - datetime(2024, 1, 1), - datetime(2024, 1, 2), - datetime(2024, 1, 3), - ]), - ([[ - datetime(2024, 1, 1), - datetime(2024, 1, 2), - ]], - [ + ( + [ + [ + datetime(2024, 1, 1), + datetime(2024, 1, 2), + ] + ], datetime(2024, 1, 3), - datetime(2024, 1, 4), - ], - [ - datetime(2024, 1, 1), - datetime(2024, 1, 2), - datetime(2024, 1, 3), - datetime(2024, 1, 4), - ]), + [ + datetime(2024, 1, 1), + datetime(2024, 1, 2), + datetime(2024, 1, 3), + ], + ), + ( + [ + [ + datetime(2024, 1, 1), + datetime(2024, 1, 2), + ] + ], + [ + datetime(2024, 1, 3), + datetime(2024, 1, 4), + ], + [ + datetime(2024, 1, 1), + datetime(2024, 1, 2), + datetime(2024, 1, 3), + datetime(2024, 1, 4), + ], + ), ], ) def test_get_options_with_current_value(self, data_columns, current_value, expected): @@ -300,15 +311,19 @@ def test_get_options_with_current_value(self, data_columns, current_value, expec [ ([[1, 2, 1]], (1, 2)), ([[1.1, 2.2, 1.1]], (1.1, 2.2)), - ([[ - datetime(2024, 1, 1), - datetime(2024, 1, 2), - datetime(2024, 1, 1), - ]], - ( - datetime(2024, 1, 1), - datetime(2024, 1, 2), - )), + ( + [ + [ + datetime(2024, 1, 1), + datetime(2024, 1, 2), + datetime(2024, 1, 1), + ] + ], + ( + datetime(2024, 1, 1), + datetime(2024, 1, 2), + ), + ), ([[1, 2], [2, 3]], (1, 3)), ([[1, 2], [3]], (1, 3)), ([[1, 2], []], (1, 2)), @@ -326,29 +341,36 @@ def test_get_min_max(self, data_columns, expected): ([[1, 2]], [3, 4], (1, 4)), ([[1.1, 2.2]], 3.3, (1.1, 3.3)), ([[1.1, 2.2]], [3.3, 4.4], (1.1, 4.4)), - ([[ - datetime(2024, 1, 1), - datetime(2024, 1, 2), - datetime(2024, 1, 1), - ]], - datetime(2024, 1, 3), - ( - datetime(2024, 1, 1), - datetime(2024, 1, 3), - )), - ([[ - datetime(2024, 1, 1), - datetime(2024, 1, 2), - datetime(2024, 1, 1), - ]], - [ + ( + [ + [ + datetime(2024, 1, 1), + datetime(2024, 1, 2), + datetime(2024, 1, 1), + ] + ], datetime(2024, 1, 3), - datetime(2024, 1, 4), - ], - ( - datetime(2024, 1, 1), - datetime(2024, 1, 4), - ) + ( + datetime(2024, 1, 1), + datetime(2024, 1, 3), + ), + ), + ( + [ + [ + datetime(2024, 1, 1), + datetime(2024, 1, 2), + datetime(2024, 1, 1), + ] + ], + [ + datetime(2024, 1, 3), + datetime(2024, 1, 4), + ], + ( + datetime(2024, 1, 1), + datetime(2024, 1, 4), + ), ), ([[1, 2], [2, 3]], 4, (1, 4)), ([[1, 2], [2, 3]], [4, 5], (1, 5)), @@ -552,10 +574,7 @@ def test_filter_is_not_dynamic(self): ], ) def test_filter_is_dynamic_with_dynamic_selectors( - self, - test_column, - test_selector, - gapminder_dynamic_first_n_last_n_function + self, test_column, test_selector, gapminder_dynamic_first_n_last_n_function ): data_manager["gapminder_dynamic_first_n_last_n"] = gapminder_dynamic_first_n_last_n_function filter = vm.Filter(column=test_column, selector=test_selector) @@ -590,10 +609,7 @@ def test_filter_is_not_dynamic_with_non_dynamic_selectors(self, gapminder_dynami ], ) def test_filter_is_not_dynamic_with_options_min_max_specified( - self, - test_column, - test_selector, - gapminder_dynamic_first_n_last_n_function + self, test_column, test_selector, gapminder_dynamic_first_n_last_n_function ): data_manager["gapminder_dynamic_first_n_last_n"] = gapminder_dynamic_first_n_last_n_function filter = vm.Filter(column=test_column, selector=test_selector) From a03aa450e7d98e616f21badf66bcd526e70a270b Mon Sep 17 00:00:00 2001 From: Antony Milne <49395058+antonymilne@users.noreply.github.com> Date: Mon, 25 Nov 2024 14:59:00 +0000 Subject: [PATCH 39/64] [Docs] Dynamic filters (#891) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Jo Stichbury Co-authored-by: Petar Pejovic <108530920+petar-qb@users.noreply.github.com> --- ...1120_154345_antony.milne_dynamic_filter.md | 46 ++++++++++ vizro-core/docs/pages/user-guides/data.md | 90 ++++++++++++++++--- vizro-core/docs/pages/user-guides/filters.md | 90 +++++++++++++++++-- .../docs/pages/user-guides/selectors.md | 89 ------------------ 4 files changed, 206 insertions(+), 109 deletions(-) create mode 100644 vizro-core/changelog.d/20241120_154345_antony.milne_dynamic_filter.md diff --git a/vizro-core/changelog.d/20241120_154345_antony.milne_dynamic_filter.md b/vizro-core/changelog.d/20241120_154345_antony.milne_dynamic_filter.md new file mode 100644 index 000000000..f7981a8b4 --- /dev/null +++ b/vizro-core/changelog.d/20241120_154345_antony.milne_dynamic_filter.md @@ -0,0 +1,46 @@ + + +### Highlights ✨ + +- Filters update automatically when underlying dynamic data changes. See the [user guide on dynamic filters](https://vizro.readthedocs.io/en/stable/pages/user-guides/data/#filters) for more information. ([#879](https://github.com/mckinsey/vizro/pull/879)) + + + + + + + diff --git a/vizro-core/docs/pages/user-guides/data.md b/vizro-core/docs/pages/user-guides/data.md index b4fdfeda1..ffd53f2fb 100644 --- a/vizro-core/docs/pages/user-guides/data.md +++ b/vizro-core/docs/pages/user-guides/data.md @@ -179,7 +179,7 @@ Since dynamic data sources must always be added to the data manager and referenc ### Configure cache -By default, each time the dashboard is refreshed a dynamic data function executes again. In fact, if there are multiple graphs on the same page using the same dynamic data source then the loading function executes _multiple_ times, once for each graph on the page. Hence, if loading your data is a slow operation, your dashboard performance may suffer. +By default, a dynamic data function executes every time the dashboard is refreshed. Data loading is batched so that a dynamic data function that supplies multiple graphs on the same page only executes _once_ on page refresh. Even with this batching, if loading your data is still a slow operation, your dashboard performance may suffer. The Vizro data manager has a server-side caching mechanism to help solve this. Vizro's cache uses [Flask-Caching](https://flask-caching.readthedocs.io/en/latest/), which supports a number of possible cache backends and [configuration options](https://flask-caching.readthedocs.io/en/latest/#configuring-flask-caching). By default, the cache is turned off. @@ -220,7 +220,7 @@ By default, when caching is turned on, dynamic data is cached in the data manage If you would like to alter some options, such as the default cache timeout, then you can specify a different cache configuration: -```py title="Simple cache with timeout set to 10 minutes" +```python title="Simple cache with timeout set to 10 minutes" data_manager.cache = Cache(config={"CACHE_TYPE": "SimpleCache", "CACHE_DEFAULT_TIMEOUT": 600}) ``` @@ -268,8 +268,12 @@ data_manager["no_expire_data"].timeout = 0 ### Parametrize data loading -You can supply arguments to your dynamic data loading function that can be modified from the dashboard. -For example, if you are handling big data then you can use an argument to specify the number of entries or size of chunk of data. +You can give arguments to your dynamic data loading function that can be modified from the dashboard. For example: + +- To load different versions of the same data. +- To handle big data you can use an argument that controls the amount of data that is loaded. This effectively pre-filters data before it reaches the Vizro dashboard. + +In general, a parametrized dynamic data source should always return a pandas DataFrame with a fixed schema (column names and types). This ensures that page components and controls continue to work as expected when the parameter is changed on screen. To add a parameter to control a dynamic data source, do the following: @@ -277,7 +281,7 @@ To add a parameter to control a dynamic data source, do the following: 2. give an `id` to all components that have the data source you wish to alter through a parameter. 3. [add a parameter](parameters.md) with `targets` of the form `.data_frame.` and a suitable [selector](selectors.md). -For example, let us extend the [dynamic data example](#dynamic-data) above to show how the `load_iris_data` can take an argument `number_of_points` controlled from the dashboard with a [`Slider`][vizro.models.Slider]. +For example, let us extend the [dynamic data example](#dynamic-data) above into a simple toy example of how parametrized dynamic data works. The `load_iris_data` can take an argument `number_of_points` controlled from the dashboard with a [`Slider`][vizro.models.Slider]. !!! example "Parametrized dynamic data" === "app.py" @@ -333,14 +337,78 @@ Parametrized data loading is compatible with [caching](#configure-cache). The ca You cannot pass [nested parameters](parameters.md#nested-parameters) to dynamic data. You can only target the top-level arguments of the data loading function, not the nested keys in a dictionary. -### Filter update limitation +### Filters + +When a [filter](filters.md) depends on dynamic data and no `selector` is explicitly defined in the `vm.Filter` model, the available selector values update on page refresh to reflect the latest dynamic data. This is called a _dynamic filter_. + +The mechanism for dynamic filters, including caching, works exactly like other non-control components such as `vm.Graph`. However, unlike such components, a filter can depend on multiple data sources. If at least one data source of the components in the filter's `targets` is dynamic then the filter is dynamic. Remember that when `targets` is not explicitly specified, a filter applies to all the components on a page that use a DataFrame including `column`. + +When the page is refreshed, the behaviour of a dynamic filter is as follows: + +- The filter's selector updates its available values: + - For [categorical selectors](selectors.md#categorical-selectors), `options` updates to give all unique values found in `column` across all the data sources of components in `targets`. + - For [numerical selectors](selectors.md#numerical-selectors), `min` and `max` update to give the overall minimum and maximum values found in `column` across all the data sources of components in `targets`. +- The value selected on screen by a dashboard user _does not_ change. If the selected value is not present in the new set of available values then it is still selected, but the filtering operation might result in an empty DataFrame. +- Even though the values present in a data source can change, the schema should not: `column` should remain present and of the same type in the data sources. The `targets` of the filter and selector type cannot change while the dashboard is running. For example, a `vm.Dropdown` selector cannot turn into `vm.RadioItems`. + +For example, let us add two filters to the [dynamic data example](#dynamic-data) above: + +!!! example "Dynamic filters" + + ```py hl_lines="10 20 21" + from vizro import Vizro + import pandas as pd + import vizro.plotly.express as px + import vizro.models as vm + + from vizro.managers import data_manager + + def load_iris_data(): + iris = pd.read_csv("iris.csv") + return iris.sample(5) # (1)! + + data_manager["iris"] = load_iris_data -If your dashboard includes a [filter](filters.md) then the values shown on a filter's [selector](selectors.md) _do not_ update while the dashboard is running. This is a known limitation that will be lifted in future releases, but if is problematic for you already then [raise an issue on our GitHub repo](https://github.com/mckinsey/vizro/issues/). + page = vm.Page( + title="Update the chart and filters on page refresh", + components=[ + vm.Graph(figure=px.box("iris", x="species", y="petal_width", color="species")) + ], + controls=[ + vm.Filter(column="species"), # (2)! + vm.Filter(column="sepal_length"), # (3)! + ], + ) -This limitation is why all arguments of your dynamic data loading function must have a default value. Regardless of the value of the `vm.Parameter` selected in the dashboard, these default parameter values are used when the `vm.Filter` is built. This determines the type of selector used in a filter and the options shown, which cannot currently be changed while the dashboard is running. + dashboard = vm.Dashboard(pages=[page]) -Although a selector is automatically chosen for you in a filter when your dashboard is built, remember that [you can change this choice](filters.md#changing-selectors). For example, we could ensure that a dropdown always contains the options "setosa", "versicolor" and "virginica" by explicitly specifying your filter as follows. + Vizro().build(dashboard).run() + ``` -```py -vm.Filter(column="species", selector=vm.Dropdown(options=["setosa", "versicolor", "virginica"]) + 1. We sample only 5 rather than 50 points so that changes to the available values in the filtered columns are more apparent when the page is refreshed. + 2. This filter implicitly controls the dynamic data source `"iris"`, which supplies the `data_frame` to the targeted `vm.Graph`. On page refresh, Vizro reloads this data, finds all the unique values in the `"species"` column and sets the categorical selector's `options` accordingly. + 3. Similarly, on page refresh, Vizro finds the minimum and maximum values of the `"sepal_length"` column in the reloaded data and sets new `min` and `max` values for the numerical selector accordingly. + +If you have a filter that depends on dynamic data but do not want the available values to change when the dynamic data changes then you should manually specify the `selector`'s `options` field (categorical selector) or `min` and `max` fields (numerical selector). In the above example, this could be achieved as follows: + +```python title="Override selector options to make a dynamic filter static" +controls = [ + vm.Filter(column="species", selector=vm.Dropdown(options=["setosa", "versicolor", "virginica"])), + vm.Filter(column="sepal_length", selector=vm.RangeSlider(min=4.3, max=7.9)), +] +``` + +If you [use a specific selector](filters.md#change-selector) for a dynamic filter without manually specifying `options` (categorical selector) or `min` and `max` (numerical selector) then the selector remains dynamic. For example: + +```python title="Dynamic filter with specific selector is still dynamic" +controls = [ + vm.Filter(column="species", selector=vm.Checklist()), + vm.Filter(column="sepal_length", selector=vm.Slider()), +] ``` + +When Vizro initially builds a filter that depends on parametrized dynamic data loading, data is loaded using the default argument values. This data is used to perform initial validation, check which data sources contain the specified `column` (unless `targets` is explicitly specified) and determine the type of selector to use (unless `selector` is explicitly specified). + +!!! note + + When a dynamic data parameter is changed on screen, the data underlying a dynamic filter can change. Currently this change affects page components such as `vm.Graph` but does not affect the available values shown in a dynamic filter, which only update on page refresh. This functionality will be coming soon! diff --git a/vizro-core/docs/pages/user-guides/filters.md b/vizro-core/docs/pages/user-guides/filters.md index bad355fab..0680c0bca 100644 --- a/vizro-core/docs/pages/user-guides/filters.md +++ b/vizro-core/docs/pages/user-guides/filters.md @@ -3,8 +3,9 @@ This guide shows you how to add filters to your dashboard. One main way to interact with the charts/components on your page is by filtering the underlying data. A filter selects a subset of rows of a component's underlying DataFrame which alters the appearance of that component on the page. The [`Page`][vizro.models.Page] model accepts the `controls` argument, where you can enter a [`Filter`][vizro.models.Filter] model. -This model enables the automatic creation of [selectors](../user-guides/selectors.md) (such as Dropdown, RadioItems, Slider, ...) that operate upon the charts/components on the screen. +This model enables the automatic creation of [selectors](selectors.md) (such as `Dropdown`, `RadioItems`, `Slider`, ...) that operate on the charts/components on the screen. +By default, filters that control components with [dynamic data](data.md#dynamic-data) are [dynamically updated](data.md#filters) when the underlying data changes while the dashboard is running. ## Basic filters @@ -13,8 +14,7 @@ To add a filter to your page, do the following: 1. add the [`Filter`][vizro.models.Filter] model into the `controls` argument of the [`Page`][vizro.models.Page] model 2. configure the `column` argument, which denotes the target column to be filtered -By default, all components on a page with such a `column` present will be filtered. The selector type will be chosen -automatically based on the target column, for example, a dropdown for categorical data, a range slider for numerical data, or a date picker for temporal data. +You can also set `targets` to specify which components on the page should be affected by the filter. If this is not explicitly set then `targets` defaults to all components on the page whose data source includes `column`. !!! example "Basic Filter" === "app.py" @@ -63,12 +63,83 @@ automatically based on the target column, for example, a dropdown for categorica [Filter]: ../../assets/user_guides/control/control1.png -## Changing selectors +The selector is configured automatically based on the target column type data as follows: + + - Categorical data uses [`vm.Dropdown(multi=True)`][vizro.models.Dropdown] where `options` is the set of unique values found in `column` across all the data sources of components in `targets`. + - [Numerical data](https://pandas.pydata.org/docs/reference/api/pandas.api.types.is_numeric_dtype.html) uses [`vm.RangeSlider`][vizro.models.RangeSlider] where `min` and `max` are the overall minimum and maximum values found in `column` across all the data sources of components in `targets`. + - [Temporal data](https://pandas.pydata.org/docs/reference/api/pandas.api.types.is_datetime64_any_dtype.html) uses [`vm.DatePicker(range=True)`][vizro.models.DatePicker] where `min` and `max` are the overall minimum and maximum values found in `column` across all the data sources of components in `targets`. A column can be converted to this type with [pandas.to_datetime](https://pandas.pydata.org/docs/reference/api/pandas.to_datetime.html). + +Below is an example demonstrating these default selector types. + +!!! example "Default Filter selectors" + === "app.py" + ```{.python pycafe-link} + import pandas as pd + from vizro import Vizro + import vizro.plotly.express as px + import vizro.models as vm + + df_stocks = px.data.stocks(datetimes=True) + + df_stocks_long = pd.melt( + df_stocks, + id_vars='date', + value_vars=['GOOG', 'AAPL', 'AMZN', 'FB', 'NFLX', 'MSFT'], + var_name='stocks', + value_name='value' + ) + + df_stocks_long['value'] = df_stocks_long['value'].round(3) + + page = vm.Page( + title="My first page", + components=[ + vm.Graph(figure=px.line(df_stocks_long, x="date", y="value", color="stocks")), + ], + controls=[ + vm.Filter(column="stocks"), + vm.Filter(column="value"), + vm.Filter(column="date"), + ], + ) + + dashboard = vm.Dashboard(pages=[page]) + + Vizro().build(dashboard).run() + ``` + === "app.yaml" + ```yaml + # Still requires a .py to add data to the data manager and parse YAML configuration + # See yaml_version example + pages: + - components: + - figure: + _target_: line + data_frame: df_stocks_long + x: date + y: value + color: stocks + type: graph + controls: + - column: stocks + type: filter + - column: value + type: filter + - column: date + type: filter + title: My first page + ``` + === "Result" + [![Filter]][Filter] + + [Filter]: ../../assets/user_guides/selectors/default_filter_selectors.png + +## Change selector If you want to have a different selector for your filter, you can give the `selector` argument of the [`Filter`][vizro.models.Filter] a different selector model. Currently available selectors are [`Checklist`][vizro.models.Checklist], [`Dropdown`][vizro.models.Dropdown], [`RadioItems`][vizro.models.RadioItems], [`RangeSlider`][vizro.models.RangeSlider], [`Slider`][vizro.models.Slider], and [`DatePicker`][vizro.models.DatePicker]. -!!! example "Filter with custom Selector" +!!! example "Filter with different selector" === "app.py" ```{.python pycafe-link} from vizro import Vizro @@ -118,11 +189,10 @@ Currently available selectors are [`Checklist`][vizro.models.Checklist], [`Dropd ## Further customization -For further customizations, you can always refer to the [`Filter`][vizro.models.Filter] reference. Some popular choices are: +For further customizations, you can always refer to the [`Filter` model][vizro.models.Filter] reference and the [guide to selectors](selectors.md). Some popular choices are: - select which component the filter will apply to by using `targets` -- select what the target column type is, hence choosing the default selector by using `column_type` -- choose options of lower level components, such as the `selector` models +- specify configuration of the `selector`, for example `multi` to switch between a multi-option and single-option selector, `options` for a categorical filter or `min` and `max` for a numerical filter Below is an advanced example where we only target one page component, and where we further customize the chosen `selector`. @@ -142,7 +212,7 @@ Below is an advanced example where we only target one page component, and where vm.Graph(figure=px.scatter(iris, x="petal_length", y="sepal_width", color="species")), ], controls=[ - vm.Filter(column="petal_length",targets=["scatter_chart"],selector=vm.RangeSlider(step=1)), + vm.Filter(column="petal_length",targets=["scatter_chart"], selector=vm.RangeSlider(step=1)), ], ) @@ -186,3 +256,5 @@ Below is an advanced example where we only target one page component, and where [![Advanced]][Advanced] [Advanced]: ../../assets/user_guides/control/control3.png + +To further customize selectors, see our [how-to-guide on creating custom components](custom-components.md). diff --git a/vizro-core/docs/pages/user-guides/selectors.md b/vizro-core/docs/pages/user-guides/selectors.md index 944515ab6..10481d2ba 100644 --- a/vizro-core/docs/pages/user-guides/selectors.md +++ b/vizro-core/docs/pages/user-guides/selectors.md @@ -53,92 +53,3 @@ For more information, refer to the API reference of the selector, or the documen When the [`DatePicker`][vizro.models.DatePicker] is configured with `range=True` (the default), the underlying component is `dmc.DateRangePicker`. When `range=False` the underlying component is `dmc.DatePicker`. When configuring the [`DatePicker`][vizro.models.DatePicker] make sure to provide your dates for `min`, `max` and `value` arguments in `"yyyy-mm-dd"` format or as `datetime` type (for example, `datetime.datetime(2024, 01, 01)`). - -## Default selectors - -If you don't specify a selector, a default selector is applied based on the data type of the provided column. - -Default selectors for: - - - categorical data: [`Dropdown`][vizro.models.Dropdown] - - numerical data: [`RangeSlider`][vizro.models.RangeSlider] - - temporal data: [`DatePicker(range=True)`][vizro.models.DatePicker] - -Categorical selectors can be used independently of the data type of the column being filtered. - -To use numerical [`Filter`][vizro.models.Filter] selectors, the filtered column must be of `numeric` format, -indicating that [pandas.api.types.is_numeric_dtype()](https://pandas.pydata.org/docs/reference/api/pandas.api.types.is_numeric_dtype.html) must return `True` for the filtered column. - -To use temporal [`Filter`][vizro.models.Filter] selectors, the filtered column must be of `datetime` format, -indicating that [pandas.api.types.is_datetime64_any_dtype()](https://pandas.pydata.org/docs/reference/api/pandas.api.types.is_datetime64_any_dtype.html) must return `True` for the filtered column. - -`pd.DataFrame` column types can be changed to `datetime` using [pandas.to_datetime()](https://pandas.pydata.org/docs/reference/api/pandas.to_datetime.html) or - - -### Example of default Filter selectors - -!!! example "Default Filter selectors" - === "app.py" - ```{.python pycafe-link} - import pandas as pd - from vizro import Vizro - import vizro.plotly.express as px - import vizro.models as vm - - df_stocks = px.data.stocks(datetimes=True) - - df_stocks_long = pd.melt( - df_stocks, - id_vars='date', - value_vars=['GOOG', 'AAPL', 'AMZN', 'FB', 'NFLX', 'MSFT'], - var_name='stocks', - value_name='value' - ) - - df_stocks_long['value'] = df_stocks_long['value'].round(3) - - page = vm.Page( - title="My first page", - components=[ - vm.Graph(figure=px.line(df_stocks_long, x="date", y="value", color="stocks")), - ], - controls=[ - vm.Filter(column="stocks"), - vm.Filter(column="value"), - vm.Filter(column="date"), - ], - ) - - dashboard = vm.Dashboard(pages=[page]) - - Vizro().build(dashboard).run() - ``` - === "app.yaml" - ```yaml - # Still requires a .py to add data to the data manager and parse YAML configuration - # See yaml_version example - pages: - - components: - - figure: - _target_: line - data_frame: df_stocks_long - x: date - y: value - color: stocks - type: graph - controls: - - column: stocks - type: filter - - column: value - type: filter - - column: date - type: filter - title: My first page - ``` - === "Result" - [![Filter]][Filter] - - [Filter]: ../../assets/user_guides/selectors/default_filter_selectors.png - - -To enhance existing selectors, see our [how-to-guide on creating custom components](custom-components.md). From 985ec4c916d1a1c37d403503596f0c350b9f4fcc Mon Sep 17 00:00:00 2001 From: Antony Milne Date: Mon, 25 Nov 2024 15:44:26 +0000 Subject: [PATCH 40/64] Add review comments and questions and small changes --- .../src/vizro/actions/_actions_utils.py | 3 ++ .../vizro/models/_components/form/dropdown.py | 16 ++++++--- .../src/vizro/models/_controls/filter.py | 36 ++++++++++--------- 3 files changed, 34 insertions(+), 21 deletions(-) diff --git a/vizro-core/src/vizro/actions/_actions_utils.py b/vizro-core/src/vizro/actions/_actions_utils.py index fb2dbf184..39e9b163e 100644 --- a/vizro-core/src/vizro/actions/_actions_utils.py +++ b/vizro-core/src/vizro/actions/_actions_utils.py @@ -282,12 +282,15 @@ def _get_modified_page_figures( **_get_parametrized_config(ctds_parameter=ctds_parameter, target=target, data_frame=False), ) + # AM comment: please check this comment I added! for target in control_targets: ctd_filter = [item for item in ctds_filter if item["id"] == model_manager[target].selector.id] # This only covers the case of cross-page actions when Filter in an output, but is not an input of the action. current_value = ctd_filter[0]["value"] if ctd_filter else None + # target_to_data_frame contains all targets, including some which might not be relevant for the filter in + # question. We filter to use just the relevant targets in Filter.__call__. outputs[target] = model_manager[target](target_to_data_frame=target_to_data_frame, current_value=current_value) return outputs diff --git a/vizro-core/src/vizro/models/_components/form/dropdown.py b/vizro-core/src/vizro/models/_components/form/dropdown.py index af22b71f4..a1c71bb13 100755 --- a/vizro-core/src/vizro/models/_components/form/dropdown.py +++ b/vizro-core/src/vizro/models/_components/form/dropdown.py @@ -88,10 +88,12 @@ def validate_multi(cls, multi, values): raise ValueError("Please set multi=True if providing a list of default values.") return multi + # AM comment: please remove build_static and change into __call__ in all places unless you think there's + # a good reason not to do so. def __call__(self, options): - return self._build_static(options) - - def _build_static(self, options): + # AM comment: this is the main confusing thing about the current approach I think: every time we run + # __call__ we override the value set below, which sounds wrong because we say that we never change the value. + # From what I remember this is a workaround for the Dash bug? Maybe add a comment explaining this. full_options, default_value = get_options_and_default(options=options, multi=self.multi) option_height = _calculate_option_height(full_options) @@ -114,13 +116,17 @@ def _build_dynamic_placeholder(self): # Setting self.value is kind of Dropdown pre_build method. It sets self.value only the first time if it's None. # We cannot create pre_build for the Dropdown because it has to be called after vm.Filter.pre_build, but nothing # guarantees that. We can call Filter.selector.pre_build() from the Filter.pre_build() method if we decide that. + # TODO: move this to pre_build once we have better control of the ordering. if self.value is None: - self.value = get_options_and_default(self.options, self.multi)[1] + _, default_value = get_options_and_default(self.options, self.multi) + self.value = default_value # TODO-NEXT: Replace this with the "universal Vizro placeholder" component. return html.Div( children=[ dbc.Label(self.title, html_for=self.id) if self.title else None, + # AM question: why do we want opacity: 0? If we don't want it to appear on screen then normally we do + # this with visibility or display. dmc.DateRangePicker( id=self.id, value=self.value, persistence=True, persistence_type="session", style={"opacity": 0} ), @@ -129,4 +135,4 @@ def _build_dynamic_placeholder(self): @_log_call def build(self): - return self._build_dynamic_placeholder() if self._dynamic else self._build_static(self.options) + return self._build_dynamic_placeholder() if self._dynamic else self.__call__(self.options) diff --git a/vizro-core/src/vizro/models/_controls/filter.py b/vizro-core/src/vizro/models/_controls/filter.py index 8920dce34..b2051bf3c 100644 --- a/vizro-core/src/vizro/models/_controls/filter.py +++ b/vizro-core/src/vizro/models/_controls/filter.py @@ -148,6 +148,7 @@ def pre_build(self): # 2. Propagate values from the model_manager and relax the limitation of requiring argument default values. # 3. Skip the pre-build and do everything in the build method (if possible). # Find more about the mentioned limitation at: https://github.com/mckinsey/vizro/pull/879/files#r1846609956 + # Even if the solution changes for dynamic data, static data should still use {} as the arguments here. multi_data_source_name_load_kwargs: list[tuple[DataSourceName, dict[str, Any]]] = [ (model_manager[target]["data_frame"], {}) for target in proposed_targets ] @@ -170,11 +171,12 @@ def pre_build(self): ) # Check if the filter is dynamic. Dynamic filter means that the filter is updated when the page is refreshed - # which causes that "options" for categorical or "min" and "max" for numerical/temporal selectors are updated. - # The filter is dynamic if mentioned attributes ("options"/"min"/"max") are not explicitly provided and if the - # filter targets at least one figure that uses the dynamic data source. + # which causes "options" for categorical or "min" and "max" for numerical/temporal selectors to be updated. + # The filter is dynamic iff mentioned attributes ("options"/"min"/"max") are not explicitly provided and + # filter targets at least one figure that uses dynamic data source. Note that min or max = 0 are Falsey values + # but should still count as manually set. if isinstance(self.selector, DYNAMIC_SELECTORS) and ( - not getattr(self.selector, "options", None) + not getattr(self.selector, "options", []) and getattr(self.selector, "min", None) is None and getattr(self.selector, "max", None) is None ): @@ -269,6 +271,12 @@ def _get_min_max(targeted_data: pd.DataFrame, current_value=None) -> tuple[float _max = targeted_data.max(axis=None) # Convert to datetime if the column is datetime64 + # AM question: I think this could be simplified: + # * column type is already stored in _column_type and validated in __call__ with validate_column_type. Hence + # we can just do if instance(self.selector, SELECTORS["temporal"] + # * why do we need to do this type conversion at all? When we set min/max for datepicker in pre_build we don't + # do any type + # conversions. if targeted_data.apply(is_datetime64_any_dtype).all(): _min = pd.to_datetime(_min) _max = pd.to_datetime(_max) @@ -280,6 +288,7 @@ def _get_min_max(targeted_data: pd.DataFrame, current_value=None) -> tuple[float # Use item() to convert to convert scalar from numpy to Python type. This isn't needed during pre_build because # pydantic will coerce the type, but it is necessary in __call__ where we don't update model field values # and instead just pass straight to the Dash component. + # AM QUESTION: why could AttributeError be raised here? with suppress(AttributeError): _min = _min.item() with suppress(AttributeError): @@ -294,20 +303,15 @@ def _get_min_max(targeted_data: pd.DataFrame, current_value=None) -> tuple[float @staticmethod def _get_options(targeted_data: pd.DataFrame, current_value=None) -> list[Any]: - # Use tolist() to convert to convert scalar from numpy to Python type. This isn't needed during pre_build - # because pydantic will coerce the type, but it is necessary in __call__ where we don't update model field - # values and instead just pass straight to the Dash component. # The dropna() isn't strictly required here but will be in future pandas versions when the behavior of stack # changes. See https://pandas.pydata.org/docs/whatsnew/v2.1.0.html#whatsnew-210-enhancements-new-stack. - # Also setting the dtype for the current_value_series to the dtype of the targeted_data_series to ensure it - # works when it's empty. See: https://pandas.pydata.org/docs/whatsnew/v2.1.0.html#other-deprecations + options = set(targeted_data.stack().dropna()) # noqa: PD013 - # Remove ALL_OPTION from the string or list of currently selected value for the categorical filters. - current_value = [] if current_value in (None, ALL_OPTION) else current_value - if isinstance(current_value, list) and ALL_OPTION in current_value: - current_value.remove(ALL_OPTION) + # AM comment: I refactored this function to work analogously to _get_min_max. + # Completely untested though so please do check!! + if current_value is not None: + current_value = set(current_value) if isinstance(current_value, list) else {current_value} + options = options | current_value - {ALL_OPTION} - targeted_data_series = targeted_data.stack().dropna() # noqa: PD013 - current_value_series = pd.Series(current_value).astype(targeted_data_series.dtypes) + return sorted(options) - return sorted(pd.concat([targeted_data_series, current_value_series]).unique()) From 66da13403b0745fa497ce5e1a84dbb431ac29505 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 25 Nov 2024 15:45:32 +0000 Subject: [PATCH 41/64] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- vizro-core/src/vizro/models/_controls/filter.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/vizro-core/src/vizro/models/_controls/filter.py b/vizro-core/src/vizro/models/_controls/filter.py index b2051bf3c..01b1907a0 100644 --- a/vizro-core/src/vizro/models/_controls/filter.py +++ b/vizro-core/src/vizro/models/_controls/filter.py @@ -305,7 +305,7 @@ def _get_min_max(targeted_data: pd.DataFrame, current_value=None) -> tuple[float def _get_options(targeted_data: pd.DataFrame, current_value=None) -> list[Any]: # The dropna() isn't strictly required here but will be in future pandas versions when the behavior of stack # changes. See https://pandas.pydata.org/docs/whatsnew/v2.1.0.html#whatsnew-210-enhancements-new-stack. - options = set(targeted_data.stack().dropna()) # noqa: PD013 + options = set(targeted_data.stack().dropna()) # noqa: PD013 # AM comment: I refactored this function to work analogously to _get_min_max. # Completely untested though so please do check!! @@ -314,4 +314,3 @@ def _get_options(targeted_data: pd.DataFrame, current_value=None) -> list[Any]: options = options | current_value - {ALL_OPTION} return sorted(options) - From 3c605020826c55d839378b801ed2547059adec5e Mon Sep 17 00:00:00 2001 From: Antony Milne Date: Mon, 25 Nov 2024 16:01:34 +0000 Subject: [PATCH 42/64] Add review comments and questions and small changes --- .../vizro/models/_controls/test_filter.py | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/vizro-core/tests/unit/vizro/models/_controls/test_filter.py b/vizro-core/tests/unit/vizro/models/_controls/test_filter.py index 6f4f5550d..e5c849f0e 100644 --- a/vizro-core/tests/unit/vizro/models/_controls/test_filter.py +++ b/vizro-core/tests/unit/vizro/models/_controls/test_filter.py @@ -248,6 +248,9 @@ class TestFilterStaticMethods: ([["A"], []], ["A"]), ], ) + # AM question: is there any way to make this a bit more "real" and do it by creating a fake page with targets + # with data sources, making an actual Filter() object properly and then checking Filter.selector.options? + # If it's too complicated then no worries though. def test_get_options(self, data_columns, expected): targeted_data = pd.DataFrame({f"target_{i}": pd.Series(data) for i, data in enumerate(data_columns)}) result = Filter._get_options(targeted_data) @@ -301,6 +304,13 @@ def test_get_options(self, data_columns, expected): ), ], ) + # AM comment: ah ok, this will get complicated to test with current_value if we do what I suggest above... Probably + # not a possibility then. + # As a compromise, how about making this TestFilterCall and testing as Filter.__call__(targeted_data, current_value)? This tests + # a higher level of interface which would be good here. Currently the logic in Filter.__call__ isn't actually + # tested anywhere including the `if isinstance(self.selector, SELECTORS["categorical"])` check and the column type + # change validation. If we make the testing higher level here it can cover everything you've done already, plus more, + # and it will be more robust to refactoring. def test_get_options_with_current_value(self, data_columns, current_value, expected): targeted_data = pd.DataFrame({f"target_{i}": pd.Series(data) for i, data in enumerate(data_columns)}) result = Filter._get_options(targeted_data, current_value) @@ -558,12 +568,12 @@ def test_filter_is_not_dynamic(self): model_manager["test_page"].controls = [filter] filter.pre_build() # Filter is not dynamic because it does not target a figure that uses dynamic data - assert filter._dynamic is False - assert filter.selector._dynamic is False + assert not filter._dynamic + assert not filter.selector._dynamic @pytest.mark.usefixtures("managers_one_page_two_graphs_with_dynamic_data") @pytest.mark.parametrize( - "test_column ,test_selector", + "test_column, test_selector", [ ("continent", vm.Checklist()), ("continent", vm.Dropdown()), @@ -581,8 +591,8 @@ def test_filter_is_dynamic_with_dynamic_selectors( model_manager["test_page"].controls = [filter] filter.pre_build() # Filter is dynamic because it targets a figure that uses dynamic data - assert filter._dynamic is True - assert filter.selector._dynamic is True + assert filter._dynamic + assert filter.selector._dynamic @pytest.mark.usefixtures("managers_one_page_two_graphs_with_dynamic_data") def test_filter_is_not_dynamic_with_non_dynamic_selectors(self, gapminder_dynamic_first_n_last_n_function): @@ -590,7 +600,7 @@ def test_filter_is_not_dynamic_with_non_dynamic_selectors(self, gapminder_dynami filter = vm.Filter(column="year", selector=vm.DatePicker()) model_manager["test_page"].controls = [filter] filter.pre_build() - assert filter._dynamic is False + assert not filter._dynamic @pytest.mark.usefixtures("managers_one_page_two_graphs_with_dynamic_data") @pytest.mark.parametrize( @@ -615,8 +625,8 @@ def test_filter_is_not_dynamic_with_options_min_max_specified( filter = vm.Filter(column=test_column, selector=test_selector) model_manager["test_page"].controls = [filter] filter.pre_build() - assert filter._dynamic is False - assert filter.selector._dynamic is False + assert not filter._dynamic + assert not filter.selector._dynamic @pytest.mark.parametrize("selector", [vm.Slider, vm.RangeSlider]) def test_numerical_min_max_default(self, selector, gapminder, managers_one_page_two_graphs): From ca6b4ad56e3cfb8446bc5ae700c8186715a7abf0 Mon Sep 17 00:00:00 2001 From: petar-qb Date: Tue, 26 Nov 2024 11:22:52 +0100 Subject: [PATCH 43/64] Add todo for filter.build() --- vizro-core/src/vizro/models/_controls/filter.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/vizro-core/src/vizro/models/_controls/filter.py b/vizro-core/src/vizro/models/_controls/filter.py index 01b1907a0..a22dca636 100644 --- a/vizro-core/src/vizro/models/_controls/filter.py +++ b/vizro-core/src/vizro/models/_controls/filter.py @@ -217,6 +217,9 @@ def pre_build(self): @_log_call def build(self): selector_build_obj = self.selector.build() + # TODO: Align the (dynamic) object's return structure with the figure's components when the Dash bug is fixed. + # This means returning an empty "html.Div(id=self.id, className="...")" as a placeholder from Filter.build(). + # Also, make selector.title visible when the filter is reloading. return dcc.Loading(id=self.id, children=selector_build_obj) if self._dynamic else selector_build_obj def _validate_targeted_data( From 9fd0f1d83ee3dddb67db64d1f00f29f29ab68473 Mon Sep 17 00:00:00 2001 From: petar-qb Date: Tue, 26 Nov 2024 18:47:28 +0100 Subject: [PATCH 44/64] First chunk of PR comments --- vizro-core/examples/scratch_dev/data.yaml | 2 +- vizro-core/src/vizro/actions/_actions_utils.py | 1 - .../src/vizro/models/_components/form/checklist.py | 9 ++------- .../src/vizro/models/_components/form/dropdown.py | 14 +++++--------- .../vizro/models/_components/form/radio_items.py | 9 ++------- .../vizro/models/_components/form/range_slider.py | 10 ++-------- .../src/vizro/models/_components/form/slider.py | 9 ++------- vizro-core/src/vizro/models/_controls/filter.py | 2 +- 8 files changed, 15 insertions(+), 41 deletions(-) diff --git a/vizro-core/examples/scratch_dev/data.yaml b/vizro-core/examples/scratch_dev/data.yaml index 63e64b40d..d8b0aea90 100644 --- a/vizro-core/examples/scratch_dev/data.yaml +++ b/vizro-core/examples/scratch_dev/data.yaml @@ -1,6 +1,6 @@ # Choose between 0-50 setosa: 5 -#versicolor: 10 +versicolor: 10 virginica: 15 # Choose between: 4.3 to 7.4 diff --git a/vizro-core/src/vizro/actions/_actions_utils.py b/vizro-core/src/vizro/actions/_actions_utils.py index 39e9b163e..8b9cb2f4c 100644 --- a/vizro-core/src/vizro/actions/_actions_utils.py +++ b/vizro-core/src/vizro/actions/_actions_utils.py @@ -282,7 +282,6 @@ def _get_modified_page_figures( **_get_parametrized_config(ctds_parameter=ctds_parameter, target=target, data_frame=False), ) - # AM comment: please check this comment I added! for target in control_targets: ctd_filter = [item for item in ctds_filter if item["id"] == model_manager[target].selector.id] diff --git a/vizro-core/src/vizro/models/_components/form/checklist.py b/vizro-core/src/vizro/models/_components/form/checklist.py index 43927629f..ed746dec3 100644 --- a/vizro-core/src/vizro/models/_components/form/checklist.py +++ b/vizro-core/src/vizro/models/_components/form/checklist.py @@ -49,9 +49,6 @@ class Checklist(VizroBaseModel): _validate_value = validator("value", allow_reuse=True, always=True)(validate_value) def __call__(self, options): - return self._build_static(options) - - def _build_static(self, options): full_options, default_value = get_options_and_default(options=options, multi=True) return html.Fieldset( @@ -71,10 +68,8 @@ def _build_dynamic_placeholder(self): if self.value is None: self.value = [get_options_and_default(self.options, multi=True)[1]] - return self._build_static(self.options) + return self.__call__(self.options) @_log_call def build(self): - # We don't have to implement _build_dynamic_placeholder, _build_static here. It's possible to: if dynamic and - # self.value is None -> set self.value + return standard build (static), but let's align it with the Dropdown. - return self._build_dynamic_placeholder() if self._dynamic else self._build_static(self.options) + return self._build_dynamic_placeholder() if self._dynamic else self.__call__(self.options) diff --git a/vizro-core/src/vizro/models/_components/form/dropdown.py b/vizro-core/src/vizro/models/_components/form/dropdown.py index a1c71bb13..0351e82f6 100755 --- a/vizro-core/src/vizro/models/_components/form/dropdown.py +++ b/vizro-core/src/vizro/models/_components/form/dropdown.py @@ -88,12 +88,7 @@ def validate_multi(cls, multi, values): raise ValueError("Please set multi=True if providing a list of default values.") return multi - # AM comment: please remove build_static and change into __call__ in all places unless you think there's - # a good reason not to do so. def __call__(self, options): - # AM comment: this is the main confusing thing about the current approach I think: every time we run - # __call__ we override the value set below, which sounds wrong because we say that we never change the value. - # From what I remember this is a workaround for the Dash bug? Maybe add a comment explaining this. full_options, default_value = get_options_and_default(options=options, multi=self.multi) option_height = _calculate_option_height(full_options) @@ -124,11 +119,12 @@ def _build_dynamic_placeholder(self): # TODO-NEXT: Replace this with the "universal Vizro placeholder" component. return html.Div( children=[ - dbc.Label(self.title, html_for=self.id) if self.title else None, - # AM question: why do we want opacity: 0? If we don't want it to appear on screen then normally we do - # this with visibility or display. dmc.DateRangePicker( - id=self.id, value=self.value, persistence=True, persistence_type="session", style={"opacity": 0} + id=self.id, + value=self.value, + persistence=True, + persistence_type="session", + style={"visibility": "hidden"}, ), ] ) diff --git a/vizro-core/src/vizro/models/_components/form/radio_items.py b/vizro-core/src/vizro/models/_components/form/radio_items.py index 6949d4992..48b8bc6bc 100644 --- a/vizro-core/src/vizro/models/_components/form/radio_items.py +++ b/vizro-core/src/vizro/models/_components/form/radio_items.py @@ -50,9 +50,6 @@ class RadioItems(VizroBaseModel): _validate_value = validator("value", allow_reuse=True, always=True)(validate_value) def __call__(self, options): - return self._build_static(options) - - def _build_static(self, options): full_options, default_value = get_options_and_default(options=options, multi=False) return html.Fieldset( @@ -72,10 +69,8 @@ def _build_dynamic_placeholder(self): if self.value is None: self.value = get_options_and_default(self.options, multi=False)[1] - return self._build_static(self.options) + return self.__call__(self.options) @_log_call def build(self): - # We don't have to implement _build_dynamic_placeholder, _build_static here. It's possible to: if dynamic and - # self.value is None -> set self.value + return standard build (static), but let's align it with the Dropdown. - return self._build_dynamic_placeholder() if self._dynamic else self._build_static(self.options) + return self._build_dynamic_placeholder() if self._dynamic else self.__call__(self.options) diff --git a/vizro-core/src/vizro/models/_components/form/range_slider.py b/vizro-core/src/vizro/models/_components/form/range_slider.py index 0800e6e75..a96521708 100644 --- a/vizro-core/src/vizro/models/_components/form/range_slider.py +++ b/vizro-core/src/vizro/models/_components/form/range_slider.py @@ -63,10 +63,6 @@ class RangeSlider(VizroBaseModel): _set_actions = _action_validator_factory("value") def __call__(self, min, max, current_value): - return self._build_static(min, max, current_value) - - @_log_call - def _build_static(self, min, max, current_value): output = [ Output(f"{self.id}_start_value", "value"), Output(f"{self.id}_end_value", "value"), @@ -142,15 +138,13 @@ def _build_static(self, min, max, current_value): ) def _build_dynamic_placeholder(self, current_value): - return self._build_static(self.min, self.max, current_value) + return self.__call__(self.min, self.max, current_value) @_log_call def build(self): - # We don't have to implement _build_dynamic_placeholder, _build_static here. It's possible to: if dynamic and - # self.value is None -> set self.value + return standard build (static), but let's align it with the Dropdown. current_value = self.value or [self.min, self.max] # type: ignore[list-item] return ( self._build_dynamic_placeholder(current_value) if self._dynamic - else self._build_static(self.min, self.max, current_value) + else self.__call__(self.min, self.max, current_value) ) diff --git a/vizro-core/src/vizro/models/_components/form/slider.py b/vizro-core/src/vizro/models/_components/form/slider.py index 8f4107b5b..2ffdb9f6a 100644 --- a/vizro-core/src/vizro/models/_components/form/slider.py +++ b/vizro-core/src/vizro/models/_components/form/slider.py @@ -61,9 +61,6 @@ class Slider(VizroBaseModel): _set_actions = _action_validator_factory("value") def __call__(self, min, max, current_value): - return self._build_static(min, max, current_value) - - def _build_static(self, min, max, current_value): output = [ Output(f"{self.id}_end_value", "value"), Output(self.id, "value"), @@ -125,15 +122,13 @@ def _build_static(self, min, max, current_value): ) def _build_dynamic_placeholder(self, current_value): - return self._build_static(self.min, self.max, current_value) + return self.__call__(self.min, self.max, current_value) @_log_call def build(self): - # We don't have to implement _build_dynamic_placeholder, _build_static here. It's possible to: if dynamic and - # self.value is None -> set self.value + return standard build (static), but let's align it with the Dropdown. current_value = self.value if self.value is not None else self.min return ( self._build_dynamic_placeholder(current_value) if self._dynamic - else self._build_static(self.min, self.max, current_value) + else self.__call__(self.min, self.max, current_value) ) diff --git a/vizro-core/src/vizro/models/_controls/filter.py b/vizro-core/src/vizro/models/_controls/filter.py index a22dca636..a74b4a88b 100644 --- a/vizro-core/src/vizro/models/_controls/filter.py +++ b/vizro-core/src/vizro/models/_controls/filter.py @@ -218,7 +218,7 @@ def pre_build(self): def build(self): selector_build_obj = self.selector.build() # TODO: Align the (dynamic) object's return structure with the figure's components when the Dash bug is fixed. - # This means returning an empty "html.Div(id=self.id, className="...")" as a placeholder from Filter.build(). + # This means returning an empty "html.Div(id=self.id, className=...)" as a placeholder from Filter.build(). # Also, make selector.title visible when the filter is reloading. return dcc.Loading(id=self.id, children=selector_build_obj) if self._dynamic else selector_build_obj From 1433734d69e82829dbb2cce2a3827b92bb8bfd41 Mon Sep 17 00:00:00 2001 From: petar-qb Date: Wed, 27 Nov 2024 14:18:04 +0100 Subject: [PATCH 45/64] _get_min_max and _get_options improvements --- vizro-core/docs/pages/user-guides/data.md | 4 +- .../src/vizro/models/_controls/filter.py | 42 +++------------- .../vizro/models/_controls/test_filter.py | 48 ++++++++----------- 3 files changed, 28 insertions(+), 66 deletions(-) diff --git a/vizro-core/docs/pages/user-guides/data.md b/vizro-core/docs/pages/user-guides/data.md index ffd53f2fb..09858afd5 100644 --- a/vizro-core/docs/pages/user-guides/data.md +++ b/vizro-core/docs/pages/user-guides/data.md @@ -281,7 +281,7 @@ To add a parameter to control a dynamic data source, do the following: 2. give an `id` to all components that have the data source you wish to alter through a parameter. 3. [add a parameter](parameters.md) with `targets` of the form `.data_frame.` and a suitable [selector](selectors.md). -For example, let us extend the [dynamic data example](#dynamic-data) above into a simple toy example of how parametrized dynamic data works. The `load_iris_data` can take an argument `number_of_points` controlled from the dashboard with a [`Slider`][vizro.models.Slider]. +For example, let us extend the [dynamic data example](#dynamic-data) above into example of how parametrized dynamic data works. The `load_iris_data` can take an argument `number_of_points` controlled from the dashboard with a [`Slider`][vizro.models.Slider]. !!! example "Parametrized dynamic data" === "app.py" @@ -343,7 +343,7 @@ When a [filter](filters.md) depends on dynamic data and no `selector` is explici The mechanism for dynamic filters, including caching, works exactly like other non-control components such as `vm.Graph`. However, unlike such components, a filter can depend on multiple data sources. If at least one data source of the components in the filter's `targets` is dynamic then the filter is dynamic. Remember that when `targets` is not explicitly specified, a filter applies to all the components on a page that use a DataFrame including `column`. -When the page is refreshed, the behaviour of a dynamic filter is as follows: +When the page is refreshed, the behavior of a dynamic filter is as follows: - The filter's selector updates its available values: - For [categorical selectors](selectors.md#categorical-selectors), `options` updates to give all unique values found in `column` across all the data sources of components in `targets`. diff --git a/vizro-core/src/vizro/models/_controls/filter.py b/vizro-core/src/vizro/models/_controls/filter.py index a74b4a88b..ae1a9d841 100644 --- a/vizro-core/src/vizro/models/_controls/filter.py +++ b/vizro-core/src/vizro/models/_controls/filter.py @@ -1,6 +1,5 @@ from __future__ import annotations -from contextlib import suppress from typing import Any, Literal, Union import pandas as pd @@ -270,37 +269,17 @@ def _validate_column_type(self, targeted_data: pd.DataFrame) -> Literal["numeric @staticmethod def _get_min_max(targeted_data: pd.DataFrame, current_value=None) -> tuple[float, float]: + targeted_data = pd.concat([targeted_data, pd.Series(current_value)]).stack().dropna() # noqa: PD013 + _min = targeted_data.min(axis=None) _max = targeted_data.max(axis=None) - # Convert to datetime if the column is datetime64 - # AM question: I think this could be simplified: - # * column type is already stored in _column_type and validated in __call__ with validate_column_type. Hence - # we can just do if instance(self.selector, SELECTORS["temporal"] - # * why do we need to do this type conversion at all? When we set min/max for datepicker in pre_build we don't - # do any type - # conversions. - if targeted_data.apply(is_datetime64_any_dtype).all(): - _min = pd.to_datetime(_min) - _max = pd.to_datetime(_max) - current_value = pd.to_datetime(current_value) - # Convert DatetimeIndex to list of Timestamp objects so that we can use min and max functions below. - with suppress(AttributeError): - current_value = current_value.tolist() - # Use item() to convert to convert scalar from numpy to Python type. This isn't needed during pre_build because # pydantic will coerce the type, but it is necessary in __call__ where we don't update model field values # and instead just pass straight to the Dash component. - # AM QUESTION: why could AttributeError be raised here? - with suppress(AttributeError): - _min = _min.item() - with suppress(AttributeError): - _max = _max.item() - - if current_value is not None: - current_value = current_value if isinstance(current_value, list) else [current_value] - _min = min(_min, *current_value) - _max = max(_max, *current_value) + # However, in some cases _min and _max are already Python types and so item() call is not required. + _min = _min if not hasattr(_min, "item") else _min.item() + _max = _max if not hasattr(_max, "item") else _max.item() return _min, _max @@ -308,12 +287,5 @@ def _get_min_max(targeted_data: pd.DataFrame, current_value=None) -> tuple[float def _get_options(targeted_data: pd.DataFrame, current_value=None) -> list[Any]: # The dropna() isn't strictly required here but will be in future pandas versions when the behavior of stack # changes. See https://pandas.pydata.org/docs/whatsnew/v2.1.0.html#whatsnew-210-enhancements-new-stack. - options = set(targeted_data.stack().dropna()) # noqa: PD013 - - # AM comment: I refactored this function to work analogously to _get_min_max. - # Completely untested though so please do check!! - if current_value is not None: - current_value = set(current_value) if isinstance(current_value, list) else {current_value} - options = options | current_value - {ALL_OPTION} - - return sorted(options) + targeted_data = pd.concat([targeted_data, pd.Series(current_value)]).stack().dropna() # noqa: PD013 + return sorted(set(targeted_data) - {ALL_OPTION}) diff --git a/vizro-core/tests/unit/vizro/models/_controls/test_filter.py b/vizro-core/tests/unit/vizro/models/_controls/test_filter.py index e5c849f0e..24906b885 100644 --- a/vizro-core/tests/unit/vizro/models/_controls/test_filter.py +++ b/vizro-core/tests/unit/vizro/models/_controls/test_filter.py @@ -243,9 +243,11 @@ class TestFilterStaticMethods: datetime(2024, 1, 2), ], ), - ([["A", "B"], ["B", "C"]], ["A", "B", "C"]), - ([["A", "B"], ["C"]], ["A", "B", "C"]), + ([[], []], []), ([["A"], []], ["A"]), + ([[], ["A"]], ["A"]), + ([["A"], ["B"]], ["A", "B"]), + ([["A", "B"], ["B", "C"]], ["A", "B", "C"]), ], ) # AM question: is there any way to make this a bit more "real" and do it by creating a fake page with targets @@ -262,55 +264,45 @@ def test_get_options(self, data_columns, expected): ([[]], None, []), ([[]], "ALL", []), ([[]], ["ALL", "A"], ["A"]), + ([["A"]], ["ALL", "B"], ["A", "B"]), ([[]], "A", ["A"]), ([[]], ["A", "B"], ["A", "B"]), - ([["A", "B"]], "C", ["A", "B", "C"]), - ([["A", "B"]], ["C", "D"], ["A", "B", "C", "D"]), - ([[1, 2]], 3, [1, 2, 3]), - ([[1, 2]], [3, 4], [1, 2, 3, 4]), - ([[1.1, 2.2]], 3.3, [1.1, 2.2, 3.3]), - ([[1.1, 2.2]], [3.3, 4.4], [1.1, 2.2, 3.3, 4.4]), + ([["A"]], "B", ["A", "B"]), + ([["A"]], ["B", "C"], ["A", "B", "C"]), + ([[1]], 2, [1, 2]), + ([[1]], [2, 3], [1, 2, 3]), + ([[1.1]], 2.2, [1.1, 2.2]), + ([[1.1]], [2.2, 3.3], [1.1, 2.2, 3.3]), ( [ [ datetime(2024, 1, 1), - datetime(2024, 1, 2), ] ], - datetime(2024, 1, 3), + datetime(2024, 1, 2), [ datetime(2024, 1, 1), datetime(2024, 1, 2), - datetime(2024, 1, 3), ], ), ( [ [ datetime(2024, 1, 1), - datetime(2024, 1, 2), ] ], [ + datetime(2024, 1, 2), datetime(2024, 1, 3), - datetime(2024, 1, 4), ], [ datetime(2024, 1, 1), datetime(2024, 1, 2), datetime(2024, 1, 3), - datetime(2024, 1, 4), ], ), ], ) - # AM comment: ah ok, this will get complicated to test with current_value if we do what I suggest above... Probably - # not a possibility then. - # As a compromise, how about making this TestFilterCall and testing as Filter.__call__(targeted_data, current_value)? This tests - # a higher level of interface which would be good here. Currently the logic in Filter.__call__ isn't actually - # tested anywhere including the `if isinstance(self.selector, SELECTORS["categorical"])` check and the column type - # change validation. If we make the testing higher level here it can cover everything you've done already, plus more, - # and it will be more robust to refactoring. def test_get_options_with_current_value(self, data_columns, current_value, expected): targeted_data = pd.DataFrame({f"target_{i}": pd.Series(data) for i, data in enumerate(data_columns)}) result = Filter._get_options(targeted_data, current_value) @@ -334,9 +326,9 @@ def test_get_options_with_current_value(self, data_columns, current_value, expec datetime(2024, 1, 2), ), ), - ([[1, 2], [2, 3]], (1, 3)), - ([[1, 2], [3]], (1, 3)), + ([[1], []], (1, 1)), ([[1, 2], []], (1, 2)), + ([[1, 2], [2, 3]], (1, 3)), ], ) def test_get_min_max(self, data_columns, expected): @@ -356,7 +348,6 @@ def test_get_min_max(self, data_columns, expected): [ datetime(2024, 1, 1), datetime(2024, 1, 2), - datetime(2024, 1, 1), ] ], datetime(2024, 1, 3), @@ -370,7 +361,6 @@ def test_get_min_max(self, data_columns, expected): [ datetime(2024, 1, 1), datetime(2024, 1, 2), - datetime(2024, 1, 1), ] ], [ @@ -382,10 +372,10 @@ def test_get_min_max(self, data_columns, expected): datetime(2024, 1, 4), ), ), - ([[1, 2], [2, 3]], 4, (1, 4)), - ([[1, 2], [2, 3]], [4, 5], (1, 5)), - ([[1, 2], []], 3, (1, 3)), - ([[1, 2], []], [3, 4], (1, 4)), + ([[1], []], 2, (1, 2)), + ([[1], []], [2, 3], (1, 3)), + ([[1], [2]], 3, (1, 3)), + ([[1], [2]], [3, 4], (1, 4)), ], ) def test_get_min_max_with_current_value(self, data_columns, current_value, expected): From fa5f1720c88e2104ceda110d73ade6fc90cb1fcf Mon Sep 17 00:00:00 2001 From: petar-qb Date: Thu, 28 Nov 2024 10:32:05 +0100 Subject: [PATCH 46/64] Simple Filter.__call__() tests --- .../vizro/models/_controls/test_filter.py | 100 +++++++++++++++++- 1 file changed, 97 insertions(+), 3 deletions(-) diff --git a/vizro-core/tests/unit/vizro/models/_controls/test_filter.py b/vizro-core/tests/unit/vizro/models/_controls/test_filter.py index 24906b885..e7a950f90 100644 --- a/vizro-core/tests/unit/vizro/models/_controls/test_filter.py +++ b/vizro-core/tests/unit/vizro/models/_controls/test_filter.py @@ -53,6 +53,32 @@ def managers_column_only_exists_in_some(): Vizro._pre_build() +@pytest.fixture +def target_to_data_frame(): + return { + "column_numerical_exists_1": pd.DataFrame( + { + "column_numerical": [1, 2], + } + ), + "column_numerical_exists_2": pd.DataFrame( + { + "column_numerical": [2, 3], + } + ), + "column_categorical_exists_1": pd.DataFrame( + { + "column_categorical": ["a", "b"], + } + ), + "column_categorical_exists_2": pd.DataFrame( + { + "column_categorical": ["b", "c"], + } + ), + } + + class TestFilterFunctions: @pytest.mark.parametrize( "data, value, expected", @@ -250,9 +276,6 @@ class TestFilterStaticMethods: ([["A", "B"], ["B", "C"]], ["A", "B", "C"]), ], ) - # AM question: is there any way to make this a bit more "real" and do it by creating a fake page with targets - # with data sources, making an actual Filter() object properly and then checking Filter.selector.options? - # If it's too complicated then no worries though. def test_get_options(self, data_columns, expected): targeted_data = pd.DataFrame({f"target_{i}": pd.Series(data) for i, data in enumerate(data_columns)}) result = Filter._get_options(targeted_data) @@ -409,6 +432,77 @@ def test_check_target_present_invalid(self): Filter(column="foo", targets=["invalid_target"]) +@pytest.mark.usefixtures("managers_column_only_exists_in_some") +class TestFilterCall: + """Test Filter.__call__() method with target_to_data_frame and current_value inputs.""" + + # TODO: three options: + # 1. enhance this solution a bit + # 2. Remove these tests completely + # 3. Merge them with the detailed _get_min_max and _get_options tests + + def test_filter_call_categorical_valid(self, target_to_data_frame): + filter = vm.Filter( + column="column_categorical", targets=["column_categorical_exists_1", "column_categorical_exists_2"] + ) + filter._column_type = "categorical" + filter.selector = vm.Dropdown(id="test_selector_id") + + selector_build = filter(target_to_data_frame=target_to_data_frame, current_value=["a", "b"])["test_selector_id"] + assert selector_build.options == ["ALL", "a", "b", "c"] + + def test_filter_call_numerical_valid(self, target_to_data_frame): + filter = vm.Filter( + column="column_numerical", targets=["column_numerical_exists_1", "column_numerical_exists_2"] + ) + filter._column_type = "numerical" + filter.selector = vm.RangeSlider(id="test_selector_id") + + selector_build = filter(target_to_data_frame=target_to_data_frame, current_value=[1, 2])["test_selector_id"] + assert selector_build.min == 1 + assert selector_build.max == 3 + + def test_filter_call_column_is_changed(self, target_to_data_frame): + filter = vm.Filter( + column="column_categorical", targets=["column_categorical_exists_1", "column_categorical_exists_2"] + ) + filter._column_type = "numerical" + filter.selector = vm.RangeSlider(id="test_selector_id") + + with pytest.raises( + ValueError, + match="column_categorical has changed type from numerical to categorical. " + "A filtered column cannot change type while the dashboard is running.", + ): + filter(target_to_data_frame=target_to_data_frame, current_value=["a", "b"]) + + def test_filter_call_selected_column_not_found_in_target(self): + filter = vm.Filter(column="column_categorical", targets=["column_categorical_exists_1"]) + filter._column_type = "categorical" + filter.selector = vm.Dropdown(id="test_selector_id") + + with pytest.raises( + ValueError, + match="Selected column column_categorical not found in dataframe for column_categorical_exists_1.", + ): + filter(target_to_data_frame={"column_categorical_exists_1": pd.DataFrame()}, current_value=["a", "b"]) + + def test_filter_call_targeted_data_empty(self): + filter = vm.Filter(column="column_categorical", targets=["column_categorical_exists_1"]) + filter._column_type = "categorical" + filter.selector = vm.Dropdown(id="test_selector_id") + + with pytest.raises( + ValueError, + match="Selected column column_categorical does not contain anything in any dataframe " + "for column_categorical_exists_1.", + ): + filter( + target_to_data_frame={"column_categorical_exists_1": pd.DataFrame({"column_categorical": []})}, + current_value=["a", "b"], + ) + + class TestPreBuildMethod: def test_targets_default_valid(self, managers_column_only_exists_in_some): # Core of tests is still interface level From d91a151381c608e15416c1639b69b5ed2b8cddaf Mon Sep 17 00:00:00 2001 From: petar-qb Date: Thu, 28 Nov 2024 11:16:39 +0100 Subject: [PATCH 47/64] Removing FILTER_COLUMN from app.py --- vizro-core/examples/scratch_dev/app.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/vizro-core/examples/scratch_dev/app.py b/vizro-core/examples/scratch_dev/app.py index 380dc1968..66b23823d 100644 --- a/vizro-core/examples/scratch_dev/app.py +++ b/vizro-core/examples/scratch_dev/app.py @@ -19,13 +19,8 @@ BAR_CHART_CONF = dict(x="species", color="species", color_discrete_map=SPECIES_COLORS) SCATTER_CHART_CONF = dict(x="sepal_length", y="petal_length", color="species", color_discrete_map=SPECIES_COLORS) -# Relevant for the "page_6" only -FILTER_COLUMN = "species" -# FILTER_COLUMN = "sepal_length" -# FILTER_COLUMN = "date_column" - -def load_from_file(filter_column=FILTER_COLUMN, parametrized_species=None): +def load_from_file(filter_column=None, parametrized_species=None): # Load the full iris dataset df = px.data.iris() df["date_column"] = pd.date_range(start=pd.to_datetime("2024-01-01"), periods=len(df), freq="D") @@ -58,7 +53,7 @@ def load_from_file(filter_column=FILTER_COLUMN, parametrized_species=None): date_max = pd.to_datetime(data["date_max"]) df = df[df[filter_column].between(date_min, date_max, inclusive="both")] else: - raise ValueError("Invalid FILTER_COLUMN") + raise ValueError("Invalid filter_column") if parametrized_species: df = df[df["species"].isin(parametrized_species)] @@ -66,7 +61,6 @@ def load_from_file(filter_column=FILTER_COLUMN, parametrized_species=None): return df -data_manager["load_from_file"] = load_from_file data_manager["load_from_file_species"] = partial(load_from_file, filter_column="species") data_manager["load_from_file_sepal_length"] = partial(load_from_file, filter_column="sepal_length") data_manager["load_from_file_date_column"] = partial(load_from_file, filter_column="date_column") @@ -207,7 +201,7 @@ def load_from_file(filter_column=FILTER_COLUMN, parametrized_species=None): page_6 = vm.Page( title="Page to test things out", components=[ - vm.Graph(id="graph_dynamic", figure=px.bar(data_frame="load_from_file", **BAR_CHART_CONF)), + vm.Graph(id="graph_dynamic", figure=px.bar(data_frame="load_from_file_species", **BAR_CHART_CONF)), vm.Graph( id="graph_static", figure=px.scatter(data_frame=px.data.iris(), **SCATTER_CHART_CONF), @@ -216,7 +210,7 @@ def load_from_file(filter_column=FILTER_COLUMN, parametrized_species=None): controls=[ vm.Filter( id="filter_container_id", - column=FILTER_COLUMN, + column="species", targets=["graph_dynamic"], # targets=["graph_static"], # selector=vm.Dropdown(id="filter_id"), From 097c02e1e23da6d4be25bad073684f212595552d Mon Sep 17 00:00:00 2001 From: petar-qb Date: Thu, 28 Nov 2024 13:43:43 +0100 Subject: [PATCH 48/64] Removing TODO --- vizro-core/tests/unit/vizro/models/_controls/test_filter.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/vizro-core/tests/unit/vizro/models/_controls/test_filter.py b/vizro-core/tests/unit/vizro/models/_controls/test_filter.py index e7a950f90..902e55200 100644 --- a/vizro-core/tests/unit/vizro/models/_controls/test_filter.py +++ b/vizro-core/tests/unit/vizro/models/_controls/test_filter.py @@ -436,11 +436,6 @@ def test_check_target_present_invalid(self): class TestFilterCall: """Test Filter.__call__() method with target_to_data_frame and current_value inputs.""" - # TODO: three options: - # 1. enhance this solution a bit - # 2. Remove these tests completely - # 3. Merge them with the detailed _get_min_max and _get_options tests - def test_filter_call_categorical_valid(self, target_to_data_frame): filter = vm.Filter( column="column_categorical", targets=["column_categorical_exists_1", "column_categorical_exists_2"] From 86171e0f4799a412c8c1d8818d237f3b938b2eea Mon Sep 17 00:00:00 2001 From: petar-qb Date: Fri, 29 Nov 2024 10:33:58 +0100 Subject: [PATCH 49/64] Hide selector during the dynamic filter reloading process --- .../vizro/models/_components/form/dropdown.py | 2 +- .../src/vizro/models/_controls/filter.py | 22 ++++++++++++++++++- .../vizro/models/_controls/test_filter.py | 9 ++++++-- 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/vizro-core/src/vizro/models/_components/form/dropdown.py b/vizro-core/src/vizro/models/_components/form/dropdown.py index 0351e82f6..b00b5eeae 100755 --- a/vizro-core/src/vizro/models/_components/form/dropdown.py +++ b/vizro-core/src/vizro/models/_components/form/dropdown.py @@ -119,12 +119,12 @@ def _build_dynamic_placeholder(self): # TODO-NEXT: Replace this with the "universal Vizro placeholder" component. return html.Div( children=[ + html.Legend(children=self.title, className="form-label") if self.title else None, dmc.DateRangePicker( id=self.id, value=self.value, persistence=True, persistence_type="session", - style={"visibility": "hidden"}, ), ] ) diff --git a/vizro-core/src/vizro/models/_controls/filter.py b/vizro-core/src/vizro/models/_controls/filter.py index ae1a9d841..61f8ca523 100644 --- a/vizro-core/src/vizro/models/_controls/filter.py +++ b/vizro-core/src/vizro/models/_controls/filter.py @@ -219,7 +219,27 @@ def build(self): # TODO: Align the (dynamic) object's return structure with the figure's components when the Dash bug is fixed. # This means returning an empty "html.Div(id=self.id, className=...)" as a placeholder from Filter.build(). # Also, make selector.title visible when the filter is reloading. - return dcc.Loading(id=self.id, children=selector_build_obj) if self._dynamic else selector_build_obj + if not self._dynamic: + return selector_build_obj + + # Temporarily hide the selector and numeric dcc.Input components during the filter reloading process. + # Other components, such as the title, remain visible because of the configuration: + # overlay_style={"visibility": "visible"} in dcc.Loading. + # Note: dcc.Slider and dcc.RangeSlider do not support the "style" property directly, + # so the "className" attribute is used to apply custom CSS for visibility control. + # Reference for Dash class names: https://dashcheatsheet.pythonanywhere.com/ + selector_build_obj[self.selector.id].className = "invisible" + if f"{self.selector.id}_start_value" in selector_build_obj: + selector_build_obj[f"{self.selector.id}_start_value"].className = "d-none" + if f"{self.selector.id}_end_value" in selector_build_obj: + selector_build_obj[f"{self.selector.id}_end_value"].className = "d-none" + + return dcc.Loading( + id=self.id, + children=selector_build_obj, + color="grey", + overlay_style={"visibility": "visible"}, + ) def _validate_targeted_data( self, target_to_data_frame: dict[ModelID, pd.DataFrame], eagerly_raise_column_not_found_error diff --git a/vizro-core/tests/unit/vizro/models/_controls/test_filter.py b/vizro-core/tests/unit/vizro/models/_controls/test_filter.py index 902e55200..e31064b33 100644 --- a/vizro-core/tests/unit/vizro/models/_controls/test_filter.py +++ b/vizro-core/tests/unit/vizro/models/_controls/test_filter.py @@ -866,9 +866,14 @@ def test_dynamic_filter_build(self, test_column, test_selector, gapminder_dynami filter.pre_build() result = filter.build() - expected = dcc.Loading(id="filter_id", children=test_selector.build()) + expected = dcc.Loading( + id="filter_id", + children=test_selector.build(), + color="grey", + overlay_style={"visibility": "visible"}, + ) - assert_component_equal(result, expected) + assert_component_equal(result, expected, keys_to_strip={"className"}) @pytest.mark.usefixtures("managers_one_page_two_graphs_with_dynamic_data") def test_dynamic_filter_build_with_non_dynamic_selectors(self, gapminder_dynamic_first_n_last_n_function): From 973269271bfecf0a01c8ad45a6f9e63ade036fa2 Mon Sep 17 00:00:00 2001 From: petar-qb Date: Fri, 29 Nov 2024 11:59:04 +0100 Subject: [PATCH 50/64] A few comments added --- vizro-core/src/vizro/_vizro.py | 2 ++ vizro-core/src/vizro/models/_page.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/vizro-core/src/vizro/_vizro.py b/vizro-core/src/vizro/_vizro.py index 09f8b9f49..81c9a46b7 100644 --- a/vizro-core/src/vizro/_vizro.py +++ b/vizro-core/src/vizro/_vizro.py @@ -149,6 +149,8 @@ def _pre_build(): for _, filter_obj in list(model_manager._items_with_type(Filter)): # Run pre_build on all filters first, then on all other models. This handles dependency between Filter # and Page pre_build and ensures that filters are pre-built before the Page objects that use them. + # This is important because the Page pre_build method checks whether filters are dynamic or not, which is + # defined in the filter's pre_build method. filter_obj.pre_build() for model_id in set(model_manager): model = model_manager[model_id] diff --git a/vizro-core/src/vizro/models/_page.py b/vizro-core/src/vizro/models/_page.py index b64e6e250..329b4a279 100644 --- a/vizro-core/src/vizro/models/_page.py +++ b/vizro-core/src/vizro/models/_page.py @@ -97,6 +97,8 @@ def __vizro_exclude_fields__(self) -> Optional[Union[set[str], Mapping[str, Any] @_log_call def pre_build(self): targets = model_manager._get_page_model_ids_with_figure(page_id=ModelID(str(self.id))) + + # TODO NEXT: make work generically for control group targets.extend(control.id for control in self.controls if getattr(control, "_dynamic", False)) if targets: From 4c0f8e8067f9dcf303695fdcacd416f0b5d6c3df Mon Sep 17 00:00:00 2001 From: Antony Milne Date: Fri, 29 Nov 2024 11:30:24 +0000 Subject: [PATCH 51/64] Docs improvements --- vizro-core/docs/pages/user-guides/data.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vizro-core/docs/pages/user-guides/data.md b/vizro-core/docs/pages/user-guides/data.md index 09858afd5..2b4892355 100644 --- a/vizro-core/docs/pages/user-guides/data.md +++ b/vizro-core/docs/pages/user-guides/data.md @@ -341,14 +341,14 @@ You cannot pass [nested parameters](parameters.md#nested-parameters) to dynamic When a [filter](filters.md) depends on dynamic data and no `selector` is explicitly defined in the `vm.Filter` model, the available selector values update on page refresh to reflect the latest dynamic data. This is called a _dynamic filter_. -The mechanism for dynamic filters, including caching, works exactly like other non-control components such as `vm.Graph`. However, unlike such components, a filter can depend on multiple data sources. If at least one data source of the components in the filter's `targets` is dynamic then the filter is dynamic. Remember that when `targets` is not explicitly specified, a filter applies to all the components on a page that use a DataFrame including `column`. +The system for updating dynamic filters works exactly like other non-control components such as `vm.Graph`. However, unlike such components, a filter can depend on multiple data sources. If at least one data source of the components in the filter's `targets` is dynamic then the filter is dynamic. Remember that when `targets` is not explicitly specified, a filter applies to all the components on a page that use a DataFrame including `column`. When the page is refreshed, the behavior of a dynamic filter is as follows: - The filter's selector updates its available values: - For [categorical selectors](selectors.md#categorical-selectors), `options` updates to give all unique values found in `column` across all the data sources of components in `targets`. - For [numerical selectors](selectors.md#numerical-selectors), `min` and `max` update to give the overall minimum and maximum values found in `column` across all the data sources of components in `targets`. -- The value selected on screen by a dashboard user _does not_ change. If the selected value is not present in the new set of available values then it is still selected, but the filtering operation might result in an empty DataFrame. +- The value selected on screen by a dashboard user _does not_ change. If the selected value is not already present in the new set of available values then the `options` or `min` and `max` are modified to include it. In this case, the filtering operation might result in an empty DataFrame. - Even though the values present in a data source can change, the schema should not: `column` should remain present and of the same type in the data sources. The `targets` of the filter and selector type cannot change while the dashboard is running. For example, a `vm.Dropdown` selector cannot turn into `vm.RadioItems`. For example, let us add two filters to the [dynamic data example](#dynamic-data) above: From ff6f89aa2e81dd91226c5b9e6e7453f4fee43375 Mon Sep 17 00:00:00 2001 From: Antony Milne <49395058+antonymilne@users.noreply.github.com> Date: Fri, 29 Nov 2024 11:31:17 +0000 Subject: [PATCH 52/64] Update vizro-core/docs/pages/user-guides/data.md Co-authored-by: Jo Stichbury --- vizro-core/docs/pages/user-guides/data.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vizro-core/docs/pages/user-guides/data.md b/vizro-core/docs/pages/user-guides/data.md index 2b4892355..8cda0d13c 100644 --- a/vizro-core/docs/pages/user-guides/data.md +++ b/vizro-core/docs/pages/user-guides/data.md @@ -179,7 +179,7 @@ Since dynamic data sources must always be added to the data manager and referenc ### Configure cache -By default, a dynamic data function executes every time the dashboard is refreshed. Data loading is batched so that a dynamic data function that supplies multiple graphs on the same page only executes _once_ on page refresh. Even with this batching, if loading your data is still a slow operation, your dashboard performance may suffer. +By default, a dynamic data function executes every time the dashboard is refreshed. Data loading is batched so that a dynamic data function that supplies multiple graphs on the same page only executes _once_ per page refresh. Even with this batching, if loading your data is a slow operation, your dashboard performance may suffer. The Vizro data manager has a server-side caching mechanism to help solve this. Vizro's cache uses [Flask-Caching](https://flask-caching.readthedocs.io/en/latest/), which supports a number of possible cache backends and [configuration options](https://flask-caching.readthedocs.io/en/latest/#configuring-flask-caching). By default, the cache is turned off. From 9da32a85df5a8a0b6f80204c7d2c7a38377bdc12 Mon Sep 17 00:00:00 2001 From: Antony Milne <49395058+antonymilne@users.noreply.github.com> Date: Fri, 29 Nov 2024 11:31:35 +0000 Subject: [PATCH 53/64] Update vizro-core/docs/pages/user-guides/data.md Co-authored-by: Jo Stichbury --- vizro-core/docs/pages/user-guides/data.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vizro-core/docs/pages/user-guides/data.md b/vizro-core/docs/pages/user-guides/data.md index 8cda0d13c..99f27490b 100644 --- a/vizro-core/docs/pages/user-guides/data.md +++ b/vizro-core/docs/pages/user-guides/data.md @@ -271,7 +271,7 @@ data_manager["no_expire_data"].timeout = 0 You can give arguments to your dynamic data loading function that can be modified from the dashboard. For example: - To load different versions of the same data. -- To handle big data you can use an argument that controls the amount of data that is loaded. This effectively pre-filters data before it reaches the Vizro dashboard. +- To handle large datasets you can use an argument that controls the amount of data that is loaded. This effectively pre-filters data before it reaches the Vizro dashboard. In general, a parametrized dynamic data source should always return a pandas DataFrame with a fixed schema (column names and types). This ensures that page components and controls continue to work as expected when the parameter is changed on screen. From 9fc39e839c409beb6fdeb3ba4b0f717c70363cb6 Mon Sep 17 00:00:00 2001 From: Antony Milne <49395058+antonymilne@users.noreply.github.com> Date: Fri, 29 Nov 2024 11:32:43 +0000 Subject: [PATCH 54/64] Update vizro-core/docs/pages/user-guides/data.md Co-authored-by: Jo Stichbury --- vizro-core/docs/pages/user-guides/data.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vizro-core/docs/pages/user-guides/data.md b/vizro-core/docs/pages/user-guides/data.md index 99f27490b..f0fa3cfe5 100644 --- a/vizro-core/docs/pages/user-guides/data.md +++ b/vizro-core/docs/pages/user-guides/data.md @@ -389,7 +389,7 @@ For example, let us add two filters to the [dynamic data example](#dynamic-data) 2. This filter implicitly controls the dynamic data source `"iris"`, which supplies the `data_frame` to the targeted `vm.Graph`. On page refresh, Vizro reloads this data, finds all the unique values in the `"species"` column and sets the categorical selector's `options` accordingly. 3. Similarly, on page refresh, Vizro finds the minimum and maximum values of the `"sepal_length"` column in the reloaded data and sets new `min` and `max` values for the numerical selector accordingly. -If you have a filter that depends on dynamic data but do not want the available values to change when the dynamic data changes then you should manually specify the `selector`'s `options` field (categorical selector) or `min` and `max` fields (numerical selector). In the above example, this could be achieved as follows: +Consider a filter that depends on dynamic data, where you do **not** want the available values to change when the dynamic data changes. You should manually specify the `selector`'s `options` field (categorical selector) or `min` and `max` fields (numerical selector). In the above example, this could be achieved as follows: ```python title="Override selector options to make a dynamic filter static" controls = [ From c0377409ec6585c7c51232d8082e6291440d2781 Mon Sep 17 00:00:00 2001 From: Antony Milne <49395058+antonymilne@users.noreply.github.com> Date: Fri, 29 Nov 2024 11:33:19 +0000 Subject: [PATCH 55/64] Update vizro-core/docs/pages/user-guides/data.md Co-authored-by: Jo Stichbury --- vizro-core/docs/pages/user-guides/data.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/vizro-core/docs/pages/user-guides/data.md b/vizro-core/docs/pages/user-guides/data.md index f0fa3cfe5..7cf9dc26e 100644 --- a/vizro-core/docs/pages/user-guides/data.md +++ b/vizro-core/docs/pages/user-guides/data.md @@ -407,7 +407,11 @@ controls = [ ] ``` -When Vizro initially builds a filter that depends on parametrized dynamic data loading, data is loaded using the default argument values. This data is used to perform initial validation, check which data sources contain the specified `column` (unless `targets` is explicitly specified) and determine the type of selector to use (unless `selector` is explicitly specified). +When Vizro initially builds a filter that depends on parametrized dynamic data loading, data is loaded using the default argument values. This data is used to: + +* perform initial validation +* check which data sources contain the specified `column` (unless `targets` is explicitly specified) and +* determine the type of selector to use (unless `selector` is explicitly specified). !!! note From 7ed7385c2cf0d6efd9ba9d80e5b5e18cfd34c4dd Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 29 Nov 2024 11:33:35 +0000 Subject: [PATCH 56/64] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- vizro-core/docs/pages/user-guides/data.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vizro-core/docs/pages/user-guides/data.md b/vizro-core/docs/pages/user-guides/data.md index 7cf9dc26e..394da4811 100644 --- a/vizro-core/docs/pages/user-guides/data.md +++ b/vizro-core/docs/pages/user-guides/data.md @@ -410,7 +410,7 @@ controls = [ When Vizro initially builds a filter that depends on parametrized dynamic data loading, data is loaded using the default argument values. This data is used to: * perform initial validation -* check which data sources contain the specified `column` (unless `targets` is explicitly specified) and +* check which data sources contain the specified `column` (unless `targets` is explicitly specified) and * determine the type of selector to use (unless `selector` is explicitly specified). !!! note From befb14d989bbd33c2fbe468d24db7ca591b9da3a Mon Sep 17 00:00:00 2001 From: Antony Milne <49395058+antonymilne@users.noreply.github.com> Date: Fri, 29 Nov 2024 11:34:21 +0000 Subject: [PATCH 57/64] Update vizro-core/docs/pages/user-guides/data.md --- vizro-core/docs/pages/user-guides/data.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vizro-core/docs/pages/user-guides/data.md b/vizro-core/docs/pages/user-guides/data.md index 394da4811..ce155df1e 100644 --- a/vizro-core/docs/pages/user-guides/data.md +++ b/vizro-core/docs/pages/user-guides/data.md @@ -415,4 +415,4 @@ When Vizro initially builds a filter that depends on parametrized dynamic data l !!! note - When a dynamic data parameter is changed on screen, the data underlying a dynamic filter can change. Currently this change affects page components such as `vm.Graph` but does not affect the available values shown in a dynamic filter, which only update on page refresh. This functionality will be coming soon! + When the value of a dynamic data parameter is changed by a dashboard user, the data underlying a dynamic filter can change. Currently this change affects page components such as `vm.Graph` but does not affect the available values shown in a dynamic filter, which only update on page refresh. This functionality will be coming soon! From 1d9f6062e2e83acf39197ed07272e476a2cfaefa Mon Sep 17 00:00:00 2001 From: Antony Milne <49395058+antonymilne@users.noreply.github.com> Date: Fri, 29 Nov 2024 11:35:40 +0000 Subject: [PATCH 58/64] Update vizro-core/docs/pages/user-guides/filters.md --- vizro-core/docs/pages/user-guides/filters.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vizro-core/docs/pages/user-guides/filters.md b/vizro-core/docs/pages/user-guides/filters.md index 0680c0bca..5fd0df70e 100644 --- a/vizro-core/docs/pages/user-guides/filters.md +++ b/vizro-core/docs/pages/user-guides/filters.md @@ -3,7 +3,7 @@ This guide shows you how to add filters to your dashboard. One main way to interact with the charts/components on your page is by filtering the underlying data. A filter selects a subset of rows of a component's underlying DataFrame which alters the appearance of that component on the page. The [`Page`][vizro.models.Page] model accepts the `controls` argument, where you can enter a [`Filter`][vizro.models.Filter] model. -This model enables the automatic creation of [selectors](selectors.md) (such as `Dropdown`, `RadioItems`, `Slider`, ...) that operate on the charts/components on the screen. +This model enables the automatic creation of [selectors](selectors.md) (for example, `Dropdown` or `RangeSlider`) that operate on the charts/components on the screen. By default, filters that control components with [dynamic data](data.md#dynamic-data) are [dynamically updated](data.md#filters) when the underlying data changes while the dashboard is running. From 97aa9d139c80e20e34fd3621c5c49d56226b4353 Mon Sep 17 00:00:00 2001 From: Antony Milne <49395058+antonymilne@users.noreply.github.com> Date: Fri, 29 Nov 2024 11:35:52 +0000 Subject: [PATCH 59/64] Update vizro-core/docs/pages/user-guides/filters.md Co-authored-by: Jo Stichbury --- vizro-core/docs/pages/user-guides/filters.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vizro-core/docs/pages/user-guides/filters.md b/vizro-core/docs/pages/user-guides/filters.md index 5fd0df70e..db9aa312e 100644 --- a/vizro-core/docs/pages/user-guides/filters.md +++ b/vizro-core/docs/pages/user-guides/filters.md @@ -69,7 +69,7 @@ The selector is configured automatically based on the target column type data as - [Numerical data](https://pandas.pydata.org/docs/reference/api/pandas.api.types.is_numeric_dtype.html) uses [`vm.RangeSlider`][vizro.models.RangeSlider] where `min` and `max` are the overall minimum and maximum values found in `column` across all the data sources of components in `targets`. - [Temporal data](https://pandas.pydata.org/docs/reference/api/pandas.api.types.is_datetime64_any_dtype.html) uses [`vm.DatePicker(range=True)`][vizro.models.DatePicker] where `min` and `max` are the overall minimum and maximum values found in `column` across all the data sources of components in `targets`. A column can be converted to this type with [pandas.to_datetime](https://pandas.pydata.org/docs/reference/api/pandas.to_datetime.html). -Below is an example demonstrating these default selector types. +The following example demonstrates these default selector types. !!! example "Default Filter selectors" === "app.py" From 28562c379eafb40ebda901c00c7fbd24220a8c57 Mon Sep 17 00:00:00 2001 From: Antony Milne <49395058+antonymilne@users.noreply.github.com> Date: Fri, 29 Nov 2024 11:37:01 +0000 Subject: [PATCH 60/64] Update vizro-core/docs/pages/user-guides/data.md Co-authored-by: Maximilian Schulz <83698606+maxschulz-COL@users.noreply.github.com> --- vizro-core/docs/pages/user-guides/data.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vizro-core/docs/pages/user-guides/data.md b/vizro-core/docs/pages/user-guides/data.md index ce155df1e..f6765f344 100644 --- a/vizro-core/docs/pages/user-guides/data.md +++ b/vizro-core/docs/pages/user-guides/data.md @@ -341,7 +341,7 @@ You cannot pass [nested parameters](parameters.md#nested-parameters) to dynamic When a [filter](filters.md) depends on dynamic data and no `selector` is explicitly defined in the `vm.Filter` model, the available selector values update on page refresh to reflect the latest dynamic data. This is called a _dynamic filter_. -The system for updating dynamic filters works exactly like other non-control components such as `vm.Graph`. However, unlike such components, a filter can depend on multiple data sources. If at least one data source of the components in the filter's `targets` is dynamic then the filter is dynamic. Remember that when `targets` is not explicitly specified, a filter applies to all the components on a page that use a DataFrame including `column`. +The mechanism behind updating dynamic filters works exactly like other non-control components such as `vm.Graph`. However, unlike such components, a filter can depend on multiple data sources. If at least one data source of the components in the filter's `targets` is dynamic then the filter is dynamic. Remember that when `targets` is not explicitly specified, a filter applies to all the components on a page that use a DataFrame including `column`. When the page is refreshed, the behavior of a dynamic filter is as follows: From ecba967e25b9d0cdc1121fcaec79cc4bb7cdae74 Mon Sep 17 00:00:00 2001 From: Antony Milne <49395058+antonymilne@users.noreply.github.com> Date: Fri, 29 Nov 2024 12:03:51 +0000 Subject: [PATCH 61/64] Update vizro-core/docs/pages/user-guides/data.md --- vizro-core/docs/pages/user-guides/data.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vizro-core/docs/pages/user-guides/data.md b/vizro-core/docs/pages/user-guides/data.md index f6765f344..8d1a2047a 100644 --- a/vizro-core/docs/pages/user-guides/data.md +++ b/vizro-core/docs/pages/user-guides/data.md @@ -281,7 +281,7 @@ To add a parameter to control a dynamic data source, do the following: 2. give an `id` to all components that have the data source you wish to alter through a parameter. 3. [add a parameter](parameters.md) with `targets` of the form `.data_frame.` and a suitable [selector](selectors.md). -For example, let us extend the [dynamic data example](#dynamic-data) above into example of how parametrized dynamic data works. The `load_iris_data` can take an argument `number_of_points` controlled from the dashboard with a [`Slider`][vizro.models.Slider]. +For example, let us extend the [dynamic data example](#dynamic-data) above into an example of how parametrized dynamic data works. The `load_iris_data` can take an argument `number_of_points` controlled from the dashboard with a [`Slider`][vizro.models.Slider]. !!! example "Parametrized dynamic data" === "app.py" From aa38a407ae8c41700124188a8385f89d8d16521c Mon Sep 17 00:00:00 2001 From: petar-qb Date: Fri, 29 Nov 2024 15:47:25 +0100 Subject: [PATCH 62/64] Remove one comment --- vizro-core/src/vizro/models/_components/form/dropdown.py | 1 - 1 file changed, 1 deletion(-) diff --git a/vizro-core/src/vizro/models/_components/form/dropdown.py b/vizro-core/src/vizro/models/_components/form/dropdown.py index b00b5eeae..a56c13c47 100755 --- a/vizro-core/src/vizro/models/_components/form/dropdown.py +++ b/vizro-core/src/vizro/models/_components/form/dropdown.py @@ -66,7 +66,6 @@ class Dropdown(VizroBaseModel): title: str = Field("", description="Title to be displayed") actions: list[Action] = [] - # A private property that allows dynamically updating components # Consider making the _dynamic public later. The same property could also be used for all other components. # For example: vm.Graph could have a dynamic that is by default set on True. _dynamic: bool = PrivateAttr(False) From 744931dd8f598a06805801b7726589b66f00f71d Mon Sep 17 00:00:00 2001 From: petar-qb Date: Fri, 29 Nov 2024 16:21:43 +0100 Subject: [PATCH 63/64] TestFilterCall refactoring --- .../vizro/models/_controls/test_filter.py | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/vizro-core/tests/unit/vizro/models/_controls/test_filter.py b/vizro-core/tests/unit/vizro/models/_controls/test_filter.py index e31064b33..823b88a91 100644 --- a/vizro-core/tests/unit/vizro/models/_controls/test_filter.py +++ b/vizro-core/tests/unit/vizro/models/_controls/test_filter.py @@ -438,20 +438,22 @@ class TestFilterCall: def test_filter_call_categorical_valid(self, target_to_data_frame): filter = vm.Filter( - column="column_categorical", targets=["column_categorical_exists_1", "column_categorical_exists_2"] + column="column_categorical", + targets=["column_categorical_exists_1", "column_categorical_exists_2"], + selector=vm.Dropdown(id="test_selector_id"), ) - filter._column_type = "categorical" - filter.selector = vm.Dropdown(id="test_selector_id") + filter.pre_build() selector_build = filter(target_to_data_frame=target_to_data_frame, current_value=["a", "b"])["test_selector_id"] assert selector_build.options == ["ALL", "a", "b", "c"] def test_filter_call_numerical_valid(self, target_to_data_frame): filter = vm.Filter( - column="column_numerical", targets=["column_numerical_exists_1", "column_numerical_exists_2"] + column="column_numerical", + targets=["column_numerical_exists_1", "column_numerical_exists_2"], + selector=vm.RangeSlider(id="test_selector_id"), ) - filter._column_type = "numerical" - filter.selector = vm.RangeSlider(id="test_selector_id") + filter.pre_build() selector_build = filter(target_to_data_frame=target_to_data_frame, current_value=[1, 2])["test_selector_id"] assert selector_build.min == 1 @@ -461,8 +463,9 @@ def test_filter_call_column_is_changed(self, target_to_data_frame): filter = vm.Filter( column="column_categorical", targets=["column_categorical_exists_1", "column_categorical_exists_2"] ) + filter.pre_build() + filter._column_type = "numerical" - filter.selector = vm.RangeSlider(id="test_selector_id") with pytest.raises( ValueError, @@ -473,8 +476,7 @@ def test_filter_call_column_is_changed(self, target_to_data_frame): def test_filter_call_selected_column_not_found_in_target(self): filter = vm.Filter(column="column_categorical", targets=["column_categorical_exists_1"]) - filter._column_type = "categorical" - filter.selector = vm.Dropdown(id="test_selector_id") + filter.pre_build() with pytest.raises( ValueError, @@ -484,8 +486,7 @@ def test_filter_call_selected_column_not_found_in_target(self): def test_filter_call_targeted_data_empty(self): filter = vm.Filter(column="column_categorical", targets=["column_categorical_exists_1"]) - filter._column_type = "categorical" - filter.selector = vm.Dropdown(id="test_selector_id") + filter.pre_build() with pytest.raises( ValueError, From 81a9bdbe161a1e58683b2613d0a21aff6a88dd8b Mon Sep 17 00:00:00 2001 From: petar-qb Date: Mon, 2 Dec 2024 10:32:19 +0100 Subject: [PATCH 64/64] PR comment referenced in the _actions_utils --- vizro-core/src/vizro/actions/_actions_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/vizro-core/src/vizro/actions/_actions_utils.py b/vizro-core/src/vizro/actions/_actions_utils.py index 8b9cb2f4c..d46803bd3 100644 --- a/vizro-core/src/vizro/actions/_actions_utils.py +++ b/vizro-core/src/vizro/actions/_actions_utils.py @@ -270,6 +270,7 @@ def _get_modified_page_figures( # TODO-NEXT: Add fetching unfiltered data for the Filter.targets as well, once dynamic filters become "targetable" # from other actions too. For example, in future, if Parameter is targeting only a single Filter. # Currently, it only works for the on_page_load because Filter.targets are indeed the part of the actions' targets. + # More about the limitation: https://github.com/mckinsey/vizro/pull/879/files#r1863535516 target_to_data_frame = _get_unfiltered_data(ctds_parameter=ctds_parameter, targets=figure_targets) # TODO: the structure here would be nicer if we could get just the ctds for a single target at one time,