Skip to content

Commit

Permalink
test: Remove test_chart_validation_benchmark
Browse files Browse the repository at this point in the history
Feeling I've squeezed out all the performance I can for now.
Will add in a collpased comment on the PR for reference
  • Loading branch information
dangotbanned committed Sep 3, 2024
1 parent 67642f0 commit 8002dab
Showing 1 changed file with 1 addition and 91 deletions.
92 changes: 1 addition & 91 deletions tests/utils/test_schemapi.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
import warnings
from collections import deque
from functools import partial
from typing import TYPE_CHECKING, Any, Callable, Iterable, Iterator, Literal, Sequence
from typing import TYPE_CHECKING, Any, Callable, Iterable, Sequence

import jsonschema
import jsonschema.exceptions
Expand Down Expand Up @@ -882,96 +882,6 @@ def test_chart_validation_errors(chart_func, expected_error_message):
chart.to_dict()


_SKIP_SLOW_BENCHMARKS: bool = True
_REPEAT_TIMES = 1000


@pytest.mark.parametrize("to_or_from", ["to_dict-validate", "to_dict", "from_dict"])
@pytest.mark.filterwarnings("ignore:.*:UserWarning")
@pytest.mark.skipif(
_SKIP_SLOW_BENCHMARKS,
reason="Should only be run in isolation to test single threaded performance.",
)
def test_chart_validation_benchmark(
to_or_from: Literal["to_dict-validate", "to_dict", "from_dict"],
) -> None:
"""
Intended to isolate `Chart.(to|from)_dict.` calls.
Repeated ``_REPEAT_TIMES`` times, non-parametric:
- in an attempt to limit the potential overhead of ``pytest``
- but enforce ``1`` thread, like a user-code would be.
Results
-------
```
_REPEAT_TIMES = 1000
pytest -k test_chart_validation_benchmark --numprocesses=3 --durations=3 tests
# Pre-`SchemaBase.from_dict` refactor (3.12.3)
108.16s call tests/utils/test_schemapi.py::test_chart_validation_benchmark[to_dict-validate]
84.62s call tests/utils/test_schemapi.py::test_chart_validation_benchmark[from_dict]
66.71s call tests/utils/test_schemapi.py::test_chart_validation_benchmark[to_dict]
# Post-`SchemaBase.from_dict` refactor (3.12.3)
107.84s call tests/utils/test_schemapi.py::test_chart_validation_benchmark[to_dict-validate]
50.43s call tests/utils/test_schemapi.py::test_chart_validation_benchmark[from_dict]
67.07s call tests/utils/test_schemapi.py::test_chart_validation_benchmark[to_dict]
# Post-`SchemaBase.__init_subclass__` addition (3.12.3)
108.24s call tests/utils/test_schemapi.py::test_chart_validation_benchmark[to_dict-validate]
50.33s call tests/utils/test_schemapi.py::test_chart_validation_benchmark[from_dict]
66.51s call tests/utils/test_schemapi.py::test_chart_validation_benchmark[to_dict]
# Post-`dict` branch micro optimization in `_FromDict.from_dict` (3.12.3)
107.90s call tests/utils/test_schemapi.py::test_chart_validation_benchmark[to_dict-validate]
49.63s call tests/utils/test_schemapi.py::test_chart_validation_benchmark[from_dict]
66.87s call tests/utils/test_schemapi.py::test_chart_validation_benchmark[to_dict]
```
"""
from itertools import chain, repeat

if TYPE_CHECKING:
from altair.typing import ChartType

def _iter_charts() -> Iterator[ChartType]:
"""
Ensures only len(chart_funcs_error_message) actual charts are constructed.
The `to_dict` calls are what gets multiplied
"""
charts: list[ChartType] = [fn() for fn, _ in chart_funcs_error_message]
yield from chain.from_iterable(repeat(charts, times=_REPEAT_TIMES))

def _iter_chart_factory() -> Iterator[ChartType]:
"""
Validation not the bottleneck, but encode is.
Ensures at least `times` * len(chart_funcs_error_message) .encode calls are made.
"""
chart_funcs: list[Callable[[], ChartType]] = [
fn for fn, _ in chart_funcs_error_message
]
for fn in chain.from_iterable(repeat(chart_funcs, times=_REPEAT_TIMES)):
yield fn()

def _to_dict(validate: bool) -> None:
if validate:
for chart in _iter_charts():
with pytest.raises(schemapi.SchemaValidationError):
chart.to_dict(validate=validate)
else:
for chart in _iter_charts():
chart.to_dict(validate=validate)

if to_or_from == "to_dict":
_to_dict(validate=False)
elif to_or_from == "to_dict-validate":
_to_dict(validate=True)
else:
assert list(_iter_chart_factory())


def test_multiple_field_strings_in_condition():
selection = alt.selection_point()
expected_error_message = "A field cannot be used for both the `if_true` and `if_false` values of a condition. One of them has to specify a `value` or `datum` definition."
Expand Down

0 comments on commit 8002dab

Please sign in to comment.