Skip to content
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 38 additions & 0 deletions tests/benchmarks/test_dataframe.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,14 @@
import random
import string

import dask
import dask.dataframe as dd
import numpy
import pandas
import pytest
from dask.sizeof import sizeof
from dask.utils import format_bytes
from packaging.version import Version

from ..utils_test import cluster_memory, timeseries_of_size, wait

Expand Down Expand Up @@ -58,3 +67,32 @@ def test_shuffle(small_client):
shuf = df.shuffle(0, shuffle="tasks")
result = shuf.size
wait(result, small_client, 20 * 60)


@pytest.mark.skipif(
Version(dask.__version__) < Version("2022.10.1"),
reason=" No support for pandas string dtypes in versions < 2022.10.1",
)
@pytest.mark.parametrize("dtype", ["object", "string[python]", "string[pyarrow]"])
def test_shuffle_string(dtype, small_client):
def make_partition(s, n=100_000):
random.seed(s)
s1 = pandas.Series(
[
"".join(
random.choices(string.ascii_letters, k=random.randint(100, 1000))
)
for _ in range(n)
],
dtype=dtype,
name="label",
)
df = pandas.DataFrame(numpy.random.randint(0, 100, size=(n, 10)))
df.insert(0, "label", s1)
return df

meta = make_partition(0, n=10)

ddf = dd.from_map(make_partition, range(100), meta=meta)
result = ddf.set_index("label").persist()
wait(result, small_client, 20 * 60)