From f470e2a9814794b338b691b3b047d2ae9e8cea20 Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Mon, 30 Oct 2023 08:56:29 +0200 Subject: [PATCH] Replace black with ruff --- .pre-commit-config.yaml | 12 +++++------ daskms/apps/fragments.py | 1 - daskms/conftest.py | 1 + daskms/dataset.py | 2 +- daskms/experimental/arrow/reads.py | 3 --- .../experimental/arrow/tests/test_parquet.py | 3 --- daskms/experimental/arrow/writes.py | 1 - daskms/experimental/zarr/__init__.py | 1 - daskms/experimental/zarr/tests/test_zarr.py | 2 -- daskms/tests/test_storage.py | 3 --- daskms/writes.py | 2 -- pyproject.toml | 20 +++++++++++++++++++ 12 files changed, 27 insertions(+), 24 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 489a5d6f..28fb60bf 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,16 +2,14 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.2.0 + rev: v4.5.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer - id: check-yaml - id: check-added-large-files -- repo: https://github.com/psf/black - rev: 22.8.0 +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.1.3 hooks: - - id: black - language_version: python3 - args: - - --target-version=py38 + - id: ruff-format + name: ruff format diff --git a/daskms/apps/fragments.py b/daskms/apps/fragments.py index 0166a30d..e746678e 100644 --- a/daskms/apps/fragments.py +++ b/daskms/apps/fragments.py @@ -21,7 +21,6 @@ def fragments(): default=False, ) def stat(fragment_path, prune): - ancestors = get_ancestry(fragment_path, only_required=prune) click.echo("Ancestry:") diff --git a/daskms/conftest.py b/daskms/conftest.py index f8f1d202..02ce2e90 100644 --- a/daskms/conftest.py +++ b/daskms/conftest.py @@ -13,6 +13,7 @@ from daskms.testing import mark_in_pytest + # content of conftest.py def pytest_configure(config): mark_in_pytest(True) diff --git a/daskms/dataset.py b/daskms/dataset.py index 494e133f..182db70a 100644 --- a/daskms/dataset.py +++ b/daskms/dataset.py @@ -491,7 +491,7 @@ def finalize_compute(results, info, coords, attrs): data_vars = OrderedDict() rev_results = list(results[::-1]) - for (dask_collection, k, v) in info: + for dask_collection, k, v in info: if dask_collection: fn, args = v r = rev_results.pop() diff --git a/daskms/experimental/arrow/reads.py b/daskms/experimental/arrow/reads.py index 7b2c0279..ae5287d2 100644 --- a/daskms/experimental/arrow/reads.py +++ b/daskms/experimental/arrow/reads.py @@ -165,7 +165,6 @@ def partition_chunking(partition, fragment_rows, chunks): it = zip(chunk_intervals, chunk_intervals[1:]) for c, (lower, upper) in enumerate(it): - si = np.searchsorted(intervals, lower, side="right") - 1 ei = np.searchsorted(intervals, upper, side="left") @@ -191,7 +190,6 @@ def partition_chunking(partition, fragment_rows, chunks): def fragment_reader(fragments, ranges, column, shape, dtype): - if len(fragments) > 1: # Reading over multiple row_groups. arr = np.empty(shape, dtype=dtype) offset = 0 @@ -277,7 +275,6 @@ def xds_from_parquet(store, columns=None, chunks=None, **kwargs): partition_chunks = partition_chunking(p, fragment_rows, chunks) for pieces in partition_chunks.values(): - chunk_fragments = [fragments[i] for i, _ in pieces] chunk_ranges = [r for _, r in pieces] chunk_metas = [f.metadata for f in chunk_fragments] diff --git a/daskms/experimental/arrow/tests/test_parquet.py b/daskms/experimental/arrow/tests/test_parquet.py index 73b63ba3..c681fcf1 100644 --- a/daskms/experimental/arrow/tests/test_parquet.py +++ b/daskms/experimental/arrow/tests/test_parquet.py @@ -166,7 +166,6 @@ def test_xds_to_parquet_local(ms, tmp_path_factory, spw_table, ant_table): def test_xds_to_parquet_s3( ms, spw_table, ant_table, py_minio_client, minio_user_key, minio_url, s3_bucket_name ): - py_minio_client.make_bucket(s3_bucket_name) store = DaskMSStore( @@ -189,7 +188,6 @@ def test_xds_to_parquet_s3( @pytest.fixture(params=[1, 2, 3, 4]) def parquet_ms(ms, tmp_path_factory, request): - parquet_store = tmp_path_factory.mktemp("parquet") / "test.parquet" # Chunk in row so we can probe chunk behaviour on reads. @@ -204,7 +202,6 @@ def parquet_ms(ms, tmp_path_factory, request): @pytest.mark.parametrize("rc", [1, 2, 3, 4]) def test_xds_from_parquet_chunks(ms, parquet_ms, rc): - xdsl = xds_from_parquet(parquet_ms, chunks={"row": rc}) chunks = chain.from_iterable([xds.chunks["row"] for xds in xdsl]) diff --git a/daskms/experimental/arrow/writes.py b/daskms/experimental/arrow/writes.py index 167a9d8e..71bf16ce 100644 --- a/daskms/experimental/arrow/writes.py +++ b/daskms/experimental/arrow/writes.py @@ -60,7 +60,6 @@ def __reduce__(self): return (ParquetFragment, (self.store, self.key, self.schema, self.dataset_id)) def write(self, chunk, *data): - table_path = ( self.key if self.store.table else self.store.join(["MAIN", self.key]) ) diff --git a/daskms/experimental/zarr/__init__.py b/daskms/experimental/zarr/__init__.py index 84049ac0..dad6335f 100644 --- a/daskms/experimental/zarr/__init__.py +++ b/daskms/experimental/zarr/__init__.py @@ -150,7 +150,6 @@ def prepare_zarr_group(dataset_id, dataset, store, rechunk=False): def get_group_chunks(group): - group_chunks = {} for array in group.values(): diff --git a/daskms/experimental/zarr/tests/test_zarr.py b/daskms/experimental/zarr/tests/test_zarr.py index e7864d56..3432a15e 100644 --- a/daskms/experimental/zarr/tests/test_zarr.py +++ b/daskms/experimental/zarr/tests/test_zarr.py @@ -113,7 +113,6 @@ def test_metadata_consolidation(ms, ant_table, tmp_path_factory, consolidated): def zarr_tester(ms, spw_table, ant_table, zarr_store, spw_store, ant_store): - ms_datasets = xds_from_ms(ms) spw_datasets = xds_from_table(spw_table, group_cols="__row__") ant_datasets = xds_from_table(ant_table) @@ -292,7 +291,6 @@ def test_fasteners(ms, tmp_path_factory): def test_basic_roundtrip(tmp_path): - path = tmp_path / "test.zarr" # We need >10 datasets to be sure roundtripping is consistent. diff --git a/daskms/tests/test_storage.py b/daskms/tests/test_storage.py index 671f2ffe..93a969fc 100644 --- a/daskms/tests/test_storage.py +++ b/daskms/tests/test_storage.py @@ -12,7 +12,6 @@ @pytest.mark.skipif(xarray is None, reason="Need xarray to check equality.") def test_storage_ms(ms): - oxdsl = xds_from_ms(ms) writes = xds_to_storage_table(oxdsl, ms) @@ -28,7 +27,6 @@ def test_storage_ms(ms): @pytest.mark.skipif(xarray is None, reason="Need xarray to check equality.") def test_storage_zarr(ms, tmp_path_factory): - zarr_store = tmp_path_factory.mktemp("zarr") / "test.zarr" oxdsl = xds_from_ms(ms) @@ -52,7 +50,6 @@ def test_storage_zarr(ms, tmp_path_factory): @pytest.mark.skipif(xarray is None, reason="Need xarray to check equality.") def test_storage_parquet(ms, tmp_path_factory): - parquet_store = tmp_path_factory.mktemp("parquet") / "test.parquet" oxdsl = xds_from_ms(ms) diff --git a/daskms/writes.py b/daskms/writes.py index 073ba1a0..f90ba917 100644 --- a/daskms/writes.py +++ b/daskms/writes.py @@ -522,7 +522,6 @@ def cached_row_order(rowid): if not layer_name.startswith("row-") and not layer_name.startswith( "group-rows-" ): - log.warning( "Unusual ROWID layer %s. " "This is probably OK but " @@ -539,7 +538,6 @@ def cached_row_order(rowid): layer_names[0].startswith("group-rows-") and layer_names[1].startswith("rechunk-merge-") ): - log.warning( "Unusual ROWID layers %s for " "the group ordering case. " diff --git a/pyproject.toml b/pyproject.toml index 68c9b0a1..ce3947f6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,9 @@ testing = ["minio", "pytest"] tbump = "^6.9.0" pre-commit = "^2.20.0" black = "^22.8.0" +ipython = "^8.16.1" +ipdb = "^0.13.13" +ruff = "^0.1.3" [tool.poetry.group.docs.dependencies] furo = "^2022.9.15" @@ -45,6 +48,23 @@ numpydoc = "^1.4.0" Pygments = "^2.13.0" sphinx-copybutton = "^0.5.0" + +[tool.ruff] +line-length = 88 +target-version = "py310" + +select = [ + # flake8-builtins + "A", + # flake8-bugbear + "B", + # isort + "I001", + "I002", + # tidy imports + "TID" +] + [build-system] requires = ["setuptools", "poetry-core"] build-backend = "poetry.core.masonry.api"