From 149d64a700f23005c826b4325a66fa6ff1663d8e Mon Sep 17 00:00:00 2001 From: "Daniel G, Krakowczyk" Date: Fri, 9 May 2025 23:59:36 +0200 Subject: [PATCH 1/9] ci: add pre-commit hook to write datasets.yaml --- .pre-commit-config.yaml | 9 +++++ .pre-commit/write_datasets_yaml.py | 59 ++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 .pre-commit/write_datasets_yaml.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 200efb398..82bb81847 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -149,3 +149,12 @@ repos: '--disable=protected-access', '--rcfile=pylintrc', ] +- repo: local + hooks: + - id: write-datasets-yaml + name: write-datasets-yaml + entry: python .pre-commit/write_datasets_yaml.py + language: python + verbose: true + additional_dependencies: + - pyyaml diff --git a/.pre-commit/write_datasets_yaml.py b/.pre-commit/write_datasets_yaml.py new file mode 100644 index 000000000..519f6989c --- /dev/null +++ b/.pre-commit/write_datasets_yaml.py @@ -0,0 +1,59 @@ +# Copyright (c) 2025 The pymovements Project Authors +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +from __future__ import annotations + +from pathlib import Path + +import yaml + + +def main( + datasets_dirpath: str | Path = './src/pymovements/datasets', + datasets_yaml_filename: str = 'datasets.yaml', +) -> int: + if isinstance(datasets_dirpath, str): + datasets_dirpath = Path(datasets_dirpath) + + dataset_filename_stems = sorted( + [ + filepath.stem for filepath in datasets_dirpath.glob('*.yaml') + if filepath.name != datasets_yaml_filename # Ignore datasets yaml file. + ], + ) + + with open(datasets_dirpath / datasets_yaml_filename) as f: + dataset_yaml_content = yaml.safe_load(f) + + # File content looks good. Exit successfully. + if dataset_filename_stems == dataset_yaml_content: + return 0 + + # We have some updates in the datasets directory. Update the datasets yaml file. + with open(datasets_dirpath / datasets_yaml_filename, 'w') as f: + yaml.dump(dataset_filename_stems, f) + + return 1 + + +if __name__ == '__main__': + # parser = argparse.ArgumentParser() + # parser.add_argument('--path') + # args = parser.parse_args() + raise SystemExit(main()) From 61662287c4eded337b41b29750604159764e0899 Mon Sep 17 00:00:00 2001 From: "Daniel G. Krakowczyk" Date: Sat, 10 May 2025 00:13:46 +0200 Subject: [PATCH 2/9] mess up datasets.yaml --- src/pymovements/datasets/datasets.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/src/pymovements/datasets/datasets.yaml b/src/pymovements/datasets/datasets.yaml index 68c5094ff..f592d7c45 100644 --- a/src/pymovements/datasets/datasets.yaml +++ b/src/pymovements/datasets/datasets.yaml @@ -1,4 +1,3 @@ -- bsc - bsc2 - codecomprehension - copco From 5e71967f24b6e8fccb852513afe486291b7fcbe4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 9 May 2025 22:14:36 +0000 Subject: [PATCH 3/9] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/pymovements/datasets/datasets.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pymovements/datasets/datasets.yaml b/src/pymovements/datasets/datasets.yaml index f592d7c45..68c5094ff 100644 --- a/src/pymovements/datasets/datasets.yaml +++ b/src/pymovements/datasets/datasets.yaml @@ -1,3 +1,4 @@ +- bsc - bsc2 - codecomprehension - copco From 4e9080c2a1ab3bef287aefbe25bc3dbae7fb3e37 Mon Sep 17 00:00:00 2001 From: "Daniel G, Krakowczyk" Date: Mon, 26 May 2025 13:50:40 +0900 Subject: [PATCH 4/9] add tests --- .pre-commit-config.yaml | 2 +- src/pymovements/_scripts/__init__.py | 20 ++++ .../_scripts}/write_datasets_yaml.py | 18 +-- .../unit/_scripts/write_datasets_yaml_test.py | 107 ++++++++++++++++++ 4 files changed, 137 insertions(+), 10 deletions(-) create mode 100644 src/pymovements/_scripts/__init__.py rename {.pre-commit => src/pymovements/_scripts}/write_datasets_yaml.py (82%) create mode 100644 tests/unit/_scripts/write_datasets_yaml_test.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 82bb81847..cc64deb31 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -153,7 +153,7 @@ repos: hooks: - id: write-datasets-yaml name: write-datasets-yaml - entry: python .pre-commit/write_datasets_yaml.py + entry: python _scripts/write_datasets_yaml.py language: python verbose: true additional_dependencies: diff --git a/src/pymovements/_scripts/__init__.py b/src/pymovements/_scripts/__init__.py new file mode 100644 index 000000000..3c9c2ae98 --- /dev/null +++ b/src/pymovements/_scripts/__init__.py @@ -0,0 +1,20 @@ +# Copyright (c) 2025 The pymovements Project Authors +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + diff --git a/.pre-commit/write_datasets_yaml.py b/src/pymovements/_scripts/write_datasets_yaml.py similarity index 82% rename from .pre-commit/write_datasets_yaml.py rename to src/pymovements/_scripts/write_datasets_yaml.py index 519f6989c..aefa13049 100644 --- a/.pre-commit/write_datasets_yaml.py +++ b/src/pymovements/_scripts/write_datasets_yaml.py @@ -19,6 +19,7 @@ # SOFTWARE. from __future__ import annotations +import io from pathlib import Path import yaml @@ -28,20 +29,22 @@ def main( datasets_dirpath: str | Path = './src/pymovements/datasets', datasets_yaml_filename: str = 'datasets.yaml', ) -> int: - if isinstance(datasets_dirpath, str): - datasets_dirpath = Path(datasets_dirpath) + datasets_dirpath = Path(datasets_dirpath) dataset_filename_stems = sorted( [ filepath.stem for filepath in datasets_dirpath.glob('*.yaml') - if filepath.name != datasets_yaml_filename # Ignore datasets yaml file. + if filepath.name != datasets_yaml_filename # Ignore datasets.yaml file. ], ) - with open(datasets_dirpath / datasets_yaml_filename) as f: - dataset_yaml_content = yaml.safe_load(f) + try: + with open(datasets_dirpath / datasets_yaml_filename) as f: + dataset_yaml_content = yaml.safe_load(f) + except FileNotFoundError: + dataset_yaml_content = None - # File content looks good. Exit successfully. + # File content matches. Exit successfully. if dataset_filename_stems == dataset_yaml_content: return 0 @@ -53,7 +56,4 @@ def main( if __name__ == '__main__': - # parser = argparse.ArgumentParser() - # parser.add_argument('--path') - # args = parser.parse_args() raise SystemExit(main()) diff --git a/tests/unit/_scripts/write_datasets_yaml_test.py b/tests/unit/_scripts/write_datasets_yaml_test.py new file mode 100644 index 000000000..076f8e4a5 --- /dev/null +++ b/tests/unit/_scripts/write_datasets_yaml_test.py @@ -0,0 +1,107 @@ +# Copyright (c) 2025 The pymovements Project Authors +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import pytest +import yaml + +from pymovements._scripts import write_datasets_yaml + +@pytest.fixture() +def make_datasets_directory(request, tmp_path): + def _make_datasets_directory(param: str, make_yaml: bool): + datasets = [] + if param in {'single', 'two'}: + filepath = tmp_path / 'first.yaml' + filepath.touch() + datasets.append('first') + if param == 'two': + filepath = tmp_path / 'second.yaml' + filepath.touch() + datasets.append('second') + if make_yaml: + with open(tmp_path / 'datasets.yaml', 'w') as f: + yaml.dump(datasets, f) + return tmp_path + + yield _make_datasets_directory + + +@pytest.mark.parametrize( + ('fixture_args', 'expected_return', 'expected_yaml'), + [ + pytest.param( + ['empty', False], + 1, + [], + id='empty_new', + ), + + pytest.param( + ['empty', True], + 0, + [], + id='empty_exist', + ), + + pytest.param( + ['single', False], + 1, + ['first'], + id='single_new', + ), + + pytest.param( + ['single', True], + 0, + ['first'], + id='single_exist', + ), + + pytest.param( + ['two', False], + 1, + ['first', 'second'], + id='two_new', + ), + + pytest.param( + ['two', True], + 0, + ['first', 'second'], + id='two_exist', + ), + ], +) +def test_write_datasets_yaml( + fixture_args, expected_return, expected_yaml, make_datasets_directory, +): + datasets_dirpath = make_datasets_directory(*fixture_args) + datasets_yaml_filename = 'datasets.yaml' + + return_value = write_datasets_yaml.main( + datasets_dirpath=datasets_dirpath, + datasets_yaml_filename=datasets_yaml_filename, + ) + + with open(datasets_dirpath / datasets_yaml_filename) as f: + yaml_content = yaml.safe_load(f) + + assert return_value == expected_return + assert yaml_content == expected_yaml From 0ff04e3118d5dad86f6f5889dc02851ed3bbb3ea Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 26 May 2025 07:44:38 +0000 Subject: [PATCH 5/9] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/pymovements/_scripts/__init__.py | 1 - src/pymovements/_scripts/write_datasets_yaml.py | 1 - tests/unit/_scripts/write_datasets_yaml_test.py | 2 +- 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/pymovements/_scripts/__init__.py b/src/pymovements/_scripts/__init__.py index 3c9c2ae98..6786db225 100644 --- a/src/pymovements/_scripts/__init__.py +++ b/src/pymovements/_scripts/__init__.py @@ -17,4 +17,3 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. - diff --git a/src/pymovements/_scripts/write_datasets_yaml.py b/src/pymovements/_scripts/write_datasets_yaml.py index aefa13049..4934f3e76 100644 --- a/src/pymovements/_scripts/write_datasets_yaml.py +++ b/src/pymovements/_scripts/write_datasets_yaml.py @@ -19,7 +19,6 @@ # SOFTWARE. from __future__ import annotations -import io from pathlib import Path import yaml diff --git a/tests/unit/_scripts/write_datasets_yaml_test.py b/tests/unit/_scripts/write_datasets_yaml_test.py index 076f8e4a5..311b0fead 100644 --- a/tests/unit/_scripts/write_datasets_yaml_test.py +++ b/tests/unit/_scripts/write_datasets_yaml_test.py @@ -17,12 +17,12 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. - import pytest import yaml from pymovements._scripts import write_datasets_yaml + @pytest.fixture() def make_datasets_directory(request, tmp_path): def _make_datasets_directory(param: str, make_yaml: bool): From a12bdd000614bab6db513750a2dc84b48a0b3998 Mon Sep 17 00:00:00 2001 From: "Daniel G, Krakowczyk" Date: Mon, 26 May 2025 16:55:15 +0900 Subject: [PATCH 6/9] make pylint happy --- .pre-commit-config.yaml | 2 +- .../_scripts/write_datasets_yaml.py | 20 +++++++++++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cc64deb31..36aae5990 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -153,7 +153,7 @@ repos: hooks: - id: write-datasets-yaml name: write-datasets-yaml - entry: python _scripts/write_datasets_yaml.py + entry: python src/pymovements/_scripts/write_datasets_yaml.py language: python verbose: true additional_dependencies: diff --git a/src/pymovements/_scripts/write_datasets_yaml.py b/src/pymovements/_scripts/write_datasets_yaml.py index 4934f3e76..7420a44f0 100644 --- a/src/pymovements/_scripts/write_datasets_yaml.py +++ b/src/pymovements/_scripts/write_datasets_yaml.py @@ -17,6 +17,7 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +"""Write datasets.yaml for DatasetLibrary.""" from __future__ import annotations from pathlib import Path @@ -28,6 +29,21 @@ def main( datasets_dirpath: str | Path = './src/pymovements/datasets', datasets_yaml_filename: str = 'datasets.yaml', ) -> int: + """Write datasets yaml file for DatasetLibrary. + + Parameters + ---------- + datasets_dirpath: str | Path + The path to the directory containing dataset definition yaml files. + (default: './src/pymovements/datasets') + datasets_yaml_filename: str + The filename of the datasets yaml file. (default: 'datasets.yaml') + + Returns + ------- + int + ``0`` if no changes needed, ``1`` otherwise. + """ datasets_dirpath = Path(datasets_dirpath) dataset_filename_stems = sorted( @@ -38,7 +54,7 @@ def main( ) try: - with open(datasets_dirpath / datasets_yaml_filename) as f: + with open(datasets_dirpath / datasets_yaml_filename, encoding='utf-8') as f: dataset_yaml_content = yaml.safe_load(f) except FileNotFoundError: dataset_yaml_content = None @@ -48,7 +64,7 @@ def main( return 0 # We have some updates in the datasets directory. Update the datasets yaml file. - with open(datasets_dirpath / datasets_yaml_filename, 'w') as f: + with open(datasets_dirpath / datasets_yaml_filename, 'w', encoding='utf-8') as f: yaml.dump(dataset_filename_stems, f) return 1 From e2d74f2ff6752b47e88f7e7adb18569881a9e9e5 Mon Sep 17 00:00:00 2001 From: "Daniel G, Krakowczyk" Date: Mon, 26 May 2025 17:11:08 +0900 Subject: [PATCH 7/9] make pylint happy --- tests/unit/_scripts/write_datasets_yaml_test.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/unit/_scripts/write_datasets_yaml_test.py b/tests/unit/_scripts/write_datasets_yaml_test.py index 311b0fead..15959f1b7 100644 --- a/tests/unit/_scripts/write_datasets_yaml_test.py +++ b/tests/unit/_scripts/write_datasets_yaml_test.py @@ -17,14 +17,15 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +"""Test write_datasets_yaml script.""" import pytest import yaml from pymovements._scripts import write_datasets_yaml -@pytest.fixture() -def make_datasets_directory(request, tmp_path): +@pytest.fixture(name='make_datasets_directory') +def make_datasets_directory_fixture(tmp_path): def _make_datasets_directory(param: str, make_yaml: bool): datasets = [] if param in {'single', 'two'}: @@ -36,7 +37,7 @@ def _make_datasets_directory(param: str, make_yaml: bool): filepath.touch() datasets.append('second') if make_yaml: - with open(tmp_path / 'datasets.yaml', 'w') as f: + with open(tmp_path / 'datasets.yaml', 'w', encoding='utf-8') as f: yaml.dump(datasets, f) return tmp_path From da91ad32e3d6b51309e275f8adba7a3a9cae5fc2 Mon Sep 17 00:00:00 2001 From: "Daniel G, Krakowczyk" Date: Mon, 26 May 2025 17:12:22 +0900 Subject: [PATCH 8/9] make mypy happy --- tests/unit/_scripts/write_datasets_yaml_test.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/unit/_scripts/write_datasets_yaml_test.py b/tests/unit/_scripts/write_datasets_yaml_test.py index 15959f1b7..b54934acf 100644 --- a/tests/unit/_scripts/write_datasets_yaml_test.py +++ b/tests/unit/_scripts/write_datasets_yaml_test.py @@ -18,6 +18,8 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. """Test write_datasets_yaml script.""" +from pathlib import Path + import pytest import yaml @@ -26,7 +28,7 @@ @pytest.fixture(name='make_datasets_directory') def make_datasets_directory_fixture(tmp_path): - def _make_datasets_directory(param: str, make_yaml: bool): + def _make_datasets_directory(param: str, make_yaml: bool) -> Path: datasets = [] if param in {'single', 'two'}: filepath = tmp_path / 'first.yaml' @@ -101,7 +103,7 @@ def test_write_datasets_yaml( datasets_yaml_filename=datasets_yaml_filename, ) - with open(datasets_dirpath / datasets_yaml_filename) as f: + with open(datasets_dirpath / datasets_yaml_filename, encoding='utf-8') as f: yaml_content = yaml.safe_load(f) assert return_value == expected_return From 01d3079e396789fb10738f805fa5c4d4299edf44 Mon Sep 17 00:00:00 2001 From: "Daniel G, Krakowczyk" Date: Mon, 26 May 2025 18:05:06 +0900 Subject: [PATCH 9/9] add coverage ignore --- src/pymovements/_scripts/write_datasets_yaml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pymovements/_scripts/write_datasets_yaml.py b/src/pymovements/_scripts/write_datasets_yaml.py index 7420a44f0..aa099c414 100644 --- a/src/pymovements/_scripts/write_datasets_yaml.py +++ b/src/pymovements/_scripts/write_datasets_yaml.py @@ -70,5 +70,5 @@ def main( return 1 -if __name__ == '__main__': +if __name__ == '__main__': # pragma: no cover raise SystemExit(main())