From 828aca96408d638e9a611a8fdc48c4c601b00be1 Mon Sep 17 00:00:00 2001 From: "Vladislav A. Proskurov" Date: Sat, 22 Jun 2024 20:39:50 +0400 Subject: [PATCH 01/12] docs: add some docs --- README.md | 126 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 124 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d84b84c..149b8cc 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,124 @@ -# iok -input / output kit +# I/O Kit Python Library + +IOKit is a Python library that offers a suite of utilities for managing a wide range of input/output operations. Central to its design is the concept of a `State`, where each state signifies a unit of data that can be loaded, saved, or transformed. Each state corresponds to a valid file state, represented as a bytes-like object. + +IOKit abstracts and unifies serialization and deserialization operations from various libraries into a single, cohesive interface. This allows for direct manipulation of the file's state in memory, eliminating the need for disk interaction. Consequently, it facilitates the (de)serialization of data in multiple formats, such as `json`, `txt`, `tar`, `gzip`, among others. This abstraction not only simplifies data handling but also enhances efficiency by reducing disk I/O operations. + +## Installation + +You can install the IOkit library using pip: + +```bash +pip install iokit +``` + +## Usage + +Here are some examples of how to use the I/O Kit library: + +### Text File Handling + +```python +from iokit import Txt + +text = "Hello, World!" +state = Txt(text, name="text") +print(state) +print(state.load()) +``` + +```plain-text +text.txt (13B) +Hello, World! +``` + +### JSON + +```python +from iokit import Json + +data = {"key": "value"} +state = Json(data, name="single") +print(state) +print(state.load()) +``` + +```plain-text +single.json (16B) +{'key': 'value'} +``` + +### GZip Compression + +```python +from iokit import Json, Gzip + +data = {"a": 1, "b": 2} +state = Gzip(Json(data, name="data")) +print(state) +print(state.load().load()) +``` + +```plain-text +data.json.gz (34B) +{'a': 1, 'b': 2} +``` + +### Tar Archive + +```python +from iokit import Tar, Txt + +state1 = Txt("First file", name="text1") +state2 = Txt("Second file", name="text2") +archive = Tar([state1, state2], name="archive") +states = archive.load() +print(states) +print(states[0].load()) +print(states[1].load()) +``` + +```plain-text +[text1.txt (10B), text2.txt (11B)] +First file +Second file +``` + +### Find State + +```python +from iokit import Tar, find_state + +state1 = Txt("First file", name="text1") +state2 = Txt("Second file", name="text2") +archive = Tar([state1, state2], name="archive") + +state = find_state(archive.load(), "?e*2.txt") +print(state.load()) +``` + +```plain-text +Second file +``` + +### Byte input handling + +```python +from iokit import State + +state = State(b"{\"first\": 1, \"second\": 2}", name="data.json") +print(state.load()) +``` + +```plain-text +{'first': 1, 'second': 2} +``` + + +## Contributing + +Contributions to the IOkit library are welcome. Please feel free to submit a pull request or open an issue on the GitHub repository. + +## License + +The IOkit library is licensed under the MIT License. You can use it for commercial and non-commercial projects without any restrictions. From 0693c463a8e5f0da856f63a700566adc608d9c86 Mon Sep 17 00:00:00 2001 From: "Vladislav A. Proskurov" Date: Sat, 22 Jun 2024 20:43:26 +0400 Subject: [PATCH 02/12] docs: use a more demonstrative example of gzip --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 149b8cc..f9b2698 100644 --- a/README.md +++ b/README.md @@ -51,17 +51,17 @@ single.json (16B) ### GZip Compression ```python -from iokit import Json, Gzip +from iokit import Txt, Gzip -data = {"a": 1, "b": 2} -state = Gzip(Json(data, name="data")) +data = "Hello, World! "* 1000 +state = Gzip(Txt(data, name="data")) print(state) -print(state.load().load()) +print(len(state.load().load())) ``` ```plain-text -data.json.gz (34B) -{'a': 1, 'b': 2} +data.txt.gz (133B) +14000 ``` ### Tar Archive From 1b29b0dfd2ede6d841b83084e56ca6042e1779c4 Mon Sep 17 00:00:00 2001 From: "Vladislav A. Proskurov" Date: Sun, 23 Jun 2024 00:48:35 +0400 Subject: [PATCH 03/12] style: autoformat --- tests/test_gzip.py | 7 +++++-- tests/test_json.py | 1 - tests/test_tar.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/test_gzip.py b/tests/test_gzip.py index c2f2e19..9291e99 100644 --- a/tests/test_gzip.py +++ b/tests/test_gzip.py @@ -1,13 +1,14 @@ -from iokit import Json, Gzip, load_file, save_temp - import os +from iokit import Gzip, Json, load_file, save_temp + def random_utf8_string(length: int) -> str: random_bytes = os.urandom(length) string = random_bytes.decode("utf-8", errors="replace") return string + def test_gzip_state() -> None: data = {"a": 1, "b": 2} state = Gzip(Json(data, name="data")) @@ -18,6 +19,7 @@ def test_gzip_state() -> None: assert state.load().load() == data assert state.size > 0 + def test_gzip_compression() -> None: string = random_utf8_string(10_000) state = Json(string, name="data") @@ -32,6 +34,7 @@ def test_gzip_compression() -> None: assert compressed3.load().load() == string assert compressed9.load().load() == string + def test_gzip_save_load_file() -> None: data = {"a": 1, "b": 2} state = Gzip(Json(data, name="data")) diff --git a/tests/test_json.py b/tests/test_json.py index e97122e..2e30ec8 100644 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -42,7 +42,6 @@ def test_json_different() -> None: assert loaded["int"] == 42 - def test_json_is_string() -> None: state = Json("hello", name="string") assert state.load() == "hello" diff --git a/tests/test_tar.py b/tests/test_tar.py index af5d6b5..0577949 100644 --- a/tests/test_tar.py +++ b/tests/test_tar.py @@ -1,4 +1,4 @@ -from iokit import Tar, Txt, find_state, Gzip +from iokit import Gzip, Tar, Txt, find_state def test_tar_state() -> None: From e747a1a00009469debaf54e22730dcc8bcf25a60 Mon Sep 17 00:00:00 2001 From: "Vladislav A. Proskurov" Date: Sun, 23 Jun 2024 00:49:23 +0400 Subject: [PATCH 04/12] ci: run pre-commit and pytest --- .github/workflows/tests.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..bf56b71 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,23 @@ +name: Tests + +on: + push: + branches: [ "main", "dev" ] + pull_request: + branches: [ "*" ] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.10" + - name: Install dependencies + run: pip install .[dev] + - name: Run pre-commit + run: pre-commit run --all-files + - name: Test with pytest + run: pytest . From 4c48ee2fee117eb8b016208b2415991141754c90 Mon Sep 17 00:00:00 2001 From: "Vladislav A. Proskurov" Date: Sun, 23 Jun 2024 00:50:01 +0400 Subject: [PATCH 05/12] ci: release by tags --- .github/workflows/publish.yml | 45 ++++++----------------------------- 1 file changed, 7 insertions(+), 38 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 0c335bb..8e5b23f 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -2,8 +2,8 @@ name: Publish to PyPI on: push: - branches: - - main + tags: + - "v*.*.*" jobs: build: @@ -25,21 +25,6 @@ jobs: name: python-package-distributions path: dist/ - test: - name: Run tests with pytest - runs-on: ubuntu-latest - needs: - - build - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.10' - - name: Run tests - run: pytest - publish-to-pypi: name: Publish Python 🐍 distribution 📦 to PyPI needs: @@ -77,27 +62,11 @@ jobs: with: name: python-package-distributions path: dist/ - - name: Sign the dists with Sigstore - uses: sigstore/gh-action-sigstore-python@v2.1.1 + + - name: Release + uses: softprops/action-gh-release@v2 + if: startsWith(github.ref, 'refs/tags/') with: - inputs: >- + files: | ./dist/*.tar.gz ./dist/*.whl - - name: Create GitHub Release - env: - GITHUB_TOKEN: ${{ github.token }} - run: >- - gh release create - '${{ github.ref_name }}' - --repo '${{ github.repository }}' - --notes "" - - name: Upload artifact signatures to GitHub Release - env: - GITHUB_TOKEN: ${{ github.token }} - # Upload to GitHub Release using the `gh` CLI. - # `dist/` contains the built packages, and the - # sigstore-produced signatures and certificates. - run: >- - gh release upload - '${{ github.ref_name }}' dist/** - --repo '${{ github.repository }}' From bedb984880538918e7ff4718a6ae48067d91d039 Mon Sep 17 00:00:00 2001 From: "Vladislav A. Proskurov" Date: Sun, 23 Jun 2024 01:03:39 +0400 Subject: [PATCH 06/12] ci: simplify tests run --- .github/workflows/tests.yml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index bf56b71..ce40615 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -8,16 +8,22 @@ on: jobs: build: + name: Run tests runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v4 - - name: Set up Python + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: - python-version: "3.10" + python-version: ${{ matrix.python-version }} - name: Install dependencies run: pip install .[dev] - - name: Run pre-commit - run: pre-commit run --all-files - - name: Test with pytest - run: pytest . + - name: Run pre-commit hooks + uses: pre-commit/action@v3.0.1 + - name: Run pytest + uses: pavelzw/pytest-action@v2 + with: + emoji: false From d5015a939dce196864d13d97bc8ba71bb517686b Mon Sep 17 00:00:00 2001 From: "Vladislav A. Proskurov" Date: Sun, 23 Jun 2024 01:07:23 +0400 Subject: [PATCH 07/12] ci: install pytest-md --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ce40615..f2a95b1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -20,7 +20,7 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Install dependencies - run: pip install .[dev] + run: pip install .[dev] pytest-md - name: Run pre-commit hooks uses: pre-commit/action@v3.0.1 - name: Run pytest From d32f95426def5e1ba345821978cb205de4012426 Mon Sep 17 00:00:00 2001 From: "Vladislav A. Proskurov" Date: Sun, 23 Jun 2024 01:15:58 +0400 Subject: [PATCH 08/12] ci: try pytest coverage comment --- .github/workflows/tests.yml | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f2a95b1..4064f85 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -20,10 +20,22 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Install dependencies - run: pip install .[dev] pytest-md + run: | + pip install --upgrade pip + + pip install .[dev] pytest-md pytest-cov - name: Run pre-commit hooks uses: pre-commit/action@v3.0.1 - - name: Run pytest - uses: pavelzw/pytest-action@v2 + - name: Build coverage file + run: | + pytest --junitxml=pytest.xml --cov-report=term-missing:skip-covered --cov=app tests/ | tee pytest-coverage.txt + - name: Pytest coverage comment + uses: MishaKav/pytest-coverage-comment@main with: - emoji: false + pytest-coverage-path: ./pytest-coverage.txt + junitxml-path: ./pytest.xml + + # - name: Run pytest + # uses: pavelzw/pytest-action@v2 + # with: + # emoji: false From 91cd99b6897e7628af5b54bb5974c7006fb8d32d Mon Sep 17 00:00:00 2001 From: "Vladislav A. Proskurov" Date: Sun, 23 Jun 2024 01:21:18 +0400 Subject: [PATCH 09/12] ci: fix module name --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4064f85..dc3490a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -28,7 +28,7 @@ jobs: uses: pre-commit/action@v3.0.1 - name: Build coverage file run: | - pytest --junitxml=pytest.xml --cov-report=term-missing:skip-covered --cov=app tests/ | tee pytest-coverage.txt + pytest --junitxml=pytest.xml --cov-report=term-missing:skip-covered --cov=iokit tests/ | tee pytest-coverage.txt - name: Pytest coverage comment uses: MishaKav/pytest-coverage-comment@main with: From f6971b45413790e198b3bfc3eccc51203b6ae2ab Mon Sep 17 00:00:00 2001 From: "Vladislav A. Proskurov" Date: Sun, 23 Jun 2024 01:23:53 +0400 Subject: [PATCH 10/12] ci: add permisions --- .github/workflows/tests.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index dc3490a..b38e324 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -6,6 +6,11 @@ on: pull_request: branches: [ "*" ] +permissions: + contents: write + checks: write + pull-requests: write + jobs: build: name: Run tests From 2a65e66d22d7fd9500e13c8654b1647333fb538e Mon Sep 17 00:00:00 2001 From: "Vladislav A. Proskurov" Date: Sun, 23 Jun 2024 01:34:12 +0400 Subject: [PATCH 11/12] fix: editable setup --- .github/workflows/tests.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b38e324..1e6fa9d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -27,8 +27,7 @@ jobs: - name: Install dependencies run: | pip install --upgrade pip - - pip install .[dev] pytest-md pytest-cov + pip install -e .[dev] - name: Run pre-commit hooks uses: pre-commit/action@v3.0.1 - name: Build coverage file @@ -39,8 +38,3 @@ jobs: with: pytest-coverage-path: ./pytest-coverage.txt junitxml-path: ./pytest.xml - - # - name: Run pytest - # uses: pavelzw/pytest-action@v2 - # with: - # emoji: false From 8bf278cfcad9f304b77d0ee35149adf8e0e728d9 Mon Sep 17 00:00:00 2001 From: "Vladislav A. Proskurov" Date: Sun, 23 Jun 2024 01:38:24 +0400 Subject: [PATCH 12/12] feat: v0,1,1 --- src/iokit/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/iokit/__init__.py b/src/iokit/__init__.py index 0cfaf36..fa96bfd 100644 --- a/src/iokit/__init__.py +++ b/src/iokit/__init__.py @@ -11,7 +11,7 @@ "save_file", "save_temp", ] -__version__ = "0.1.0" +__version__ = "0.1.1" from .extensions import Gzip, Json, Jsonl, Tar, Txt from .state import State, filter_states, find_state