Skip to content

Commit

Permalink
refactor: use buffer
Browse files Browse the repository at this point in the history
  • Loading branch information
rilshok committed Nov 29, 2024
1 parent 4fdb26d commit e6e169b
Show file tree
Hide file tree
Showing 23 changed files with 47 additions and 45 deletions.
4 changes: 2 additions & 2 deletions src/iokit/__init__.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
__all__ = [
"Dat",
"Encryption",
"Enc",
"Env",
"Gzip",
"Json",
Expand All @@ -27,7 +27,7 @@

from .extensions import (
Dat,
Encryption,
Enc,
Env,
Flac,
Gzip,
Expand Down
4 changes: 2 additions & 2 deletions src/iokit/extensions/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
"Wav",
"Waveform",
"Dat",
"Encryption",
"Enc",
"Env",
"Gzip",
"Json",
Expand All @@ -19,7 +19,7 @@

from .audio import Flac, Mp3, Ogg, Wav, Waveform
from .dat import Dat
from .enc import Encryption
from .enc import Enc
from .env import Env
from .gz import Gzip
from .json import Json
Expand Down
2 changes: 1 addition & 1 deletion src/iokit/extensions/audio.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ def __init__(self, waveform: "Waveform", **kwargs: Any):
super().__init__(data=buffer.getvalue(), **kwargs)

def load(self) -> "Waveform":
wave, freq = soundfile.read(self.data, always_2d=True)
wave, freq = soundfile.read(self.buffer, always_2d=True)
return Waveform(wave=wave, freq=freq)


Expand Down
2 changes: 1 addition & 1 deletion src/iokit/extensions/dat.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,4 +12,4 @@ def __init__(self, data: bytes, **kwargs: Any):
super().__init__(data=data, **kwargs)

def load(self) -> bytes:
return self._data.getvalue()
return self.data
6 changes: 3 additions & 3 deletions src/iokit/extensions/enc.py
Original file line number Diff line number Diff line change
Expand Up @@ -91,12 +91,12 @@ def __repr__(self) -> str:

@classmethod
def pack(cls, state: State, password: bytes | str, salt: bytes | str = b"42") -> Self:
payload = _pack_arrays(str(state.name).encode("utf-8"), state.data.getvalue())
payload = _pack_arrays(str(state.name).encode("utf-8"), state.data)
data = encrypt(data=payload, password=_to_bytes(password), salt=_to_bytes(salt))
return cls(data=data)


class Encryption(State, suffix="enc"):
class Enc(State, suffix="enc"):
def __init__(
self,
state: State,
Expand All @@ -112,4 +112,4 @@ def __init__(
super().__init__(data=data, name=name, **kwargs)

def load(self) -> SecretState:
return SecretState(data=self.data.getvalue())
return SecretState(data=self.data)
2 changes: 1 addition & 1 deletion src/iokit/extensions/env.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,5 +20,5 @@ def __init__(self, data: dict[str, str], **kwargs: Any):
super().__init__(data=data_bytes, **kwargs)

def load(self) -> dict[str, str | None]:
with StringIO(self.data.getvalue().decode()) as stream:
with StringIO(self.data.decode()) as stream:
return dict(dotenv.dotenv_values(stream=stream))
9 changes: 4 additions & 5 deletions src/iokit/extensions/gz.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,10 @@ def __init__(self, state: State, *, compression: int = 1, **kwargs: Any):
with BytesIO() as buffer:
gzip_file = gzip.GzipFile(fileobj=buffer, mode="wb", compresslevel=compression, mtime=0)
with gzip_file as gzip_buffer:
gzip_buffer.write(state.data.getvalue())
gzip_buffer.write(state.data)
super().__init__(data=buffer.getvalue(), name=state.name, **kwargs)

def load(self) -> State:
gzip_file = gzip.GzipFile(fileobj=self.data, mode="rb")
with gzip_file as file:
data = file.read()
return State(data=data, name=str(self.name).removesuffix(".gz")).cast()
# gzip_file =
with gzip.GzipFile(fileobj=self.buffer, mode="rb") as file:
return State(data=file.read(), name=str(self.name).removesuffix(".gz")).cast()
2 changes: 1 addition & 1 deletion src/iokit/extensions/json.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,4 +42,4 @@ def __init__(
super().__init__(data=data_, **kwargs)

def load(self) -> Any:
return json.load(self.data)
return json.load(self.buffer)
2 changes: 1 addition & 1 deletion src/iokit/extensions/jsonl.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,5 +31,5 @@ def __init__(
super().__init__(data=buffer.getvalue(), **kwargs)

def load(self) -> list[Any]:
with Reader(self.data) as reader:
with Reader(self.buffer) as reader:
return list(reader)
2 changes: 1 addition & 1 deletion src/iokit/extensions/npy.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,4 +14,4 @@ def __init__(self, array: NDArray[Any], **kwargs: Any) -> None:
super().__init__(data=buffer.getvalue(), **kwargs)

def load(self) -> NDArray[Any]:
return np.load(self.data, allow_pickle=False, fix_imports=False)
return np.load(self.buffer, allow_pickle=False, fix_imports=False)
4 changes: 2 additions & 2 deletions src/iokit/extensions/tar.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,13 @@ def __init__(self, states: Iterable[State], **kwargs: Any):
file_data = tarfile.TarInfo(name=str(state.name))
file_data.size = state.size
file_data.mtime = int(state.time.timestamp())
tar_buffer.addfile(fileobj=state.data, tarinfo=file_data)
tar_buffer.addfile(fileobj=state.buffer, tarinfo=file_data)

super().__init__(data=buffer.getvalue(), **kwargs)

def load(self) -> list[State]:
states: list[State] = []
with tarfile.open(fileobj=self.data, mode="r") as tar_buffer:
with tarfile.open(fileobj=self.buffer, mode="r") as tar_buffer:
assert tar_buffer is not None
for member in tar_buffer.getmembers():
if not member.isfile():
Expand Down
2 changes: 1 addition & 1 deletion src/iokit/extensions/txt.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,4 @@ def __init__(self, data: str, **kwargs: Any):
super().__init__(data=data.encode("utf-8"), **kwargs)

def load(self) -> str:
return self.data.getvalue().decode("utf-8")
return self.data.decode("utf-8")
2 changes: 1 addition & 1 deletion src/iokit/extensions/yaml.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,4 +15,4 @@ def __init__(self, data: Any, **kwargs: Any):
super().__init__(data=data, **kwargs)

def load(self) -> Any:
return yaml.safe_load(self.data)
return yaml.safe_load(self.buffer)
4 changes: 2 additions & 2 deletions src/iokit/extensions/zip.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,13 +12,13 @@ def __init__(self, states: Iterable[State], **kwargs: Any):
with BytesIO() as buffer:
with zipfile.ZipFile(buffer, mode="w") as zip_buffer:
for state in states:
zip_buffer.writestr(str(state.name), data=state.data.getvalue())
zip_buffer.writestr(str(state.name), data=state.data)

super().__init__(data=buffer.getvalue(), **kwargs)

def load(self) -> list[State]:
states: list[State] = []
with zipfile.ZipFile(self.data, mode="r") as zip_buffer:
with zipfile.ZipFile(self.buffer, mode="r") as zip_buffer:
for file in zip_buffer.namelist():
with zip_buffer.open(file) as member_buffer:
state = State(
Expand Down
15 changes: 9 additions & 6 deletions src/iokit/state.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ class State:
_suffixes: tuple[str, ...] = ("",)

def __init__(self, data: bytes, name: str | StateName = "", time: datetime | None = None):
self._data = BytesIO(data)
self._data = data
self._name = StateName.make(name, self._suffix)
self._time = time or now()

Expand Down Expand Up @@ -113,13 +113,16 @@ def time(self, value: datetime) -> None:
self._time = value

@property
def data(self) -> BytesIO:
self._data.seek(0)
return BytesIO(self._data.getvalue())
def data(self) -> bytes:
return self._data

@property
def buffer(self) -> BytesIO:
return BytesIO(self._data)

@property
def size(self) -> int:
return self._data.getbuffer().nbytes
return len(self._data)

def __repr__(self) -> str:
size = naturalsize(self.size, gnu=True)
Expand All @@ -146,7 +149,7 @@ def cast(self) -> "State":

def load(self) -> Any:
if not self.name.suffix:
return self.data.getvalue()
return self.data
state = self.cast()
if type(state) is State: # pylint: disable=unidiomatic-typecheck
msg = f"Cannot load state with suffix '{self.name.suffix}'"
Expand Down
2 changes: 1 addition & 1 deletion src/iokit/storage/local.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ def save_file(
raise FileExistsError(msg)
root.mkdir(parents=parents, exist_ok=True)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(state.data.getvalue())
path.write_bytes(state.data)
return path


Expand Down
2 changes: 1 addition & 1 deletion tests/test_download_file.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,4 @@
def test_download_file() -> None:
uri = "https://raw.githubusercontent.com/rilshok/iokit/main/LICENSE"
state = download_file(uri)
assert "MIT License" in state.data.getvalue().decode("utf-8")
assert "MIT License" in state.data.decode("utf-8")
4 changes: 2 additions & 2 deletions tests/test_encryption.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

import pytest

from iokit import Encryption, Json
from iokit import Enc, Json


def test_encryption() -> None:
Expand All @@ -14,7 +14,7 @@ def test_encryption() -> None:
"int": 42,
}
json = Json(data, name="different")
state = Encryption(json, password="pA$sw0Rd", salt="s@lt")
state = Enc(json, password="pA$sw0Rd", salt="s@lt")
state_secret = state.load()
with pytest.raises(ValueError) as excinfo:
state_secret.load(password="pA$sw0Rd")
Expand Down
10 changes: 5 additions & 5 deletions tests/test_json.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,21 +7,21 @@ def test_json_empty() -> None:
state = Json({}, name="empty")
assert state.name == "empty.json"
assert state.size == 2
assert state.data.getvalue() == b"{}"
assert state.data == b"{}"
assert not state.load()


def test_json_single() -> None:
state = Json({"key": "value"}, name="single")
assert state.name == "single.json"
assert state.data.getvalue() == b'{"key": "value"}'
assert state.data == b'{"key": "value"}'
assert state.load() == {"key": "value"}


def test_json_multiple() -> None:
state = Json({"first": 1, "second": 2}, name="multiple")
assert state.name == "multiple.json"
assert state.data.getvalue() == b'{"first": 1, "second": 2}'
assert state.data == b'{"first": 1, "second": 2}'
assert state.load() == {"first": 1, "second": 2}
assert 20 < state.size < 30

Expand All @@ -48,11 +48,11 @@ def test_json_is_string() -> None:
state = Json("hello", name="string")
assert state.load() == "hello"
assert state.size == 7
assert state.data.getvalue() == b'"hello"'
assert state.data == b'"hello"'


def test_json_is_sequence() -> None:
state = Json([1, 2, 3], name="sequence")
assert state.load() == [1, 2, 3]
assert state.size == 9
assert state.data.getvalue() == b"[1, 2, 3]"
assert state.data == b"[1, 2, 3]"
6 changes: 3 additions & 3 deletions tests/test_jsonl.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,21 +5,21 @@ def test_jsonl_empty() -> None:
state = Jsonl([], name="empty")
assert state.name == "empty.jsonl"
assert state.size == 0
assert state.data.getvalue() == b""
assert state.data == b""
assert not state.load()


def test_jsonl_single() -> None:
state = Jsonl([{"key": "value"}], name="single")
assert state.name == "single.jsonl"
assert state.data.getvalue() == b'{"key":"value"}\n'
assert state.data == b'{"key":"value"}\n'
assert state.load() == [{"key": "value"}]


def test_jsonl_multiple_repeat() -> None:
state = Jsonl([{"key": "value"}] * 2, name="multiple")
assert state.name == "multiple.jsonl"
assert state.data.getvalue() == b'{"key":"value"}\n{"key":"value"}\n'
assert state.data == b'{"key":"value"}\n{"key":"value"}\n'
assert state.load() == [{"key": "value"}] * 2


Expand Down
2 changes: 1 addition & 1 deletion tests/test_state_inheritance.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,5 +10,5 @@ def test_state_inheritance_json() -> None:
assert MyJson._suffixes == ("myjson",)
myjson = MyJson({"a": 1}, name="test")
assert myjson.name == "test.myjson"
loaded = State(data=myjson.data.getvalue(), name="test.myjson")
loaded = State(data=myjson.data, name="test.myjson")
assert loaded.load() == myjson.load() == {"a": 1}
2 changes: 1 addition & 1 deletion tests/test_tar.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ def test_tar_compress() -> None:
archive2_gz = Gzip(archive)

assert archive1_gz.size == archive2_gz.size
assert archive1_gz.data.getvalue() == archive2_gz.data.getvalue()
assert archive1_gz.data == archive2_gz.data
loaded = archive1_gz.load().load()
assert find_state(loaded, "text1.txt").load() == "First file"
assert find_state(loaded, "text2.txt").load() == "Second file"
2 changes: 1 addition & 1 deletion tests/test_yaml.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ def test_yaml_empty() -> None:
state = Yaml([], name="empty")
assert state.name == "empty.yaml"
assert state.size == 3
assert state.data.getvalue() == b"[]\n"
assert state.data == b"[]\n"
assert not state.load()


Expand Down

0 comments on commit e6e169b

Please sign in to comment.