Skip to content

Commit

Permalink
[UPDATE] JSONSerializer: Removed indent and separators options from J…
Browse files Browse the repository at this point in the history
…SONEncoderConfig
  • Loading branch information
francis-clairicia committed Sep 29, 2023
1 parent d6682df commit 6e41a0c
Show file tree
Hide file tree
Showing 3 changed files with 91 additions and 92 deletions.
175 changes: 88 additions & 87 deletions src/easynetwork/serializers/json.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,11 +33,6 @@
from ..tools._utils import iter_bytes
from .abc import AbstractIncrementalPacketSerializer

_JSON_VALUE_BYTES: frozenset[int] = frozenset(bytes(string.digits + string.ascii_letters + string.punctuation, "ascii"))
_ESCAPE_BYTE: int = b"\\"[0]

_whitespaces_match: Callable[[bytes, int], re.Match[bytes]] = re.compile(rb"[ \t\n\r]*", re.MULTILINE | re.DOTALL).match # type: ignore[assignment]


@dataclass(kw_only=True)
class JSONEncoderConfig:
Expand All @@ -51,8 +46,6 @@ class JSONEncoderConfig:
check_circular: bool = True
ensure_ascii: bool = True
allow_nan: bool = True
indent: int | None = None
separators: tuple[str, str] | None = (",", ":") # Compact JSON (w/o whitespaces)
default: Callable[..., Any] | None = None


Expand All @@ -72,85 +65,6 @@ class JSONDecoderConfig:
strict: bool = True


class _JSONParser:
class _PlainValueError(Exception):
pass

@staticmethod
def _escaped(partial_document_view: memoryview) -> bool:
escaped = False
for byte in reversed(partial_document_view):
if byte == _ESCAPE_BYTE:
escaped = not escaped
else:
break
return escaped

@staticmethod
def raw_parse() -> Generator[None, bytes, tuple[bytes, bytes]]:
escaped = _JSONParser._escaped
split_partial_document = _JSONParser._split_partial_document
enclosure_counter: Counter[bytes] = Counter()
partial_document: bytes = yield
first_enclosure: bytes = b""
start: int = 0
try:
while True:
with memoryview(partial_document) as partial_document_view:
for nb_chars, char in enumerate(iter_bytes(partial_document_view[start:]), start=start + 1):
match char:
case b'"' if not escaped(partial_document_view[: nb_chars - 1]):
enclosure_counter[b'"'] = 0 if enclosure_counter[b'"'] == 1 else 1
case _ if enclosure_counter[b'"'] > 0: # We are within a JSON string, move on.
continue
case b"{" | b"[":
enclosure_counter[char] += 1
case b"}":
enclosure_counter[b"{"] -= 1
case b"]":
enclosure_counter[b"["] -= 1
case b" " | b"\t" | b"\n" | b"\r": # Optimization: Skip spaces
continue
case _ if len(enclosure_counter) == 0: # No enclosure, only value
partial_document = partial_document[nb_chars - 1 :] if nb_chars > 1 else partial_document
del char, nb_chars
raise _JSONParser._PlainValueError
case _: # JSON character, quickly go to next character
continue
assert len(enclosure_counter) > 0 # nosec assert_used
if not first_enclosure:
first_enclosure = next(iter(enclosure_counter))
if enclosure_counter[first_enclosure] <= 0: # 1st found is closed
return split_partial_document(partial_document, nb_chars)

# partial_document not complete
start = partial_document_view.nbytes

# yield outside view scope
partial_document += yield

except _JSONParser._PlainValueError:
pass

# The document is a plain value (null, true, false, or a number)

del enclosure_counter, first_enclosure

while (nprint_idx := next((idx for idx, byte in enumerate(partial_document) if byte not in _JSON_VALUE_BYTES), -1)) < 0:
partial_document += yield

return split_partial_document(partial_document, nprint_idx)

@staticmethod
def _split_partial_document(partial_document: bytes, index: int) -> tuple[bytes, bytes]:
index = _whitespaces_match(partial_document, index).end()
if index == len(partial_document):
# The following bytes are only spaces
# Do not slice the document, the trailing spaces will be ignored by JSONDecoder
return partial_document, b""
return partial_document[:index], partial_document[index:]


class JSONSerializer(AbstractIncrementalPacketSerializer[Any]):
"""
A :term:`serializer` built on top of the :mod:`json` module.
Expand Down Expand Up @@ -193,7 +107,7 @@ def __init__(
elif not isinstance(decoder_config, JSONDecoderConfig):
raise TypeError(f"Invalid decoder config: expected {JSONDecoderConfig.__name__}, got {type(decoder_config).__name__}")

self.__encoder = JSONEncoder(**dataclass_asdict(encoder_config))
self.__encoder = JSONEncoder(**dataclass_asdict(encoder_config), indent=None, separators=(",", ":"))
self.__decoder = JSONDecoder(**dataclass_asdict(decoder_config))
self.__decoder_error_cls = JSONDecodeError

Expand Down Expand Up @@ -342,3 +256,90 @@ def incremental_deserialize(self) -> Generator[None, bytes, tuple[Any, bytes]]:
},
) from exc
return packet, remaining_data


class _JSONParser:
_JSON_VALUE_BYTES: frozenset[int] = frozenset(bytes(string.digits + string.ascii_letters + string.punctuation, "ascii"))
_ESCAPE_BYTE: int = ord(b"\\")

_whitespaces_match: Callable[[bytes, int], re.Match[bytes]] = re.compile(rb"[ \t\n\r]*", re.MULTILINE | re.DOTALL).match # type: ignore[assignment]

class _PlainValueError(Exception):
pass

@staticmethod
def _escaped(partial_document_view: memoryview) -> bool:
escaped = False
_ESCAPE_BYTE = _JSONParser._ESCAPE_BYTE
for byte in reversed(partial_document_view):
if byte == _ESCAPE_BYTE:
escaped = not escaped
else:
break
return escaped

@staticmethod
def raw_parse() -> Generator[None, bytes, tuple[bytes, bytes]]:
escaped = _JSONParser._escaped
split_partial_document = _JSONParser._split_partial_document
enclosure_counter: Counter[bytes] = Counter()
partial_document: bytes = yield
first_enclosure: bytes = b""
start: int = 0
try:
while True:
with memoryview(partial_document) as partial_document_view:
for nb_chars, char in enumerate(iter_bytes(partial_document_view[start:]), start=start + 1):
match char:
case b'"' if not escaped(partial_document_view[: nb_chars - 1]):
enclosure_counter[b'"'] = 0 if enclosure_counter[b'"'] == 1 else 1
case _ if enclosure_counter[b'"'] > 0: # We are within a JSON string, move on.
continue
case b"{" | b"[":
enclosure_counter[char] += 1
case b"}":
enclosure_counter[b"{"] -= 1
case b"]":
enclosure_counter[b"["] -= 1
case b" " | b"\t" | b"\n" | b"\r": # Optimization: Skip spaces
continue
case _ if len(enclosure_counter) == 0: # No enclosure, only value
partial_document = partial_document[nb_chars - 1 :] if nb_chars > 1 else partial_document
del char, nb_chars
raise _JSONParser._PlainValueError
case _: # JSON character, quickly go to next character
continue
assert len(enclosure_counter) > 0 # nosec assert_used
if not first_enclosure:
first_enclosure = next(iter(enclosure_counter))
if enclosure_counter[first_enclosure] <= 0: # 1st found is closed
return split_partial_document(partial_document, nb_chars)

# partial_document not complete
start = partial_document_view.nbytes

# yield outside view scope
partial_document += yield

except _JSONParser._PlainValueError:
pass

# The document is a plain value (null, true, false, or a number)

del enclosure_counter, first_enclosure

_JSON_VALUE_BYTES = _JSONParser._JSON_VALUE_BYTES

while (nprint_idx := next((idx for idx, byte in enumerate(partial_document) if byte not in _JSON_VALUE_BYTES), -1)) < 0:
partial_document += yield

return split_partial_document(partial_document, nprint_idx)

@staticmethod
def _split_partial_document(partial_document: bytes, index: int) -> tuple[bytes, bytes]:
index = _JSONParser._whitespaces_match(partial_document, index).end()
if index == len(partial_document):
# The following bytes are only spaces
# Do not slice the document, the trailing spaces will be ignored by JSONDecoder
return partial_document, b""
return partial_document[:index], partial_document[index:]
2 changes: 1 addition & 1 deletion tests/functional_test/test_serializers/test_json.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ def packet_to_serialize(request: Any) -> Any:
def expected_complete_data(cls, packet_to_serialize: Any) -> bytes:
import json

return json.dumps(packet_to_serialize, **dataclasses.asdict(cls.ENCODER_CONFIG)).encode("utf-8")
return json.dumps(packet_to_serialize, **dataclasses.asdict(cls.ENCODER_CONFIG), separators=(",", ":")).encode("utf-8")

#### Incremental Serialize

Expand Down
6 changes: 2 additions & 4 deletions tests/unit_test/test_serializers/test_json.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,8 +66,6 @@ def encoder_config(request: Any, mocker: MockerFixture) -> JSONEncoderConfig | N
check_circular=mocker.sentinel.check_circular,
ensure_ascii=mocker.sentinel.ensure_ascii,
allow_nan=mocker.sentinel.allow_nan,
indent=mocker.sentinel.indent,
separators=mocker.sentinel.separators,
default=mocker.sentinel.object_default,
)

Expand Down Expand Up @@ -108,8 +106,8 @@ def test____dunder_init____with_encoder_config(
check_circular=mocker.sentinel.check_circular if encoder_config is not None else True,
ensure_ascii=mocker.sentinel.ensure_ascii if encoder_config is not None else True,
allow_nan=mocker.sentinel.allow_nan if encoder_config is not None else True,
indent=mocker.sentinel.indent if encoder_config is not None else None,
separators=mocker.sentinel.separators if encoder_config is not None else (",", ":"),
indent=None,
separators=(",", ":"),
default=mocker.sentinel.object_default if encoder_config is not None else None,
)

Expand Down

0 comments on commit 6e41a0c

Please sign in to comment.