From dea2012c7eea6e0f03fde0444ecf1a1421012425 Mon Sep 17 00:00:00 2001 From: Nick Macholl Date: Fri, 10 May 2024 13:12:44 -0700 Subject: [PATCH 1/7] MOD: Python mock live server enhancement --- databento/common/cram.py | 3 +- databento/live/gateway.py | 21 +- pyproject.toml | 3 +- tests/conftest.py | 125 ++--- tests/mock_live_server.py | 793 ----------------------------- tests/mockliveserver/__init__.py | 0 tests/mockliveserver/__main__.py | 111 ++++ tests/mockliveserver/controller.py | 137 +++++ tests/mockliveserver/fixture.py | 240 +++++++++ tests/mockliveserver/server.py | 306 +++++++++++ tests/mockliveserver/source.py | 43 ++ tests/test_live_client.py | 175 +++---- tests/test_live_protocol.py | 14 +- 13 files changed, 972 insertions(+), 999 deletions(-) delete mode 100644 tests/mock_live_server.py create mode 100644 tests/mockliveserver/__init__.py create mode 100644 tests/mockliveserver/__main__.py create mode 100644 tests/mockliveserver/controller.py create mode 100644 tests/mockliveserver/fixture.py create mode 100644 tests/mockliveserver/server.py create mode 100644 tests/mockliveserver/source.py diff --git a/databento/common/cram.py b/databento/common/cram.py index 24a9a27..e2db817 100644 --- a/databento/common/cram.py +++ b/databento/common/cram.py @@ -6,9 +6,10 @@ import hashlib import os import sys +from typing import Final -BUCKET_ID_LENGTH = 5 +BUCKET_ID_LENGTH: Final = 5 def get_challenge_response(challenge: str, key: str) -> str: diff --git a/databento/live/gateway.py b/databento/live/gateway.py index ef19c31..d109536 100644 --- a/databento/live/gateway.py +++ b/databento/live/gateway.py @@ -4,6 +4,7 @@ import logging from io import BytesIO from operator import attrgetter +from typing import SupportsBytes from typing import TypeVar from databento_dbn import Encoding @@ -20,16 +21,21 @@ @dataclasses.dataclass -class GatewayControl: +class GatewayControl(SupportsBytes): """ Base class for gateway control messages. """ @classmethod - def parse(cls: type[T], line: str) -> T: + def parse(cls: type[T], line: str | bytes) -> T: """ Parse a message of type `T` from a string. + Parameters + ---------- + line : str | bytes + The data to parse into a GatewayControl message. + Returns ------- T @@ -40,17 +46,20 @@ def parse(cls: type[T], line: str) -> T: If the line fails to parse. """ + if isinstance(line, bytes): + line = line.decode("utf-8") + if not line.endswith("\n"): - raise ValueError(f"`{line.strip()}` does not end with a newline") + raise ValueError(f"'{line!r}' does not end with a newline") - split_tokens = [t.partition("=") for t in line[:-1].split("|")] + split_tokens = [t.partition("=") for t in line.strip().split("|")] data_dict = {k: v for k, _, v in split_tokens} try: return cls(**data_dict) except TypeError: raise ValueError( - f"`{line.strip()} is not a parsible {cls.__name__}", + f"'{line!r}'is not a parsible {cls.__name__}", ) from None def __str__(self) -> str: @@ -154,7 +163,7 @@ def parse_gateway_message(line: str) -> GatewayControl: return message_cls.parse(line) except ValueError: continue - raise ValueError(f"`{line.strip()}` is not a parsible gateway message") + raise ValueError(f"'{line.strip()}' is not a parsible gateway message") class GatewayDecoder: diff --git a/pyproject.toml b/pyproject.toml index 4980665..8ebd784 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,12 +47,13 @@ zstandard = ">=0.21.0" black = "^23.9.1" mypy = "1.5.1" pytest = "^7.4.2" -pytest-asyncio = ">=0.21.0" +pytest-asyncio = "==0.21.1" ruff = "^0.0.291" types-requests = "^2.30.0.0" tomli = "^2.0.1" teamcity-messages = "^1.32" types-pytz = "^2024.1.0.20240203" +types-aiofiles = "^23.2.0.20240403" [build-system] requires = ["poetry-core"] diff --git a/tests/conftest.py b/tests/conftest.py index 35b39ad..28c1496 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,23 +5,25 @@ from __future__ import annotations import asyncio +import logging import pathlib import random import string -import threading +from collections.abc import AsyncGenerator from collections.abc import Generator from collections.abc import Iterable from typing import Callable +import databento.live.session import pytest from databento import historical from databento import live from databento.common.publishers import Dataset -from databento.live import session from databento_dbn import Schema from tests import TESTS_ROOT -from tests.mock_live_server import MockLiveServer +from tests.mockliveserver.fixture import MockLiveServerInterface +from tests.mockliveserver.fixture import fixture_mock_live_server # noqa def pytest_addoption(parser: pytest.Parser) -> None: @@ -88,6 +90,14 @@ def pytest_collection_modifyitems( item.add_marker(skip_release) +@pytest.fixture(name="event_loop", scope="module") +def fixture_event_loop() -> Generator[asyncio.AbstractEventLoop, None, None]: + policy = asyncio.get_event_loop_policy() + loop = policy.new_event_loop() + yield loop + loop.close() + + @pytest.fixture(name="live_test_data_path") def fixture_live_test_data_path() -> pathlib.Path: """ @@ -199,74 +209,13 @@ def fixture_test_api_key() -> str: return f"db-{random_str}" -@pytest.fixture(name="thread_loop", scope="session") -def fixture_thread_loop() -> Generator[asyncio.AbstractEventLoop, None, None]: - """ - Fixture for a threaded event loop. - - Yields - ------ - asyncio.AbstractEventLoop - - """ - loop = asyncio.new_event_loop() - thread = threading.Thread( - name="MockLiveServer", - target=loop.run_forever, - args=(), - daemon=True, - ) - thread.start() - yield loop - loop.stop() - - -@pytest.fixture(name="mock_live_server") -def fixture_mock_live_server( - thread_loop: asyncio.AbstractEventLoop, +@pytest.fixture(name="test_live_api_key") +async def fixture_test_live_api_key( test_api_key: str, - caplog: pytest.LogCaptureFixture, - unused_tcp_port: int, - monkeypatch: pytest.MonkeyPatch, -) -> Generator[MockLiveServer, None, None]: - """ - Fixture for a MockLiveServer instance. - - Yields - ------ - MockLiveServer - - """ - monkeypatch.setenv( - name="DATABENTO_API_KEY", - value=test_api_key, - ) - monkeypatch.setattr( - session, - "AUTH_TIMEOUT_SECONDS", - 1, - ) - monkeypatch.setattr( - session, - "CONNECT_TIMEOUT_SECONDS", - 1, - ) - with caplog.at_level("DEBUG"): - mock_live_server = asyncio.run_coroutine_threadsafe( - coro=MockLiveServer.create( - host="127.0.0.1", - port=unused_tcp_port, - dbn_path=TESTS_ROOT / "data", - ), - loop=thread_loop, - ).result() - - yield mock_live_server - - asyncio.run_coroutine_threadsafe( - coro=mock_live_server.stop(), - loop=thread_loop, - ).result() + mock_live_server: MockLiveServerInterface, +) -> AsyncGenerator[str, None]: + async with mock_live_server.api_key_context(test_api_key): + yield test_api_key @pytest.fixture(name="historical_client") @@ -289,10 +238,12 @@ def fixture_historical_client( @pytest.fixture(name="live_client") -def fixture_live_client( - test_api_key: str, - mock_live_server: MockLiveServer, -) -> Generator[live.client.Live, None, None]: +async def fixture_live_client( + test_live_api_key: str, + mock_live_server: MockLiveServerInterface, + caplog: pytest.LogCaptureFixture, + monkeypatch: pytest.MonkeyPatch, +) -> AsyncGenerator[live.client.Live, None]: """ Fixture for a Live client to connect to the MockLiveServer. @@ -301,11 +252,25 @@ def fixture_live_client( Live """ - test_client = live.client.Live( - key=test_api_key, - gateway=mock_live_server.host, - port=mock_live_server.port, + monkeypatch.setattr( + databento.live.session, + "AUTH_TIMEOUT_SECONDS", + 0.5, ) - yield test_client - if test_client.is_connected(): + monkeypatch.setattr( + databento.live.session, + "CONNECT_TIMEOUT_SECONDS", + 0.5, + ) + + with caplog.at_level(logging.DEBUG): + test_client = live.client.Live( + key=test_live_api_key, + gateway=mock_live_server.host, + port=mock_live_server.port, + ) + + with mock_live_server.test_context(): + yield test_client + test_client.stop() diff --git a/tests/mock_live_server.py b/tests/mock_live_server.py deleted file mode 100644 index 2cfdff3..0000000 --- a/tests/mock_live_server.py +++ /dev/null @@ -1,793 +0,0 @@ -from __future__ import annotations - -import argparse -import asyncio -import enum -import logging -import os -import pathlib -import queue -import random -import string -import sys -import threading -import time -from concurrent import futures -from functools import singledispatchmethod -from io import BytesIO -from os import PathLike -from typing import Any -from typing import Callable -from typing import NewType -from typing import TypeVar - -import zstandard -from databento.common import cram -from databento.common.constants import SCHEMA_STRUCT_MAP -from databento.common.publishers import Dataset -from databento.live.gateway import AuthenticationRequest -from databento.live.gateway import AuthenticationResponse -from databento.live.gateway import ChallengeRequest -from databento.live.gateway import GatewayControl -from databento.live.gateway import Greeting -from databento.live.gateway import SessionStart -from databento.live.gateway import SubscriptionRequest -from databento.live.gateway import parse_gateway_message -from databento_dbn import Schema - - -LIVE_SERVER_VERSION: str = "1.0.0" - -G = TypeVar("G", bound=GatewayControl) -MessageQueue = NewType("MessageQueue", "queue.Queue[GatewayControl]") - - -logger = logging.getLogger(__name__) - - -class MockLiveMode(enum.Enum): - REPLAY = "replay" - REPEAT = "repeat" - - -class MockLiveServerProtocol(asyncio.BufferedProtocol): - """ - The connection protocol to mock the Databento Live Subscription Gateway. - - Attributes - ---------- - cram_challenge : str - The CRAM challenge string that will be used - to authenticate users. - is_authenticated : bool - Flag indicating the user has been authenticated. - is_streaming : bool - Flag indicating streaming has begun. - peer : str - The peer ip and port in the format - {ip}:{port}. - version : str - The server version string. - mode : MockLiveMode - The mode for the mock lsg; defaults to "replay" - - See Also - -------- - `asyncio.BufferedProtocol` - - """ - - def __init__( - self, - version: str, - user_api_keys: dict[str, str], - message_queue: MessageQueue, - dbn_path: PathLike[str], - mode: MockLiveMode = MockLiveMode.REPLAY, - ) -> None: - self.__transport: asyncio.Transport - self._buffer: bytearray - self._data: BytesIO - self._dataset: Dataset | None = None - self._message_queue: MessageQueue = message_queue - self._cram_challenge: str = "".join( - random.choice(string.ascii_letters) for _ in range(32) # noqa: S311 - ) - self._mode = mode - self._message_queue = message_queue - self._peer: str = "" - self._version: str = version - self._is_authenticated: bool = False - self._is_streaming: bool = False - self._subscriptions: list[SubscriptionRequest] = [] - self._tasks: set[asyncio.Task[None]] = set() - - self._dbn_path = pathlib.Path(dbn_path) - self._user_api_keys = user_api_keys - - @property - def cram_challenge(self) -> str: - """ - Return the CRAM challenge string that will be used to authenticate - users. - - Returns - ------- - str - - """ - return self._cram_challenge - - @property - def dataset(self) -> Dataset | None: - """ - Return the session Dataset. If `None`, a dataset has not yet been - specified. This is done on authentication. - - Returns - ------- - Dataset | None - - """ - return self._dataset - - @property - def is_authenticated(self) -> bool: - """ - Return True if the user has been authenticated. - - Returns - ------- - bool - - """ - return self._is_authenticated - - @property - def is_streaming(self) -> bool: - """ - Return True if the streaming has begun. - - Returns - ------- - bool - - """ - return self._is_streaming - - @property - def dataset_path(self) -> pathlib.Path: - """ - The path to the DBN files for serving. - - Returns - ------- - Path - - """ - return self._dbn_path / (self._dataset or "") - - @property - def mode(self) -> MockLiveMode: - """ - Return the mock live server replay mode. - - Returns - ------- - MockLiveMode - - """ - return self._mode - - @property - def peer(self) -> str: - """ - Return the peer IP and port in the format {ip}:{port}. - - Returns - ------- - str - - """ - return self._peer - - @property - def user_api_keys(self) -> dict[str, str]: - """ - Return a dictionary of user api keys for testing. The keys to this - dictionary are the bucket_ids. The value should be a single user API - key. - - Returns - ------- - dict[str, str] - - """ - return self._user_api_keys - - @property - def session_id(self) -> str: - """ - A mock session_id for this protocol. - - Returns - ------- - str - - """ - return str(hash(self)) - - @property - def subscriptions(self) -> tuple[SubscriptionRequest, ...]: - """ - The received subscriptions. - - Returns - ------- - tuple[SubscriptionRequest, ...] - - """ - return tuple(self._subscriptions) - - @property - def version(self) -> str: - """ - Return the server version string. - - Returns - ------- - str - - """ - return self._version - - def connection_lost( - self, - exception: Exception, - ) -> None: - """ - Event handler when the connection is lost. - - Parameters - ---------- - exception : Exception - The exception that closed the connection. - - See Also - -------- - asyncio.BufferedProtocol - - """ - logger.info("%s disconnected", self._peer) - return super().connection_lost(exception) - - def connection_made( - self, - transport: asyncio.BaseTransport, - ) -> None: - """ - Event handler when the connection is made. - - Parameters - ---------- - transport : asyncio.BaseTransport - The transport for the new connection. - - See Also - -------- - asyncio.BufferedProtocol - - """ - if not isinstance(transport, asyncio.Transport): - raise RuntimeError(f"cannot write to {transport}") - - self.__transport = transport - self._buffer = bytearray(2**16) - self._data = BytesIO() - self._schemas: list[Schema] = [] - - peer_host, peer_port, *_ = transport.get_extra_info("peername") - self._peer = f"{peer_host}:{peer_port}" - logger.info("%s connected to %s", type(self).__name__, self._peer) - - # Print server version - greeting = Greeting(lsg_version=self.version) - logger.debug("sending greeting to %s", self._peer) - self.__transport.write(bytes(greeting)) - - # Print CRAM challenge - logger.debug("sending authentication challenge to %s", self._peer) - cram_challenge = ChallengeRequest(cram=self.cram_challenge) - self.__transport.write(bytes(cram_challenge)) - - def get_buffer(self, _: int) -> bytearray: - """ - Get the receive buffer. This protocol allocates the buffer at - initialization, because of this the size_hint is unused. - - Parameters - ---------- - size_hint : int - (unused) - - Returns - ------- - bytearray - - See Also - -------- - asyncio.BufferedProtocol - - """ - return self._buffer - - def buffer_updated(self, nbytes: int) -> None: - """ - Call when the buffer has data to read. - - Parameters - ---------- - nbytes : int - The number of bytes available for reading. - - See Also - -------- - asyncio.BufferedProtocol - - """ - logger.debug("%d bytes from %s", nbytes, self.peer) - - self._data.write(self._buffer[:nbytes]) - buffer_lines = (self._data.getvalue()).splitlines(keepends=True) - - if not buffer_lines[-1].endswith(b"\n"): - # Save this for the next call - self._data = BytesIO(buffer_lines.pop(-1)) - else: - self._data = BytesIO() - - for line in buffer_lines: - try: - message = parse_gateway_message(line.decode("utf-8")) - except ValueError as val_err: - logger.exception(val_err) - continue - else: - self._message_queue.put(message) - self.handle_client_message(message) - - def eof_received(self) -> bool: - """ - Call when the EOF has been received. - - See Also - -------- - asyncio.BufferedProtocol - - """ - logger.info("received eof from %s", self.peer) - return bool(super().eof_received()) - - @singledispatchmethod - def handle_client_message(self, message: GatewayControl) -> None: - raise TypeError(f"Unhandled client message {message}") - - @handle_client_message.register(AuthenticationRequest) - def _(self, message: AuthenticationRequest) -> None: - logger.info("received CRAM response: %s", message.auth) - if self.is_authenticated: - logger.error("authentication request sent when already authenticated") - self.__transport.write_eof() - return - if self.is_streaming: - logger.error("authentication request sent while streaming") - self.__transport.write_eof() - return - - _, bucket_id = message.auth.split("-") - - try: - # First, get the user's API key - user_api_key = self.user_api_keys.get(bucket_id) - if user_api_key is None: - raise KeyError("Could not resolve API key.") - - # Next, compute the expected response - expected_response = cram.get_challenge_response( - self.cram_challenge, - user_api_key, - ) - if message.auth != expected_response: - raise ValueError( - f"Expected `{expected_response}` but was `{message.auth}`", - ) - self._dataset = Dataset(message.dataset) - except (KeyError, ValueError) as exc: - logger.error( - "could not authenticate user", - exc_info=exc, - ) - auth_fail = AuthenticationResponse(success="0", error=str(exc)) - self.__transport.write(bytes(auth_fail)) - self.__transport.write_eof() - else: - # Establish a new user session - self._is_authenticated = True - auth_success = AuthenticationResponse( - success="1", - session_id=self.session_id, - ) - self.__transport.write(bytes(auth_success)) - - @handle_client_message.register(SubscriptionRequest) - def _(self, message: SubscriptionRequest) -> None: - logger.info("received subscription request: %s", str(message).strip()) - if not self.is_authenticated: - logger.error("subscription request sent while unauthenticated") - self.__transport.write_eof() - - self._subscriptions.append(message) - - if self.is_streaming: - self.create_server_task(message) - - @handle_client_message.register(SessionStart) - def _(self, message: SessionStart) -> None: - logger.info("received session start request: %s", str(message).strip()) - self._is_streaming = True - - for sub in self.subscriptions: - self.create_server_task(sub) - - def create_server_task(self, message: SubscriptionRequest) -> None: - if self.mode is MockLiveMode.REPLAY: - task = asyncio.create_task(self.replay_task(schema=Schema(message.schema))) - else: - task = asyncio.create_task(self.repeater_task(schema=Schema(message.schema))) - - self._tasks.add(task) - task.add_done_callback(self._tasks.remove) - task.add_done_callback(self.check_done) - - def check_done(self, _: Any) -> None: - if not self._tasks: - logger.info("streaming tasks completed") - self.__transport.write_eof() - - async def replay_task(self, schema: Schema) -> None: - for test_data_path in self.dataset_path.glob(f"*{schema}.dbn.zst"): - decompressor = zstandard.ZstdDecompressor().stream_reader( - test_data_path.read_bytes(), - ) - logger.info( - "streaming %s for %s schema", - test_data_path.name, - schema, - ) - self.__transport.write(decompressor.readall()) - - async def repeater_task(self, schema: Schema) -> None: - struct = SCHEMA_STRUCT_MAP[schema] - repeated = bytes(struct(*[0] * 12)) # for now we only support MBP_1 - - logger.info("repeating %d bytes for %s", len(repeated), schema) - while not self.__transport.is_closing(): - self.__transport.write(16 * repeated) - await asyncio.sleep(0) - - -class MockLiveServer: - """ - A mock of the Databento Live Subscription Gateway. This is used for unit - testing instead of connecting to the actual gateway. - - Attributes - ---------- - host : str - The host of the mock server. - port : int - The port of the mock server. - server : asyncio.base_events.Server - The mock server object. - mode : MockLiveMode - The mock server mode; defaults to "replay". - - Methods - ------- - create(host="localhost", port=0) - Factory method to create a new MockLiveServer instance. - This is the preferred way to create an instance of - this class. - - See Also - -------- - `asyncio.create_server` - - """ - - def __init__(self) -> None: - self._server: asyncio.base_events.Server - self._host: str - self._port: int - self._dbn_path: pathlib.Path - self._user_api_keys: dict[str, str] - self._message_queue: MessageQueue - self._thread: threading.Thread - self._mode: MockLiveMode - - @property - def host(self) -> str: - """ - Return the host of the mock server. - - Returns - ------- - str - - """ - return self._host - - @property - def mode(self) -> MockLiveMode: - """ - Return the mock live server mode. - - Returns - ------- - MockLiveMode - - """ - return self._mode - - @property - def port(self) -> int: - """ - Return the port of the mock server. - - Returns - ------- - int - - """ - return self._port - - @property - def server(self) -> asyncio.base_events.Server: - """ - Return the mock server object. - - Returns - ------- - asyncio.base_events.Server - - """ - return self._server - - @classmethod - def _protocol_factory( - cls, - user_api_keys: dict[str, str], - message_queue: MessageQueue, - version: str, - dbn_path: PathLike[str], - mode: MockLiveMode, - ) -> Callable[[], MockLiveServerProtocol]: - def factory() -> MockLiveServerProtocol: - return MockLiveServerProtocol( - version=version, - user_api_keys=user_api_keys, - message_queue=message_queue, - dbn_path=dbn_path, - mode=mode, - ) - - return factory - - @classmethod - async def create( - cls, - host: str = "localhost", - port: int = 0, - dbn_path: PathLike[str] = pathlib.Path.cwd(), - mode: MockLiveMode = MockLiveMode.REPLAY, - ) -> MockLiveServer: - """ - Create a mock server instance. This factory method is the preferred way - to create an instance of MockLiveServer. - - Parameters - ---------- - host : str - The hostname for the mock server. - Defaults to "localhost" - port : int - The port to bind for the mock server. - Defaults to 0 which will bind to an open port. - dbn_path : PathLike[str] (default: cwd) - A path to DBN files for streaming. - The files must contain the schema name and end with - `.dbn.zst`. - See `tests/data` for examples. - - Returns - ------- - MockLiveServer - - """ - logger.info( - "creating %s with host=%s port=%s dbn_path=%s mode=%s", - cls.__name__, - host, - port, - dbn_path, - mode, - ) - - user_api_keys: dict[str, str] = {} - message_queue: MessageQueue = queue.Queue() # type: ignore - - # We will add an API key from DATABENTO_API_KEY if it exists - env_key = os.environ.get("DATABENTO_API_KEY") - if env_key is not None: - bucket_id = env_key[-cram.BUCKET_ID_LENGTH :] - user_api_keys[bucket_id] = env_key - - loop = asyncio.get_event_loop() - server = await loop.create_server( - protocol_factory=cls._protocol_factory( - user_api_keys=user_api_keys, - message_queue=message_queue, - version=LIVE_SERVER_VERSION, - dbn_path=dbn_path, - mode=mode, - ), - host=host, - port=port, - start_serving=True, - ) - - mock_live_server = cls() - - # Initialize the MockLiveServer instance - mock_live_server._server = server - mock_live_server._host, mock_live_server._port, *_ = server.sockets[-1].getsockname() - mock_live_server._user_api_keys = user_api_keys - mock_live_server._message_queue = message_queue - - return mock_live_server - - def get_message(self, timeout: float | None) -> GatewayControl: - """ - Return the next gateway message received from the client. - - Parameters - ---------- - timeout : float, optional - Duration in seconds to wait before timing out. - - Returns - ------- - GatewayControl - - Raises - ------ - asyncio.TimeoutError - If the timeout duration is reached, if specified. - - """ - return self._message_queue.get(timeout=timeout) - - def get_message_of_type( - self, - message_type: type[G], - timeout: float, - ) -> G: - """ - Return the next gateway message that is an instance of message_type - received from the client. Messages that are removed from the queue - until a match is found or the timeout expires, if specified. - - Parameters - ---------- - message_type : type[GatewayControl] - The type of GatewayControl message to wait for. - timeout : float, optional - Duration in seconds to wait before timing out. - - Returns - ------- - GatewayControl - - Raises - ------ - futures.TimeoutError - If the timeout duration is reached, if specified. - - """ - start_time = time.perf_counter() - end_time = time.perf_counter() + timeout - while start_time < end_time: - remaining_time = abs(end_time - time.perf_counter()) - try: - message = self._message_queue.get(timeout=remaining_time) - except queue.Empty: - break - - if isinstance(message, message_type): - return message - - raise futures.TimeoutError - - async def stop(self) -> None: - """ - Stop the mock server. - """ - logger.info( - "stopping %s on %s:%s", - self.__class__.__name__, - self.host, - self.port, - ) - - self.server.close() - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument( - "host", - help="the hostname to bind; defaults to `localhost`", - nargs="?", - default="localhost", - ) - parser.add_argument( - "port", - help="the port to bind; defaults to an open port", - nargs="?", - default=0, - ) - parser.add_argument( - "-d", - "--dbn-path", - metavar="DBN", - action="store", - help="path to a directory containing DBN files to stream", - ) - parser.add_argument( - "-m", - "--mode", - metavar="mode", - default="replay", - choices=(x.value for x in MockLiveMode), - action="store", - help="the mock server live mode", - ) - - params = parser.parse_args(sys.argv[1:]) - - # Setup console logging handler - logger.setLevel(logging.INFO) - handler = logging.StreamHandler() - handler.setLevel(logging.INFO) - formatter = logging.Formatter("%(asctime)s %(levelname)s %(message)s") - handler.setFormatter(formatter) - logger.addHandler(handler) - - # Start MockLiveServer - loop = asyncio.get_event_loop() - mock_live_server = loop.run_until_complete( - MockLiveServer.create( - host=params.host, - port=params.port, - dbn_path=pathlib.Path(params.dbn_path), - mode=MockLiveMode(params.mode), - ), - ) - - # Serve Forever - try: - loop.run_forever() - except (KeyboardInterrupt, SystemExit) as exit_exc: - logger.fatal("Terminating on %s", type(exit_exc).__name__) - finally: - exit(0) diff --git a/tests/mockliveserver/__init__.py b/tests/mockliveserver/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/mockliveserver/__main__.py b/tests/mockliveserver/__main__.py new file mode 100644 index 0000000..0f03b16 --- /dev/null +++ b/tests/mockliveserver/__main__.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +import argparse +import asyncio +import logging +import os +import sys +from collections import defaultdict +from socket import AF_INET + +from databento.common.publishers import Dataset +from databento_dbn import Schema + +from tests.mockliveserver.controller import CommandProtocol +from tests.mockliveserver.source import ReplayProtocol + +from .server import MockLiveServerProtocol +from .server import SessionMode + + +logger = logging.getLogger(__name__) + + +parser = argparse.ArgumentParser() +parser.add_argument( + "host", + help="the hostname to bind; defaults to `localhost`", + nargs="?", + default="localhost", +) +parser.add_argument( + "-p", + "--port", + help="the port to bind; defaults to an open port", + type=int, + default=0, +) +parser.add_argument( + "-e", + "--echo", + help="a file to write echos of gateway control messages to", + default=os.devnull, +) +parser.add_argument( + "-v", + "--verbose", + action="store_true", + help="enabled debug logging", +) + + +async def main() -> None: + params = parser.parse_args(sys.argv[1:]) + + # Setup console logging handler + log_level = logging.DEBUG if params.verbose else logging.INFO + logging.basicConfig( + level=log_level, + format="%(asctime)s %(levelname)s %(message)s", + stream=sys.stderr, + ) + + logger.info("mockliveserver starting") + loop = asyncio.get_running_loop() + + api_key_table: dict[str, set[str]] = defaultdict(set) + file_replay_table: dict[tuple[Dataset, Schema], ReplayProtocol] = {} + echo_stream = open(params.echo, "wb", buffering=0) + + # Create server for incoming connections + server = await loop.create_server( + protocol_factory=lambda: MockLiveServerProtocol( + loop=loop, + mode=SessionMode.FILE_REPLAY, + api_key_table=api_key_table, + file_replay_table=file_replay_table, + echo_stream=echo_stream, + ), + family=AF_INET, # force ipv4 + host=params.host, + port=params.port, + start_serving=True, + ) + ip, port, *_ = server._sockets[-1].getsockname() # type: ignore [attr-defined] + + # Create command interface for stdin + await loop.connect_read_pipe( + protocol_factory=lambda: CommandProtocol( + api_key_table=api_key_table, + file_replay_table=file_replay_table, + server=server, + ), + pipe=sys.stdin, + ) + + # Log Arguments + logger.info("host: %s (%s)", params.host, ip) + logger.info("port: %d", port) + logger.info("echo: %s", params.echo) + logger.info("verbose: %s", params.verbose) + logger.info("mockliveserver now serving") + + try: + await server.serve_forever() + except asyncio.CancelledError: + logger.info("terminating mock live server") + + echo_stream.close() + + +asyncio.run(main()) diff --git a/tests/mockliveserver/controller.py b/tests/mockliveserver/controller.py new file mode 100644 index 0000000..e387b6c --- /dev/null +++ b/tests/mockliveserver/controller.py @@ -0,0 +1,137 @@ +from __future__ import annotations + +import argparse +import asyncio +import logging +from collections.abc import Mapping +from collections.abc import MutableMapping +from pathlib import Path + +from databento.common.cram import BUCKET_ID_LENGTH +from databento.common.publishers import Dataset +from databento_dbn import Schema + +from tests.mockliveserver.source import FileReplay +from tests.mockliveserver.source import ReplayProtocol + + +logger = logging.getLogger(__name__) + + +class CommandProtocol(asyncio.Protocol): + command_parser = argparse.ArgumentParser(prog="mockliveserver") + subparsers = command_parser.add_subparsers(dest="command") + + close_command = subparsers.add_parser("close", help="close the mock live server") + + active_count = subparsers.add_parser( + "active_count", + help="log the number of active connections", + ) + + add_key = subparsers.add_parser("add_key", help="add an API key to the mock live server") + add_key.add_argument("key", type=str) + + del_key = subparsers.add_parser("del_key", help="delete an API key from the mock live server") + del_key.add_argument("key", type=str) + + add_dbn = subparsers.add_parser("add_dbn", help="add a dbn file for replay") + add_dbn.add_argument("dataset", type=str) + add_dbn.add_argument("schema", type=str) + add_dbn.add_argument("dbn_file", type=str) + + def __init__( + self, + server: asyncio.base_events.Server, + api_key_table: Mapping[str, set[str]], + file_replay_table: MutableMapping[tuple[Dataset, Schema], ReplayProtocol], + ) -> None: + self._server = server + self._api_key_table = api_key_table + self._file_replay_table = file_replay_table + + def eof_received(self) -> bool | None: + self._server.close() + return super().eof_received() + + def data_received(self, data: bytes) -> None: + logger.debug("%d bytes from stdin", len(data)) + try: + command_str = data.decode("utf-8") + except Exception: + logger.error("error parsing command") + raise + + for command in command_str.splitlines(): + params = self.command_parser.parse_args(command.split()) + command_func = getattr(self, f"_command_{params.command}", None) + if command_func is None: + raise ValueError(f"{params.command} does not have a command handler") + else: + logger.info("received command: %s", command) + command_params = dict(params._get_kwargs()) + command_params.pop("command") + try: + command_func(**command_params) + except Exception: + logger.exception("error processing command: %s", params.command) + print(f"nack: {command}", flush=True) + else: + print(f"ack: {command}", flush=True) + + return super().data_received(data) + + def _command_close(self, *_: str) -> None: + """ + Close the server. + """ + self._server.close() + + def _command_active_count(self, *_: str) -> None: + """ + Log the number of active connections. + """ + count = self._server._active_count # type: ignore [attr-defined] + logger.info("active connections: %d", count) + + def _command_add_key(self, key: str) -> None: + """ + Add an API key to the server. + """ + if len(key) < BUCKET_ID_LENGTH: + logger.error("api key must be at least %d characters long", BUCKET_ID_LENGTH) + return + + bucket_id = key[-BUCKET_ID_LENGTH:] + self._api_key_table[bucket_id].add(key) + logger.info("added api key '%s'", key) + + def _command_del_key(self, key: str) -> None: + """ + Remove API key from the server. + """ + if len(key) < BUCKET_ID_LENGTH: + logger.error("api key must be at least %d characters long", BUCKET_ID_LENGTH) + return + + bucket_id = key[-BUCKET_ID_LENGTH:] + self._api_key_table[bucket_id].remove(key) + logger.info("deleted api key '%s'", key) + + def _command_add_dbn(self, dataset: str, schema: str, dbn_file: str) -> None: + """ + Add a DBN file for streaming. + """ + try: + dataset_valid = Dataset(dataset) + schema_valid = Schema(schema) + except ValueError as exc: + logger.error("invalid parameter value: %s", exc) + return + + dbn_path = Path(dbn_file) + if not dbn_path.exists() or not dbn_path.is_file(): + logger.error("invalid file path: %s", dbn_path) + return + + self._file_replay_table[(dataset_valid, schema_valid)] = FileReplay(dbn_path) diff --git a/tests/mockliveserver/fixture.py b/tests/mockliveserver/fixture.py new file mode 100644 index 0000000..c31e75a --- /dev/null +++ b/tests/mockliveserver/fixture.py @@ -0,0 +1,240 @@ +from __future__ import annotations + +import asyncio +import contextlib +import os +import pathlib +import signal +import sys +from asyncio.subprocess import Process +from collections.abc import AsyncGenerator +from collections.abc import Generator +from typing import Callable +from typing import TypeVar + +import pytest +import pytest_asyncio +from databento.common.publishers import Dataset +from databento.live.gateway import GatewayControl +from databento_dbn import Schema + +from tests import TESTS_ROOT + + +class MockLiveServerInterface: + """ + Process wrapper for communicating with the mock live server. + """ + + _GC = TypeVar("_GC", bound=GatewayControl) + + def __init__( + self, + process: Process, + host: str, + port: int, + echo_file: pathlib.Path, + ): + self._process = process + self._host = host + self._port = port + self._echo_fd = open(echo_file) + + @property + def host(self) -> str: + """ + The mock live server host. + + Returns + ------- + str + + """ + return self._host + + @property + def port(self) -> int: + """ + The mock live server port. + + Returns + ------- + int + + """ + return self._port + + @property + def stdout(self) -> asyncio.StreamReader: + if self._process.stdout is not None: + return self._process.stdout + raise RuntimeError("no stream reader for stdout") + + async def _send_command(self, command: str) -> None: + if self._process.stdin is None: + raise RuntimeError("cannot write command to mock live server") + self._process.stdin.write( + f"{command.strip()}\n".encode(), + ) + line = await self.stdout.readline() + line_str = line.decode("utf-8") + + if line_str.startswith(f"ack: {command}"): + return + elif line_str.startswith(f"nack: {command}"): + raise RuntimeError(f"received nack for command: {command}") + + raise RuntimeError(f"invalid response from server: {line_str!r}") + + async def active_count(self) -> None: + """ + Send the "active_count" command. + """ + await self._send_command("active_count") + + async def add_key(self, api_key: str) -> None: + """ + Send the "add_key" command. + + Parameters + ---------- + api_key : str + The API key to add. + + """ + await self._send_command(f"add_key {api_key}") + + async def add_dbn(self, dataset: Dataset, schema: Schema, path: pathlib.Path) -> None: + """ + Send the "add_dbn" command. + + Parameters + ---------- + dataset : Dataset + The DBN dataset. + schema : Schema + The DBN schema. + path : pathlib.Path + The path to the DBN file. + + """ + await self._send_command(f"add_dbn {dataset} {schema} {path.resolve()}") + + async def del_key(self, api_key: str) -> None: + """ + Send the "del_key" command. + + Parameters + ---------- + api_key : str + The API key to delete. + + """ + await self._send_command(f"del_key {api_key}") + + async def close(self) -> None: + """ + Send the "close" command. + """ + await self._send_command("close") + + def kill(self) -> None: + """ + Kill the mock live server by sending SIGKILL. + """ + self._process.send_signal(signal.SIGKILL) + + @contextlib.contextmanager + def test_context(self) -> Generator[None, None, None]: + self._echo_fd.seek(0, os.SEEK_END) + yield + + @contextlib.asynccontextmanager + async def api_key_context(self, api_key: str) -> AsyncGenerator[str, None]: + await self.add_key(api_key) + yield api_key + await self.del_key(api_key) + + async def wait_for_start(self) -> None: + await self.active_count() + + async def wait_for_message_of_type( + self, + message_type: type[_GC], + timeout: float = 1.0, + ) -> _GC: + """ + Wait for a message of a given type. + + Parameters + ---------- + message_type : type[_GC] + The type of GatewayControl message to wait for. + timeout: float, default 1.0 + The maximum number of seconds to wait. + + Returns + ------- + _GC + + """ + loop = asyncio.get_running_loop() + deadline = loop.time() + timeout + while self._process.returncode is None: + line = await asyncio.wait_for( + loop.run_in_executor(None, self._echo_fd.readline), + timeout=max( + 0, + deadline - loop.time(), + ), + ) + try: + return message_type.parse(line) + except ValueError: + continue + raise RuntimeError("Mock server is closed.") + + +@pytest_asyncio.fixture(name="mock_live_server", scope="module") +async def fixture_mock_live_server( + unused_tcp_port_factory: Callable[[], int], + tmp_path_factory: pytest.TempPathFactory, +) -> AsyncGenerator[MockLiveServerInterface, None]: + port = unused_tcp_port_factory() + echo_file = tmp_path_factory.mktemp("mockliveserver") / "echo.txt" + echo_file.touch() + + process = await asyncio.subprocess.create_subprocess_exec( + "python3", + "-m", + "tests.mockliveserver", + "127.0.0.1", + "--port", + str(port), + "--echo", + echo_file.resolve(), + "--verbose", + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=sys.stderr, + ) + + interface = MockLiveServerInterface( + process=process, + host="127.0.0.1", + port=port, + echo_file=echo_file, + ) + + await interface.wait_for_start() + + for dataset in Dataset: + for schema in Schema.variants(): + path = TESTS_ROOT / "data" / dataset / f"test_data.{schema}.dbn.zst" + if path.exists(): + await interface.add_dbn(dataset, schema, path) + + yield interface + + interface.kill() + await asyncio.wait_for(process.wait(), timeout=1) diff --git a/tests/mockliveserver/server.py b/tests/mockliveserver/server.py new file mode 100644 index 0000000..868a019 --- /dev/null +++ b/tests/mockliveserver/server.py @@ -0,0 +1,306 @@ +from __future__ import annotations + +import asyncio +import enum +import logging +import random +import string +from collections.abc import Mapping +from functools import singledispatchmethod +from io import BytesIO +from io import FileIO +from typing import Any +from typing import Final +from typing import Generator + +from databento.common import cram +from databento.common.publishers import Dataset +from databento.live.gateway import AuthenticationRequest +from databento.live.gateway import AuthenticationResponse +from databento.live.gateway import ChallengeRequest +from databento.live.gateway import GatewayControl +from databento.live.gateway import Greeting +from databento.live.gateway import SessionStart +from databento.live.gateway import SubscriptionRequest +from databento.live.gateway import parse_gateway_message +from databento_dbn import Schema + +from .source import ReplayProtocol + + +SERVER_VERSION: Final = "0.4.2" +READ_BUFFER_SIZE: Final = 2**10 + +logger = logging.getLogger(__name__) + + +class SessionState(enum.Enum): + NEW = enum.auto() + NOT_AUTHENTICATED = enum.auto() + AUTHENTICATED = enum.auto() + STREAMING = enum.auto() + CLOSED = enum.auto() + + +class SessionMode(enum.Enum): + FILE_REPLAY = enum.auto() + + +def session_id_generator(start: int = 0) -> Generator[int, None, None]: + while True: + yield start + start += 1 + + +class MockLiveServerProtocol(asyncio.BufferedProtocol): + session_id_generator = session_id_generator(0) + + def __init__( + self, + loop: asyncio.AbstractEventLoop, + mode: SessionMode, + api_key_table: Mapping[str, set[str]], + file_replay_table: Mapping[tuple[Dataset, Schema], ReplayProtocol], + echo_stream: FileIO, + ) -> None: + self._loop = loop + self._mode = mode + self._api_key_table = api_key_table + self._file_replay_table = file_replay_table + self._echo_stream = echo_stream + + self._transport: asyncio.Transport | None = None + self._buffer = bytearray(READ_BUFFER_SIZE) + self._cram = "".join(random.choice(string.ascii_letters) for _ in range(32)) # noqa: S311 + self._peer: str | None = None + self._session_id: str | None = None + + self._data = BytesIO() + self._state = SessionState.NEW + self._dataset: Dataset | None = None + self._subscriptions: list[SubscriptionRequest] = [] + self._replay_tasks: set[asyncio.Task[None]] = set() + + @property + def mode(self) -> SessionMode: + return self._mode + + @property + def dataset(self) -> Dataset: + if self._dataset is None: + raise RuntimeError("No dataset set") + return self._dataset + + @property + def state(self) -> SessionState: + return self._state + + @state.setter + def state(self, value: SessionState) -> None: + logger.debug( + "session state changed from %s to %s for %s", + self._state.name, + value.name, + self.peer, + ) + self._state = value + + @property + def session_id(self) -> str: + if self._session_id is None: + self._session_id = f"mock-{next(self.session_id_generator)}" + logger.info("assigned session id %s for %s", self._session_id, self.peer) + return self._session_id + + @property + def transport(self) -> asyncio.Transport: + if self._transport is None: + raise RuntimeError("No transport set") + return self._transport + + @property + def buffer(self) -> bytearray: + return self._buffer + + @property + def peer(self) -> str | None: + return self._peer + + def get_authentication_response( + self, + success: bool, + session_id: str = "0", + ) -> AuthenticationResponse: + return AuthenticationResponse( + success="0" if not success else "1", + error="Authentication failed." if not success else None, + session_id=None if not success else str(session_id), + ) + + def get_challenge(self) -> ChallengeRequest: + return ChallengeRequest( + cram=self._cram, + ) + + def get_greeting(self) -> Greeting: + return Greeting(lsg_version=SERVER_VERSION) + + def send_gateway_message(self, message: GatewayControl) -> None: + logger.info( + "sending %s message to %s", + message.__class__.__name__, + self.peer, + ) + self.transport.write(bytes(message)) + + def hangup(self, reason: str | None = None, is_error: bool = False) -> None: + if reason is not None: + if is_error: + logger.error(reason) + else: + logger.info(reason) + logger.info("sending eof to %s", self.peer) + self.transport.write_eof() + + @singledispatchmethod + def handle_client_message(self, message: GatewayControl) -> None: + logger.error("unhandled client message %s", message.__class__.__name__) + + @handle_client_message.register(AuthenticationRequest) + def _(self, message: AuthenticationRequest) -> None: + logger.debug("received challenge response %s from %s", message.auth, self.peer) + if self.state != SessionState.NOT_AUTHENTICATED: + self.hangup( + reason="authentication request sent when already authenticated", + is_error=True, + ) + return + + _, bucket_id = message.auth.split("-") + + for api_key in self._api_key_table.get(bucket_id, []): + logger.debug("checking key %s", api_key) + expected_response = cram.get_challenge_response( + self._cram, + api_key, + ) + if message.auth == expected_response: + break + else: + logger.error("failed authentication for %s", self.peer) + self.send_gateway_message(self.get_authentication_response(success=False)) + return + + self.state = SessionState.AUTHENTICATED + self._dataset = Dataset(message.dataset) + self.send_gateway_message( + self.get_authentication_response( + success=True, + session_id=self.session_id, + ), + ) + + @handle_client_message.register(SubscriptionRequest) + def _(self, message: SubscriptionRequest) -> None: + logger.info("received subscription request %s from %s", str(message).strip(), self.peer) + if self.state == SessionState.NOT_AUTHENTICATED: + self.hangup( + reason="subscription received while unauthenticated", + is_error=True, + ) + + self._subscriptions.append(message) + + @handle_client_message.register(SessionStart) + def _(self, message: SessionStart) -> None: + logger.info("received session start request %s from %s", str(message).strip(), self.peer) + if self.state == SessionState.NOT_AUTHENTICATED: + self.hangup( + reason="session start received while unauthenticated", + is_error=True, + ) + + if self.mode == SessionMode.FILE_REPLAY: + task = self._loop.create_task(self._file_replay_task()) + self._replay_tasks.add(task) + task.add_done_callback(self._replay_done_callback) + else: + logger.error("unsupported session mode %s", self.mode) + + def buffer_updated(self, nbytes: int) -> None: + logger.debug("%d bytes from %s", nbytes, self.peer) + + self._data.write(self._buffer[:nbytes]) + buffer_lines = (self._data.getvalue()).splitlines(keepends=True) + + if not buffer_lines[-1].endswith(b"\n"): + self._data = BytesIO(buffer_lines.pop(-1)) + else: + self._data = BytesIO() + + for line in buffer_lines: + try: + message = parse_gateway_message(line.decode("utf-8")) + except ValueError as val_err: + self.hangup( + reason=str(val_err), + is_error=True, + ) + else: + self._echo_stream.write(bytes(message)) + self.handle_client_message(message) + + return super().buffer_updated(nbytes) + + def connection_made(self, transport: asyncio.transports.BaseTransport) -> None: + if not isinstance(transport, asyncio.Transport): + raise RuntimeError(f"cannot write to {transport}") + + self._transport = transport + + peer_host, peer_port, *_ = transport.get_extra_info("peername") + self._peer = f"{peer_host}:{peer_port}" + + logger.info("incoming connection from %s", self.peer) + self.send_gateway_message(self.get_greeting()) + self.send_gateway_message(self.get_challenge()) + + self.state = SessionState.NOT_AUTHENTICATED + + return super().connection_made(transport) + + def connection_lost(self, exc: Exception | None) -> None: + logger.info("disconnected %s", self.peer) + self.state = SessionState.CLOSED + return super().connection_lost(exc) + + def eof_received(self) -> bool | None: + logger.info("eof received from %s", self.peer) + return super().eof_received() + + def get_buffer(self, sizehint: int) -> bytearray: + if sizehint > len(self.buffer): + logger.warning("requested buffer size %d is larger than current size", sizehint) + return self.buffer + + def _replay_done_callback(self, task: asyncio.Task[Any]) -> None: + self._replay_tasks.remove(task) + + replay_exception = task.exception() + if replay_exception is not None: + logger.error("exception while replaying DBN files: %s", replay_exception) + + if self._replay_tasks: + logger.debug("%d replay tasks remain", len(self._replay_tasks)) + else: + self.hangup(reason="all replay tasks completed") + + async def _file_replay_task(self) -> None: + for subscription in self._subscriptions: + schema = Schema(subscription.schema) + replay = self._file_replay_table[(self.dataset, schema)] + logger.info("starting replay %s for %s", replay.name, self.peer) + for chunk in replay: + self.transport.write(chunk) + await asyncio.sleep(0) + logger.info("replay of %s completed for %s", replay.name, self.peer) diff --git a/tests/mockliveserver/source.py b/tests/mockliveserver/source.py new file mode 100644 index 0000000..7a0948b --- /dev/null +++ b/tests/mockliveserver/source.py @@ -0,0 +1,43 @@ +from pathlib import Path +from typing import IO +from typing import Final +from typing import Generator +from typing import Iterator +from typing import Protocol + +import zstandard +from databento.common.dbnstore import is_zstandard +from databento_dbn import Compression + + +FILE_READ_SIZE: Final = 2**10 + + +class ReplayProtocol(Protocol): + @property + def name(self) -> str: ... + + def __iter__(self) -> Iterator[bytes]: ... + + +class FileReplay(ReplayProtocol): + def __init__(self, dbn_file: Path): + self._dbn_file = dbn_file + self._compression = Compression.NONE + + with self._dbn_file.open("rb") as dbn: + if is_zstandard(dbn): + self._compression = Compression.ZSTD + + @property + def name(self) -> str: + return self._dbn_file.name + + def __iter__(self) -> Generator[bytes, None, None]: + with self._dbn_file.open("rb") as dbn: + if self._compression == Compression.ZSTD: + reader: IO[bytes] = zstandard.ZstdDecompressor().stream_reader(dbn) + else: + reader = dbn + while next_bytes := reader.read(FILE_READ_SIZE): + yield next_bytes diff --git a/tests/test_live_client.py b/tests/test_live_client.py index 03df56e..7b9ca1b 100644 --- a/tests/test_live_client.py +++ b/tests/test_live_client.py @@ -30,7 +30,7 @@ from databento_dbn import Schema from databento_dbn import SType -from tests.mock_live_server import MockLiveServer +from tests.mockliveserver.fixture import MockLiveServerInterface def test_live_connection_refused( @@ -59,8 +59,7 @@ def test_live_connection_refused( def test_live_connection_timeout( monkeypatch: pytest.MonkeyPatch, - mock_live_server: MockLiveServer, - test_api_key: str, + live_client: client.Live, ) -> None: """ Test that a timeout raises a BentoError. @@ -76,12 +75,6 @@ def test_live_connection_timeout( 0, ) - live_client = client.Live( - key=test_api_key, - gateway=mock_live_server.host, - port=mock_live_server.port, - ) - # Act, Assert with pytest.raises(BentoError) as exc: live_client.subscribe( @@ -101,7 +94,7 @@ def test_live_connection_timeout( ], ) def test_live_invalid_gateway( - mock_live_server: MockLiveServer, + mock_live_server: MockLiveServerInterface, test_api_key: str, gateway: str, ) -> None: @@ -125,7 +118,7 @@ def test_live_invalid_gateway( ], ) def test_live_invalid_port( - mock_live_server: MockLiveServer, + mock_live_server: MockLiveServerInterface, test_api_key: str, port: object, ) -> None: @@ -142,22 +135,16 @@ def test_live_invalid_port( def test_live_connection_cram_failure( - mock_live_server: MockLiveServer, - monkeypatch: pytest.MonkeyPatch, - test_api_key: str, + mock_live_server: MockLiveServerInterface, ) -> None: """ Test that a failed auth message due to an incorrect CRAM raises a BentoError. """ - # Arrange - # Dork up the API key in the mock client to fail CRAM - bucket_id = test_api_key[-BUCKET_ID_LENGTH:] invalid_key = "db-invalidkey00000000000000FFFFF" - monkeypatch.setitem(mock_live_server._user_api_keys, bucket_id, invalid_key) live_client = client.Live( - key=test_api_key, + key=invalid_key, gateway=mock_live_server.host, port=mock_live_server.port, ) @@ -182,7 +169,7 @@ def test_live_connection_cram_failure( ], ) def test_live_subscription_with_snapshot_failed( - mock_live_server: MockLiveServer, + mock_live_server: MockLiveServerInterface, test_api_key: str, start: str | int, ) -> None: @@ -215,22 +202,16 @@ def test_live_subscription_with_snapshot_failed( [pytest.param(dataset, id=str(dataset)) for dataset in Dataset], ) def test_live_creation( + mock_live_server: MockLiveServerInterface, + live_client: client.Live, test_api_key: str, - mock_live_server: MockLiveServer, dataset: Dataset, ) -> None: """ - Test the live constructor and successful connection to the MockLiveServer. + Test the live constructor and successful connection to the + MockLiveServerInterface. """ - # Arrange - live_client = client.Live( - key=test_api_key, - gateway=mock_live_server.host, - port=mock_live_server.port, - ) - - # Act - # Subscribe to connect + # Arrange, Act live_client.subscribe( dataset=dataset, schema=Schema.MBO, @@ -245,8 +226,8 @@ def test_live_creation( assert live_client._map_symbol in live_client._user_callbacks -def test_live_connect_auth( - mock_live_server: MockLiveServer, +async def test_live_connect_auth( + mock_live_server: MockLiveServerInterface, live_client: client.Live, ) -> None: """ @@ -260,9 +241,8 @@ def test_live_connect_auth( ) # Act - message = mock_live_server.get_message_of_type( - gateway.AuthenticationRequest, - timeout=1, + message = await mock_live_server.wait_for_message_of_type( + message_type=gateway.AuthenticationRequest, ) # Assert @@ -271,9 +251,9 @@ def test_live_connect_auth( assert message.encoding == Encoding.DBN -def test_live_connect_auth_two_clients( - mock_live_server: MockLiveServer, - test_api_key: str, +async def test_live_connect_auth_two_clients( + mock_live_server: MockLiveServerInterface, + test_live_api_key: str, ) -> None: """ Test the live sent a correct AuthenticationRequest message after connecting @@ -281,13 +261,13 @@ def test_live_connect_auth_two_clients( """ # Arrange first = client.Live( - key=test_api_key, + key=test_live_api_key, gateway=mock_live_server.host, port=mock_live_server.port, ) second = client.Live( - key=test_api_key, + key=test_live_api_key, gateway=mock_live_server.host, port=mock_live_server.port, ) @@ -298,9 +278,8 @@ def test_live_connect_auth_two_clients( schema=Schema.MBO, ) - first_auth = mock_live_server.get_message_of_type( - gateway.AuthenticationRequest, - timeout=1, + first_auth = await mock_live_server.wait_for_message_of_type( + message_type=gateway.AuthenticationRequest, ) # Assert @@ -313,9 +292,8 @@ def test_live_connect_auth_two_clients( schema=Schema.MBO, ) - second_auth = mock_live_server.get_message_of_type( - gateway.AuthenticationRequest, - timeout=1, + second_auth = await mock_live_server.wait_for_message_of_type( + message_type=gateway.AuthenticationRequest, ) assert second_auth.auth.endswith(second.key[-BUCKET_ID_LENGTH:]) @@ -323,9 +301,9 @@ def test_live_connect_auth_two_clients( assert second_auth.encoding == Encoding.DBN -def test_live_start( +async def test_live_start( live_client: client.Live, - mock_live_server: MockLiveServer, + mock_live_server: MockLiveServerInterface, ) -> None: """ Test the live sends a SesssionStart message upon calling start(). @@ -341,11 +319,8 @@ def test_live_start( # Act live_client.start() - live_client.block_for_close() - - message = mock_live_server.get_message_of_type( - gateway.SessionStart, - timeout=1, + message = await mock_live_server.wait_for_message_of_type( + message_type=gateway.SessionStart, ) # Assert @@ -448,9 +423,9 @@ def test_live_async_iteration_after_start( "1680736543000000000", ], ) -def test_live_subscribe( +async def test_live_subscribe( live_client: client.Live, - mock_live_server: MockLiveServer, + mock_live_server: MockLiveServerInterface, schema: Schema, stype_in: SType, symbols: str, @@ -458,7 +433,7 @@ def test_live_subscribe( ) -> None: """ Test various combination of subscription messages are serialized and - correctly deserialized by the MockLiveServer. + correctly deserialized by the MockLiveServerInterface. """ # Arrange live_client.subscribe( @@ -470,9 +445,8 @@ def test_live_subscribe( ) # Act - message = mock_live_server.get_message_of_type( - gateway.SubscriptionRequest, - timeout=1, + message = await mock_live_server.wait_for_message_of_type( + message_type=gateway.SubscriptionRequest, ) if symbols is None: @@ -493,9 +467,9 @@ def test_live_subscribe( True, ], ) -def test_live_subscribe_snapshot( +async def test_live_subscribe_snapshot( live_client: client.Live, - mock_live_server: MockLiveServer, + mock_live_server: MockLiveServerInterface, snapshot: bool, ) -> None: """ @@ -518,7 +492,7 @@ def test_live_subscribe_snapshot( ) # Act - message = mock_live_server.get_message_of_type( + message = await mock_live_server.wait_for_message_of_type( gateway.SubscriptionRequest, timeout=1, ) @@ -556,7 +530,7 @@ async def test_live_subscribe_session_id( async def test_live_subscribe_large_symbol_list( live_client: client.Live, - mock_live_server: MockLiveServer, + mock_live_server: MockLiveServerInterface, ) -> None: """ Test that sending a subscription with a large symbol list breaks that list @@ -577,11 +551,10 @@ async def test_live_subscribe_large_symbol_list( reconstructed: list[str] = [] for _ in range(8): - message = mock_live_server.get_message_of_type( - gateway.SubscriptionRequest, - timeout=1, - ).symbols.split(",") - reconstructed.extend(message) + message = await mock_live_server.wait_for_message_of_type( + message_type=gateway.SubscriptionRequest, + ) + reconstructed.extend(message.symbols.split(",")) # Assert assert reconstructed == large_symbol_list @@ -589,7 +562,7 @@ async def test_live_subscribe_large_symbol_list( async def test_live_subscribe_from_callback( live_client: client.Live, - mock_live_server: MockLiveServer, + mock_live_server: MockLiveServerInterface, ) -> None: """ Test that `Live.subscribe` can be called from a callback. @@ -613,18 +586,16 @@ def cb_sub(_: DBNRecord) -> None: live_client.add_callback(cb_sub) # Act - first_sub = mock_live_server.get_message_of_type( - gateway.SubscriptionRequest, - timeout=1, + first_sub = await mock_live_server.wait_for_message_of_type( + message_type=gateway.SubscriptionRequest, ) live_client.start() await live_client.wait_for_close() - second_sub = mock_live_server.get_message_of_type( - gateway.SubscriptionRequest, - timeout=1, + second_sub = await mock_live_server.wait_for_message_of_type( + message_type=gateway.SubscriptionRequest, ) # Assert @@ -954,8 +925,7 @@ async def test_live_async_iteration( async def test_live_async_iteration_backpressure( monkeypatch: pytest.MonkeyPatch, - mock_live_server: MockLiveServer, - test_api_key: str, + live_client: client.Live, ) -> None: """ Test that a full queue disables reading on the transport but will resume it @@ -964,12 +934,6 @@ async def test_live_async_iteration_backpressure( # Arrange monkeypatch.setattr(session, "DBN_QUEUE_CAPACITY", 2) - live_client = client.Live( - key=test_api_key, - gateway=mock_live_server.host, - port=mock_live_server.port, - ) - live_client.subscribe( dataset=Dataset.GLBX_MDP3, schema=Schema.MBO, @@ -998,7 +962,7 @@ async def test_live_async_iteration_backpressure( async def test_live_async_iteration_dropped( monkeypatch: pytest.MonkeyPatch, - mock_live_server: MockLiveServer, + live_client: client.Live, test_api_key: str, ) -> None: """ @@ -1008,12 +972,6 @@ async def test_live_async_iteration_dropped( # Arrange monkeypatch.setattr(session, "DBN_QUEUE_CAPACITY", 1) - live_client = client.Live( - key=test_api_key, - gateway=mock_live_server.host, - port=mock_live_server.port, - ) - live_client.subscribe( dataset=Dataset.GLBX_MDP3, schema=Schema.MBO, @@ -1152,7 +1110,7 @@ async def test_live_stream_to_dbn( schema: Schema, ) -> None: """ - Test that DBN data streamed by the MockLiveServer is properly re- + Test that DBN data streamed by the MockLiveServerInterface is properly re- constructed client side. """ # Arrange @@ -1205,7 +1163,7 @@ async def test_live_stream_to_dbn_from_path( schema: Schema, ) -> None: """ - Test that DBN data streamed by the MockLiveServer is properly re- + Test that DBN data streamed by the MockLiveServerInterface is properly re- constructed client side when specifying a file as a path. """ # Arrange @@ -1256,7 +1214,7 @@ async def test_live_stream_to_dbn_with_tiny_buffer( buffer_size: int, ) -> None: """ - Test that DBN data streamed by the MockLiveServer is properly re- + Test that DBN data streamed by the MockLiveServerInterface is properly re- constructed client side when using the small values for RECV_BUFFER_SIZE. """ # Arrange @@ -1521,10 +1479,10 @@ async def test_live_stream_with_reconnect( pytest.skip("no stub data for tcbbo schema") output = tmp_path / "output.dbn" - live_client.add_stream(output.open("wb", buffering=0)) + live_client.add_stream(output.open("wb")) # Act - for _ in range(5): + for _ in range(3): live_client.subscribe( dataset=Dataset.GLBX_MDP3, schema=schema, @@ -1548,20 +1506,14 @@ async def test_live_stream_with_reconnect( assert isinstance(record, SCHEMA_STRUCT_MAP[schema]) -def test_live_connection_reconnect_cram_failure( - mock_live_server: MockLiveServer, - monkeypatch: pytest.MonkeyPatch, +async def test_live_connection_reconnect_cram_failure( + mock_live_server: MockLiveServerInterface, test_api_key: str, ) -> None: """ Test that a failed connection can reconnect. """ # Arrange - # Dork up the API key in the mock client to fail CRAM - bucket_id = test_api_key[-BUCKET_ID_LENGTH:] - invalid_key = "db-invalidkey00000000000000FFFFF" - monkeypatch.setitem(mock_live_server._user_api_keys, bucket_id, invalid_key) - live_client = client.Live( key=test_api_key, gateway=mock_live_server.host, @@ -1578,12 +1530,13 @@ def test_live_connection_reconnect_cram_failure( # Ensure this was an authentication error exc.match(r"User authentication failed:") - # Fix the key in the mock live server to connect - monkeypatch.setitem(mock_live_server._user_api_keys, bucket_id, test_api_key) - live_client.subscribe( - dataset=Dataset.GLBX_MDP3, - schema=Schema.MBO, - ) + async with mock_live_server.api_key_context(test_api_key): + live_client.subscribe( + dataset=Dataset.GLBX_MDP3, + schema=Schema.MBO, + ) + + assert live_client.is_connected() async def test_live_callback_exception_handler( @@ -1642,4 +1595,4 @@ async def test_live_stream_exception_handler( # Assert await live_client.wait_for_close() - assert len(exceptions) == 1 + assert exceptions diff --git a/tests/test_live_protocol.py b/tests/test_live_protocol.py index 7f43ae6..4a1273b 100644 --- a/tests/test_live_protocol.py +++ b/tests/test_live_protocol.py @@ -7,7 +7,7 @@ from databento_dbn import Schema from databento_dbn import SType -from tests.mock_live_server import MockLiveServer +from tests.mockliveserver.fixture import MockLiveServerInterface @pytest.mark.parametrize( @@ -22,8 +22,8 @@ ], ) async def test_protocol_connection( - mock_live_server: MockLiveServer, - test_api_key: str, + mock_live_server: MockLiveServerInterface, + test_live_api_key: str, dataset: Dataset, ) -> None: """ @@ -33,7 +33,7 @@ async def test_protocol_connection( # Arrange transport, protocol = await asyncio.get_event_loop().create_connection( protocol_factory=lambda: DatabentoLiveProtocol( - api_key=test_api_key, + api_key=test_live_api_key, dataset=dataset, ), host=mock_live_server.host, @@ -59,8 +59,8 @@ async def test_protocol_connection( ) async def test_protocol_connection_streaming( monkeypatch: pytest.MonkeyPatch, - mock_live_server: MockLiveServer, - test_api_key: str, + mock_live_server: MockLiveServerInterface, + test_live_api_key: str, dataset: Dataset, ) -> None: """ @@ -81,7 +81,7 @@ async def test_protocol_connection_streaming( _, protocol = await asyncio.get_event_loop().create_connection( protocol_factory=lambda: DatabentoLiveProtocol( - api_key=test_api_key, + api_key=test_live_api_key, dataset=dataset, ), host=mock_live_server.host, From 244e9f7c855611016e15b962b78a9c361fa6ece2 Mon Sep 17 00:00:00 2001 From: Nick Macholl Date: Tue, 4 Jun 2024 15:04:31 -0700 Subject: [PATCH 2/7] FIX: Python mockliveserver on Windows --- pyproject.toml | 2 +- tests/mockliveserver/__main__.py | 14 +++---- tests/mockliveserver/controller.py | 61 ++++++++++++++---------------- tests/mockliveserver/fixture.py | 23 +++++++---- tests/test_live_client.py | 4 -- 5 files changed, 52 insertions(+), 52 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8ebd784..d087a07 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,4 +76,4 @@ warn_unused_ignores = true [tool.pytest.ini_options] testpaths = ["tests"] -addopts = "--asyncio-mode auto" +asyncio_mode = "auto" diff --git a/tests/mockliveserver/__main__.py b/tests/mockliveserver/__main__.py index 0f03b16..81a0b09 100644 --- a/tests/mockliveserver/__main__.py +++ b/tests/mockliveserver/__main__.py @@ -11,7 +11,7 @@ from databento.common.publishers import Dataset from databento_dbn import Schema -from tests.mockliveserver.controller import CommandProtocol +from tests.mockliveserver.controller import Controller from tests.mockliveserver.source import ReplayProtocol from .server import MockLiveServerProtocol @@ -84,13 +84,11 @@ async def main() -> None: ip, port, *_ = server._sockets[-1].getsockname() # type: ignore [attr-defined] # Create command interface for stdin - await loop.connect_read_pipe( - protocol_factory=lambda: CommandProtocol( - api_key_table=api_key_table, - file_replay_table=file_replay_table, - server=server, - ), - pipe=sys.stdin, + _ = Controller( + server=server, + api_key_table=api_key_table, + file_replay_table=file_replay_table, + loop=loop, ) # Log Arguments diff --git a/tests/mockliveserver/controller.py b/tests/mockliveserver/controller.py index e387b6c..e735533 100644 --- a/tests/mockliveserver/controller.py +++ b/tests/mockliveserver/controller.py @@ -3,6 +3,7 @@ import argparse import asyncio import logging +import sys from collections.abc import Mapping from collections.abc import MutableMapping from pathlib import Path @@ -18,7 +19,7 @@ logger = logging.getLogger(__name__) -class CommandProtocol(asyncio.Protocol): +class Controller: command_parser = argparse.ArgumentParser(prog="mockliveserver") subparsers = command_parser.add_subparsers(dest="command") @@ -45,47 +46,43 @@ def __init__( server: asyncio.base_events.Server, api_key_table: Mapping[str, set[str]], file_replay_table: MutableMapping[tuple[Dataset, Schema], ReplayProtocol], + loop: asyncio.AbstractEventLoop, ) -> None: self._server = server self._api_key_table = api_key_table self._file_replay_table = file_replay_table - - def eof_received(self) -> bool | None: - self._server.close() - return super().eof_received() - - def data_received(self, data: bytes) -> None: - logger.debug("%d bytes from stdin", len(data)) - try: - command_str = data.decode("utf-8") - except Exception: - logger.error("error parsing command") - raise - - for command in command_str.splitlines(): - params = self.command_parser.parse_args(command.split()) - command_func = getattr(self, f"_command_{params.command}", None) - if command_func is None: - raise ValueError(f"{params.command} does not have a command handler") + self._loop = loop + + self._read_task = loop.create_task(self._read_commands()) + + async def _read_commands(self) -> None: + while self._server.is_serving(): + line = await self._loop.run_in_executor(None, sys.stdin.readline) + self.data_received(line.strip()) + + def data_received(self, command_str: str) -> None: + params = self.command_parser.parse_args(command_str.split()) + command_func = getattr(self, f"_command_{params.command}", None) + if command_func is None: + raise ValueError(f"{params.command} does not have a command handler") + else: + logger.info("received command: %s", command_str) + command_params = dict(params._get_kwargs()) + command_params.pop("command") + try: + command_func(**command_params) + except Exception: + logger.exception("error processing command: %s", params.command) + print(f"nack: {command_str}", flush=True) else: - logger.info("received command: %s", command) - command_params = dict(params._get_kwargs()) - command_params.pop("command") - try: - command_func(**command_params) - except Exception: - logger.exception("error processing command: %s", params.command) - print(f"nack: {command}", flush=True) - else: - print(f"ack: {command}", flush=True) - - return super().data_received(data) + print(f"ack: {command_str}", flush=True) def _command_close(self, *_: str) -> None: """ Close the server. """ - self._server.close() + self._read_task.cancel() + self._loop.call_soon(self._server.close) def _command_active_count(self, *_: str) -> None: """ diff --git a/tests/mockliveserver/fixture.py b/tests/mockliveserver/fixture.py index c31e75a..6ff712d 100644 --- a/tests/mockliveserver/fixture.py +++ b/tests/mockliveserver/fixture.py @@ -4,7 +4,6 @@ import contextlib import os import pathlib -import signal import sys from asyncio.subprocess import Process from collections.abc import AsyncGenerator @@ -70,13 +69,22 @@ def stdout(self) -> asyncio.StreamReader: return self._process.stdout raise RuntimeError("no stream reader for stdout") - async def _send_command(self, command: str) -> None: + async def _send_command( + self, + command: str, + timeout: float = 1.0, + ) -> None: if self._process.stdin is None: raise RuntimeError("cannot write command to mock live server") self._process.stdin.write( f"{command.strip()}\n".encode(), ) - line = await self.stdout.readline() + + try: + line = await asyncio.wait_for(self.stdout.readline(), timeout) + except asyncio.TimeoutError: + raise RuntimeError("timeout waiting for command acknowledgement") + line_str = line.decode("utf-8") if line_str.startswith(f"ack: {command}"): @@ -140,9 +148,9 @@ async def close(self) -> None: def kill(self) -> None: """ - Kill the mock live server by sending SIGKILL. + Kill the mock live server. """ - self._process.send_signal(signal.SIGKILL) + self._process.kill() @contextlib.contextmanager def test_context(self) -> Generator[None, None, None]: @@ -214,6 +222,7 @@ async def fixture_mock_live_server( "--echo", echo_file.resolve(), "--verbose", + executable=sys.executable, stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, stderr=sys.stderr, @@ -236,5 +245,5 @@ async def fixture_mock_live_server( yield interface - interface.kill() - await asyncio.wait_for(process.wait(), timeout=1) + process.terminate() + await process.wait() diff --git a/tests/test_live_client.py b/tests/test_live_client.py index 7b9ca1b..a5d61f6 100644 --- a/tests/test_live_client.py +++ b/tests/test_live_client.py @@ -5,7 +5,6 @@ from __future__ import annotations import pathlib -import platform import random import string from io import BytesIO @@ -894,7 +893,6 @@ def test_live_add_stream_path_directory( live_client.add_stream(tmp_path) -@pytest.mark.skipif(platform.system() == "Windows", reason="flaky on windows") async def test_live_async_iteration( live_client: client.Live, ) -> None: @@ -998,7 +996,6 @@ async def test_live_async_iteration_dropped( assert live_client._dbn_queue.empty() -@pytest.mark.skipif(platform.system() == "Windows", reason="flaky on windows") async def test_live_async_iteration_stop( live_client: client.Live, ) -> None: @@ -1025,7 +1022,6 @@ async def test_live_async_iteration_stop( assert live_client._dbn_queue.empty() -@pytest.mark.skipif(platform.system() == "Windows", reason="flaky on windows") def test_live_sync_iteration( live_client: client.Live, ) -> None: From d8df53582c6cd6a831d618f5ebd7bef05e852eaf Mon Sep 17 00:00:00 2001 From: Nick Macholl Date: Wed, 5 Jun 2024 16:26:21 -0700 Subject: [PATCH 3/7] FIX: Touchup change notes --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 406f54f..bb35842 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ - Added new off-market publisher values for `IFEU.IMPACT` and `NDEX.IMPACT` #### Breaking changes -- Renamed `CbboMsg` to `CBBOMsg`. +- Renamed `CbboMsg` to `CBBOMsg` - Renamed `use_snapshot` parameter in `Live.subscribe` function to `snapshot` - All Python exceptions raised by `databento-dbn` have been changed to use the `DBNError` type @@ -244,7 +244,7 @@ In some cases, DBN v1 records will be converted to their v2 counterparts: - Fixed an issue where `DBNStore.from_bytes` did not rewind seekable buffers - Fixed an issue where the `DBNStore` would not map symbols with input symbology of `SType.INSTRUMENT_ID` - Fixed an issue with `DBNStore.request_symbology` when the DBN metadata's start date and end date were the same -- Fixed an issue where closed streams were not removed from a `Live` client on shutdown. +- Fixed an issue where closed streams were not removed from a `Live` client on shutdown ## 0.20.0 - 2023-09-21 From 9d97a0b1929b63c4e4331ee7e57c510fcf1cffff Mon Sep 17 00:00:00 2001 From: Nick Macholl Date: Wed, 5 Jun 2024 17:48:28 -0700 Subject: [PATCH 4/7] FIX: Fix sending of heartbeat_interval_s --- CHANGELOG.md | 5 +++++ databento/live/session.py | 2 +- tests/test_live_client.py | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bb35842..88cccbe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## 0.35.1 - TBD + +#### Bug fixes +- Fixed an issue where heartbeat_interval_s was not being sent to the gateway + ## 0.35.0 - 2024-06-04 #### Enhancements diff --git a/databento/live/session.py b/databento/live/session.py index 79676bf..edd83c0 100644 --- a/databento/live/session.py +++ b/databento/live/session.py @@ -188,7 +188,7 @@ def __init__( ts_out: bool = False, heartbeat_interval_s: int | None = None, ): - super().__init__(api_key, dataset, ts_out) + super().__init__(api_key, dataset, ts_out, heartbeat_interval_s) self._dbn_queue = dbn_queue self._loop = loop diff --git a/tests/test_live_client.py b/tests/test_live_client.py index a5d61f6..c42c418 100644 --- a/tests/test_live_client.py +++ b/tests/test_live_client.py @@ -250,6 +250,39 @@ async def test_live_connect_auth( assert message.encoding == Encoding.DBN +async def test_live_connect_auth_with_heartbeat_interval( + mock_live_server: MockLiveServerInterface, + test_live_api_key: str, +) -> None: + """ + Test that setting `heartbeat_interval_s` on a Live client sends that field + to the gateway. + """ + # Arrange + live_client = client.Live( + key=test_live_api_key, + gateway=mock_live_server.host, + port=mock_live_server.port, + heartbeat_interval_s=10, + ) + + live_client.subscribe( + dataset=Dataset.GLBX_MDP3, + schema=Schema.MBO, + ) + + # Act + message = await mock_live_server.wait_for_message_of_type( + message_type=gateway.AuthenticationRequest, + ) + + # Assert + assert message.auth.endswith(live_client.key[-BUCKET_ID_LENGTH:]) + assert message.dataset == live_client.dataset + assert message.encoding == Encoding.DBN + assert message.heartbeat_interval_s == "10" + + async def test_live_connect_auth_two_clients( mock_live_server: MockLiveServerInterface, test_live_api_key: str, From dc652df8087fb2827b27e317faa30521ffc37f96 Mon Sep 17 00:00:00 2001 From: Nick Macholl Date: Fri, 7 Jun 2024 12:28:06 -0700 Subject: [PATCH 5/7] MOD: Update databento_dbn to 0.18.1 --- CHANGELOG.md | 7 +++++-- README.md | 2 +- pyproject.toml | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 88cccbe..abee3b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,12 @@ # Changelog -## 0.35.1 - TBD +## 0.36.0 - TBD + +#### Enhancements +- Upgraded `databento-dbn` to 0.18.1 #### Bug fixes -- Fixed an issue where heartbeat_interval_s was not being sent to the gateway +- Fixed an issue where `heartbeat_interval_s` was not being sent to the gateway ## 0.35.0 - 2024-06-04 diff --git a/README.md b/README.md index ac627bc..e957d79 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ The library is fully compatible with the latest distribution of Anaconda 3.8 and The minimum dependencies as found in the `pyproject.toml` are also listed below: - python = "^3.8" - aiohttp = "^3.8.3" -- databento-dbn = "0.18.0" +- databento-dbn = "0.18.1" - numpy= ">=1.23.5" - pandas = ">=1.5.3" - pip-system-certs = ">=4.0" (Windows only) diff --git a/pyproject.toml b/pyproject.toml index d087a07..0a220e0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,7 @@ aiohttp = [ {version = "^3.8.3", python = "<3.12"}, {version = "^3.9.0", python = "^3.12"} ] -databento-dbn = "0.18.0" +databento-dbn = "0.18.1" numpy = [ {version = ">=1.23.5", python = "<3.12"}, {version = "^1.26.0", python = "^3.12"} From 84186ada830ab3c0287f71c7ab206305cee7e367 Mon Sep 17 00:00:00 2001 From: Nick Macholl Date: Fri, 7 Jun 2024 11:39:00 -0700 Subject: [PATCH 6/7] MOD: Decode Live client DBN streams before write --- CHANGELOG.md | 4 + databento/live/session.py | 89 +++++++++++------- .../DBEQ.BASIC/test_data.definition.dbn.zst | Bin 385 -> 363 bytes tests/data/DBEQ.BASIC/test_data.mbp-1.dbn.zst | Bin 268 -> 259 bytes .../data/DBEQ.BASIC/test_data.mbp-10.dbn.zst | Bin 320 -> 302 bytes .../DBEQ.BASIC/test_data.ohlcv-1d.dbn.zst | Bin 245 -> 225 bytes .../DBEQ.BASIC/test_data.ohlcv-1h.dbn.zst | Bin 253 -> 237 bytes .../DBEQ.BASIC/test_data.ohlcv-1m.dbn.zst | Bin 194 -> 180 bytes .../DBEQ.BASIC/test_data.ohlcv-1s.dbn.zst | Bin 194 -> 180 bytes tests/data/DBEQ.BASIC/test_data.tbbo.dbn.zst | Bin 276 -> 273 bytes .../data/DBEQ.BASIC/test_data.trades.dbn.zst | Bin 235 -> 223 bytes .../GLBX.MDP3/test_data.definition.dbn.zst | Bin 327 -> 290 bytes tests/data/GLBX.MDP3/test_data.mbo.dbn.zst | Bin 209 -> 189 bytes tests/data/GLBX.MDP3/test_data.mbp-1.dbn.zst | Bin 272 -> 254 bytes tests/data/GLBX.MDP3/test_data.mbp-10.dbn.zst | Bin 443 -> 420 bytes .../data/GLBX.MDP3/test_data.ohlcv-1d.dbn.zst | Bin 162 -> 145 bytes .../data/GLBX.MDP3/test_data.ohlcv-1h.dbn.zst | Bin 274 -> 258 bytes .../data/GLBX.MDP3/test_data.ohlcv-1m.dbn.zst | Bin 233 -> 215 bytes .../data/GLBX.MDP3/test_data.ohlcv-1s.dbn.zst | Bin 189 -> 169 bytes .../GLBX.MDP3/test_data.statistics.dbn.zst | Bin 234 -> 222 bytes tests/data/GLBX.MDP3/test_data.tbbo.dbn.zst | Bin 278 -> 258 bytes tests/data/GLBX.MDP3/test_data.trades.dbn.zst | Bin 230 -> 215 bytes .../IFEU.IMPACT/test_data.definition.dbn.zst | Bin 299 -> 260 bytes tests/data/IFEU.IMPACT/test_data.mbo.dbn.zst | Bin 242 -> 218 bytes .../data/IFEU.IMPACT/test_data.mbp-1.dbn.zst | Bin 234 -> 214 bytes .../data/IFEU.IMPACT/test_data.mbp-10.dbn.zst | Bin 277 -> 256 bytes .../IFEU.IMPACT/test_data.ohlcv-1d.dbn.zst | Bin 183 -> 163 bytes .../IFEU.IMPACT/test_data.ohlcv-1h.dbn.zst | Bin 278 -> 259 bytes .../IFEU.IMPACT/test_data.ohlcv-1m.dbn.zst | Bin 258 -> 238 bytes .../IFEU.IMPACT/test_data.ohlcv-1s.dbn.zst | Bin 251 -> 235 bytes .../IFEU.IMPACT/test_data.statistics.dbn.zst | Bin 246 -> 229 bytes tests/data/IFEU.IMPACT/test_data.tbbo.dbn.zst | Bin 234 -> 213 bytes .../data/IFEU.IMPACT/test_data.trades.dbn.zst | Bin 223 -> 201 bytes tests/data/LIVE/test_data.live.dbn.zst | Bin 1074 -> 1101 bytes .../NDEX.IMPACT/test_data.definition.dbn.zst | Bin 303 -> 265 bytes tests/data/NDEX.IMPACT/test_data.mbo.dbn.zst | Bin 256 -> 236 bytes .../data/NDEX.IMPACT/test_data.mbp-1.dbn.zst | Bin 294 -> 277 bytes .../data/NDEX.IMPACT/test_data.mbp-10.dbn.zst | Bin 366 -> 351 bytes .../NDEX.IMPACT/test_data.ohlcv-1d.dbn.zst | Bin 183 -> 162 bytes .../NDEX.IMPACT/test_data.ohlcv-1h.dbn.zst | Bin 282 -> 267 bytes .../NDEX.IMPACT/test_data.ohlcv-1m.dbn.zst | Bin 238 -> 224 bytes .../NDEX.IMPACT/test_data.ohlcv-1s.dbn.zst | Bin 230 -> 215 bytes .../NDEX.IMPACT/test_data.statistics.dbn.zst | Bin 243 -> 223 bytes tests/data/NDEX.IMPACT/test_data.tbbo.dbn.zst | Bin 269 -> 244 bytes .../data/NDEX.IMPACT/test_data.trades.dbn.zst | Bin 226 -> 207 bytes .../OPRA.PILLAR/test_data.definition.dbn.zst | Bin 298 -> 256 bytes .../data/OPRA.PILLAR/test_data.mbp-1.dbn.zst | Bin 224 -> 218 bytes .../OPRA.PILLAR/test_data.ohlcv-1d.dbn.zst | Bin 253 -> 237 bytes .../OPRA.PILLAR/test_data.ohlcv-1h.dbn.zst | Bin 229 -> 217 bytes .../OPRA.PILLAR/test_data.ohlcv-1m.dbn.zst | Bin 226 -> 214 bytes .../OPRA.PILLAR/test_data.ohlcv-1s.dbn.zst | Bin 233 -> 220 bytes .../OPRA.PILLAR/test_data.statistics.dbn.zst | Bin 203 -> 191 bytes tests/data/OPRA.PILLAR/test_data.tbbo.dbn.zst | Bin 301 -> 283 bytes .../data/OPRA.PILLAR/test_data.trades.dbn.zst | Bin 241 -> 229 bytes .../XNAS.ITCH/test_data.definition.dbn.zst | Bin 245 -> 215 bytes .../XNAS.ITCH/test_data.imbalance.dbn.zst | Bin 230 -> 217 bytes tests/data/XNAS.ITCH/test_data.mbo.dbn.zst | Bin 260 -> 246 bytes tests/data/XNAS.ITCH/test_data.mbp-1.dbn.zst | Bin 283 -> 268 bytes tests/data/XNAS.ITCH/test_data.mbp-10.dbn.zst | Bin 311 -> 289 bytes .../data/XNAS.ITCH/test_data.ohlcv-1d.dbn.zst | Bin 165 -> 144 bytes .../data/XNAS.ITCH/test_data.ohlcv-1h.dbn.zst | Bin 249 -> 235 bytes .../data/XNAS.ITCH/test_data.ohlcv-1m.dbn.zst | Bin 204 -> 190 bytes .../data/XNAS.ITCH/test_data.ohlcv-1s.dbn.zst | Bin 183 -> 168 bytes tests/data/XNAS.ITCH/test_data.tbbo.dbn.zst | Bin 286 -> 268 bytes tests/data/XNAS.ITCH/test_data.trades.dbn.zst | Bin 239 -> 228 bytes tests/test_historical_bento.py | 10 +- 66 files changed, 65 insertions(+), 38 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index abee3b7..60b14be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ #### Bug fixes - Fixed an issue where `heartbeat_interval_s` was not being sent to the gateway +- Fixed an issue where a truncated DBN stream could be written by the `Live` client in the event of an ungraceful disconnect + +#### Breaking changes +- Output streams of the `Live` client added with `Live.add_stream` will now upgrade to the latest DBN version before being written ## 0.35.0 - 2024-06-04 diff --git a/databento/live/session.py b/databento/live/session.py index edd83c0..2d297e5 100644 --- a/databento/live/session.py +++ b/databento/live/session.py @@ -4,6 +4,7 @@ import dataclasses import logging import queue +import struct import threading from collections.abc import Iterable from typing import IO @@ -11,7 +12,6 @@ from typing import Final import databento_dbn -from databento_dbn import Metadata from databento_dbn import Schema from databento_dbn import SType @@ -196,33 +196,37 @@ def __init__( self._user_callbacks = user_callbacks self._user_streams = user_streams - def _process_dbn(self, data: bytes) -> None: - start_index = 0 - if data.startswith(b"DBN") and self._metadata: - # We have already received metata for the stream - # Set start index to metadata length - start_index = int.from_bytes(data[4:8], byteorder="little") + 8 - self._metadata.check(Metadata.decode(bytes(data[:start_index]))) - for stream, exc_callback in self._user_streams.items(): - try: - stream.write(data[start_index:]) - except Exception as exc: - stream_name = getattr(stream, "name", str(stream)) - logger.error( - "error writing %d bytes to `%s` stream", - len(data[start_index:]), - stream_name, - exc_info=exc, - ) - if exc_callback is not None: - exc_callback(exc) - return super()._process_dbn(data) - def received_metadata(self, metadata: databento_dbn.Metadata) -> None: - self._metadata.data = metadata + if self._metadata: + self._metadata.check(metadata) + else: + metadata_bytes = metadata.encode() + for stream, exc_callback in self._user_streams.items(): + try: + stream.write(metadata_bytes) + except Exception as exc: + stream_name = getattr(stream, "name", str(stream)) + logger.error( + "error writing %d bytes to `%s` stream", + len(metadata_bytes), + stream_name, + exc_info=exc, + ) + if exc_callback is not None: + exc_callback(exc) + + self._metadata.data = metadata return super().received_metadata(metadata) def received_record(self, record: DBNRecord) -> None: + self._dispatch_writes(record) + self._dispatch_callbacks(record) + if self._dbn_queue.is_enabled(): + self._queue_for_iteration(record) + + return super().received_record(record) + + def _dispatch_callbacks(self, record: DBNRecord) -> None: for callback, exc_callback in self._user_callbacks.items(): try: callback(record) @@ -236,18 +240,37 @@ def received_record(self, record: DBNRecord) -> None: if exc_callback is not None: exc_callback(exc) - if self._dbn_queue.is_enabled(): - self._dbn_queue.put(record) + def _dispatch_writes(self, record: DBNRecord) -> None: + if hasattr(record, "ts_out"): + ts_out_bytes = struct.pack("Q", record.ts_out) + else: + ts_out_bytes = b"" - # DBNQueue has no max size; so check if it's above capacity, and if so, pause reading - if self._dbn_queue.is_full(): - logger.warning( - "record queue is full; %d record(s) to be processed", - self._dbn_queue.qsize(), + record_bytes = bytes(record) + ts_out_bytes + + for stream, exc_callback in self._user_streams.items(): + try: + stream.write(record_bytes) + except Exception as exc: + stream_name = getattr(stream, "name", str(stream)) + logger.error( + "error writing %d bytes to `%s` stream", + len(record_bytes), + stream_name, + exc_info=exc, ) - self.transport.pause_reading() + if exc_callback is not None: + exc_callback(exc) - return super().received_record(record) + def _queue_for_iteration(self, record: DBNRecord) -> None: + self._dbn_queue.put(record) + # DBNQueue has no max size; so check if it's above capacity, and if so, pause reading + if self._dbn_queue.is_full(): + logger.warning( + "record queue is full; %d record(s) to be processed", + self._dbn_queue.qsize(), + ) + self.transport.pause_reading() class Session: diff --git a/tests/data/DBEQ.BASIC/test_data.definition.dbn.zst b/tests/data/DBEQ.BASIC/test_data.definition.dbn.zst index 7d1df91758536bfabc9a7de79b0c62a875d2bce3..b33616454efae1484eeba7542c8823ef265b9b71 100644 GIT binary patch literal 363 zcmV-x0hInIwJ-eySlkK#VvaXCFmclXz(9Ir)ZzjT)`j|;PDthO;OM!XHGj?1<-27b5x#v;wC8!Y zrA4;1!0cRDScyHm(gKQwZ30VBe3{1JZCc2d833FX3OXPlgPGa17>Z%JP;Ws9l)z{M z)6J4BHc4{#Bw>=UXa@q?1465;l+nO)| J002K;Pt&acz literal 385 zcmV-{0e=1{wJ-f7&ZPnXqyt1kP65UM002ZnMNuw7K~qUX00{s908FGTY)}^f0GlJ0RMmg0JewYM}Gjy znmYhDQ3gd34Sx${e*^$gQBeSRV`ynDH8lWOPC-)uMF2}i5di=H0RKV%|Nl+_e_9hL z)^PB6dQcaGyK@QvX>%=fa$#g;b1gI6uE7CfVs9-nDvZi{pmd8+7h+9&=mR_e02~cv zaB^%dF*E>we*gdg05>2I0~w-C2Ov;&fkW@ncn-k8tN}YDqOWpc$AGZ`A#ns9fWV$_ znJU>h)*&`Rn?VdvPRZbuK*@;<#U}1luruKhV@MFZ z5#e<}WZRC*d~W9t%xz-kbJS*e(&V}9KwH8ER%Y&F9nuGySu$9!C^_src#&a~E(bHv JZkdz%3jhl_G;RO@ delta 171 zcmV;c095~j0*nF?D77#BB+jJ*0E7cXLQVn30FeMrXAni%0K{aq!XTh13m{)$*b9UKat`zyD77#BWIzDz1OUVakx(8Mc}@cWjs*Y! z02BczVei9uf&hcNbMTRm76B-cq#-jD0D$1B01#j)0aMuW&Ne_0r$EY1VE`{^?M&kR!6-^AyyNf?m041BBGBF(Cow!;*%Y?&a z#lv$wXPQ0*9GlRr;>j+wpdo;z$DLDc&deYGXRh4BT>C&K^fw#ZheXrOYMU-?y}yHh zIu~AW?>zM)c$|>sR1Q8y}3ExTCo5C diff --git a/tests/data/DBEQ.BASIC/test_data.ohlcv-1d.dbn.zst b/tests/data/DBEQ.BASIC/test_data.ohlcv-1d.dbn.zst index e0aec4493c33d1deb5a40ea14bc7585b716d2df7..0e0672a8e8ce9f25c46ae5f58733e4f52e5921a5 100644 GIT binary patch delta 170 zcmey$_>fUhW2^pOmWUoU1`|#fCqJf0#)*Q8o{S6(?hHVJAuuoyOz+rW!nhNN4ULUW z4EU6_7-UQs#FF;T{d&NCx9>MMpvs1=dX4TtI>wuaPg%V|++j1ri@NU<=PNr)GYF)m zaxgGGkW$e9zumPSi(DEG4Mok1 Q8B?MK*nn;@nsQkP0Egx>FaQ7m delta 208 zcmaFJ_?1ymW2^pOm2*p(7?v=*IQcOiW0)wYSpT1qAuup-#|9I|oj`18Y;0n{#n8a8 zU|+g`>rI9uDO;NwnhzKu5v1O9Nh8xXWGFU|v9CjYO$goM5gBfVWnu20O0CX@bC;$Ke delta 155 zcmaFM_?J;oW2^pOm2*p(7?v=*IQcOiW0)wYX!W0wAuup-#|9I|oj`18Y;0n{#n8a8 zU|+g`>rI9uDO;NwnhzK4cGE78(g|7!pguf zG0)aeltF>bf+?rr8N=qaD;iHI2q-W#F@DPXTJSZIvwK<DjycamCPfPH9 Hlu`o#UO+bd diff --git a/tests/data/DBEQ.BASIC/test_data.ohlcv-1m.dbn.zst b/tests/data/DBEQ.BASIC/test_data.ohlcv-1m.dbn.zst index 7eb1e6032b73cecf8a5e249309704ebee1023230..770a3c888c1e4171ffcf12e6179ff631a1ff1c55 100644 GIT binary patch delta 130 zcmX@axP?(rW2^pOmIxJAh9&GSPJT?0j1vVFO&J*&+!=rbLttPanBK9$gmEVj8yXv% z81N}-FvyqyWh0IzES(e}&hTaSb@vI36FcPe1sN7FDKN9Jq(v|?gvl3VtY#~dC|K%r cU=w@hMJ^48hN9-hj49CqY(TST_P1>T036mLx&QzG delta 127 zcmdnOc!*I@W2^pOm2*p(7?v=*IQcOiW0)wYX!W0wAuup-#|9I|oj`18Y;0n{#n8a8 zU|+g`>rI9uDO;NwnhzKxXFO|v_$@F#*F};vndb& diff --git a/tests/data/DBEQ.BASIC/test_data.ohlcv-1s.dbn.zst b/tests/data/DBEQ.BASIC/test_data.ohlcv-1s.dbn.zst index 7f6f49b70a02d31734229cc23de276312d409b76..ef646065bb32a10dca3081cd428baede89408064 100644 GIT binary patch delta 130 zcmX@axP?(rW2^pOmIxJAh9&GSPJT?0j1vVFO&J*&+!=rbLttPanBK9$gmEVj8yXv% z81N}*FvyqyWkdE`uA3Af&hTaSb@vI36FcPe1sN7FDKN9Jq(v|?gvl3VtY#~dC|K%r cU=w@hMJ^48hN9-hj49CqY(TSpmRv6e01^r!X#fBK delta 127 zcmdnOc!*I@W2^pOm2*p(7?v=*IQcOiW0)wYX!W0wAuup-#|9I|oj`18Y;0n{#n8a8 zU|+g`>rI9uDO;NwnhzK<(-Lo3nlaJn;tjo;M2vpA?yzHVokeUFrMn{sNIC;y`jJJE3dD;$`8@as~VP@gs zNpwg|NamVU7RM~)8`y9tCnhT~C-&C4Z?+!Y!eI`KOjD#~col9p830YFl8X2W0M)!f A3IG5A delta 169 zcmV;a09OBz0+a#~D77#BB+jJ*0Hgy%LQVn30Feqa3=Hldf*~+45KQmbV8XZ)hz*U6 zO$;XHXil7NY%IiJeUzcukw;*{%82(l><|8gtSR5dtYS2E>f1f{nzv-IiYPejJa~~| OlP(7{&@8!ta$^7rOd@>% delta 110 zcmcc5_?l5rW2^pOm2*p(7+RQJoctJ%F-#PcxB1V=5EvM^V}l9fP9QckHa0QfW@uno zurFP}^(MoSl&wt-%?Bjcgci3lZD0<#W-bdh<-l8Jh9|5O12mZ!7$#O4GxAJaE6y4E N@vZ6nRr{A@0|0HgD53xW diff --git a/tests/data/GLBX.MDP3/test_data.definition.dbn.zst b/tests/data/GLBX.MDP3/test_data.definition.dbn.zst index 9ec89b32fea00e57cbdd345593147a320687cd3b..235ea8a61c591ed5b04dadf8205ce92f889bdd47 100644 GIT binary patch literal 290 zcmV+-0p0#6wJ-eySd<6=0**8|Fptw1h*cqMP9GR7azXrftT8V`z=tZ_12l5gte_Pm zfJN{J^jrPh6Z*1Oe3DiKj2%GVo;G9(JEymMT?j0;yHbK4DFlXc@e@N*b|Vb&>$orc zT?|GkK&v@*jq2b20*8dTbN?}s()MdkEJproL*oIZ9wn>)YG|=q{wqZyb9(c_ph^)b zoS9hT%1p>9)cB7OqcUco2?|Jy!}Obvb)iF|LYq*nbD&zs;MXHSU-#>u*8{R}z=h@X z8K@F*wx3`Qi5o$Po{Z0H3BdDk0U7|X+E%UvP8|UO^&|)!00R>4!T<{ZPJz3xFb2-R oShR~cFdDeQ1wr6WT8jVxP(Ul7_5gYejs`F@fG`070LC`x(S-bd4FCWD literal 327 zcmdPcs{dEz+*~GxDa^|5IDtF|WA7ayVhjxZNA9zQim@>K zXJl{<_AunJH(}%jVp9_fBQAyph7C6_H!aFx=-HIyc);<2M8kT<8;pKg<##lI=BH#Z zoMmI!!WkvZ$dDzpjFIA`TTkf9L68#X=T1KPyUFl`1y!S8(y2Xn<31iaVmka{G_ z(EqJ&1~X9Z^1;XQdlsZJPSf;{DY0I{4RTpzG5h~|egzf=E+B6QNT_~~hZ*Bj_W%DG z7(#g$1u8KfCP*?Z2G z6&V<$vanp4EMUOE=x|_)U;=|d$Er7uN4yv~wwE3fes>_VD&zn|{SHB^!{>Q6g$RTl M`pD$3v*;WX0A<#0DF6Tf diff --git a/tests/data/GLBX.MDP3/test_data.mbo.dbn.zst b/tests/data/GLBX.MDP3/test_data.mbo.dbn.zst index 6f9f4589923c18e0aebf38cd4c7ad376a3a728d3..e836c74e69f88f5918e58ff8275d223fc0160d2d 100644 GIT binary patch delta 113 zcmcb}xR+5-W2^pOmWUiy1``e!CqJf0#)*P5a*PZN?hHVJ!8O>!5X|7QH(}%jVp9_f zqlxKSOmkQ!&KK7gWM~jHQ1FmokmO*V#C*~rrJ#51JcqnoRtByN)+=%j{Rb~H#F+`O L0gd`xw&V{0Wpx{e delta 152 zcmdnXc#%<1W2^pOm2Ya1Hh_U^5=LGBbQ(ndq(M;9%^%BSee=2|w}bZ^Fn6#HJ<|M%)Yy3>$7< zZd#PX(6cEiQ6OLn>jZs+8H_EA@!T)mL1v^lFo?1;sIZGLGBRX|GBD&$n`!L5BSfs> gocI4&W(EdE1_p+S31N()6Sr%zYoAf?o}Ty$02B8ofB*mh diff --git a/tests/data/GLBX.MDP3/test_data.mbp-10.dbn.zst b/tests/data/GLBX.MDP3/test_data.mbp-10.dbn.zst index 40b34ce7cf679bf2a48130aa69f624a9f6fb8e9c..4c086005bc65962703c7b75f34adb9a146f03f30 100644 GIT binary patch literal 420 zcmV;V0bBkkwJ-eySfmU9rcy>d@MNnsKxm6(sHaz%dP&q zOamn2zw`dCT7LmgMaEx4fO@lw|9s}bc0>So9K63Gr*Fpw?O*Vx|H1!yR0z>r{%CNj zB;kRd^}8MwD!^v#XNAn0v~Zea6L&f^*z74=o1@ABzYF1KfElVsrqr*M7ZRPQoVMjU zx#96i`gj;H8GxuqrTzr}`nNWgdQ3Dwr>GFOtZ5$&Lb;#>s?f9dru-$9oQ!E2^sL1V zY3E_0?sjyds~7SyhBjtQ%m!eH7yBE22V|{f7K!2pFUs@UyUto}&?29N^WZVuz`kt^n-*|){TVLjGZ>8G_HGwVk$QP>bkTx>iyj{> OW(FJv0RRBehC4^~1H&c& literal 443 zcmV;s0Yv^NwJ-f7&Yc1PlmkRUP65UM002i!LRc6YR2BdLkHp^w zRTcyQ{{a9+Q%ErkKQsXiKQsX~G&nK}0Du6n+0%leYygb1VnM(`;240f0oVbw)%Kn! zwJ-f-zyxgy020nMIRJOl7*J44U={f4w~gBMGremr!%!_NUAvZjr8nSTywi`I8Oa9A z+A8d$z1~R0p>(e*@qq6TAt_bpTbxPrm?bsIP#W|t3-f3P;up&_>b!wa_82ae4Te-n z4P}tO!~YaC$cb>dsh+jB_9sc#IFn1 z&%GCNJ83I+0G*{xEafZ8TmVQpAOHgqv6>DXK!ph+SLzke6gw~sN_AbfUbaibE(x^6 zQXQ9(PmlwpxjnG#$C6kyCU*}IXJpvulm&;WZDh10c?T@E1~4*YblhOsg1oUYpho9` lbHr@y;r^^KAfXAcyw3t7ZfwlRfGj!WH5l>jb)JGQ5#?t>z=r?; diff --git a/tests/data/GLBX.MDP3/test_data.ohlcv-1d.dbn.zst b/tests/data/GLBX.MDP3/test_data.ohlcv-1d.dbn.zst index 745663816659f74e304bc699534d56541f3b2b8b..3369763f176f37ffcfffe35b238460fcddb643ab 100644 GIT binary patch delta 97 zcmZ3)IFV6MW2^pOmIxjeh88v#CqJf0#)*Oos*DT_?hHVJ!8O>!5X|7QH(}%jVp9_f uBR*wDhAdHriPf@l91Q6^lNYrzHt0n$va(-DE--1B$=JdO)SB~0HW&bu6%-x- delta 139 zcmbQpxQJ0uW2^pOm2-2M7^X10IQcOiW0)wYQ23vb!8O>!kjLJHkr#+fO)QMK7#bKh z+`QbhD2JhEQltq_ZhZDF4Qz(RT1JK|Onl0W3|XQKK;U5Py(2`d!B|o@ jjv2^Ri=3Ydq#N$IUX6+S$^}xvz{nt*Y{yu5T6hlt@vteo diff --git a/tests/data/GLBX.MDP3/test_data.ohlcv-1h.dbn.zst b/tests/data/GLBX.MDP3/test_data.ohlcv-1h.dbn.zst index 04c24c5e00373be30684d5c2537bb8a9d9c5a722..f8828a338ef8aaf100b024c0372d0d55de8fb0c5 100644 GIT binary patch literal 258 zcmV+d0sa0cwJ-eySd<3Uw z{aq!HVE;}HfvaaD!v<`E)Qwfr!!^|5*nvC;WA7ayVhjxZNA9zQim@>K zXJl{<_AunJH(}%jVp9_fBQAyph7C6_H!aFx=-HIyc);<2M8kT<8;o%hfh`&!^B?H4 zG8|#&Q(|Pu5(OFwHM7B3QZ|km$mPqHiU-mScU-T=OniC1SdU@BkK~EFLADmUB!Exov~Kx{1(O@2IT`@(Wdh`5 FA^?$PRS5t9 diff --git a/tests/data/GLBX.MDP3/test_data.ohlcv-1m.dbn.zst b/tests/data/GLBX.MDP3/test_data.ohlcv-1m.dbn.zst index 72df690878b8966a5c6fccb03858740980d3acda..1d3db3ae60e21e9ec74b32214ecee3f4eb2bfc99 100644 GIT binary patch delta 154 zcmaFKc%4yDW2^pOmIxa*hAkW}PJT?0j1vVFtQi>?+!=rbgKMydA(+8qZ^Fn6#HJ<| zMtq8l3|XQK4aSnPam)-14afbafM|wBBR(U>iTzrRk_-*!y%?PZ**-KGI3)0~I`9Zg ySZlQ|fWgsW#!8L`hiL8{ha1iosJU`8GsJFT;^JFy{6PkTF*64<&`$mPODh0Ok0r1G delta 172 zcmcc4_>xgjW2^pOm2-2M7^X10IQcOiW0)wY5ci*v!8O>!kjLJHkr#+fO)QMK7#bKh z+`QbhD2JhEQltq_cJ?~UfX#UDl$qfP3!fq*LzXB55I7in?+6iVFqV{! zV`gAzIPN!vq0xxXh>?L|Vvd%92t&hpFGgoUwhv7P0SP=T4m<)A)>^F#U~qJpv67>~ WVVl;UkOMPC90bHf)MMALivj>S2{p6; diff --git a/tests/data/GLBX.MDP3/test_data.ohlcv-1s.dbn.zst b/tests/data/GLBX.MDP3/test_data.ohlcv-1s.dbn.zst index 22c92cbe05d279ce1e26efbbf47092fcc48cf342..7e960659927da315d948b9f7d11cfff23484223e 100644 GIT binary patch delta 146 zcmdnXxROy&W2^pOmWVSf3`f{pocx#~87B%VR4_6yxHA9=2G?K@LokEK-h`1Ch)qo_ zjQA878L~tf8jK}n!kjLJHkr#+fO)QMK7#bKh z+`QbhD2JhEQltq_>h#&a0Gsikm5ITHnNNX{Axo422po*PcZ7&F7)#2= zF$1}cMtreQ_~#wn$%_}Z$k_<)2#fSZBAlVK;LC4*(dksuBSiD=%d7NRi+IF(OH J+~nqX004g-Fckm* diff --git a/tests/data/GLBX.MDP3/test_data.statistics.dbn.zst b/tests/data/GLBX.MDP3/test_data.statistics.dbn.zst index 4a97f43c934d50a9126ee5756f98d2d38c273d42..0bb2c8124ef9940f0c3dbd2f6d27d41c38f6388e 100644 GIT binary patch delta 112 zcmaFGc#ly~W2^pOmWUEIhA*5hPJT?0j1vVF!kjLJHkr#+fO)QMK7#bKh z+`QbhD2JhEQltq_O7UNo0-2G*!0?orVGHX-|7b?OiL0gAx2p1#-HzA` E0EwL;`v3p{ diff --git a/tests/data/GLBX.MDP3/test_data.tbbo.dbn.zst b/tests/data/GLBX.MDP3/test_data.tbbo.dbn.zst index 1d71aa1fa165a99e0b6b17043e670a9952e67193..a870bfea2acfa5b21a1997b22507ff3abd2eb73a 100644 GIT binary patch literal 258 zcmV+d0sa0cwJ-eySd<3<0t+fLKu^>dm_j%Z_+%lKOWLEl6IYUdBR{2g2M;$CH@LGw zdgNU(HU$gQW z3jIht%8@ZeXFgV`@_1(<7#RGIy_l zVUQYx!G(b#0D-13_X(K>sX;)1(i9%7xVyvHQnYri;o_kIKo_lmf&=I=I2ypr0Kx$P I05CaY3VMofaR2}S literal 278 zcmdPcs{dEz+*~GxDa^|5n1MV8WA7ayVhjxZNA9zQim@>K zXJl{<_AunJH(}%jVp9_fBQAyph7C6_H!aFx=-HIyc);<2M8kT<8;rMS7fWaW%};S) zaAjpEVH06wWXKX_U|>%B=m;{j!B|o@4rm7>(3%h@28I&jZwK8N817m!F)%b5@o_M) zi$)c?hpj1bzvjxo@SYQ>jJ35>-NTWAVaI$ekH29I48PdIx7o!3_3oeQ6t^F!iqDEm zhC$sp(utu#Jb{gO+A~I-S&t5${Il-HjTxd1R}^^kcz75&Ojw&j8T?O}c*z$ux1Ap|LQ7z=b0U42=wz t6(j;!SeSg88yLKn1#P~}Bc9C5uq%UAM8={2;6;WsHUT!EWf`6gi~y5fBMSfk delta 146 zcmcc4_>56dW2^pOm2-2M7)qF3octJ%F-#Pcv-!`+;2P{<$YXE9$P2`#CKg8A3=Iq$ zZeDI$l*7=oDarAG;{%>+t9CN+++a*}=u8Bga^Ng8!x7er{$ipG42%p63?WVo3?;_j x4!SWg+_hw4m{@Em&%+>a;fMl5Bg16{i2xQBCZFa82CrpRHQHy?Yk3WS003a7(oYF(0%k#V8B z5Fs7^)xr|rhJ#>}xOlo1-Nf9mRseW-APC_(Fo4lF0;5mlhYv)M<1A!RKx)$uzaBnm zypAH_;~Vb+(ZKjCZ-WzLk8 z;Z;axR8x|#=F|Y$IDsLSm7#)d z`u}na{To^xEjH-2MjElWG&Ee!@G@09WToG6;24*s!5>ozA&)ZxeJbux08~!nc=-D}1prjCk*p*E6_L>wAk~(k&Dg;;SlNn9 Y3bX=hA4tpKXaFw+U;+UE0I=NOmr3s`&j0`b delta 161 zcmcb`_=%B6W2^q(ltczrW`-#&6S=L@zFjk6{0_vXW`^cQMur>=4Gdi2ervkJ9Nsc# zO3Ls)Xl3x>R=u{L&zouhKY45O1umTf*1Hx*mxLEHlA}3+jgt>X50amh)tp&d%dSJ Ju6z4H9RO)8HUa$Xsa&5;4Oth`r?H_Uq_C^T`Fwv`wIgQkMz1;YjqxsgG^fFp`=&N2sv n3x*OB=IPI8rA*lqEF^@`nuX$LPd7>Ok?0gY!fKe`J5d5tM# delta 134 zcmcb{_==H7W2^q(ltczqW`-7)iQKx0PC#W3eg|SxGedJDBSS8R z1_rKhzct-q4s3-lgr-MFG#nB`hM0mJF`jLK!wk2Hdjp iUM=1*@0Fks0|Nsi5Kk=BX5^nZSC+l{ta_;Q`hEaL$SR5e diff --git a/tests/data/IFEU.IMPACT/test_data.mbp-10.dbn.zst b/tests/data/IFEU.IMPACT/test_data.mbp-10.dbn.zst index af3a3d12e6566e08bafde63c51e97456ec93a3be..e44499e195b6f0a93c1afb077d1c4b77b7b93bd4 100644 GIT binary patch delta 164 zcmbQr)WF28u~q*sOGFDhg9ztD9y<+21_p+2*Gw3{1F@-@p}CQfVGNfggX^|XhRu-y zx2(KZi#N=BB`7p;mUfDj!iMH07Z@E@sa!Wy0TLS*bBT+vJT-A|aA0ELP;dZK0uBuf zjVv4>ijj#!Kw$!easX0&2Y`x>XC=Iuqk6rh{8*UEQU~p<_$a%m6rBSsK!@<0?O6-} D;VUsv delta 185 zcmZo*n##nZu~q+XN+N?QGs6{@iQIMp->#W3eg|SxGedJDBSQ{`1_rKhzct-q4sV$= zC1rROk{Q(^{GPl6>pQ@*mW82&C5FqA!F5|G!{*3g$WFdKw1IFWn=eH75W7 diff --git a/tests/data/IFEU.IMPACT/test_data.ohlcv-1d.dbn.zst b/tests/data/IFEU.IMPACT/test_data.ohlcv-1d.dbn.zst index 6867268f3ab27f4a1faaea16067d3c3a35b63d9b..85c7d55c66c7aa9b9b469f10997478f253863edc 100644 GIT binary patch literal 163 zcmV;U09^klwJ-eySeyg^0>~aKaBUg`z;?`?{C~ZX9&iHU5UT+sSP+RCK#GPv{b6*A z-2ts9g(^S8M=}Nf2S%Yk*39MmE0WxNf5d1C|FJ50K2MbGKoI1d47pG&4O2@))W09- zye4IKshckOq91$H2N literal 183 zcmdPcs{c17kwKN2;R=h3lOIzgBLjn{n`@|^r*D9xa|i-WzLk8 z;Z;axRNEcr*{cDxQN`jQBLknZC4=j>P`CvR^Ii!G0eKA4IWZu*q1^C=Ac&5Na|7|$ OJ!A%w3=CYnELi|_880^g diff --git a/tests/data/IFEU.IMPACT/test_data.ohlcv-1h.dbn.zst b/tests/data/IFEU.IMPACT/test_data.ohlcv-1h.dbn.zst index b7c6731bd70e494f419f966ad753ae2733a55ad1..b9673c3774121446ce3d8418129b2dcacdf71108 100644 GIT binary patch delta 116 zcmbQn)Xc=Ku~q*sOT-*@h7#_HJa)#63=9n4u9+}?2Vzq*Lvte|Lp~)-2G?z&3=9im zZdiG*7H^pMN>HedNlAz!Xpfj!9{0q>Zj4eBU&*N4k2)m7)@+c*$L8pCAS@+lfoQ;Q UhbT68<`WSOoIr!i?mgNE0Q_ww2><{9 delta 159 zcmZo>n#RPVu~q+XN+N?QGs6{@iQIOP->#W3eg|SxGedJDBSQ{`1_rKhzct-q4sV$= zC1rROk{Q*CPaj_o(x>uZFAKvFHa;aw2G?z&3=9imZdiG*7H^pMN>GRa27KKGcZi7anluMP~hNTYHB#peCzJ#Ot*j77P{C~}DW-{wmHk1&H=U%UfVx3Ik|EmoOkhGXN5g^bPJ1Sa zZDDL_IMB?;oYx?&aN!!$GXseh!(sytA*O9glbO~w++x3EWFh5njL8RRef;ZXhX8W} BF{=Oo delta 174 zcmaFI*u=!6u~q+XN+N?QGs6{@iQIOP->#W3eg|SxGedJDBSQ{`1_rKhzct-q4sV$= zC1rROk{Q+R==)}X^r<|MWnm~`$~HHhq(D5791sXIL>r%JWZ1!I8UY)vDQ3e g!_&jkz4pWs2kiyl8n^KDu}#>>APjUH-_v9+0LI!n3jhEB delta 199 zcmaFO_?wYOW2^q(ltczqW`-*)6S?i`zg;t7{0_vXW`^cQMur>=4Gdi2ervkJ9Nsc# zO3LsmBr~dI%vv4`(x>v^DKoGRa2wu92a|8)$ z16f($g{{RZB^rX(i}9FD@^4rpC@Hi@Ol-kFo$iME2jUH?b)u{w9SlYc48jcBJ`GEk vax^4tb9%?szi18LRK_ivMc)fC`MNT&Fc)kJJ$t^uTC+jojh0{HAD%h@DdIsi diff --git a/tests/data/IFEU.IMPACT/test_data.statistics.dbn.zst b/tests/data/IFEU.IMPACT/test_data.statistics.dbn.zst index 3319de6796fd5a2eae8aebbc7496055124fe212b..08ae05abbdddad73c4859c3bc8c17bc827ac6e8f 100644 GIT binary patch delta 115 zcmeyy_>_@bW2^pOmWVZM3|lxS^4OU&GB7ZFyJo`p9f(cM49$&<3{AlCyoFB delta 132 zcmaFL_>GZAW2^q(ltczqW`-*)6S?gozFjk6{0_vXW`^cQMur>=4Gdi2ervkJ9Nsc# zO3LsmBr~f0nQyQTtdD_VD>K6p76A!M2G?z&3=IeWiI;n=7C*AnSfXLxD?uTKCEctH iObiSRj0_X&G#CXZE|=wFF%ZbR?!zP|qSoiPp$z~)iz%G| diff --git a/tests/data/IFEU.IMPACT/test_data.tbbo.dbn.zst b/tests/data/IFEU.IMPACT/test_data.tbbo.dbn.zst index 9640cc94c285788ba6469545578b27f9cea8cf2f..c08410f65b92bd48f84046c1925fd2b0019fec3c 100644 GIT binary patch delta 121 zcmaFGc$JY`W2^pOmIxg-1`*DQJa!t43=9n4u9+}?2Vzq*Lvte|LlH(x2G?z&44WeZ zZdrM+7H^pMN>FIxENx3M1_n(9%L|4LAaWyvf&oVqOR~B+S#F&(cfFan)yy XV%*BVB(=b#p@($_(DbdBe|rD`qd_Jh delta 117 zcmcc0_==H7W2^q(ltczqW`-*)6S?gIzFjk6{0_vXW`^cQMur>=4Gdi2ervkJ9Nsc# zO3LsmBr~cVUYMZ;*5|+w%FIy0BEo3N;JPi8VRK}_Ei3QU;tlg&2?{YVFfan~#6oRG R{)uyC*{jd0hf1&S2LOluBj5l4 diff --git a/tests/data/IFEU.IMPACT/test_data.trades.dbn.zst b/tests/data/IFEU.IMPACT/test_data.trades.dbn.zst index 6e5b2f6dfece3aaa495e2f9dfcc964278c6d6562..8c0798647d24cfc5d01374e6b0734fd8db4c3950 100644 GIT binary patch delta 173 zcmcc5c#@G@W2^pOmWVT~3|BZN@~G9bFfcMOxPu4=ryxHCH(xIU10xefu)w!#CXC;K z*woC>+{nm~hryD;bz3OI=E#6sR^F?{8|J+d6bkWcVt8Ef{L&&Bd-fcIS7#NgH zuf4j(!ph)&`45Z7UuMtC3=BdH3W68-C3sYrez8V3h-T+6S>vu85tPd8Gr8^PXXy{N^2UxW7s_?~sG*tm$6&|8M_K`aecP_ul_u2x%@t)SDv} zDUjj6W2M-9gBC3go1LJ~m0CNx^8Ulp2tm$%b1$VgTIzR=dYcYV2x$J<@@0EAcuaW4 zb|Y5ge9t67I`_@bhOjE$RQXqmlYd~+GynVd@BNpe%IAS<(#ib)>kCaP0|1(yXd2o( zPsa6I?|r$PLZi(ob@^3kwYgl* _(R!XV=z3a%hMrRkAc4H&|FZKUC{|^g&=&l31 zlKTG`$5Yt`3hd*n@qW*_lnCQW!28sYt0ZvxK2Y<@AWkDZ|7j@J=x8F+!kHZ89kB*AJ-8g1GD*u zow7|oAVEY(hm)57m`?P_&{`8`tysPDsnKEbpZ^C~i+vYkmB_Bv7lJf1#E5?cc$5rV8vLnjoIw@Y07iUR%_w2_!lV!5VdswN7sc@ZmS3u@q875hFB&L=ulgA|e#YL_SjB z{EcyM9+sJ5l-LfW?Ipu)SA3qP*{D6cfLYrH9WhGoJ@U4`ql9ZOTob3E)LNhg)1HcY zXZWGy{L@_n6SB&In?mHke^uv5F*DcqZZnjg%x-kD)+lmns>>N`lU%NzS_-gv7?pl|$Ea3#KA3Vk_D8GNhrQcj>Q=mB8lA3NA&IQ*VRu80f>2 z{}|&NXo)A%3K zVVD*{+dv<>+w(QBt8IQci-6rFwocBSm3NF_R{a&yN@&GjA+3Z~{1uW4)J#=2*aB+# T{tX5~MTJBv4FLcER0dHv7Uv|$ literal 1074 zcmV-21kL*>wJ-f-s1PL~0E8+;LQVmA0000-OhQ;LO+-*L0RR60{{a606czvg#LWT# z)%cKm9mts%MN?Wd0RIpMk%tI|3c(Db4ucS65=9go7X23vAprn@+xRkVnHK=auvb$A z00062007t`@jw9(7zy+G%-?mH7mr>4003_)w=0LTtoAa+a_NieJy-dYm@<4z3iqXd~3B~=XcfU@QefR>m501z0n0qGJL0C0o( zVSa@cfZeN$lLP<&fRa{&0KA}Ofa+%!N_0D<3jhEBKv{DKe-9uE0002E3;+NBf8IP^ z;78oyF#!RK@F68PR6+m%WdSAMJOBVT_Dui)p0So~Dr65U3p`IhQI{*Ou6HWm| z0NKl*Cz%)P4`2{Nh34E{LjVBZ2=d^iVD~j26eD|?7h^>wfM+AQQ2h%40NXgG9JYv0 ze;2a}e3=*Hpd9`S004(P16&I4Rw=G$W0@BKc-neRPJ#ioYb*ji007ntIsgDm^|VZY zm=2}^007BB7U~}|7%M$DXk(cdfW`A`S^GaUHUI!E4ZDqo<$EZZ7aAUbL%#7wiCc*G zIRF48C=-;x<{JR1#b-@HFGs@Wkv;$bB`DEz(a;-$nHL>k+2{rU00ImE0L(-H011Et z2I=Q+RUn3$7ezjF0IokjQ3wnG0O3J^xTql|V~&xC$Vif;G4KKznUUxL5&+{QPDM&F zVu&GvAtHgqkBA6`UJeSCs}jVxI>x~If9Z2bx7}bfTPC(Dr1xz%8WOpx%#$vYE7j(?s%AYsDSxh_JvHRv=*HJd;fgZiN9o zAvAy6K-ccr57HT^iBFWtm#xBgEm?Grb>Z*r{zfvYrBh=tKGc+nf2G7-Hp34Y8aiBh z?gj&{kUI%U&bwcNriXmo2b82D?Odebt!{(etWL0CjLp7K^y3?`j*16PYq_1w=?j+a z7+eV|vZ%Pw0?{)NKu{jR%~fBP4lGdnhHSnV6pyQnVVK`4EY+ag+0Q~UmkUhDsTaE% sTC^T!d7FZ^e=ke4|Ch6jwtjz9b!sMBT{Dh2j}rxFMFDvf6)h%;T5-dp?*IS* diff --git a/tests/data/NDEX.IMPACT/test_data.definition.dbn.zst b/tests/data/NDEX.IMPACT/test_data.definition.dbn.zst index be18b655aad81c86c4d8544a2bd408901f7abac0..40187ad16e865096620006f99d6018d364bd3ff0 100644 GIT binary patch literal 265 zcmV+k0rvhVwJ-eySj-0i5=$yFU`^8)gn0rkX-lx6a8YoOh0Ql00)CMDFfo|squFlr zX{of-(x@hiF`I7gCCnLXG6ky|x4^0P+^zR8E|7chZDX)2LKPhLy8)eZqNxD;>+Y&U z$H=<6-3Gzrza#k=c%P6G5ftrpC$xu42FDEq7Yq?4NeWa>nw6Y1Dt~@3MGzDLp8z>bbZX&A>m zM=Pm*CghwZF9d PXppfm69E7KlxgrY{hWE4 literal 303 zcmdPcs{c17kwKN2;R=h3lOIzgBLjn-WzLk8 z;Z;axRMSnDysiPVaRNgvE5jAG6k#idnKj{18yviS82;BYFnGV+e|Nk1e+Gf2S6)0) z{`{gID0pg?b^U+VddB+aOfT5~|7T!u4Gm&2Kz0Dg)gTwSg@*WsX8>L2=H|@spON9e zBa_U3zxw+BvJCwjS{*Gm_;I-jEO=_-6@0g2!xP_}grf&}SPnZ(SkK{bNS>!{Q}jkg fqc;Kx2V$BV6PP%HLM|vM&){ad#UO4dw#5kmTfJsW diff --git a/tests/data/NDEX.IMPACT/test_data.mbo.dbn.zst b/tests/data/NDEX.IMPACT/test_data.mbo.dbn.zst index df458f7d0002c69c758db7be0f00f95f5ff62008..d333ecea35f41ee44542db962f177dc6b9ac19cc 100644 GIT binary patch delta 139 zcmZo*dc(-Au~q*sOT-g4hA&(bd8`~685kJ8T{B_)4#cMB#>S>bmV67W7-rUlGdP}E z;d>@-wfL_OGQ0;mXZf%Jm2x>cX(qWcJOB#M{L8{1+QKw(t%;E^!-1oxB$z52RTE++ rSe)SK;l9N9^Uf-<9n3o0lqNH+ZMemL$;d+L05{_dpnV?$baw*)9(*xM delta 183 zcmaFE*ucc2u~q+XN+JU*Gs6^?iQHE8->#W3eg|Sxb7NyuBTEj31_rKhzct-q4sV$= zC1rRYv@-a_%|9~_q)+96E(=2o8{YyehM6_t431}3_@0SdE&l6+3@-x^9O#_o!v>_e z7#y86lUx}d0Oe->WnmC)VPcrsqb=bAuV_1?}3RKzN#__;*o(!_Z(@;nR&j-HZW es%%tEh?!td_4>dj7$_6Kmy;cnJ|6_VpDTt zV^br`iFGE7tP?j`Fv?AQt)+NK=IN;%%}I$GjNvIk3q%8cJ4CU$GoOfP-~?*TKg{<8 E0KaP#W3eg|Sxb7NyuBTFuZ z1_rKhzct-q4s3-lgr-MFG#lIufrAkezXqj9#$9J=thmO;yNB7$Zp_ z=>}v9e=`^?;*oJY#JzBR6(`RP)d_B!#fe;u-z!i}cXhH3lmLDxLmSF+`001!w?eyFurAPn( literal 366 zcmdPcs{c17kwKN2;R=h3lOIzgBLjn-WzLk8 z;Z;axRAV?kzefXP;{ldZc7_zr7%nS@nKj`I3+~MQcRvnj$AiEOHlV?*K5Dy&+t|Jq_=hr}6)< zZ+Pt_&H?29P;%}pki{$^`98yyuiQcrwW F1^^B$e4qdT diff --git a/tests/data/NDEX.IMPACT/test_data.ohlcv-1d.dbn.zst b/tests/data/NDEX.IMPACT/test_data.ohlcv-1d.dbn.zst index ce8b42c4e14e85d49c1f07486c1c5f41e2ba6614..acfd89272baf5cd9600943d62d36231c23d6b647 100644 GIT binary patch delta 109 zcmdnaxQLNkW2^pOmWU}V3?=LndF&z?85kJ8T{B_)4#cMB#>S>bmVC-q3^QxO9XM|^ zu>oZpm5{rB`XZHt28fgt*BMdZC=cnqH}-+XyUq6 GO_Kok5+Te0 delta 130 zcmZ3)xSf$lW2^q(ltczqW`-*)6S?hjzg;t7{0_vX=ElaRMwT254Gdi2ervkJ9Nsc# zO3LsmBr~dA`_VoLq))}-AR_~xvK7P3ns5dNh9%3txUUv>;Jneq2IMs`6>b304iV=w V!Tgu^dO`Hp0w4hd=fB!y0|29iC^`TD diff --git a/tests/data/NDEX.IMPACT/test_data.ohlcv-1h.dbn.zst b/tests/data/NDEX.IMPACT/test_data.ohlcv-1h.dbn.zst index ed5dd99b8bc1b5b3d851800ad6128886b6c9afb0..d91570c1429bca6dac0afaef3bead4f30cc04f31 100644 GIT binary patch delta 84 zcmbQm)Xl`Lu~q*sOT-;^1{I!(Ja)p23=9n4u9+}?2VzrmV`Ebz%ZasW6PI{1N>2PB nqd3i={h$Iz$H7-c+D4iexK`9E=r%8AOwl>O0@S$kPs>gKY@Hmq delta 98 zcmeBXn#IJUu~q+XN+N?QGs6{@iQIO^->#W3eg|Sxb7NyuBTEj31_rKhzct-q4sV$= zC1rROk{Q+d*-mc*=~H=dmxV!vePWE-#6}-R;fd#D_*OOSX6#{*UZd;x`ha*n02JRL A0RR91 diff --git a/tests/data/NDEX.IMPACT/test_data.ohlcv-1m.dbn.zst b/tests/data/NDEX.IMPACT/test_data.ohlcv-1m.dbn.zst index d49fa63d53cb024807ae2499ad83c99ad2e75b84..90012ee1c8d1924181ab3151c9308c0ac4516281 100644 GIT binary patch delta 104 zcmaFI_<)gHW2^pOmWUQM1`)1_Ja)p23=9n4u9+}?2VzrmV`Ebz%ZarztZYDG-HEgH zO@tX37!??p?sFwFh^^TtdWhk?D3`*IYeGdQ^wK75Uo@R*ZNn}0OGXw_4#${$fQH5X Hnvn|tD}*3u delta 118 zcmaFB_>PfBW2^q(ltczqW`-*)6S?h-zg;t7{0_vX=ElaRMwT254Gdi2ervkJ9Nsc# zO3LsmBr~erxpD0fNT14sTClbl875tZiM9GFd<+bX3Jgs5xe^(~*6b5K#Bg4eOW{YD S>I1eposFzVIhS5aQ3e1Sr6+Iz diff --git a/tests/data/NDEX.IMPACT/test_data.ohlcv-1s.dbn.zst b/tests/data/NDEX.IMPACT/test_data.ohlcv-1s.dbn.zst index f5d2631769f1b359a8bd8e0111ba53704e4f645e..b1124e1dceb32dd3d1e07f16dc7ca58460d8e6db 100644 GIT binary patch delta 98 zcmaFHc%6}3W2^pOmIxa*h8E6=Ja)p23=9n4u9+}?2VzrmV`Ebz%ZatJ6K89g2m@sz zWX~`%G+f>8RKcnfu`je;=2N4LG3SO|ce;GIwlHPMdz^ld!637T9W2^q(ltczqW`-*)6S?h-zg;t7{0_vX=ElaRMwT254Gdi2ervkJ9Nsc# zO3LsmBr~cNtZ-il(x>vkmYJc2WnzphBg4c>Ek#}i28IaPGmH!kSGPM=uO%<003FptK9@jDs=z= delta 170 zcmcc5_?eMMW2^q(ltczqW`-*)6S?h7zFjk6{0_vX=ElaRMwT254Gdi2ervkJ9Nsc# zO3LsmBr~c_+qm-^SRVt!TxNzPEE8iSVi}IGu`)0*FfcGOdKFD z&ZulCQe08Up^+vOu~Q&`O<~4Nh8(_y T3#W3eg|Sxb7NyuBTEj31_rKhzct-q4sV$= zC1rROk{Q+5qva2Q^*J!)vM@|xn;4@k`}0}cYVqo~EDVAn%nS@Enhhch3=E7w4CFCP aoT$mjKXJDtkH8iYeW_`;m_M?KKLP-orzD#I diff --git a/tests/data/NDEX.IMPACT/test_data.trades.dbn.zst b/tests/data/NDEX.IMPACT/test_data.trades.dbn.zst index 7179fbb82eea61617f8eb9e45eb31e280accb666..c7e44c48eb31409f7dfe7056fd17039d462b5813 100644 GIT binary patch delta 121 zcmaFFc%G44W2^pOmWV&B3{yBK@~COEFfcMOxPu6W5I0{1H(w6}10xefu)w!#CXC;K z*wozE*wn~!Vy!ZJ^;;GO!4T$&Yvr{085}eY@Eh+6S>vA85tPd8Gr;sh?}p1o3Dp~fsu(K!?$ZDjNgIS z)ZEzE)X0*Pp@D%b+;2^Hn8RD>i80E&KcB^| q7O#HG!XOyJ%rLP{PMm|mLE`|w0iOcT5q1{#ea)ULY*+t05DNfkzAT~u diff --git a/tests/data/OPRA.PILLAR/test_data.definition.dbn.zst b/tests/data/OPRA.PILLAR/test_data.definition.dbn.zst index 33852e14731527ac15489dc0bc8008cbf8525b07..c60a902c6020d379b800b182f30758ace51ec16c 100644 GIT binary patch literal 256 zcmV+b0ssCewJ-eyScC@vQi~}tfJ@RCFacx+R(=cvN0R{qAazuSo@2w9hDxGrYrA+h zK4t?UglrcXkcN-z{43chuWDs?ai7VR6!he&I@(t-{sFftdJ%5r&Cm0;LAy1ReD z#s}qx2oVW|*VP?xSZJxyjI@wnf|)>|BATFxO`8rjZ8(0mb5sHaIX8hM$w|@kyhQxj zlxG99^Z=nUzNLkM#`^=l@V$maz$>k16HbMp3LgNV4!8_J6y|U+hz3$|5ws|ba2J84 zi3%PAPJjcf+vCL>BJhI6m10Bb00itL91O$_yK)VTr4~B@t$o<2T~K@6NgzRwb!oB(kKhS{bEcLj*E zF#Km^aC8jtQBW{4HZV3ebT%+BG&MIhFxaudgmEVj8yT7yaWU-VUA5?qq(h(>%c;mn zg$D`+JO@rQm7m*e=AZ$zG9`n-mz7}&Ta>UILmUUtk{9BC`uYOIt-YoH*Z;3)V6cpM z6&_g6#K6G#pMjwsh(m*2uv^PekY7-gsA~vzH;@Q%1{&wSd$o$f|NoBv|Nr-^2We-J zV(=Gc(Wtp9V8*MyO=12)E~d0i%qOA>Wb$tNJeDin%-z6HEMU=eK!WpKf~K}w+a(5$ T8=e*t9nxjt8<`h(ZhQa$ShroY diff --git a/tests/data/OPRA.PILLAR/test_data.mbp-1.dbn.zst b/tests/data/OPRA.PILLAR/test_data.mbp-1.dbn.zst index bff5cedbd9e6e3c50ce74dc6258593cb4d484fc0..d834885e3dbbd5bd0ccf18a20547e1d5d4b24d6c 100644 GIT binary patch delta 148 zcmaFBc#Ba`W2^pOmWU8G1`|#fCqJf0#)*PT`iu+=?hHVJ!O=0mM?t~J*udD>(AmJi z(A3=2zyK_>V}l9fP9QciG%=c3FUt;8%;0XqIB|i5jX1*rL6;kqyc0Nh%oQZ|uv}n# u^GW1F1B1FmgkplmBkTMwX>PWF?z!Dz4vlQj4J`N%oMtKq+V<{AfFl4c+$X01 delta 183 zcmcb`_<&JRW2^pOm2;-d3``iovdIDB|LcKl21ma`4B=ZIx|=XEFo-b9sl?3syDKJV-+%2Q@ mT>Cqdoi&$YH+E*Qiewb@slW2^pOm2;-d3`ba8octJ%F-#OxO8?Ku;OH3Oqo819Y+!6`=xktMXlib1 zV6bC@3FA&6HZn9Z;$qmzyK2!JNryl&mQ#_D3J(+tcn+LqD*u%9`6k%V2f9E*nfa7N z7~(h>7#I}R$nKpMAkLuqp^-s9BQhN*G_gv7QE1|98NRQF;tr&@9$+wF+$?eaF92jI BG93T_ diff --git a/tests/data/OPRA.PILLAR/test_data.ohlcv-1m.dbn.zst b/tests/data/OPRA.PILLAR/test_data.ohlcv-1m.dbn.zst index 7b5fcd60d599373ec6250697b558ed131019c3e0..367f8be7b9a37d28999af7bdfc884735e33a4da8 100644 GIT binary patch delta 140 zcmaFFc#Tm|W2^pOmIxCzh9?{@PJT?0j1vWw+!+}d+!=rbgQH`BkAi}cv4OF%p|gR3 zp{cp4fdN=%#|9I|oj`14Xkx^tD8UfN0aRYVVU{o>K%C*vz6Ot9)l3s-$umk!+$E(n nXZ;=SA8P~{^qLv3Bt2gw8nED?CC@qL51S6i0&S7?iqioAos=hl delta 152 zcmcb{_=r(ZW2^pOm2;-d3`ba8octJ%F-#OxO8?Ku;OH3Oqo819Y+!6`=xktMXlib1 zV6bC@3FA&6HZn9Z;$qmzyK2!JNryl&mQ#_D3J(+tcn+LqDt|jCXgb)?2cpajTbTJ2 zB^cs37#J7|ILs1e1c)>I+1J3}@vE8%C^WH3o>6e(EGfP#+|`WpMVXinurzik{09Ib CDlcdN diff --git a/tests/data/OPRA.PILLAR/test_data.ohlcv-1s.dbn.zst b/tests/data/OPRA.PILLAR/test_data.ohlcv-1s.dbn.zst index 0aa041a41236bdf58794851c7137586857d6efe1..95743579391bd018b93ebe23bfee8853e1273d6c 100644 GIT binary patch delta 140 zcmaFKc!yC?W2^pOmWUKK1{F>hCqJf0#)*PT?u-l!?hHVJ!O=0mM?t~J*udD>(AmJi z(A3=2zyK_>V}l9fP9QciG%?~+P+*AT04guwFiV&bAl|@zq;^qE9nZvB%8Zf|_sJ+F m?>m*J#wNhLgNH4kdv14_LnGUB0}K8GrxgjW2^pOm2;-d3`ba8octJ%F-#OxO8?Ku;OH3Oqo819Y+!6`=xktMXlib1 zV6bC@3FA&6HZn9Z;$qmzyK2!JNryl&mQ#_D3J(+tcn+LqDrZvPcMELj17Bu_FU))j z3Jh@^3=9kf9A*hK0>m4*kJK_OimBrP3Qeq1W)zw@Pli9(qIq^_qUtdQ1I91^MVZ0J>Etj{pDw delta 179 zcmdnbc$!gAW2^pOm2;-d3`ba8octJ%F-#Oxs{hZ(;OH3Oqo819Y+!6`=xktMXlib1 zV6bC@3FA&6HZn9Z;$qmzyK2!JNryl&mQ#_D3J(+tcn+LqDt}Pq9t$!wg@GZIiQx*9 zfP_3l90vmf!wd00eSHDq*51

yN54Fu2q*Z~}!H83ZJxf4D@uD!Qq-^Drm~GFVA| dFk}$=5St)qAbqGHL1BlIUzf0##COJu;{fjHH6Q>0 diff --git a/tests/data/OPRA.PILLAR/test_data.tbbo.dbn.zst b/tests/data/OPRA.PILLAR/test_data.tbbo.dbn.zst index ad109c2704345364fb3d175148b125321aa4a16e..0a65a315864005e0ccfb12c9171725bb33188954 100644 GIT binary patch delta 227 zcmZ3>G@D6KW2^pOmWUV*1{EF`CqJf0#)*PT&WsEU?hHVJ!O=0mM?t~J*udD>(AmJi z(A3=2zyK_>V}l9fP9QciG%*rkRA7kX04i@j;hQicK)iwbNbSVwD)m4S2DXsqXof{G zb-tV0qHlyTGc=^CHOK+QMHm&Ic^6FzZ)AMs<6qRg8YrWGa5tB-fU~>Jj#Tp+HP#y+dCvrjC(b=yB;lZIDXyz= U;rWKo2Tli-1Kl1M!K8W$04J`uI1UB|hUOE#2{Qu38@P|u0(lG*%T$WkLYku)7RA*0Zfc9Z5ys5WkgC?e2$Yip zVi86~@1jZJjjw$Ci<(yh<@68k=8|SeI3mI!*(Km0aDZ7#Zl-p?9=$_K&bCZ(Ofwqh pYKoi?teBP0&*jP?BfxZ%)h<|r;e`ZGkMv&2#Okx^ub-BD0sv=wOWFVc diff --git a/tests/data/OPRA.PILLAR/test_data.trades.dbn.zst b/tests/data/OPRA.PILLAR/test_data.trades.dbn.zst index 316f267ffb3e3460611c15b7b40ed9fa9746e573..d3a85d694b84a705c1ce74f88600d219e1b558ad 100644 GIT binary patch delta 150 zcmey!_>@slW2^pOmWVZM3?f`EPJT?0j1vVFf>{_C85rC_1cRevfRBQLk+FfXv7xhp zfuX6nseu7lX2%8-#+^WHWN2coaiW2^pOm2;-d3`RT#kvL&evZKm-fn*X9smFT?^j>{Uz#C4f=_1=gJ$1h@fN{%P7K_? z*^UJE)Dltq_KB`)+2{I!kgTa=C;R#FB!~nG<28JjFkO2i1d5Oik z1`G@levZKm-V6+Ge*Yc+|Nrk-U;m$hfkA>npkx!b@@fkvO@^B{p6IAE7(_-gao2Ip n$b6^Ky_E4xB9Foc(Z;o$?=A!#U}#8OAYi_ZSC!|Z!rXNLLaISv diff --git a/tests/data/XNAS.ITCH/test_data.imbalance.dbn.zst b/tests/data/XNAS.ITCH/test_data.imbalance.dbn.zst index d90b32519d06f87872d969f7e08f244d084d1056..8d47c7f5e904cc9c03badf57654bf1a1c5b6e024 100644 GIT binary patch delta 165 zcmaFHc#~03W2^pOmIxmviHZe6d z3IKoru-VgsqHF++vSLBNLEspGuL0NrLg9gKD77#BWWWH$0{}z>kxv&c0H%wTeiH!z zMH^2+0000^OaQ>fRYG*^fDiMO&=~Ymg#s{0#wcf#7zv2PB*9dXfiOo$Y7cHWaamUX*y=* z6BpWAi7_~{&uhMzuwZSOWEPJBp9IGYHuHqW(V!f>zKZ7&-yylAu3)YrNX7L#CNpQ?yD{rV#(9cL- PeVo75n|lVs3&kt|SVK1L diff --git a/tests/data/XNAS.ITCH/test_data.mbp-1.dbn.zst b/tests/data/XNAS.ITCH/test_data.mbp-1.dbn.zst index cdaafa4167664348441259af0523d0a6be7e4479..10e37031e3b2e1e82a1f1ab2ba751c0b5902206e 100644 GIT binary patch literal 268 zcmV+n0rUPSwJ-eySmXx)q6;lFKvUBg2qU`YT7?;{>**B}F&Flo9~n4-I}~xY=y*c2 zXPM;VsP6>rmPfRf*}0@ZblWXcM-$Lbn8rfLhhy1f^=>B546W3$@dhniNcu%acIdKt zH8Cj}*&a2}PZ><~eN+eC_M)@$4oP~B+V|5$XLAX?e)xOjoD?jpM+4%*f${_a!yTA2 zp<2p{ma+u?{lYBhzd!sQWd#7vjLOR@BcZ_X?+^|(0)gV97rz*gHedlB08;>9sK<9O zHhSiPVIT!qSW3VYcG)>_kS(YOUKr|eAOivzDmAp3!UF(fFa|g7P=sse@BpBTRzSf4 S^cWlsU}gYe0RRA+DCRdFfObOw literal 283 zcmdPcs{dEz+*~Gx5@r`CKgMGW3=9!|j=_4KA>`Xz4BQe7486H4Dz0Az8Y@U996O1%V1F^0$VZNB06ysj1~A^5F%zB&U#TD~CTH`70+cY%gB%xz|_ zw2F6WWnqwF2xdrlasI@b-OU>>8uRloYaTch#I$U~>2r;WKTkw(X*H*}SIKlResY+t ZX~Q7Ucr@r+K{K!GqOut&>s)TL0{~)QUV;Ds diff --git a/tests/data/XNAS.ITCH/test_data.mbp-10.dbn.zst b/tests/data/XNAS.ITCH/test_data.mbp-10.dbn.zst index 6e961a7d4f80e229300fee5a37cfb56cb976a1ec..bc7d230abed83dd09ed46cf0fd5e61324280dc55 100644 GIT binary patch delta 229 zcmdnaw2(h8AuYCqJf0#)*OonT!k!?hHVJ!7t3k5zOGRH(}%jVlzWi z(-ltq_s>!ieg3L%cz~alsu!Mc0zo-oZ0|OHfJNh*-+??j4 z$-wX;gN5OLJtGkRuV?r_@yW`DX*y=*KqZVXcvpo36}$v0U<{Ax+k9iCjsr5pfQ~6{0iN|Ke fwFm8jEMjD(GYl@btd)?ENNOl}`(Xb6An~IB@Dx?x diff --git a/tests/data/XNAS.ITCH/test_data.ohlcv-1d.dbn.zst b/tests/data/XNAS.ITCH/test_data.ohlcv-1d.dbn.zst index 8983b90e730f0e3af68a78d8e90088effa2bbfb6..07f8ea6a03d884776386685628119900bd03533d 100644 GIT binary patch literal 144 zcmV;B0B`>&wJ-eySOf$Bl13RMfG;!#01S}=B8Gr9?JP1>{&cy2VTi4uWO3f@A#II( zWL#UO{~t`D|HBp}PfEl%tvdgp4J1YoUJAo|4v0EfPzTwsN2%Fgr@Rhm8G#Bl`Zi0K yN8}$|nM_Lq6L<5>9->UycQ835>_6;N;hJqAYum>EDA0RRAnfEPLEpEhm) literal 165 zcmdPcs{dEz+*~GxDaK zXJqgTb8+OcH(}%jVlzWiQ!a)Eh7C6_H!aFx=-HIyc);<2M8kT<8;o<-+_KgHnQw8B wk%3Q{iGf=JXdui)DOSHq5V!cz9x$!lcdrb@U!u-a0i+!=rbgI}17BbdQsZ^Fn6#Ab%3 zrhH0F4BQex*#&!F9=aML#vsM&SLvC%yt1LFGs=X~p-hVNW{6nB4xc)Py!REAK!p?M zY1>OM1gG4X$)v!+QKNlCJH?TwfF+_%`-#udlL1Gzi50XTRJd@4<$y)mkW2^pOm2-2M7^X10IQcOiW0)wYVD+Do!7t3kk;mSIkr#-~3{6eB7#bKh z+`QbhD2JhEQltq_O8>j^0BpvCR2GIOtP}l3A1BLjm& znH1;E5V3|GK6MOv?<*!YY8wbM1gG4X$)v!+QKNlCJH?TwfF)v|sF6UCTI*US$*vh| UxgPWIFrD=|aO)9E`%fzu06i2t0RR91 diff --git a/tests/data/XNAS.ITCH/test_data.ohlcv-1m.dbn.zst b/tests/data/XNAS.ITCH/test_data.ohlcv-1m.dbn.zst index 3050538b9298d8792e9c5dfe5fd6f8397e699ecc..fcb1e3876d895c432ed2ff21fccc1d720d0f664d 100644 GIT binary patch delta 168 zcmX@ZxQ|g#W2^pOmWUEo1{Mw%CqJf0#)*Oo^^6P*?hHVJ!7t3k5zOGRH(}%jVlzWi zQ$9r|25t$UY=qKbj%y)e3{tFqm6{1ZRvQ614IAb%?yz6t01|&Pzmr*@eU6`J?sAY} zARx@JfJuRwWs7JL15-q;c1w5zE7Nwd4bElSqD<2m9mLl*G%>&6GqjL8aE9R!P}8$1 GFZuz-bSu69 delta 164 zcmdnTc!p6>W2^pOm2-2M7^X10IQcOiW0)wYVD+Do!7t3kk;mSIkr#-~3{6eB7#bKh z+`QbhD2JhEQltq_-ZFXE3^wDzUnYhp%oF`ZeHa)RG=Vtb$7-W%Az}?1 z<}&WEU*hm&ekZd)`y4;d+~tfwSrFi5Siq#f%(6waNZYib#}dp?U})U7>ZfE3+cd-C Heg-c9yg@cS diff --git a/tests/data/XNAS.ITCH/test_data.ohlcv-1s.dbn.zst b/tests/data/XNAS.ITCH/test_data.ohlcv-1s.dbn.zst index 729b0a46f7c10064ae5d8ab51caf7d7434b944ef..1dbdefb8250ba911a9bfee63409ac42f22ce58c5 100644 GIT binary patch delta 145 zcmdnaxPnnoW2^pOmWU%P3{Tixocx#~87B%Vlru6gxHA9=2EQ;DM=*oO-h`1Ch|LU5 zP5Bg<7`P>XvKNy3xUYqXF-WocRjPO_jpSoF@JvOJ@t@))E1+=0hPi?a4Gp3|%&?Qe n6NneCm3lMbu2*YAD#Ha=?FA1Qs(AX?42&BLfL3Y0lWqb4BflsI delta 143 zcmZ3%xSdf@W2^pOm2-2M7^X10IQcOiW0)wYVD+Do!7t3kk;mSIkr#-~3{6eB7#bKh z+`QbhD2JhEQltq_CciSy2b=LAl!<|bd7{6lIRgWO3J`lNjpSoF@JvOJ p@t@))t7{=*4IAbHO=%DXVuqazo>L_4KSezoqs`7y5Ew8XBw`Ax>>L93g3&1q^O*cBMSyk=1ON(X1=Jou SkHOIZW(E)j0RR9}j|$8Z1$g=Z literal 286 zcmV+(0pb2AwJ-f7&Yc1PlmkRUP65UM003A{K~pYCR6|Gr0{{R3Kr>6YR2BdLkHp^w zRTcyQ{{a9_RzyJzKQsXiKQsY0F*P*`0Du6n+0%leYygb1VnM(`;240f0oVb)P>;2?Au_sAdWmZ1da)JgscV!ymv%`r&R8 zM5cfI<>8W6*JVA;D6Q=_@;I%6kbpoDUjOH02|Z1qr=`eZD~W@Ls=$$=$~(9(e3@BK zmx+Jl#}MX4J>7Jv?WF#Uzq8H6S_Luh(_9{b4Ty#n08dD<)Q%}^vvX+J{1o+Uj5a$* kL14gmkccU$vU3RB3r43v!Y+t9CN+++eIv>O2ZI<$x#)g9+P2e@#Y)i4~TLJPg+DY`6Wl fo<0{SAYqWdinrXa>2yP7@Wd6eYLfF0GOq#v(Sj`I diff --git a/tests/test_historical_bento.py b/tests/test_historical_bento.py index b5057a7..4c6c1ce 100644 --- a/tests/test_historical_bento.py +++ b/tests/test_historical_bento.py @@ -90,7 +90,7 @@ def test_sources_metadata_returns_expected_json_as_dict( dbnstore = DBNStore.from_bytes(data=stub_data) # Assert - assert dbnstore.metadata.version == 1 + assert dbnstore.metadata.version == 2 assert dbnstore.metadata.dataset == "GLBX.MDP3" assert dbnstore.metadata.schema == Schema.MBO assert dbnstore.metadata.stype_in == SType.RAW_SYMBOL @@ -123,7 +123,7 @@ def test_dbnstore_given_initial_nbytes_returns_expected_metadata( dbnstore = DBNStore.from_bytes(data=stub_data) # Assert - assert dbnstore.nbytes == 209 + assert dbnstore.nbytes == 189 assert dbnstore.dataset == "GLBX.MDP3" assert dbnstore.schema == Schema.MBO assert dbnstore.symbols == ["ESH1"] @@ -171,7 +171,7 @@ def test_file_dbnstore_given_valid_path_initialized_expected_data( # Assert assert dbnstore.dataset == "GLBX.MDP3" - assert dbnstore.nbytes == 209 + assert dbnstore.nbytes == 189 def test_to_file_persists_to_disk( @@ -188,7 +188,7 @@ def test_to_file_persists_to_disk( # Assert assert dbn_path.exists() - assert dbn_path.stat().st_size == 209 + assert dbn_path.stat().st_size == 189 def test_to_ndarray_with_stub_data_returns_expected_array( @@ -925,7 +925,7 @@ def test_dbnstore_buffer_long( ) # Act - dbn_stub_data += b"\xF0\xFF" + dbn_stub_data += b"\xf0\xff" dbnstore = DBNStore.from_bytes(data=dbn_stub_data) # Assert From 1fac92c80c8d247b394240bd9fe3003c9e4d547a Mon Sep 17 00:00:00 2001 From: Nick Macholl Date: Mon, 10 Jun 2024 12:07:31 -0700 Subject: [PATCH 7/7] VER: Release 0.36.0 --- CHANGELOG.md | 2 +- databento/version.py | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 60b14be..a0c9a48 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## 0.36.0 - TBD +## 0.36.0 - 2024-06-11 #### Enhancements - Upgraded `databento-dbn` to 0.18.1 diff --git a/databento/version.py b/databento/version.py index e0d0ed6..d9f2629 100644 --- a/databento/version.py +++ b/databento/version.py @@ -1 +1 @@ -__version__ = "0.35.0" +__version__ = "0.36.0" diff --git a/pyproject.toml b/pyproject.toml index 0a220e0..0321f0a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "databento" -version = "0.35.0" +version = "0.36.0" description = "Official Python client library for Databento" authors = [ "Databento ",