diff --git a/CHANGELOG.md b/CHANGELOG.md index 54454f6..957da2a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ # Changelog +## 0.17.0 - 2023-08-10 + +This release includes improvements to the ergonomics of the clients metadata API, you can read more about the changes [here](https://databento.com/blog/api-improvements-august-2023). + +#### Enhancements +- Upgraded `databento-dbn` to 0.8.2 + +#### Breaking changes +- Changed `metadata.list_publishers()` to return a list of publisher details objects +- Changed `metadata.list_fields(...)` to return a list of field detail objects for a particular schema and encoding +- Changed `metadata.list_fields(...)` to require the `schema` and `encoding` parameters +- Changed `metadata.list_unit_prices(...)` to return a list of unit prices for each feed mode and data schema +- Changed `metadata.list_unit_prices(...)` to require the `dataset` parameter +- Removed `metadata.list_unit_prices(...)` `mode` and `schema` parameters +- Removed `metadata.list_fields(...)` `dataset` parameter + ## 0.16.1 - 2023-08-03 #### Bug fixes diff --git a/README.md b/README.md index 9aba4d9..7da45a0 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ The library is fully compatible with the latest distribution of Anaconda 3.8 and The minimum dependencies as found in the `pyproject.toml` are also listed below: - python = "^3.8" - aiohttp = "^3.8.3" -- databento-dbn = "0.7.1" +- databento-dbn = "0.8.2" - numpy= ">=1.23.5" - pandas = ">=1.5.3" - requests = ">=2.24.0" diff --git a/databento/common/data.py b/databento/common/data.py index 0e2aebb..39580a2 100644 --- a/databento/common/data.py +++ b/databento/common/data.py @@ -29,22 +29,6 @@ Schema.TRADES: TradeMsg, } -################################################################################ -# DBN struct schema -################################################################################ - - -def get_deriv_ba_types(level: int) -> list[tuple[str, type | str]]: - return [ - (f"bid_px_{level:02d}", np.int64), - (f"ask_px_{level:02d}", np.int64), - (f"bid_sz_{level:02d}", np.uint32), - (f"ask_sz_{level:02d}", np.uint32), - (f"bid_ct_{level:02d}", np.uint32), - (f"ask_ct_{level:02d}", np.uint32), - ] - - DERIV_SCHEMAS = ( Schema.MBP_1, Schema.MBP_10, @@ -52,184 +36,21 @@ def get_deriv_ba_types(level: int) -> list[tuple[str, type | str]]: Schema.TRADES, ) -RECORD_HEADER: list[tuple[str, type | str]] = [ - ("length", np.uint8), - ("rtype", np.uint8), - ("publisher_id", np.uint16), - ("instrument_id", np.uint32), - ("ts_event", np.uint64), -] - -MBO_MSG: list[tuple[str, type | str]] = RECORD_HEADER + [ - ("order_id", np.uint64), - ("price", np.int64), - ("size", np.uint32), - ("flags", np.uint8), - ("channel_id", np.uint8), - ("action", "S1"), # 1 byte chararray - ("side", "S1"), # 1 byte chararray - ("ts_recv", np.uint64), - ("ts_in_delta", np.int32), - ("sequence", np.uint32), -] - -MBP_MSG: list[tuple[str, type | str]] = RECORD_HEADER + [ - ("price", np.int64), - ("size", np.uint32), - ("action", "S1"), # 1 byte chararray - ("side", "S1"), # 1 byte chararray - ("flags", np.uint8), - ("depth", np.uint8), - ("ts_recv", np.uint64), - ("ts_in_delta", np.int32), - ("sequence", np.uint32), -] - - -OHLCV_MSG: list[tuple[str, type | str]] = [ - *RECORD_HEADER, - ("open", np.int64), - ("high", np.int64), - ("low", np.int64), - ("close", np.int64), - ("volume", np.int64), -] - -DEFINITION_MSG: list[tuple[str, type | str]] = RECORD_HEADER + [ - ("ts_recv", np.uint64), - ("min_price_increment", np.int64), - ("display_factor", np.int64), - ("expiration", np.uint64), - ("activation", np.uint64), - ("high_limit_price", np.int64), - ("low_limit_price", np.int64), - ("max_price_variation", np.int64), - ("trading_reference_price", np.int64), - ("unit_of_measure_qty", np.int64), - ("min_price_increment_amount", np.int64), - ("price_ratio", np.int64), - ("inst_attrib_value", np.int32), - ("underlying_id", np.uint32), - ("_reserved1", "S4"), - ("market_depth_implied", np.int32), - ("market_depth", np.int32), - ("market_segment_id", np.uint32), - ("max_trade_vol", np.uint32), - ("min_lot_size", np.int32), - ("min_lot_size_block", np.int32), - ("min_lot_size_round_lot", np.int32), - ("min_trade_vol", np.uint32), - ("_reserved2", "S4"), - ("contract_multiplier", np.int32), - ("decay_quantity", np.int32), - ("original_contract_size", np.int32), - ("_reserved3", "S4"), - ("trading_reference_date", np.uint16), - ("appl_id", np.int16), - ("maturity_year", np.uint16), - ("decay_start_date", np.uint16), - ("channel_id", np.uint16), - ("currency", "S4"), # 4 byte chararray - ("settl_currency", "S4"), # 4 byte chararray - ("secsubtype", "S6"), # 6 byte chararray - ("raw_symbol", "S22"), # 22 byte chararray - ("group", "S21"), # 21 byte chararray - ("exchange", "S5"), # 5 byte chararray - ("asset", "S7"), # 7 byte chararray - ("cfi", "S7"), # 7 byte chararray - ("security_type", "S7"), # 7 byte chararray - ("unit_of_measure", "S31"), # 31 byte chararray - ("underlying", "S21"), # 21 byte chararray - ("strike_price_currency", "S4"), - ("instrument_class", "S1"), - ("_reserved4", "S2"), - ("strike_price", np.int64), - ("_reserved5", "S6"), - ("match_algorithm", "S1"), # 1 byte chararray - ("md_security_trading_status", np.uint8), - ("main_fraction", np.uint8), - ("price_display_format", np.uint8), - ("settl_price_type", np.uint8), - ("sub_fraction", np.uint8), - ("underlying_product", np.uint8), - ("security_update_action", "S1"), # 1 byte chararray - ("maturity_month", np.uint8), - ("maturity_day", np.uint8), - ("maturity_week", np.uint8), - ("user_defined_instrument", "S1"), # 1 byte chararray - ("contract_multiplier_unit", np.int8), - ("flow_schedule_type", np.int8), - ("tick_rule", np.uint8), - ("dummy", "S3"), # 3 byte chararray (Adjustment filler for 8-bytes alignment) -] - -IMBALANCE_MSG: list[tuple[str, type | str]] = [ - *RECORD_HEADER, - ("ts_recv", np.uint64), - ("ref_price", np.int64), - ("auction_time", np.uint64), - ("cont_book_clr_price", np.int64), - ("auct_interest_clr_price", np.int64), - ("ssr_filling_price", np.int64), - ("ind_match_price", np.int64), - ("upper_collar", np.int64), - ("lower_collar", np.int64), - ("paired_qty", np.uint32), - ("total_imbalance_qty", np.uint32), - ("market_imbalance_qty", np.uint32), - ("unpaired_qty", np.uint32), - ("auction_type", "S1"), - ("side", "S1"), - ("auction_status", np.uint8), - ("freeze_status", np.uint8), - ("num_extensions", np.uint8), - ("unpaired_side", "S1"), - ("significant_imbalance", "S1"), - ("dummy", "S1"), -] - -STATISTICS_MSG: list[tuple[str, type | str]] = [ - *RECORD_HEADER, - ("ts_recv", np.uint64), - ("ts_ref", np.uint64), - ("price", np.int64), - ("quantity", np.int32), - ("sequence", np.uint32), - ("ts_in_delta", np.int32), - ("stat_type", np.uint16), - ("channel_id", np.uint16), - ("update_action", np.uint8), - ("stat_flags", np.uint8), - ("dummy", "S6"), -] - - -STRUCT_MAP: dict[Schema, list[tuple[str, type | str]]] = { - Schema.MBO: MBO_MSG, - Schema.MBP_1: MBP_MSG + get_deriv_ba_types(0), # 1 - Schema.MBP_10: MBP_MSG - + get_deriv_ba_types(0) # 1 - + get_deriv_ba_types(1) # 2 - + get_deriv_ba_types(2) # 3 - + get_deriv_ba_types(3) # 4 - + get_deriv_ba_types(4) # 5 - + get_deriv_ba_types(5) # 6 - + get_deriv_ba_types(6) # 7 - + get_deriv_ba_types(7) # 8 - + get_deriv_ba_types(8) # 9 - + get_deriv_ba_types(9), # 10 - Schema.TBBO: MBP_MSG + get_deriv_ba_types(0), - Schema.TRADES: MBP_MSG, - Schema.OHLCV_1S: OHLCV_MSG, - Schema.OHLCV_1M: OHLCV_MSG, - Schema.OHLCV_1H: OHLCV_MSG, - Schema.OHLCV_1D: OHLCV_MSG, - Schema.DEFINITION: DEFINITION_MSG, - Schema.IMBALANCE: IMBALANCE_MSG, - Schema.STATISTICS: STATISTICS_MSG, +SCHEMA_DTYPES_MAP: dict[Schema, list[tuple[str, str]]] = { + Schema.MBO: MBOMsg._dtypes, + Schema.MBP_1: MBP1Msg._dtypes, + Schema.MBP_10: MBP10Msg._dtypes, + Schema.TBBO: MBP1Msg._dtypes, + Schema.TRADES: TradeMsg._dtypes, + Schema.OHLCV_1S: OHLCVMsg._dtypes, + Schema.OHLCV_1M: OHLCVMsg._dtypes, + Schema.OHLCV_1H: OHLCVMsg._dtypes, + Schema.OHLCV_1D: OHLCVMsg._dtypes, + Schema.DEFINITION: InstrumentDefMsg._dtypes, + Schema.IMBALANCE: ImbalanceMsg._dtypes, + Schema.STATISTICS: StatMsg._dtypes, } - DEFINITION_CHARARRAY_COLUMNS = [ "currency", "settl_currency", @@ -262,202 +83,21 @@ def get_deriv_ba_types(level: int) -> list[tuple[str, type | str]]: DEFINITION_TYPE_MAX_MAP = { x[0]: np.iinfo(x[1]).max - for x in STRUCT_MAP[Schema.DEFINITION] + for x in InstrumentDefMsg._dtypes if not isinstance(x[1], str) } -################################################################################ -# DBN fields -################################################################################ - - -def get_deriv_ba_fields(level: int) -> list[str]: - return [ - f"bid_px_{level:02d}", - f"ask_px_{level:02d}", - f"bid_sz_{level:02d}", - f"ask_sz_{level:02d}", - f"bid_ct_{level:02d}", - f"ask_ct_{level:02d}", - ] - - -MBP_COLUMNS = [ - "ts_recv", - "ts_event", - "rtype", - "publisher_id", - "instrument_id", - "action", - "side", - "depth", - "price", - "size", - "flags", - "ts_in_delta", - "sequence", -] - -OHLCV_COLUMNS = [ - "ts_event", - "rtype", - "publisher_id", - "instrument_id", - "open", - "high", - "low", - "close", - "volume", -] - -DEFINITION_COLUMNS = [ - "ts_recv", - "ts_event", - "rtype", - "publisher_id", - "instrument_id", - "raw_symbol", - "security_update_action", - "instrument_class", - "min_price_increment", - "display_factor", - "expiration", - "activation", - "high_limit_price", - "low_limit_price", - "max_price_variation", - "trading_reference_price", - "unit_of_measure_qty", - "min_price_increment_amount", - "price_ratio", - "inst_attrib_value", - "underlying_id", - "market_depth_implied", - "market_depth", - "market_segment_id", - "max_trade_vol", - "min_lot_size", - "min_lot_size_block", - "min_lot_size_round_lot", - "min_trade_vol", - "contract_multiplier", - "decay_quantity", - "original_contract_size", - "trading_reference_date", - "appl_id", - "maturity_year", - "decay_start_date", - "channel_id", - "currency", - "settl_currency", - "secsubtype", - "group", - "exchange", - "asset", - "cfi", - "security_type", - "unit_of_measure", - "underlying", - "strike_price_currency", - "strike_price", - "match_algorithm", - "md_security_trading_status", - "main_fraction", - "price_display_format", - "settl_price_type", - "sub_fraction", - "underlying_product", - "maturity_month", - "maturity_day", - "maturity_week", - "user_defined_instrument", - "contract_multiplier_unit", - "flow_schedule_type", - "tick_rule", -] - -STATISTICS_COLUMNS = [ - "ts_recv", - "ts_event", - "rtype", - "publisher_id", - "instrument_id", - "ts_ref", - "price", - "quantity", - "sequence", - "ts_in_delta", - "stat_type", - "channel_id", - "update_action", - "stat_flags", -] - -IMBALANCE_COLUMNS = [ - "ts_recv", - "ts_event", - "rtype", - "publisher_id", - "instrument_id", - "ref_price", - "auction_time", - "cont_book_clr_price", - "auct_interest_clr_price", - "ssr_filling_price", - "ind_match_price", - "upper_collar", - "lower_collar", - "paired_qty", - "total_imbalance_qty", - "market_imbalance_qty", - "unpaired_qty", - "auction_type", - "side", - "auction_status", - "freeze_status", - "num_extensions", - "unpaired_side", - "significant_imbalance", -] - -MBO_COLUMNS = [ - "ts_recv", - "ts_event", - "rtype", - "publisher_id", - "instrument_id", - "action", - "side", - "price", - "size", - "channel_id", - "order_id", - "flags", - "ts_in_delta", - "sequence", -] - -COLUMNS = { - Schema.MBO: MBO_COLUMNS, - Schema.MBP_1: MBP_COLUMNS + get_deriv_ba_fields(0), - Schema.MBP_10: MBP_COLUMNS - + get_deriv_ba_fields(0) - + get_deriv_ba_fields(1) - + get_deriv_ba_fields(2) - + get_deriv_ba_fields(3) - + get_deriv_ba_fields(4) - + get_deriv_ba_fields(5) - + get_deriv_ba_fields(6) - + get_deriv_ba_fields(7) - + get_deriv_ba_fields(8) - + get_deriv_ba_fields(9), - Schema.TBBO: MBP_COLUMNS + get_deriv_ba_fields(0), - Schema.TRADES: MBP_COLUMNS, - Schema.OHLCV_1S: OHLCV_COLUMNS, - Schema.OHLCV_1M: OHLCV_COLUMNS, - Schema.OHLCV_1H: OHLCV_COLUMNS, - Schema.OHLCV_1D: OHLCV_COLUMNS, - Schema.DEFINITION: DEFINITION_COLUMNS, - Schema.IMBALANCE: IMBALANCE_COLUMNS, - Schema.STATISTICS: STATISTICS_COLUMNS, +SCHEMA_COLUMNS = { + Schema.MBO: MBOMsg._ordered_fields, + Schema.MBP_1: MBP1Msg._ordered_fields, + Schema.MBP_10: MBP10Msg._ordered_fields, + Schema.TBBO: MBP1Msg._ordered_fields, + Schema.TRADES: TradeMsg._ordered_fields, + Schema.OHLCV_1S: OHLCVMsg._ordered_fields, + Schema.OHLCV_1M: OHLCVMsg._ordered_fields, + Schema.OHLCV_1H: OHLCVMsg._ordered_fields, + Schema.OHLCV_1D: OHLCVMsg._ordered_fields, + Schema.DEFINITION: InstrumentDefMsg._ordered_fields, + Schema.IMBALANCE: ImbalanceMsg._ordered_fields, + Schema.STATISTICS: StatMsg._ordered_fields, } diff --git a/databento/common/dbnstore.py b/databento/common/dbnstore.py index 557421a..1f5aff1 100644 --- a/databento/common/dbnstore.py +++ b/databento/common/dbnstore.py @@ -28,13 +28,13 @@ from databento_dbn import SymbolMappingMsg from databento_dbn import SystemMsg -from databento.common.data import COLUMNS from databento.common.data import DEFINITION_CHARARRAY_COLUMNS from databento.common.data import DEFINITION_PRICE_COLUMNS from databento.common.data import DEFINITION_TYPE_MAX_MAP from databento.common.data import DERIV_SCHEMAS +from databento.common.data import SCHEMA_COLUMNS +from databento.common.data import SCHEMA_DTYPES_MAP from databento.common.data import SCHEMA_STRUCT_MAP -from databento.common.data import STRUCT_MAP from databento.common.error import BentoError from databento.common.symbology import InstrumentIdMappingInterval from databento.common.validation import validate_file_write_path @@ -953,7 +953,7 @@ def to_df( df = pd.DataFrame( self.to_ndarray(schema), - columns=COLUMNS[schema], + columns=SCHEMA_COLUMNS[schema], ) df.set_index(self._get_index_column(schema), inplace=True) @@ -1074,10 +1074,10 @@ def to_ndarray( self, ) - decoder = functools.partial(np.frombuffer, dtype=STRUCT_MAP[schema]) + decoder = functools.partial(np.frombuffer, dtype=SCHEMA_DTYPES_MAP[schema]) result = tuple(map(decoder, map(bytes, schema_records))) if not result: - return np.empty(shape=(0, 1), dtype=STRUCT_MAP[schema]) + return np.empty(shape=(0, 1), dtype=SCHEMA_DTYPES_MAP[schema]) return np.ravel(result) diff --git a/databento/historical/api/metadata.py b/databento/historical/api/metadata.py index 609741c..3526448 100644 --- a/databento/historical/api/metadata.py +++ b/databento/historical/api/metadata.py @@ -16,7 +16,6 @@ from databento.common.parsing import optional_datetime_to_string from databento.common.parsing import optional_symbols_list_to_list from databento.common.validation import validate_enum -from databento.common.validation import validate_maybe_enum from databento.common.validation import validate_semantic_string from databento.historical.api import API_VERSION from databento.historical.http import BentoHttpAPI @@ -31,17 +30,17 @@ def __init__(self, key: str, gateway: str) -> None: super().__init__(key=key, gateway=gateway) self._base_url = gateway + f"/v{API_VERSION}/metadata" - def list_publishers(self) -> dict[str, int]: + def list_publishers(self) -> list[dict[str, Any]]: """ Request all publishers from Databento. Makes a `GET /metadata.list_publishers` HTTP request. - Use this method to list the mappings of publisher names to publisher IDs. + Use this method to list the details of publishers, including their dataset and venue mappings. Returns ------- - dict[str, int] + list[dict[str, Any]] """ response: Response = self._get( @@ -56,11 +55,11 @@ def list_datasets( end_date: date | str | None = None, ) -> list[str]: """ - Request all available datasets from Databento. + Request all available dataset codes from Databento. Makes a `GET /metadata.list_datasets` HTTP request. - Use this method to list the _names_ of all available datasets, so you + Use this method to list the available dataset _codes (string identifiers), so you can use other methods which take the `dataset` parameter. Parameters @@ -118,42 +117,35 @@ def list_schemas(self, dataset: Dataset | str) -> list[str]: def list_fields( self, - dataset: Dataset | str, - schema: Schema | str | None = None, - encoding: Encoding | str | None = None, - ) -> dict[str, dict[str, str]]: + schema: Schema | str, + encoding: Encoding | str, + ) -> list[dict[str, Any]]: """ - Request all fields for a dataset, schema and encoding from Databento. + List all fields for a particular schema and encoding from Databento. Makes a `GET /metadata.list_fields` HTTP request. - The `schema` and `encoding` parameters act as optional filters. All - metadata for that parameter is returned if they are not specified. - Parameters ---------- - dataset : Dataset or str - The dataset code (string identifier) for the request. - schema : Schema or str {'mbo', 'mbp-1', 'mbp-10', 'trades', 'tbbo', 'ohlcv-1s', 'ohlcv-1m', 'ohlcv-1h', 'ohlcv-1d', 'definition', 'statistics', 'status'}, optional # noqa + schema : Schema or str {'mbo', 'mbp-1', 'mbp-10', 'trades', 'tbbo', 'ohlcv-1s', 'ohlcv-1m', 'ohlcv-1h', 'ohlcv-1d', 'definition', 'statistics', 'status'}, The data record schema for the request. - encoding : Encoding or str {'dbn', 'csv', 'json'}, optional + encoding : Encoding or str {'dbn', 'csv', 'json'} The data encoding. Returns ------- - dict[str, dict[str, str]] - A mapping of dataset to encoding to schema to field to data type. + list[dict[str, Any]] + A list of field details. """ - params: list[tuple[str, Dataset | Schema | Encoding | str | None]] = [ - ("dataset", validate_semantic_string(dataset, "dataset")), - ("schema", validate_maybe_enum(schema, Schema, "schema")), - ("encoding", validate_maybe_enum(encoding, Encoding, "encoding")), + params: list[tuple[str, str | Any]] = [ + ("schema", validate_enum(schema, Schema, "schema")), + ("encoding", validate_enum(encoding, Encoding, "encoding")), ] response: Response = self._get( url=self._base_url + ".list_fields", - params=params, # type: ignore [arg-type] + params=params, basic_auth=True, ) return response.json() @@ -161,11 +153,10 @@ def list_fields( def list_unit_prices( self, dataset: Dataset | str, - mode: FeedMode | str | None = None, - schema: Schema | str | None = None, - ) -> float | dict[str, Any]: + ) -> list[dict[str, Any]]: """ - List unit prices for each data schema in US dollars per gigabyte. + List unit prices for each feed mode and data schema in US dollars per + gigabyte. Makes a `GET /metadata.list_unit_prices` HTTP request. @@ -173,27 +164,20 @@ def list_unit_prices( ---------- dataset : Dataset or str The dataset code for the request. - mode : FeedMode or str {'live', 'historical-streaming', 'historical'}, optional - The data feed mode for the request. - schema : Schema or str {'mbo', 'mbp-1', 'mbp-10', 'trades', 'tbbo', 'ohlcv-1s', 'ohlcv-1m', 'ohlcv-1h', 'ohlcv-1d', 'definition', 'statistics', 'status'}, optional # noqa - The data record schema for the request. Returns ------- - float or dict[str, Any] - If both `mode` and `schema` are specified, the unit price is returned as a single number. - Otherwise, return a map of feed mode to schema to unit price. + list[dict[str, Any]] + A list of maps of feed mode to schema to unit price. """ - params: list[tuple[str, Dataset | FeedMode | Schema | str | None]] = [ + params: list[tuple[str, Dataset | str]] = [ ("dataset", validate_semantic_string(dataset, "dataset")), - ("mode", validate_maybe_enum(mode, FeedMode, "mode")), - ("schema", validate_maybe_enum(schema, Schema, "schema")), ] response: Response = self._get( url=self._base_url + ".list_unit_prices", - params=params, # type: ignore [arg-type] + params=params, basic_auth=True, ) return response.json() diff --git a/databento/live/gateway.py b/databento/live/gateway.py index 6f2f3ca..6f3e3d2 100644 --- a/databento/live/gateway.py +++ b/databento/live/gateway.py @@ -12,6 +12,7 @@ from databento_dbn import SType from databento.common.enums import Dataset +from databento.version import __version__ logger = logging.getLogger(__name__) @@ -108,6 +109,7 @@ class AuthenticationRequest(GatewayControl): encoding: Encoding = Encoding.DBN details: str | None = None ts_out: str = "0" + client: str = f"Python {__version__}" @dataclasses.dataclass diff --git a/databento/version.py b/databento/version.py index 113af05..fd86b3e 100644 --- a/databento/version.py +++ b/databento/version.py @@ -1 +1 @@ -__version__ = "0.16.1" +__version__ = "0.17.0" diff --git a/examples/historical_metadata.py b/examples/historical_metadata.py index 5e35fa5..2babdc0 100644 --- a/examples/historical_metadata.py +++ b/examples/historical_metadata.py @@ -8,4 +8,4 @@ print(client.metadata.list_publishers()) print(client.metadata.list_datasets()) print(client.metadata.list_schemas(dataset="GLBX.MDP3")) - print(client.metadata.list_fields(dataset="GLBX.MDP3")) + print(client.metadata.list_fields(schema="trades", encoding="dbn")) diff --git a/examples/historical_metadata_list_unit_price.py b/examples/historical_metadata_list_unit_price.py index 5f9177a..f768c4b 100644 --- a/examples/historical_metadata_list_unit_price.py +++ b/examples/historical_metadata_list_unit_price.py @@ -7,9 +7,6 @@ key = "YOUR_API_KEY" client = Historical(key=key) - unit_prices = client.metadata.list_unit_prices( - dataset="GLBX.MDP3", - mode="historical-streaming", - ) + unit_prices = client.metadata.list_unit_prices(dataset="GLBX.MDP3") pprint(unit_prices) diff --git a/pyproject.toml b/pyproject.toml index 825594d..0805d94 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "databento" -version = "0.16.1" +version = "0.17.0" description = "Official Python client library for Databento" authors = [ "Databento ", @@ -26,7 +26,7 @@ repository = "https://github.com/databento/databento-python" [tool.poetry.dependencies] python = "^3.8" aiohttp = "^3.8.3" -databento-dbn = "0.7.1" +databento-dbn = "0.8.2" numpy= ">=1.23.5" pandas = ">=1.5.3" requests = ">=2.24.0" diff --git a/tests/test_historical_data.py b/tests/test_historical_data.py index a542f14..a65dab6 100644 --- a/tests/test_historical_data.py +++ b/tests/test_historical_data.py @@ -1,6 +1,6 @@ import databento import pytest -from databento.common.data import COLUMNS +from databento.common.data import SCHEMA_COLUMNS from databento.common.data import SCHEMA_STRUCT_MAP @@ -9,9 +9,9 @@ def test_mbo_fields() -> None: Test that columns match the MBO struct. """ struct = SCHEMA_STRUCT_MAP[databento.Schema.MBO] - columns = COLUMNS[databento.Schema.MBO] + columns = SCHEMA_COLUMNS[databento.Schema.MBO] - fields = set(f for f in dir(struct) if not f.startswith("_")) + fields = set(f for f in dir(struct) if not f.startswith(("_", "pretty_"))) fields.remove("hd") fields.remove("record_size") fields.remove("size_hint") @@ -36,9 +36,9 @@ def test_mbp_fields( Test that columns match the MBP structs. """ struct = SCHEMA_STRUCT_MAP[schema] - columns = COLUMNS[schema] + columns = SCHEMA_COLUMNS[schema] - fields = set(f for f in dir(struct) if not f.startswith("_")) + fields = set(f for f in dir(struct) if not f.startswith(("_", "pretty_"))) fields.remove("hd") fields.remove("record_size") fields.remove("size_hint") @@ -67,9 +67,9 @@ def test_ohlcv_fields( Test that columns match the OHLCV structs. """ struct = SCHEMA_STRUCT_MAP[schema] - columns = COLUMNS[schema] + columns = SCHEMA_COLUMNS[schema] - fields = set(f for f in dir(struct) if not f.startswith("_")) + fields = set(f for f in dir(struct) if not f.startswith(("_", "pretty_"))) fields.remove("hd") fields.remove("record_size") fields.remove("size_hint") @@ -83,9 +83,9 @@ def test_trades_struct() -> None: Test that columns match the Trades struct. """ struct = SCHEMA_STRUCT_MAP[databento.Schema.TRADES] - columns = COLUMNS[databento.Schema.TRADES] + columns = SCHEMA_COLUMNS[databento.Schema.TRADES] - fields = set(f for f in dir(struct) if not f.startswith("_")) + fields = set(f for f in dir(struct) if not f.startswith(("_", "pretty_"))) fields.remove("hd") fields.remove("record_size") fields.remove("size_hint") @@ -99,13 +99,15 @@ def test_definition_struct() -> None: Test that columns match the Definition struct. """ struct = SCHEMA_STRUCT_MAP[databento.Schema.DEFINITION] - columns = COLUMNS[databento.Schema.DEFINITION] + columns = SCHEMA_COLUMNS[databento.Schema.DEFINITION] - fields = set(f for f in dir(struct) if not f.startswith("_")) + fields = set(f for f in dir(struct) if not f.startswith(("_", "pretty_"))) fields.remove("hd") fields.remove("record_size") fields.remove("size_hint") + columns.remove("raw_instrument_id") # TODO: Remove after databento_dbn 0.8.3 + difference = fields.symmetric_difference(set(columns)) assert not difference @@ -115,9 +117,9 @@ def test_imbalance_struct() -> None: Test that columns match the Imbalance struct. """ struct = SCHEMA_STRUCT_MAP[databento.Schema.IMBALANCE] - columns = COLUMNS[databento.Schema.IMBALANCE] + columns = SCHEMA_COLUMNS[databento.Schema.IMBALANCE] - fields = set(f for f in dir(struct) if not f.startswith("_")) + fields = set(f for f in dir(struct) if not f.startswith(("_", "pretty_"))) fields.remove("hd") fields.remove("record_size") fields.remove("size_hint") @@ -131,9 +133,9 @@ def test_statistics_struct() -> None: Test that columns match the Statistics struct. """ struct = SCHEMA_STRUCT_MAP[databento.Schema.STATISTICS] - columns = COLUMNS[databento.Schema.STATISTICS] + columns = SCHEMA_COLUMNS[databento.Schema.STATISTICS] - fields = set(f for f in dir(struct) if not f.startswith("_")) + fields = set(f for f in dir(struct) if not f.startswith(("_", "pretty_"))) fields.remove("hd") fields.remove("record_size") fields.remove("size_hint") diff --git a/tests/test_historical_metadata.py b/tests/test_historical_metadata.py index ccb607e..c359ea6 100644 --- a/tests/test_historical_metadata.py +++ b/tests/test_historical_metadata.py @@ -6,9 +6,7 @@ import pytest import requests from databento.common.enums import Dataset -from databento.common.enums import FeedMode from databento.historical.client import Historical -from databento_dbn import Schema def test_list_publishers_sends_expected_request( @@ -95,7 +93,6 @@ def test_list_fields_sends_expected_request( # Act historical_client.metadata.list_fields( - dataset="GLBX.MDP3", schema="mbo", encoding="dbn", ) @@ -106,7 +103,6 @@ def test_list_fields_sends_expected_request( call["url"] == f"{historical_client.gateway}/v{db.API_VERSION}/metadata.list_fields" ) - assert ("dataset", "GLBX.MDP3") in call["params"] assert ("schema", "mbo") in call["params"] assert ("encoding", "dbn") in call["params"] assert sorted(call["headers"].keys()) == ["accept", "user-agent"] @@ -117,28 +113,22 @@ def test_list_fields_sends_expected_request( @pytest.mark.parametrize( - "dataset, schema, mode", + "dataset", [ - ["GLBX.MDP3", "mbo", "live"], - [Dataset.GLBX_MDP3, Schema.MBO, FeedMode.LIVE], + "GLBX.MDP3", + Dataset.GLBX_MDP3, ], ) def test_list_unit_price_sends_expected_request( monkeypatch: pytest.MonkeyPatch, historical_client: Historical, dataset: Dataset | str, - schema: Schema | str, - mode: FeedMode | str, ) -> None: # Arrange monkeypatch.setattr(requests, "get", mocked_get := MagicMock()) # Act - historical_client.metadata.list_unit_prices( - dataset=dataset, - schema=schema, - mode=mode, - ) + historical_client.metadata.list_unit_prices(dataset=dataset) # Assert call = mocked_get.call_args.kwargs @@ -151,8 +141,6 @@ def test_list_unit_price_sends_expected_request( assert all(v in call["headers"]["user-agent"] for v in ("Databento/", "Python/")) assert call["params"] == [ ("dataset", "GLBX.MDP3"), - ("mode", "live"), - ("schema", "mbo"), ] assert call["timeout"] == (100, 100) assert isinstance(call["auth"], requests.auth.HTTPBasicAuth) diff --git a/tests/test_live_client.py b/tests/test_live_client.py index 7085613..62bd6d5 100644 --- a/tests/test_live_client.py +++ b/tests/test_live_client.py @@ -1051,6 +1051,8 @@ async def test_live_stream_with_reconnect( # TODO: Remove when status schema is available if schema == "status": pytest.skip("no stub data for status schema") + if schema == "ohlcv-eod": + pytest.skip("no stub data for ohlcv-eod schema") output = tmp_path / "output.dbn" live_client.add_stream(output.open("wb", buffering=0)) diff --git a/tests/test_live_gateway_messages.py b/tests/test_live_gateway_messages.py index 6529942..eb343e6 100644 --- a/tests/test_live_gateway_messages.py +++ b/tests/test_live_gateway_messages.py @@ -9,6 +9,7 @@ from databento.live.gateway import Greeting from databento.live.gateway import SessionStart from databento.live.gateway import SubscriptionRequest +from databento.version import __version__ from databento_dbn import Encoding from databento_dbn import Schema from databento_dbn import SType @@ -29,27 +30,29 @@ "line, expected", [ pytest.param( - "auth=abcd1234|dataset=GLBX.MDP3|encoding=json\n", - ("abcd1234", "GLBX.MDP3", "json", None, "0"), + f"auth=abcd1234|dataset=GLBX.MDP3|encoding=json|client=Python {__version__}\n", + ("abcd1234", "GLBX.MDP3", "json", None, "0", f"Python {__version__}"), ), pytest.param( - "auth=abcd1234|dataset=GLBX.MDP3|ts_out=1\n", + f"auth=abcd1234|dataset=GLBX.MDP3|ts_out=1|client=Python {__version__}\n", ( "abcd1234", "GLBX.MDP3", str(Encoding.DBN), None, "1", + f"Python {__version__}", ), ), pytest.param( - "auth=abcd1234|dataset=XNAS.ITCH\n", + f"auth=abcd1234|dataset=XNAS.ITCH|client=Python {__version__}\n", ( "abcd1234", "XNAS.ITCH", str(Encoding.DBN), None, "0", + f"Python {__version__}", ), ), pytest.param( @@ -73,6 +76,7 @@ def test_parse_authentication_request( msg.encoding, msg.details, msg.ts_out, + msg.client, ) == expected else: with pytest.raises(expected): @@ -87,7 +91,7 @@ def test_parse_authentication_request( auth="abcd1234", dataset=Dataset.GLBX_MDP3, ), - b"auth=abcd1234|dataset=GLBX.MDP3|encoding=dbn|ts_out=0\n", + f"auth=abcd1234|dataset=GLBX.MDP3|encoding=dbn|ts_out=0|client=Python {__version__}\n".encode(), ), pytest.param( AuthenticationRequest( @@ -95,7 +99,7 @@ def test_parse_authentication_request( dataset=Dataset.XNAS_ITCH, ts_out="1", ), - b"auth=abcd1234|dataset=XNAS.ITCH|encoding=dbn|ts_out=1\n", + f"auth=abcd1234|dataset=XNAS.ITCH|encoding=dbn|ts_out=1|client=Python {__version__}\n".encode(), ), ], )