VER: Release 0.15.1

See release notes.

Author: nmacholl

.github/workflows/release.yml

@@ -27,7 +27,7 @@ jobs:
           echo "RELEASE_NAME=$(poetry version)" >> $GITHUB_ENV
           echo "TAG_NAME=v$(poetry version -s)" >> $GITHUB_ENV
           echo "## Release notes" > NOTES.md
-          sed -n '4,/^$/p' CHANGELOG.md >> NOTES.md
+          sed -n '/^## /{n; :a; /^## /q; p; n; ba}' CHANGELOG.md >> NOTES.md
 
       - name: Release
         uses: softprops/action-gh-release@v1

CHANGELOG.md

@@ -1,31 +1,67 @@
 # Changelog
 
+## 0.15.1 - 2023-07-05
+
+#### Bug Fixes
+- Fixed an `ImportError` observed in Python 3.8
+
+## 0.15.0 - 2023-07-03
+
+#### Enhancements
+- Added `symbology_map` property to `Live` client
+- Added `optional_symbols_list_to_list` parsing function
+- Changed `Live.add_callback` and `Live.add_stream` to accept an exception callback
+- Changed `Live.__iter__()` and `Live.__aiter__()` to send the session start message if the session is connected but not started
+- Upgraded `databento-dbn` to 0.7.1
+- Removed exception chaining from exceptions emitted by the library
+
+#### Bug fixes
+- Fixed issue where a large unreadable symbol subscription message could be sent
+
+#### Breaking changes
+- Removed `Encoding`, `Compression`, `Schema`, and `SType` enums as they are now exposed by `databento-dbn`
+- Renamed `func` parameter to `record_callback` for `Live.add_callback` and `Live.add_stream`
+- Removed `optional_symbols_list_to_string` parsing function
+
 ## 0.14.1 - 2023-06-16
+
 - Fixed issue where `DBNStore.to_df()` would raise an exception if no records were present
 - Fixed exception message when creating a DBNStore from an empty data source
 
 ## 0.14.0 - 2023-06-14
-- Added support for reusing a `Live` client to reconnect
-- Added `metadata` property to `Live`
+
+#### Enhancements
 - Added `DatatbentoLiveProtocol` class
+- Added `metadata` property to `Live`
+- Added support for reusing a `Live` client to reconnect
 - Added support for emitting warnings in API response headers
+- Relaxed 10 minute minimum request time range restriction
+- Upgraded `aiohttp` to 3.8.3
+- Upgraded `numpy` to 1.23.5
+- Upgraded `pandas` to 1.5.3
+- Upgraded `requests` to 2.28.1
+- Upgraded `zstandard` to 0.21.0
+
+#### Breaking changes
+- Removed support for Python 3.7
+- Renamed `symbol` to `raw_symbol` in definition schema when converting to a DataFrame
 - Changed iteration of `Live` to no longer yield DBN metadata
 - Changed `Live` callbacks to no longer yield DBN metadata
+
+#### Bug fixes
 - Fixed issue where `Historical.timeseries.get_range` would write empty files on error
 - Fixed issue with `numpy` types not being handled in symbols field
-- Upgraded `aiohttp` to 3.8.3
-- Upgraded `numpy` to to 1.23.5
-- Upgraded `pandas` to to 1.5.3
-- Upgraded `requests` to to 2.28.1
-- Upgraded `zstandard` to to 0.21.0
-- Removed 10 minute minimum request time range restriction
-- Removed support for Python 3.7
+- Fixed optional `end` parameter for `batch.submit_job(...)`
 
 ## 0.13.0 - 2023-06-02
+
+#### Enhancements
 - Added support for `statistics` schema
 - Added batch download support data files (`condition.json` and `symbology.json`)
-- Upgraded `databento-dbn` to 0.6.1
 - Renamed `booklevel` MBP field to `levels` for brevity and consistent naming
+- Upgraded `databento-dbn` to 0.6.1
+
+#### Breaking changes
 - Changed `flags` field to an unsigned int
 - Changed default of `ts_out` to `False` for `Live` client
 - Changed `instrument_class` DataFrame representation to be consistent with other `char` types
@@ -34,85 +70,118 @@
 - Removed support for legacy stypes
 
 ## 0.12.0 - 2023-05-01
+
+#### Enhancements
 - Added `Live` client for connecting to Databento's live service
+- Added `degraded`, `pending` and `missing` condition variants for `batch.get_dataset_condition`
+- Added `last_modified_date` field to `batch.get_dataset_condition` response
 - Upgraded `databento-dbn` to 0.5.0
 - Upgraded `DBNStore` to support mixed schema types to support live data
+
+#### Breaking changes
 - Changed iteration `DBNStore` to return record types from `databento-dbn` instead of numpy arrays
-- Removed `dtype` property from `DBNStore`
-- Removed `record_size` property from `DBNStore`
 - Renamed the `cost` field to `cost_usd` for `batch.submit_job` and `batch.list_jobs` (value now expressed as US dollars)
-- Removed `bad` condition variant from `batch.get_dataset_condition`
-- Added `degraded`, `pending` and `missing` condition variants for `batch.get_dataset_condition`
-- Added `last_modified_date` field to `batch.get_dataset_condition` response
 - Renamed `product_id` field to `instrument_id`
 - Renamed `symbol` field in definitions to `raw_symbol`
-- Deprecated `SType.PRODUCT_ID` to `SType.INSTRUMENT_ID`
-- Deprecated `SType.NATIVE` to `SType.RAW_SYMBOL`
-- Deprecated `SType.SMART` to `SType.PARENT` and `SType.CONTINUOUS`
+- Removed `dtype` property from `DBNStore`
+- Removed `record_size` property from `DBNStore`
+- Removed `bad` condition variant from `batch.get_dataset_condition`
 - Removed unused `LiveGateway` enum
 - Removed `STATSTICS` from `Schema` enum
 - Removed `STATUS` from `Schema` enum
 - Removed `GATEWAY_ERROR` from `Schema` enum
 - Removed `SYMBOL_MAPPING` from `Schema` enum
 
+#### Deprecations
+- Deprecated `SType.PRODUCT_ID` to `SType.INSTRUMENT_ID`
+- Deprecated `SType.NATIVE` to `SType.RAW_SYMBOL`
+- Deprecated `SType.SMART` to `SType.PARENT` and `SType.CONTINUOUS`
+
 ## 0.11.0 - 2023-04-13
+
 - Changed `end` and `end_date` to optional to support new forward-fill behaviour
 - Upgraded `zstandard` to 0.20.0
 
 ## 0.10.0 - 2023-04-07
+
+#### Enhancements
+- Added support for `imbalance` schema
+- Added `instrument_class`, `strike_price`, and `strike_price_currency` to definition
+  schema
+- Changed parsing of `end` and `end_date` params throughout the API
+- Improved exception messages for server and client timeouts
 - Upgraded `databento-dbn` to 0.4.3
+
+#### Breaking changes
 - Renamed `Bento` class to `DBNStore`
 - Removed `metadata.list_compressions` (redundant with docs)
 - Removed `metadata.list_encodings` (redundant with docs)
 - Removed optional `start` and `end` params from `metadata.list_schemas` (redundant)
 - Removed `related` and `related_security_id` from definition schema
-- Added `instrument_class`, `strike_price`, and `strike_price_currency` to definition
-  schema
-- Added support for `imbalance` schema
-- Improved exception messages for server and client timeouts
 
 ## 0.9.0 - 2023-03-10
-- Removed `record_count` property from Bento class
-- Fixed bug in `Bento` where invalid metadata would prevent iteration
+
+#### Enhancements
 - Improved use of the logging module
+
+#### Breaking changes
+- Removed `record_count` property from Bento class
 - Changed `metadata.get_dataset_condition` response to a list of condition per date
 
+#### Bug fixes
+- Fixed bug in `Bento` where invalid metadata would prevent iteration
+
 ## 0.8.1 - 2023-03-05
-- Fixed bug in `Bento` iteration where multiple readers were created
+
+#### Enhancements
 - Added `from_dbn` convenience alias for loading DBN files
 
+#### Bug fixes
+- Fixed bug in `Bento` iteration where multiple readers were created
+
 ## 0.8.0 - 2023-03-03
-- Integrated DBN encoding 0.3.2
-- Renamed `timeseries.stream` to `timeseries.get_range`
-- Renamed `timeseries.stream_async` to `timeseries.get_range_async`
-- Deprecated `timeseries.stream(...)` method
-- Deprecated `timeseries.stream_async(...)` method
+
+#### Enhancements
 - Added `batch.list_files(...)` method
 - Added `batch.download(...)` method
 - Added `batch.download_async(...)` method
+- Integrated DBN encoding 0.3.2
+
+#### Breaking changes
+- Dropped support for DBZ encoding
+- Renamed `timeseries.stream` to `timeseries.get_range`
+- Renamed `timeseries.stream_async` to `timeseries.get_range_async`
 - Changed `.to_df(...)` `pretty_ts` default argument to `True`
 - Changed `.to_df(...)` `pretty_px` default argument to `True`
 - Changed `.to_df(...)` `map_symbols` default argument to `True`
-- Drop support for DBZ encoding
+
+#### Deprecations
+- Deprecated `timeseries.stream(...)` method
+- Deprecated `timeseries.stream_async(...)` method
 
 ## 0.7.0 - 2023-01-10
+
 - Added support for `definition` schema
 - Updated `Flags` enum
 - Upgraded `dbz-python` to 0.2.1
 - Upgraded `zstandard` to 0.19.0
 
 ## 0.6.0 - 2022-12-02
+
 - Added `metadata.get_dataset_condition` method to `Historical` client
 - Upgraded `dbz-python` to 0.2.0
 
 ## 0.5.0 - 2022-11-07
+
  - Fixed dataframe columns for derived data schemas (dropped `channel_id`)
  - Fixed `batch.submit_job` requests for `dbz` encoding
  - Updated `quickstart.ipynb` jupyter notebook
 
 ## 0.4.0 - 2022-09-14
+
  - Upgraded `dbz-python` to 0.1.5
  - Added `map_symbols` option for `.to_df()` (experimental)
 
 ## 0.3.0 - 2022-08-30
+
  - Initial release

README.md

@@ -32,10 +32,10 @@ The library is fully compatible with the latest distribution of Anaconda 3.8 and
 The minimum dependencies as found in the `pyproject.toml` are also listed below:
 - python = "^3.8"
 - aiohttp = "^3.8.3"
-- databento-dbn = "0.6.1"
+- databento-dbn = "0.7.1"
 - numpy= ">=1.23.5"
 - pandas = ">=1.5.3"
-- requests = ">=2.28.1"
+- requests = ">=2.24.0"
 - zstandard = ">=0.21.0"
 
 ## Installation

databento/__init__.py

@@ -1,6 +1,8 @@
 import logging
 import warnings
 
+from databento_dbn import Compression
+from databento_dbn import Encoding
 from databento_dbn import ErrorMsg
 from databento_dbn import ImbalanceMsg
 from databento_dbn import InstrumentDefMsg
@@ -9,25 +11,23 @@
 from databento_dbn import MBP10Msg
 from databento_dbn import Metadata
 from databento_dbn import OHLCVMsg
+from databento_dbn import Schema
 from databento_dbn import StatMsg
+from databento_dbn import SType
 from databento_dbn import SymbolMappingMsg
 from databento_dbn import SystemMsg
 from databento_dbn import TradeMsg
 
 from databento.common import bentologging
 from databento.common.dbnstore import DBNStore
-from databento.common.enums import Compression
 from databento.common.enums import Dataset
 from databento.common.enums import Delivery
-from databento.common.enums import Encoding
 from databento.common.enums import FeedMode
 from databento.common.enums import HistoricalGateway
 from databento.common.enums import Packaging
 from databento.common.enums import RecordFlags
 from databento.common.enums import RollRule
-from databento.common.enums import Schema
 from databento.common.enums import SplitDuration
-from databento.common.enums import SType
 from databento.common.enums import SymbologyResolution
 from databento.common.error import BentoClientError
 from databento.common.error import BentoError

databento/common/data.py

@@ -1,9 +1,33 @@
 from __future__ import annotations
 
 import numpy as np
-
-from databento.common.enums import Schema
-
+from databento_dbn import ImbalanceMsg
+from databento_dbn import InstrumentDefMsg
+from databento_dbn import MBOMsg
+from databento_dbn import MBP1Msg
+from databento_dbn import MBP10Msg
+from databento_dbn import OHLCVMsg
+from databento_dbn import Schema
+from databento_dbn import StatMsg
+from databento_dbn import TradeMsg
+
+from databento.live import DBNRecord
+
+
+SCHEMA_STRUCT_MAP: dict[Schema, type[DBNRecord]] = {
+    Schema.DEFINITION: InstrumentDefMsg,
+    Schema.IMBALANCE: ImbalanceMsg,
+    Schema.MBO: MBOMsg,
+    Schema.MBP_1: MBP1Msg,
+    Schema.MBP_10: MBP10Msg,
+    Schema.OHLCV_1S: OHLCVMsg,
+    Schema.OHLCV_1M: OHLCVMsg,
+    Schema.OHLCV_1H: OHLCVMsg,
+    Schema.OHLCV_1D: OHLCVMsg,
+    Schema.STATISTICS: StatMsg,
+    Schema.TBBO: MBP1Msg,
+    Schema.TRADES: TradeMsg,
+}
 
 ################################################################################
 # DBN struct schema
@@ -284,9 +308,10 @@ def get_deriv_ba_fields(level: int) -> list[str]:
 DEFINITION_DROP_COLUMNS = [
     "length",
     "rtype",
-    "reserved1",
-    "reserved2",
-    "reserved3",
+    "_reserved1",
+    "_reserved2",
+    "_reserved3",
+    "_reserved4",
     "dummy",
 ]
 

databento/common/dbnstore.py

@@ -19,9 +19,12 @@
 import numpy as np
 import pandas as pd
 import zstandard
+from databento_dbn import Compression
 from databento_dbn import DBNDecoder
 from databento_dbn import ErrorMsg
 from databento_dbn import Metadata
+from databento_dbn import Schema
+from databento_dbn import SType
 from databento_dbn import SymbolMappingMsg
 from databento_dbn import SystemMsg
 
@@ -30,10 +33,8 @@
 from databento.common.data import DEFINITION_PRICE_COLUMNS
 from databento.common.data import DEFINITION_TYPE_MAX_MAP
 from databento.common.data import DERIV_SCHEMAS
+from databento.common.data import SCHEMA_STRUCT_MAP
 from databento.common.data import STRUCT_MAP
-from databento.common.enums import Compression
-from databento.common.enums import Schema
-from databento.common.enums import SType
 from databento.common.error import BentoError
 from databento.common.symbology import InstrumentIdMappingInterval
 from databento.common.validation import validate_maybe_enum
@@ -788,10 +789,9 @@ def replay(self, callback: Callable[[Any], None]) -> None:
         for record in self:
             try:
                 callback(record)
-            except Exception as exc:
+            except Exception:
                 logger.exception(
                     "exception while replaying to user callback",
-                    exc_info=exc,
                 )
                 raise
 
@@ -1059,7 +1059,7 @@ def to_ndarray(
             schema = self.schema
 
         schema_records = filter(
-            lambda r: isinstance(r, schema.get_record_type()),  # type: ignore
+            lambda r: isinstance(r, SCHEMA_STRUCT_MAP[schema]),  # type: ignore
             self,
         )