From 30b8641323801c936fdb466a94ea735dcdd713b4 Mon Sep 17 00:00:00 2001 From: Nick Macholl Date: Mon, 20 Nov 2023 16:47:41 -0800 Subject: [PATCH 01/21] FIX: Python size_hint for DBN v1 structs --- CHANGELOG.md | 6 ++++++ rust/dbn/src/python/record.rs | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dec0c47..a220386 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## 0.14.3 - TBD + +### Bug fixes +- Fixed Python `size_hint` return value for `InstrumentDefMsgV1` and + `SymbolMappingMsgV1` + ## 0.14.2 - 2023-11-17 ### Enhancements - Added `set_upgrade_policy` setters to `DbnDecoder`, `DbnRecordDecoder`, diff --git a/rust/dbn/src/python/record.rs b/rust/dbn/src/python/record.rs index 632be8a..18b3b80 100644 --- a/rust/dbn/src/python/record.rs +++ b/rust/dbn/src/python/record.rs @@ -1571,7 +1571,7 @@ impl InstrumentDefMsgV1 { #[classattr] fn size_hint() -> PyResult { - Ok(mem::size_of::()) + Ok(mem::size_of::()) } #[getter] @@ -2346,7 +2346,7 @@ impl SymbolMappingMsgV1 { #[classattr] fn size_hint() -> PyResult { - Ok(mem::size_of::()) + Ok(mem::size_of::()) } #[getter] From a2631a3c480cebda50a688a00092e1b5262ff596 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Tue, 21 Nov 2023 15:26:29 +1100 Subject: [PATCH 02/21] REF: Standardize Nasdaq TotalView-ITCH --- rust/dbn/src/lib.rs | 2 +- rust/dbn/src/publishers.rs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/rust/dbn/src/lib.rs b/rust/dbn/src/lib.rs index c8c60e0..bf8bf04 100644 --- a/rust/dbn/src/lib.rs +++ b/rust/dbn/src/lib.rs @@ -110,6 +110,6 @@ pub mod datasets { pub const GLBX_MDP3: &str = Dataset::GlbxMdp3.as_str(); /// The dataset code for OPRA PILLAR. pub const OPRA_PILLAR: &str = Dataset::OpraPillar.as_str(); - /// The dataset code for Nasdaq TotalView ITCH. + /// The dataset code for Nasdaq TotalView-ITCH. pub const XNAS_ITCH: &str = Dataset::XnasItch.as_str(); } diff --git a/rust/dbn/src/publishers.rs b/rust/dbn/src/publishers.rs index 76b4ec3..dfaee75 100644 --- a/rust/dbn/src/publishers.rs +++ b/rust/dbn/src/publishers.rs @@ -366,11 +366,11 @@ impl std::str::FromStr for Dataset { pub enum Publisher { /// CME Globex MDP 3.0 GlbxMdp3Glbx = 1, - /// Nasdaq TotalView ITCH + /// Nasdaq TotalView-ITCH XnasItchXnas = 2, - /// Nasdaq BX TotalView ITCH + /// Nasdaq BX TotalView-ITCH XbosItchXbos = 3, - /// Nasdaq PSX TotalView ITCH + /// Nasdaq PSX TotalView-ITCH XpsxItchXpsx = 4, /// Cboe BZX Depth Pitch BatsPitchBats = 5, From ec4170a937d21b26062f15a4ca759efa733311a4 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Tue, 28 Nov 2023 13:38:57 +1100 Subject: [PATCH 03/21] MOD: Upgrade neko to DBN v2 via symfs-py --- CHANGELOG.md | 2 ++ rust/dbn/src/python/metadata.rs | 2 ++ 2 files changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a220386..327d5a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ # Changelog ## 0.14.3 - TBD +### Enhancements +- Added `version` param to `Metadata::builder` to choose between DBNv1 and DBNv2 specs ### Bug fixes - Fixed Python `size_hint` return value for `InstrumentDefMsgV1` and diff --git a/rust/dbn/src/python/metadata.rs b/rust/dbn/src/python/metadata.rs index bdfa0db..066d845 100644 --- a/rust/dbn/src/python/metadata.rs +++ b/rust/dbn/src/python/metadata.rs @@ -32,6 +32,7 @@ impl Metadata { end: Option, limit: Option, ts_out: Option, + version: Option, ) -> Metadata { Metadata::builder() .dataset(dataset) @@ -46,6 +47,7 @@ impl Metadata { .end(NonZeroU64::new(end.unwrap_or_default())) .limit(NonZeroU64::new(limit.unwrap_or_default())) .ts_out(ts_out.unwrap_or_default()) + .version(version.unwrap_or(crate::DBN_VERSION)) .build() } From df41399b84ecddd65a79e86e4c084427872cdf44 Mon Sep 17 00:00:00 2001 From: Zach Banks Date: Tue, 28 Nov 2023 14:31:39 -0500 Subject: [PATCH 04/21] ADD: Add richcmp to python DBN objects --- python/databento_dbn.pyi | 2 +- rust/dbn/src/compat.rs | 4 ++-- rust/dbn/src/enums.rs | 8 ++++---- rust/dbn/src/python/record.rs | 8 ++++++++ rust/dbn/src/record.rs | 30 +++++++++++++++--------------- 5 files changed, 30 insertions(+), 22 deletions(-) diff --git a/python/databento_dbn.pyi b/python/databento_dbn.pyi index 6d00d36..603b9a1 100644 --- a/python/databento_dbn.pyi +++ b/python/databento_dbn.pyi @@ -814,7 +814,7 @@ class BidAskPair: """ @property - def bid_ask_ct(self) -> int: + def ask_ct(self) -> int: """ The ask order count. diff --git a/rust/dbn/src/compat.rs b/rust/dbn/src/compat.rs index 3968a70..013a072 100644 --- a/rust/dbn/src/compat.rs +++ b/rust/dbn/src/compat.rs @@ -73,7 +73,7 @@ pub unsafe fn decode_record_ref<'a>( /// /// Note: This will be renamed to `InstrumentDefMsg` in DBN version 2. #[repr(C)] -#[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq)] +#[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] #[cfg_attr( feature = "python", @@ -300,7 +300,7 @@ pub struct InstrumentDefMsgV1 { /// /// Note: This will be renamed to `SymbolMappingMsg` in DBN version 2. #[repr(C)] -#[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq)] +#[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] #[cfg_attr( feature = "python", diff --git a/rust/dbn/src/enums.rs b/rust/dbn/src/enums.rs index c258338..93a2c4b 100644 --- a/rust/dbn/src/enums.rs +++ b/rust/dbn/src/enums.rs @@ -650,7 +650,7 @@ pub mod flags { /// The type of [`InstrumentDefMsg`](crate::record::InstrumentDefMsg) update. #[repr(u8)] -#[derive(Clone, Copy, Debug, PartialEq, Eq, IntoPrimitive, TryFromPrimitive)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, IntoPrimitive, TryFromPrimitive)] pub enum SecurityUpdateAction { /// A new instrument definition. Add = b'A', @@ -666,7 +666,7 @@ pub enum SecurityUpdateAction { /// The type of statistic contained in a [`StatMsg`](crate::record::StatMsg). #[repr(u16)] -#[derive(Clone, Copy, Debug, PartialEq, Eq, IntoPrimitive, TryFromPrimitive)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, IntoPrimitive, TryFromPrimitive)] pub enum StatType { /// The price of the first trade of an instrument. `price` will be set. OpeningPrice = 1, @@ -708,7 +708,7 @@ pub enum StatType { /// The type of [`StatMsg`](crate::record::StatMsg) update. #[repr(u8)] -#[derive(Clone, Copy, Debug, PartialEq, Eq, IntoPrimitive, TryFromPrimitive)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, IntoPrimitive, TryFromPrimitive)] pub enum StatUpdateAction { /// A new statistic. New = 1, @@ -717,7 +717,7 @@ pub enum StatUpdateAction { } /// How to handle decoding DBN data from a prior version. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] #[cfg_attr( feature = "python", pyo3::pyclass(module = "databento_dbn", rename_all = "SCREAMING_SNAKE_CASE") diff --git a/rust/dbn/src/python/record.rs b/rust/dbn/src/python/record.rs index 18b3b80..6c28e29 100644 --- a/rust/dbn/src/python/record.rs +++ b/rust/dbn/src/python/record.rs @@ -197,6 +197,14 @@ impl BidAskPair { } } + fn __richcmp__(&self, other: &Self, op: CompareOp, py: Python<'_>) -> Py { + match op { + CompareOp::Eq => self.eq(other).into_py(py), + CompareOp::Ne => self.ne(other).into_py(py), + _ => py.NotImplemented(), + } + } + fn __repr__(&self) -> String { format!("{self:?}") } diff --git a/rust/dbn/src/record.rs b/rust/dbn/src/record.rs index 1198297..f8e6121 100644 --- a/rust/dbn/src/record.rs +++ b/rust/dbn/src/record.rs @@ -31,7 +31,7 @@ pub use conv::{ /// Common data for all Databento records. Always found at the beginning of a record /// struct. #[repr(C)] -#[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq)] +#[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] #[cfg_attr( feature = "python", @@ -61,7 +61,7 @@ pub struct RecordHeader { /// A market-by-order (MBO) tick message. The record of the /// [`Mbo`](crate::enums::Schema::Mbo) schema. #[repr(C)] -#[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq)] +#[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] #[cfg_attr( feature = "python", @@ -117,7 +117,7 @@ pub struct MboMsg { /// A level. #[repr(C)] -#[derive(Clone, Debug, JsonSerialize, PartialEq, Eq)] +#[derive(Clone, Debug, JsonSerialize, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] #[cfg_attr( feature = "python", @@ -145,7 +145,7 @@ pub struct BidAskPair { /// Market by price implementation with a book depth of 0. Equivalent to /// MBP-0. The record of the [`Trades`](crate::enums::Schema::Trades) schema. #[repr(C)] -#[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq)] +#[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] #[cfg_attr( feature = "python", @@ -197,7 +197,7 @@ pub struct TradeMsg { /// Market by price implementation with a known book depth of 1. The record of the /// [`Mbp1`](crate::enums::Schema::Mbp1) schema. #[repr(C)] -#[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq)] +#[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] #[cfg_attr( feature = "python", @@ -253,7 +253,7 @@ pub struct Mbp1Msg { /// Market by price implementation with a known book depth of 10. The record of the /// [`Mbp10`](crate::enums::Schema::Mbp10) schema. #[repr(C)] -#[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq)] +#[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] #[cfg_attr( feature = "python", @@ -316,7 +316,7 @@ pub type TbboMsg = Mbp1Msg; /// - [`Ohlcv1D`](crate::enums::Schema::Ohlcv1D) /// - [`OhlcvEod`](crate::enums::Schema::OhlcvEod) #[repr(C)] -#[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq)] +#[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] #[cfg_attr( feature = "python", @@ -355,7 +355,7 @@ pub struct OhlcvMsg { /// [`Status`](crate::enums::Schema::Status) schema. #[doc(hidden)] #[repr(C)] -#[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq)] +#[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] #[cfg_attr( feature = "python", @@ -386,7 +386,7 @@ pub struct StatusMsg { /// Definition of an instrument. The record of the /// [`Definition`](crate::enums::Schema::Definition) schema. #[repr(C)] -#[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq)] +#[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] #[cfg_attr( feature = "python", @@ -604,7 +604,7 @@ pub struct InstrumentDefMsg { /// An auction imbalance message. #[repr(C)] -#[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq)] +#[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] #[cfg_attr( feature = "python", @@ -696,7 +696,7 @@ pub struct ImbalanceMsg { /// A statistics message. A catchall for various data disseminated by publishers. /// The [`stat_type`](Self::stat_type) indicates the statistic contained in the message. #[repr(C)] -#[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq)] +#[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] #[cfg_attr( feature = "python", @@ -747,7 +747,7 @@ pub struct StatMsg { /// An error message from the Databento Live Subscription Gateway (LSG). #[repr(C)] -#[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq)] +#[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] #[cfg_attr( feature = "python", @@ -768,7 +768,7 @@ pub struct ErrorMsg { /// A symbol mapping message which maps a symbol of one [`SType`](crate::enums::SType) /// to another. #[repr(C)] -#[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq)] +#[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] #[cfg_attr( feature = "python", @@ -808,7 +808,7 @@ pub struct SymbolMappingMsg { /// A non-error message from the Databento Live Subscription Gateway (LSG). Also used /// for heartbeating. #[repr(C)] -#[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq)] +#[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] #[cfg_attr( feature = "python", @@ -899,7 +899,7 @@ pub trait HasRType: Record + RecordMut { /// Wrapper object for records that include the live gateway send timestamp (`ts_out`). #[repr(C)] -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] pub struct WithTsOut { /// The inner record. From 76dac327cb00f9753b5bb86a079da36c85b3f5a2 Mon Sep 17 00:00:00 2001 From: Carter Green Date: Mon, 4 Dec 2023 09:10:28 -0600 Subject: [PATCH 05/21] ADD: Add type hint for `Metadata.__init__` --- CHANGELOG.md | 3 +- python/databento_dbn.pyi | 46 ++++++++++++++++++++----- python/pyproject.toml | 2 ++ python/python/databento_dbn/__init__.py | 33 ++++++++++++++++++ python/src/dbn_decoder.rs | 6 ++-- python/src/lib.rs | 5 +-- 6 files changed, 81 insertions(+), 14 deletions(-) create mode 100644 python/python/databento_dbn/__init__.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 327d5a9..75ffd3f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,8 @@ ## 0.14.3 - TBD ### Enhancements -- Added `version` param to `Metadata::builder` to choose between DBNv1 and DBNv2 specs +- Added type definition for `Metadata.__init__` +- Added `version` param to Python `Metadata` contructor choose between DBNv1 and DBNv2 ### Bug fixes - Fixed Python `size_hint` return value for `InstrumentDefMsgV1` and diff --git a/python/databento_dbn.pyi b/python/databento_dbn.pyi index 603b9a1..d12a9c3 100644 --- a/python/databento_dbn.pyi +++ b/python/databento_dbn.pyi @@ -5,7 +5,9 @@ import datetime as dt from collections.abc import Iterable from collections.abc import Sequence from enum import Enum -from typing import Any, BinaryIO, ClassVar, SupportsBytes, TextIO, Union +from typing import BinaryIO, ClassVar, SupportsBytes, TextIO, TypedDict, Union + +from . import SymbolMapping FIXED_PRICE_SCALE: int @@ -227,18 +229,46 @@ class VersionUpgradePolicy(Enum): @classmethod def variants(cls) -> Iterable[SType]: ... +class MappingIntervalDict(TypedDict): + """ + Represents a symbol mapping over a start and end date range interval. + + Parameters + ---------- + start_date : dt.date + The start of the mapping period. + end_date : dt.date + The end of the mapping period. + symbol : str + The symbol value. + """ + + start_date: dt.date + end_date: dt.date + symbol: str + class Metadata(SupportsBytes): """ Information about the data contained in a DBN file or stream. DBN requires the Metadata to be included at the start of the encoded data. - - See Also - -------- - decode_metadata - encode_metadata - """ + def __init__( + self, + dataset: str, + start: int, + stype_out: SType, + symbols: list[str], + partial: list[str], + not_found: list[str], + mappings: Sequence[SymbolMapping], + schema: Schema | None = None, + stype_in: SType | None = None, + end: int | None = None, + limit: int | None = None, + ts_out: bool | None = None, + version: int | None = None, + ) -> None: ... def __bytes__(self) -> bytes: ... def __eq__(self, other) -> bool: ... def __ne__(self, other) -> bool: ... @@ -368,7 +398,7 @@ class Metadata(SupportsBytes): """ @property - def mappings(self) -> dict[str, list[dict[str, Any]]]: + def mappings(self) -> dict[str, list[MappingIntervalDict]]: """ Symbol mappings containing a native symbol and its mapping intervals. diff --git a/python/pyproject.toml b/python/pyproject.toml index a9e53c1..9ac0c97 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -32,3 +32,5 @@ classifiers = [ [tool.maturin] features = ["pyo3/extension-module"] +python-source = "python" +module-name = "databento_dbn._lib" diff --git a/python/python/databento_dbn/__init__.py b/python/python/databento_dbn/__init__.py new file mode 100644 index 0000000..ffe6582 --- /dev/null +++ b/python/python/databento_dbn/__init__.py @@ -0,0 +1,33 @@ +import datetime as dt +from typing import Protocol, Sequence + +# Import native module +from ._lib import * # noqa: F403 + + +class MappingInterval(Protocol): + """ + Represents a symbol mapping over a start and end date range interval. + + Parameters + ---------- + start_date : dt.date + The start of the mapping period. + end_date : dt.date + The end of the mapping period. + symbol : str + The symbol value. + + """ + + start_date: dt.date + end_date: dt.date + symbol: str + +class SymbolMapping(Protocol): + """ + Represents the mappings for one native symbol. + """ + + raw_symbol: str + intervals: Sequence[MappingInterval] diff --git a/python/src/dbn_decoder.rs b/python/src/dbn_decoder.rs index 0a7a40e..0e76aff 100644 --- a/python/src/dbn_decoder.rs +++ b/python/src/dbn_decoder.rs @@ -241,7 +241,7 @@ mod tests { py_run!( py, path, - r#"from databento_dbn import DBNDecoder + r#"from _lib import DBNDecoder decoder = DBNDecoder() with open(path, 'rb') as fin: @@ -260,7 +260,7 @@ for record in records[1:]: setup(); Python::with_gil(|py| { py.run( - r#"from databento_dbn import DBNDecoder, Metadata, Schema, SType + r#"from _lib import DBNDecoder, Metadata, Schema, SType metadata = Metadata( dataset="GLBX.MDP3", @@ -296,7 +296,7 @@ except Exception as ex: setup(); Python::with_gil(|py| { py.run( - r#"from databento_dbn import DBNDecoder, OHLCVMsg + r#"from _lib import DBNDecoder, OHLCVMsg decoder = DBNDecoder(has_metadata=False) record = OHLCVMsg(0x20, 1, 10, 0, 0, 0, 0, 0, 0) diff --git a/python/src/lib.rs b/python/src/lib.rs index 0d87c85..54291b0 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -21,6 +21,7 @@ mod transcoder; /// A Python module wrapping dbn functions #[pymodule] // The name of the function must match `lib.name` in `Cargo.toml` +#[pyo3(name = "_lib")] fn databento_dbn(_py: Python<'_>, m: &PyModule) -> PyResult<()> { fn checked_add_class(m: &PyModule) -> PyResult<()> { // ensure a module was specified, otherwise it defaults to builtins @@ -100,7 +101,7 @@ mod tests { pyo3::py_run!( py, stype_in stype_out, - r#"from databento_dbn import Metadata, Schema, SType + r#"from _lib import Metadata, Schema, SType metadata = Metadata( dataset="GLBX.MDP3", @@ -133,7 +134,7 @@ assert metadata.ts_out is False"# setup(); Python::with_gil(|py| { py.run( - r#"from databento_dbn import DBNDecoder + r#"from _lib import DBNDecoder decoder = DBNDecoder() try: From 704eb1e6b36390f98107ddd5248925d87ed48f92 Mon Sep 17 00:00:00 2001 From: Carter Green Date: Tue, 5 Dec 2023 12:09:57 -0600 Subject: [PATCH 06/21] MOD: Update DBN changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 75ffd3f..de26f85 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,8 +4,10 @@ ### Enhancements - Added type definition for `Metadata.__init__` - Added `version` param to Python `Metadata` contructor choose between DBNv1 and DBNv2 +- Implemented `Hash` for all record types ### Bug fixes +- Added missing Python `__eq__` and `__ne__` implementations for `BidAskPair` - Fixed Python `size_hint` return value for `InstrumentDefMsgV1` and `SymbolMappingMsgV1` @@ -25,7 +27,6 @@ - Changed `PitSymbolMap::on_symbol_mapping` to accept either version of `SymbolMappingMsg` - ### Bug fixes - Fixed missing DBNv1 compatibility in `PitSymbolMap::on_record` - Fixed missing Python export for `VersionUpgradePolicy` From 2bbb76de273f84eb16c87e797804aae5bb3117c5 Mon Sep 17 00:00:00 2001 From: Carter Green Date: Mon, 11 Dec 2023 11:53:45 -0600 Subject: [PATCH 07/21] ADD: Add Deserialize and Serialize to records --- CHANGELOG.md | 3 +++ rust/dbn/src/compat.rs | 5 +++++ rust/dbn/src/enums.rs | 3 +++ rust/dbn/src/record.rs | 38 ++++++++++++++++++++++++++++++++++++- rust/dbn/src/record/conv.rs | 27 ++++++++++++++++++++++++++ 5 files changed, 75 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index de26f85..9407121 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ - Added type definition for `Metadata.__init__` - Added `version` param to Python `Metadata` contructor choose between DBNv1 and DBNv2 - Implemented `Hash` for all record types +- Implemented `Deserialize` and `Serialize` for all records and enums (with `serde` + feature enabled). This allows serializing records with additional encodings not + supported by the DBN crate ### Bug fixes - Added missing Python `__eq__` and `__ne__` implementations for `BidAskPair` diff --git a/rust/dbn/src/compat.rs b/rust/dbn/src/compat.rs index 013a072..4eb90be 100644 --- a/rust/dbn/src/compat.rs +++ b/rust/dbn/src/compat.rs @@ -75,6 +75,7 @@ pub unsafe fn decode_record_ref<'a>( #[repr(C)] #[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr( feature = "python", pyo3::pyclass(dict, module = "databento_dbn"), @@ -207,10 +208,13 @@ pub struct InstrumentDefMsgV1 { #[pyo3(get, set)] pub channel_id: u16, /// The currency used for price fields. + #[cfg_attr(feature = "serde", serde(with = "crate::record::cstr_serde"))] pub currency: [c_char; 4], /// The currency used for settlement, if different from `currency`. + #[cfg_attr(feature = "serde", serde(with = "crate::record::cstr_serde"))] pub settl_currency: [c_char; 4], /// The strategy type of the spread. + #[cfg_attr(feature = "serde", serde(with = "crate::record::cstr_serde"))] pub secsubtype: [c_char; 6], /// The instrument raw symbol assigned by the publisher. #[dbn(encode_order(2))] @@ -302,6 +306,7 @@ pub struct InstrumentDefMsgV1 { #[repr(C)] #[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr( feature = "python", pyo3::pyclass(dict, module = "databento_dbn"), diff --git a/rust/dbn/src/enums.rs b/rust/dbn/src/enums.rs index 93a2c4b..d6594d4 100644 --- a/rust/dbn/src/enums.rs +++ b/rust/dbn/src/enums.rs @@ -87,6 +87,7 @@ impl From for char { /// The type of matching algorithm used for the instrument at the exchange. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, TryFromPrimitive, IntoPrimitive)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[repr(u8)] pub enum MatchAlgorithm { /// First-in-first-out matching. @@ -120,6 +121,7 @@ impl From for char { /// Whether the instrument is user-defined. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, TryFromPrimitive, IntoPrimitive, Default)] #[repr(u8)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub enum UserDefinedInstrument { /// The instrument is not user-defined. #[default] @@ -651,6 +653,7 @@ pub mod flags { /// The type of [`InstrumentDefMsg`](crate::record::InstrumentDefMsg) update. #[repr(u8)] #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, IntoPrimitive, TryFromPrimitive)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub enum SecurityUpdateAction { /// A new instrument definition. Add = b'A', diff --git a/rust/dbn/src/record.rs b/rust/dbn/src/record.rs index f8e6121..1702fbb 100644 --- a/rust/dbn/src/record.rs +++ b/rust/dbn/src/record.rs @@ -23,6 +23,8 @@ use crate::{ Error, Result, SYMBOL_CSTR_LEN, }; pub(crate) use conv::as_u8_slice; +#[cfg(feature = "serde")] +pub(crate) use conv::cstr_serde; pub use conv::{ c_chars_to_str, str_to_c_chars, transmute_header_bytes, transmute_record, transmute_record_bytes, transmute_record_mut, ts_to_dt, @@ -33,6 +35,7 @@ pub use conv::{ #[repr(C)] #[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr( feature = "python", pyo3::pyclass(get_all, set_all, dict, module = "databento_dbn"), @@ -63,6 +66,7 @@ pub struct RecordHeader { #[repr(C)] #[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr( feature = "python", pyo3::pyclass(set_all, dict, module = "databento_dbn", name = "MBOMsg"), @@ -119,6 +123,7 @@ pub struct MboMsg { #[repr(C)] #[derive(Clone, Debug, JsonSerialize, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr( feature = "python", pyo3::pyclass(get_all, set_all, dict, module = "databento_dbn"), @@ -147,6 +152,7 @@ pub struct BidAskPair { #[repr(C)] #[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr( feature = "python", pyo3::pyclass(set_all, dict, module = "databento_dbn"), @@ -199,6 +205,7 @@ pub struct TradeMsg { #[repr(C)] #[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr( feature = "python", pyo3::pyclass(set_all, dict, module = "databento_dbn", name = "MBP1Msg"), @@ -255,6 +262,7 @@ pub struct Mbp1Msg { #[repr(C)] #[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr( feature = "python", pyo3::pyclass(set_all, dict, module = "databento_dbn", name = "MBP10Msg"), @@ -318,6 +326,7 @@ pub type TbboMsg = Mbp1Msg; #[repr(C)] #[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr( feature = "python", pyo3::pyclass(get_all, set_all, dict, module = "databento_dbn", name = "OHLCVMsg"), @@ -357,6 +366,7 @@ pub struct OhlcvMsg { #[repr(C)] #[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr( feature = "python", pyo3::pyclass(dict, module = "databento_dbn"), @@ -374,6 +384,7 @@ pub struct StatusMsg { #[dbn(unix_nanos)] #[pyo3(get, set)] pub ts_recv: u64, + #[cfg_attr(feature = "serde", serde(with = "conv::cstr_serde"))] pub group: [c_char; 21], #[pyo3(get, set)] pub trading_status: u8, @@ -388,6 +399,7 @@ pub struct StatusMsg { #[repr(C)] #[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr( feature = "python", pyo3::pyclass(dict, module = "databento_dbn"), @@ -521,30 +533,41 @@ pub struct InstrumentDefMsg { #[pyo3(get, set)] pub channel_id: u16, /// The currency used for price fields. + #[cfg_attr(feature = "serde", serde(with = "conv::cstr_serde"))] pub currency: [c_char; 4], /// The currency used for settlement, if different from `currency`. + #[cfg_attr(feature = "serde", serde(with = "conv::cstr_serde"))] pub settl_currency: [c_char; 4], /// The strategy type of the spread. + #[cfg_attr(feature = "serde", serde(with = "conv::cstr_serde"))] pub secsubtype: [c_char; 6], /// The instrument raw symbol assigned by the publisher. #[dbn(encode_order(2))] + #[cfg_attr(feature = "serde", serde(with = "conv::cstr_serde"))] pub raw_symbol: [c_char; SYMBOL_CSTR_LEN], /// The security group code of the instrument. + #[cfg_attr(feature = "serde", serde(with = "conv::cstr_serde"))] pub group: [c_char; 21], /// The exchange used to identify the instrument. + #[cfg_attr(feature = "serde", serde(with = "conv::cstr_serde"))] pub exchange: [c_char; 5], /// The underlying asset code (product code) of the instrument. + #[cfg_attr(feature = "serde", serde(with = "conv::cstr_serde"))] pub asset: [c_char; 7], /// The ISO standard instrument categorization code. + #[cfg_attr(feature = "serde", serde(with = "conv::cstr_serde"))] pub cfi: [c_char; 7], /// The type of the instrument, e.g. FUT for future or future spread. + #[cfg_attr(feature = "serde", serde(with = "conv::cstr_serde"))] pub security_type: [c_char; 7], /// The unit of measure for the instrument’s original contract size, e.g. USD or LBS. + #[cfg_attr(feature = "serde", serde(with = "conv::cstr_serde"))] pub unit_of_measure: [c_char; 31], /// The symbol of the first underlying instrument. - // TODO(carter): finalize if this size also needs to be increased + #[cfg_attr(feature = "serde", serde(with = "conv::cstr_serde"))] pub underlying: [c_char; 21], /// The currency of [`strike_price`](Self::strike_price). + #[cfg_attr(feature = "serde", serde(with = "conv::cstr_serde"))] pub strike_price_currency: [c_char; 4], /// The classification of the instrument. #[dbn(c_char, encode_order(4))] @@ -599,6 +622,7 @@ pub struct InstrumentDefMsg { pub tick_rule: u8, // Filler for alignment. #[doc(hidden)] + #[cfg_attr(feature = "serde", serde(skip))] pub _reserved: [u8; 10], } @@ -606,6 +630,7 @@ pub struct InstrumentDefMsg { #[repr(C)] #[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr( feature = "python", pyo3::pyclass(set_all, dict, module = "databento_dbn"), @@ -690,6 +715,7 @@ pub struct ImbalanceMsg { pub significant_imbalance: c_char, // Filler for alignment. #[doc(hidden)] + #[cfg_attr(feature = "serde", serde(skip))] pub _dummy: [u8; 1], } @@ -698,6 +724,7 @@ pub struct ImbalanceMsg { #[repr(C)] #[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr( feature = "python", pyo3::pyclass(get_all, set_all, dict, module = "databento_dbn"), @@ -742,6 +769,7 @@ pub struct StatMsg { pub stat_flags: u8, // Filler for alignment #[doc(hidden)] + #[cfg_attr(feature = "serde", serde(skip))] pub _dummy: [u8; 6], } @@ -749,6 +777,7 @@ pub struct StatMsg { #[repr(C)] #[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr( feature = "python", pyo3::pyclass(dict, module = "databento_dbn"), @@ -762,6 +791,7 @@ pub struct ErrorMsg { #[pyo3(get, set)] pub hd: RecordHeader, /// The error message. + #[cfg_attr(feature = "serde", serde(with = "conv::cstr_serde"))] pub err: [c_char; 64], } @@ -770,6 +800,7 @@ pub struct ErrorMsg { #[repr(C)] #[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr( feature = "python", pyo3::pyclass(dict, module = "databento_dbn"), @@ -787,11 +818,13 @@ pub struct SymbolMappingMsg { #[pyo3(get, set)] pub stype_in: u8, /// The input symbol. + #[cfg_attr(feature = "serde", serde(with = "conv::cstr_serde"))] pub stype_in_symbol: [c_char; SYMBOL_CSTR_LEN], /// The output symbology type of `stype_out_symbol`. #[pyo3(get, set)] pub stype_out: u8, /// The output symbol. + #[cfg_attr(feature = "serde", serde(with = "conv::cstr_serde"))] pub stype_out_symbol: [c_char; SYMBOL_CSTR_LEN], /// The start of the mapping interval expressed as the number of nanoseconds since /// the UNIX epoch. @@ -810,6 +843,7 @@ pub struct SymbolMappingMsg { #[repr(C)] #[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr( feature = "python", pyo3::pyclass(dict, module = "databento_dbn"), @@ -823,6 +857,7 @@ pub struct SystemMsg { #[pyo3(get, set)] pub hd: RecordHeader, /// The message from the Databento Live Subscription Gateway (LSG). + #[cfg_attr(feature = "serde", serde(with = "conv::cstr_serde"))] pub msg: [c_char; 64], } @@ -901,6 +936,7 @@ pub trait HasRType: Record + RecordMut { #[repr(C)] #[derive(Clone, Debug, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub struct WithTsOut { /// The inner record. pub rec: T, diff --git a/rust/dbn/src/record/conv.rs b/rust/dbn/src/record/conv.rs index 770a604..a3c9083 100644 --- a/rust/dbn/src/record/conv.rs +++ b/rust/dbn/src/record/conv.rs @@ -141,3 +141,30 @@ pub fn ts_to_dt(ts: u64) -> Option { Some(time::OffsetDateTime::from_unix_timestamp_nanos(ts as i128).unwrap()) } } + +#[cfg(feature = "serde")] +pub(crate) mod cstr_serde { + use std::ffi::c_char; + + use serde::{de, ser, Deserialize, Deserializer, Serializer}; + + use super::{c_chars_to_str, str_to_c_chars}; + + pub fn serialize( + chars: &[c_char; N], + serializer: S, + ) -> Result + where + S: Serializer, + { + serializer.serialize_str(c_chars_to_str(chars).map_err(ser::Error::custom)?) + } + + pub fn deserialize<'de, D, const N: usize>(deserializer: D) -> Result<[c_char; N], D::Error> + where + D: Deserializer<'de>, + { + let str = String::deserialize(deserializer)?; + str_to_c_chars(&str).map_err(de::Error::custom) + } +} From 52e1e42b703db18841181e430688dd4d48f84180 Mon Sep 17 00:00:00 2001 From: Jack Culhane Date: Tue, 12 Dec 2023 15:41:31 +0000 Subject: [PATCH 08/21] ADD: Add OPRA participant MIAX Sapphire --- CHANGELOG.md | 1 + rust/dbn/src/publishers.rs | 15 +++++++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9407121..979cd6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - Implemented `Deserialize` and `Serialize` for all records and enums (with `serde` feature enabled). This allows serializing records with additional encodings not supported by the DBN crate +- Added new publisher value for OPRA MIAX Sapphire ### Bug fixes - Added missing Python `__eq__` and `__ne__` implementations for `BidAskPair` diff --git a/rust/dbn/src/publishers.rs b/rust/dbn/src/publishers.rs index dfaee75..316f9c0 100644 --- a/rust/dbn/src/publishers.rs +++ b/rust/dbn/src/publishers.rs @@ -90,10 +90,12 @@ pub enum Venue { Ndex = 39, /// Databento Equities - Consolidated Dbeq = 40, + /// MIAX Sapphire + Sphr = 41, } /// The number of Venue variants. -pub const VENUE_COUNT: usize = 40; +pub const VENUE_COUNT: usize = 41; impl Venue { /// Convert a Venue to its `str` representation. @@ -139,6 +141,7 @@ impl Venue { Self::Ifeu => "IFEU", Self::Ndex => "NDEX", Self::Dbeq => "DBEQ", + Self::Sphr => "SPHR", } } } @@ -200,6 +203,7 @@ impl std::str::FromStr for Venue { "IFEU" => Ok(Self::Ifeu), "NDEX" => Ok(Self::Ndex), "DBEQ" => Ok(Self::Dbeq), + "SPHR" => Ok(Self::Sphr), _ => Err(Error::conversion::(s)), } } @@ -484,10 +488,12 @@ pub enum Publisher { DbeqBasicDbeq = 59, /// DBEQ Plus - Consolidated DbeqPlusDbeq = 60, + /// OPRA - MIAX Sapphire + OpraPillarSphr = 61, } /// The number of Publisher variants. -pub const PUBLISHER_COUNT: usize = 60; +pub const PUBLISHER_COUNT: usize = 61; impl Publisher { /// Convert a Publisher to its `str` representation. @@ -553,6 +559,7 @@ impl Publisher { Self::NdexImpactNdex => "NDEX.IMPACT.NDEX", Self::DbeqBasicDbeq => "DBEQ.BASIC.DBEQ", Self::DbeqPlusDbeq => "DBEQ.PLUS.DBEQ", + Self::OpraPillarSphr => "OPRA.PILLAR.SPHR", } } @@ -619,6 +626,7 @@ impl Publisher { Self::NdexImpactNdex => Venue::Ndex, Self::DbeqBasicDbeq => Venue::Dbeq, Self::DbeqPlusDbeq => Venue::Dbeq, + Self::OpraPillarSphr => Venue::Sphr, } } @@ -685,6 +693,7 @@ impl Publisher { Self::NdexImpactNdex => Dataset::NdexImpact, Self::DbeqBasicDbeq => Dataset::DbeqBasic, Self::DbeqPlusDbeq => Dataset::DbeqPlus, + Self::OpraPillarSphr => Dataset::OpraPillar, } } @@ -753,6 +762,7 @@ impl Publisher { (Dataset::NdexImpact, Venue::Ndex) => Ok(Self::NdexImpactNdex), (Dataset::DbeqBasic, Venue::Dbeq) => Ok(Self::DbeqBasicDbeq), (Dataset::DbeqPlus, Venue::Dbeq) => Ok(Self::DbeqPlusDbeq), + (Dataset::OpraPillar, Venue::Sphr) => Ok(Self::OpraPillarSphr), _ => Err(Error::conversion::(format!("({dataset}, {venue})"))), } } @@ -835,6 +845,7 @@ impl std::str::FromStr for Publisher { "NDEX.IMPACT.NDEX" => Ok(Self::NdexImpactNdex), "DBEQ.BASIC.DBEQ" => Ok(Self::DbeqBasicDbeq), "DBEQ.PLUS.DBEQ" => Ok(Self::DbeqPlusDbeq), + "OPRA.PILLAR.SPHR" => Ok(Self::OpraPillarSphr), _ => Err(Error::conversion::(s)), } } From 79a21e3e481b899da5df6819b65ba0cd5e642634 Mon Sep 17 00:00:00 2001 From: Carter Green Date: Wed, 13 Dec 2023 17:47:02 -0600 Subject: [PATCH 09/21] FIX: Fix TsSymbolMap panic with invalid interval --- CHANGELOG.md | 1 + rust/dbn/src/symbol_map.rs | 49 ++++++++++++++++++++++++++++---------- 2 files changed, 37 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 979cd6b..a20abe4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ - Added new publisher value for OPRA MIAX Sapphire ### Bug fixes +- Fixed panic in `TsSymbolMap` when `start_date` == `end_date` - Added missing Python `__eq__` and `__ne__` implementations for `BidAskPair` - Fixed Python `size_hint` return value for `InstrumentDefMsgV1` and `SymbolMappingMsgV1` diff --git a/rust/dbn/src/symbol_map.rs b/rust/dbn/src/symbol_map.rs index 6a669de..d470f54 100644 --- a/rust/dbn/src/symbol_map.rs +++ b/rust/dbn/src/symbol_map.rs @@ -1,6 +1,6 @@ //! Maps for mapping instrument IDs to human-readable symbols. -use std::{collections::HashMap, ops::Deref, sync::Arc}; +use std::{cmp::Ordering, collections::HashMap, ops::Deref, sync::Arc}; use time::{macros::time, PrimitiveDateTime}; @@ -73,21 +73,27 @@ impl TsSymbolMap { end_date: time::Date, symbol: Arc, ) -> crate::Result<()> { - if start_date > end_date { - return Err(Error::BadArgument { + match start_date.cmp(&end_date) { + Ordering::Less => { + let mut day = start_date; + loop { + self.0.insert((day, instrument_id), symbol.clone()); + day = day.next_day().unwrap(); + if day >= end_date { + break; + } + } + Ok(()) + } + Ordering::Equal => { + // Shouldn't happen but better to just ignore + Ok(()) + } + Ordering::Greater => Err(Error::BadArgument { param_name: "start_date".to_owned(), desc: "start_date cannot come after end_date".to_owned(), - }); + }), } - let mut day = start_date; - loop { - self.0.insert((day, instrument_id), symbol.clone()); - day = day.next_day().unwrap(); - if day == end_date { - break; - } - } - Ok(()) } /// Returns the symbol mapping for the given date and instrument ID. Returns `None` @@ -1001,4 +1007,21 @@ mod tests { Ok(()) } + + // start_date == end_date is generally invalid and + // previously caused a panic + #[test] + fn test_insert_start_end_date_same() { + let mut target = TsSymbolMap::new(); + target + .insert( + 1, + date!(2023 - 12 - 03), + date!(2023 - 12 - 03), + Arc::new("test".to_owned()), + ) + .unwrap(); + // should have no effect + assert!(target.is_empty()); + } } From e9a8e724d5c555b4f2b50d7366199f3acf15212c Mon Sep 17 00:00:00 2001 From: Carter Green Date: Thu, 14 Dec 2023 10:12:42 -0600 Subject: [PATCH 10/21] ADD: Add pre-commit for non-core Rust projects --- rust/dbn/src/decode/compat.rs | 1 - 1 file changed, 1 deletion(-) delete mode 100644 rust/dbn/src/decode/compat.rs diff --git a/rust/dbn/src/decode/compat.rs b/rust/dbn/src/decode/compat.rs deleted file mode 100644 index 9407185..0000000 --- a/rust/dbn/src/decode/compat.rs +++ /dev/null @@ -1 +0,0 @@ -/// Decoder wrapper for converting from one DBN version to another. From b932e2f212a8d88eece6325c5d6a0fd1b683d8e4 Mon Sep 17 00:00:00 2001 From: Carter Green Date: Thu, 14 Dec 2023 08:34:02 -0600 Subject: [PATCH 11/21] ADD: Add schema filter to dbn CLI --- CHANGELOG.md | 19 +++- c/src/decode.rs | 2 +- python/src/encode.rs | 17 ++-- rust/dbn-cli/src/encode.rs | 44 +++------ rust/dbn-cli/src/filter.rs | 122 ++++++++++++++++++++++++ rust/dbn-cli/src/lib.rs | 17 +++- rust/dbn-cli/src/main.rs | 64 ++++++++----- rust/dbn-cli/tests/integration_tests.rs | 11 ++- rust/dbn/src/decode.rs | 51 ++++++++-- rust/dbn/src/decode/dbn/sync.rs | 33 ++++--- rust/dbn/src/decode/dbz.rs | 14 ++- rust/dbn/src/decode/stream.rs | 12 +-- rust/dbn/src/encode.rs | 13 +-- rust/dbn/src/encode/csv/sync.rs | 6 +- rust/dbn/src/python/metadata.rs | 2 +- 15 files changed, 306 insertions(+), 121 deletions(-) create mode 100644 rust/dbn-cli/src/filter.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index a20abe4..9965a76 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,14 +1,27 @@ # Changelog -## 0.14.3 - TBD +## 0.15.0 - TBD ### Enhancements -- Added type definition for `Metadata.__init__` +- Added `--schema` option to `dbn` CLI tool to filter a DBN to a particular schema. This + allows outputting saved live data to CSV +- Allowed passing `--limit` option to `dbn` CLI tool with `--metadata` flag +- Improved performance of decoding uncompressed DBN fragments with the `dbn` CLI tool - Added `version` param to Python `Metadata` contructor choose between DBNv1 and DBNv2 -- Implemented `Hash` for all record types - Implemented `Deserialize` and `Serialize` for all records and enums (with `serde` feature enabled). This allows serializing records with additional encodings not supported by the DBN crate +- Implemented `Hash` for all record types - Added new publisher value for OPRA MIAX Sapphire +- Added Python type definition for `Metadata.__init__` +- Added `metadata_mut` method to decoders to get a mutable reference to the decoded + metadata + +### Breaking changes +- Split `DecodeDbn` trait into `DecodeRecord` and `DbnMetadata` traits for more + flexibility. `DecodeDbn` continues to exist as a trait alias +- Moved `decode_stream` out of `DecodeDbn` to its own separate trait `DecodeStream` +- Changed trait bounds of `EncodeDbn::encode_decoded` and `encode_decoded_with_limit` to + `DecodeRecordRef + DbnMetadata` ### Bug fixes - Fixed panic in `TsSymbolMap` when `start_date` == `end_date` diff --git a/c/src/decode.rs b/c/src/decode.rs index f99b9a0..45beb58 100644 --- a/c/src/decode.rs +++ b/c/src/decode.rs @@ -9,7 +9,7 @@ use std::{ }; use dbn::{ - decode::{DecodeDbn, DecodeRecordRef, DynDecoder}, + decode::{DbnMetadata, DecodeRecordRef, DynDecoder}, Compression, Metadata, Record, RecordHeader, VersionUpgradePolicy, }; diff --git a/python/src/encode.rs b/python/src/encode.rs index a280954..ab40034 100644 --- a/python/src/encode.rs +++ b/python/src/encode.rs @@ -210,16 +210,15 @@ fn py_to_rs_io_err(e: PyErr) -> io::Error { #[cfg(test)] pub mod tests { - use std::io::{Cursor, Seek, Write}; - - use std::sync::{Arc, Mutex}; + use std::{ + io::{Cursor, Seek, Write}, + sync::{Arc, Mutex}, + }; - use dbn::datasets::GLBX_MDP3; use dbn::{ - decode::{dbn::Decoder as DbnDecoder, DecodeDbn}, - enums::SType, - metadata::MetadataBuilder, - record::TbboMsg, + datasets::GLBX_MDP3, + decode::{dbn::Decoder as DbnDecoder, DbnMetadata, DecodeRecord}, + SType, TbboMsg, }; use super::*; @@ -298,7 +297,7 @@ pub mod tests { let mock_file = MockPyFile::new(); let output_buf = mock_file.inner(); let mock_file = Py::new(py, mock_file).unwrap().into_py(py); - let metadata = MetadataBuilder::new() + let metadata = Metadata::builder() .dataset(DATASET.to_owned()) .schema(Some($schema)) .start(0) diff --git a/rust/dbn-cli/src/encode.rs b/rust/dbn-cli/src/encode.rs index a21693c..7f9f8c9 100644 --- a/rust/dbn-cli/src/encode.rs +++ b/rust/dbn-cli/src/encode.rs @@ -1,7 +1,7 @@ use std::io; use dbn::{ - decode::{DbnRecordDecoder, DecodeDbn, DecodeRecordRef, DynDecoder}, + decode::{DbnMetadata, DecodeRecordRef}, encode::{ json, DbnEncodable, DbnRecordEncoder, DynEncoder, DynWriter, EncodeDbn, EncodeRecordRef, }, @@ -10,7 +10,10 @@ use dbn::{ use crate::{infer_encoding_and_compression, output_from_args, Args}; -pub fn encode_from_dbn(decoder: DynDecoder, args: &Args) -> anyhow::Result<()> { +pub fn encode_from_dbn(decoder: D, args: &Args) -> anyhow::Result<()> +where + D: DecodeRecordRef + DbnMetadata, +{ let writer = output_from_args(args)?; let (encoding, compression) = infer_encoding_and_compression(args)?; let encode_res = if args.should_output_metadata { @@ -23,21 +26,7 @@ pub fn encode_from_dbn(decoder: DynDecoder, args: &Args) -> a ) .encode_metadata(decoder.metadata()) } else if args.fragment { - encode_fragment(decoder, writer, compression, args) - } else if let Some(limit) = args.limit { - let mut metadata = decoder.metadata().clone(); - // Update metadata - metadata.limit = args.limit; - DynEncoder::new( - writer, - encoding, - compression, - &metadata, - args.should_pretty_print, - args.should_pretty_print, - args.should_pretty_print, - )? - .encode_decoded_with_limit(decoder, limit) + encode_fragment(decoder, writer, compression) } else { DynEncoder::new( writer, @@ -59,14 +48,14 @@ pub fn encode_from_dbn(decoder: DynDecoder, args: &Args) -> a } } -pub fn encode_from_frag( - mut decoder: DbnRecordDecoder, - args: &Args, -) -> anyhow::Result<()> { +pub fn encode_from_frag(mut decoder: D, args: &Args) -> anyhow::Result<()> +where + D: DecodeRecordRef, +{ let writer = output_from_args(args)?; let (encoding, compression) = infer_encoding_and_compression(args)?; if args.fragment { - encode_fragment(decoder, writer, compression, args)?; + encode_fragment(decoder, writer, compression)?; return Ok(()); } assert!(!args.should_output_metadata); @@ -87,7 +76,6 @@ pub fn encode_from_frag( args.should_pretty_print, args.should_pretty_print, )?; - let mut n = 0; let mut has_written_header = encoding != Encoding::Csv; fn write_header( _record: &T, @@ -115,10 +103,6 @@ pub fn encode_from_frag( } res => res?, }; - n += 1; - if args.limit.map_or(false, |l| n >= l.get()) { - break; - } } Ok(()) } @@ -127,16 +111,10 @@ fn encode_fragment( mut decoder: D, writer: Box, compression: Compression, - args: &Args, ) -> dbn::Result<()> { let mut encoder = DbnRecordEncoder::new(DynWriter::new(writer, compression)?); - let mut n = 0; while let Some(record) = decoder.decode_record_ref()? { encoder.encode_record_ref(record)?; - n += 1; - if args.limit.map_or(false, |l| n >= l.get()) { - break; - } } Ok(()) } diff --git a/rust/dbn-cli/src/filter.rs b/rust/dbn-cli/src/filter.rs new file mode 100644 index 0000000..ab3521c --- /dev/null +++ b/rust/dbn-cli/src/filter.rs @@ -0,0 +1,122 @@ +use std::num::NonZeroU64; + +use dbn::{ + decode::{DbnMetadata, DecodeRecordRef}, + RType, Record, RecordRef, Schema, +}; + +#[derive(Debug)] +pub struct SchemaFilter { + decoder: D, + rtype: Option, +} + +impl SchemaFilter +where + D: DbnMetadata, +{ + pub fn new(mut decoder: D, schema: Option) -> Self { + if let Some(schema) = schema { + decoder.metadata_mut().schema = Some(schema); + } + Self::new_no_metadata(decoder, schema) + } +} + +impl SchemaFilter { + pub fn new_no_metadata(decoder: D, schema: Option) -> Self { + Self { + decoder, + rtype: schema.map(RType::from), + } + } +} + +impl DbnMetadata for SchemaFilter { + fn metadata(&self) -> &dbn::Metadata { + self.decoder.metadata() + } + + fn metadata_mut(&mut self) -> &mut dbn::Metadata { + self.decoder.metadata_mut() + } +} + +impl DecodeRecordRef for SchemaFilter { + fn decode_record_ref(&mut self) -> dbn::Result> { + while let Some(record) = self.decoder.decode_record_ref()? { + if self + .rtype + .map(|rtype| rtype as u8 == record.header().rtype) + .unwrap_or(true) + { + // Safe: casting reference to pointer so the pointer will always be valid. + // Getting around borrow checker limitation. + return Ok(Some(unsafe { + RecordRef::unchecked_from_header(record.header()) + })); + } + } + Ok(None) + } +} + +#[derive(Debug)] +pub struct LimitFilter { + decoder: D, + limit: Option, + record_count: u64, +} + +impl LimitFilter +where + D: DbnMetadata, +{ + pub fn new(mut decoder: D, limit: Option) -> Self { + if let Some(limit) = limit { + let metadata_limit = &mut decoder.metadata_mut().limit; + if let Some(metadata_limit) = metadata_limit { + *metadata_limit = (*metadata_limit).min(limit); + } else { + *metadata_limit = Some(limit); + } + } + Self::new_no_metadata(decoder, limit) + } +} + +impl LimitFilter { + pub fn new_no_metadata(decoder: D, limit: Option) -> Self { + Self { + decoder, + limit, + record_count: 0, + } + } +} + +impl DbnMetadata for LimitFilter { + fn metadata(&self) -> &dbn::Metadata { + self.decoder.metadata() + } + + fn metadata_mut(&mut self) -> &mut dbn::Metadata { + self.decoder.metadata_mut() + } +} + +impl DecodeRecordRef for LimitFilter { + fn decode_record_ref(&mut self) -> dbn::Result> { + if self + .limit + .map(|limit| self.record_count >= limit.get()) + .unwrap_or(false) + { + return Ok(None); + } + Ok(self.decoder.decode_record_ref()?.map(|rec| { + self.record_count += 1; + rec + })) + } +} diff --git a/rust/dbn-cli/src/lib.rs b/rust/dbn-cli/src/lib.rs index cd235d3..1bbe991 100644 --- a/rust/dbn-cli/src/lib.rs +++ b/rust/dbn-cli/src/lib.rs @@ -10,10 +10,11 @@ use clap::{ArgAction, Parser, ValueEnum}; use dbn::{ enums::{Compression, Encoding}, - VersionUpgradePolicy, + Schema, VersionUpgradePolicy, }; pub mod encode; +pub mod filter; /// How the output of the `dbn` command will be encoded. #[derive(Clone, Copy, Debug, ValueEnum)] @@ -118,7 +119,6 @@ pub struct Args { short = 'l', long = "limit", value_name = "NUM_RECORDS", - conflicts_with = "should_output_metadata", help = "Limit the number of records in the output to the specified number" )] pub limit: Option, @@ -149,6 +149,12 @@ pub struct Args { requires = "input_fragment" )] pub input_dbn_version_override: Option, + #[clap( + long = "schema", + help = "Only encode records of this schema. This is particularly useful for transcoding mixed-schema DBN to CSV, which doesn't support mixing schemas", + value_name = "SCHEMA" + )] + pub schema_filter: Option, } impl Args { @@ -174,6 +180,10 @@ impl Args { VersionUpgradePolicy::AsIs } } + + pub fn input_version(&self) -> u8 { + self.input_dbn_version_override.unwrap_or(dbn::DBN_VERSION) + } } /// Infer the [`Encoding`] and [`Compression`] from `args` if they aren't already explicitly @@ -228,8 +238,7 @@ pub fn output_from_args(args: &Args) -> anyhow::Result> { fn open_output_file(path: &PathBuf, force: bool) -> anyhow::Result { let mut options = File::options(); - options.write(true); - options.truncate(true); + options.write(true).truncate(true); if force { options.create(true); } else if path.exists() { diff --git a/rust/dbn-cli/src/main.rs b/rust/dbn-cli/src/main.rs index d57e365..13005ff 100644 --- a/rust/dbn-cli/src/main.rs +++ b/rust/dbn-cli/src/main.rs @@ -1,65 +1,81 @@ -use std::{fs::File, io}; +use std::{ + fs::File, + io::{self, BufReader}, +}; use clap::Parser; -use dbn::decode::{DbnRecordDecoder, DynDecoder}; +use dbn::decode::{DbnMetadata, DbnRecordDecoder, DecodeRecordRef, DynDecoder}; use dbn_cli::{ encode::{encode_from_dbn, encode_from_frag}, + filter::{LimitFilter, SchemaFilter}, Args, }; const STDIN_SENTINEL: &str = "-"; +fn wrap_frag(args: &Args, reader: impl io::Read) -> anyhow::Result { + Ok(LimitFilter::new_no_metadata( + SchemaFilter::new_no_metadata( + DbnRecordDecoder::with_version(reader, args.input_version(), args.upgrade_policy())?, + args.schema_filter, + ), + args.limit, + )) +} + +fn wrap( + args: &Args, + decoder: DynDecoder<'static, R>, +) -> impl DecodeRecordRef + DbnMetadata { + LimitFilter::new(SchemaFilter::new(decoder, args.schema_filter), args.limit) +} + fn main() -> anyhow::Result<()> { let args = Args::parse(); - let input_version = args.input_dbn_version_override.unwrap_or(dbn::DBN_VERSION); + // DBN fragment if args.is_input_fragment { if args.input.as_os_str() == STDIN_SENTINEL { - encode_from_frag( - DbnRecordDecoder::with_version( - io::stdin().lock(), - input_version, - args.upgrade_policy(), - )?, - &args, - ) + encode_from_frag(wrap_frag(&args, io::stdin().lock())?, &args) } else { encode_from_frag( - DbnRecordDecoder::with_version( - File::open(args.input.clone())?, - input_version, - args.upgrade_policy(), - )?, + wrap_frag(&args, BufReader::new(File::open(args.input.clone())?))?, &args, ) } + // Zstd-compressed DBN fragment } else if args.is_input_zstd_fragment { if args.input.as_os_str() == STDIN_SENTINEL { encode_from_frag( - DbnRecordDecoder::with_version( + wrap_frag( + &args, zstd::stream::Decoder::with_buffer(io::stdin().lock())?, - input_version, - args.upgrade_policy(), )?, &args, ) } else { encode_from_frag( - DbnRecordDecoder::with_version( + wrap_frag( + &args, zstd::stream::Decoder::new(File::open(args.input.clone())?)?, - input_version, - args.upgrade_policy(), )?, &args, ) } + // DBN stream (with metadata) } else if args.input.as_os_str() == STDIN_SENTINEL { encode_from_dbn( - DynDecoder::inferred_with_buffer(io::stdin().lock(), args.upgrade_policy())?, + wrap( + &args, + DynDecoder::inferred_with_buffer(io::stdin().lock(), args.upgrade_policy())?, + ), &args, ) } else { encode_from_dbn( - DynDecoder::from_file(&args.input, args.upgrade_policy())?, + wrap( + &args, + DynDecoder::from_file(&args.input, args.upgrade_policy())?, + ), &args, ) } diff --git a/rust/dbn-cli/tests/integration_tests.rs b/rust/dbn-cli/tests/integration_tests.rs index 0105851..77b9365 100644 --- a/rust/dbn-cli/tests/integration_tests.rs +++ b/rust/dbn-cli/tests/integration_tests.rs @@ -351,18 +351,21 @@ fn convert_dbz_to_dbn() { } #[test] -fn metadata_conflicts_with_limit() { +fn limit_and_schema_filter_update_metadata() { cmd() .args([ - &format!("{TEST_DATA_PATH}/test_data.definition.dbn.zst"), + &format!("{TEST_DATA_PATH}/test_data.ohlcv-1m.dbn.zst"), "--json", "--metadata", "--limit", "1", + "--schema", + "ohlcv-1d", ]) .assert() - .failure() - .stderr(contains("'--metadata' cannot be used with '--limit")); + .success() + .stdout(contains(r#""limit":"1""#)) + .stdout(contains(r#""schema":"ohlcv-1d""#)); } #[rstest] diff --git a/rust/dbn/src/decode.rs b/rust/dbn/src/decode.rs index 648c575..3f931a9 100644 --- a/rust/dbn/src/decode.rs +++ b/rust/dbn/src/decode.rs @@ -51,11 +51,17 @@ pub trait DecodeRecordRef { fn decode_record_ref(&mut self) -> crate::Result>; } -/// Trait for types that decode DBN records of a particular type. -pub trait DecodeDbn: DecodeRecordRef + private::BufferSlice { - /// Returns a reference to the decoded [`Metadata`]. +/// Trait for decoders with metadata about what's being decoded. +pub trait DbnMetadata { + /// Returns an immutable reference to the decoded [`Metadata`]. fn metadata(&self) -> &Metadata; + /// Returns a mutable reference to the decoded [`Metadata`]. + fn metadata_mut(&mut self) -> &mut Metadata; +} + +/// Trait for types that decode DBN records of a particular type. +pub trait DecodeRecord { /// Tries to decode a reference to a single record of type `T`. Returns `Ok(None)` /// if the input has been exhausted. /// @@ -70,12 +76,6 @@ pub trait DecodeDbn: DecodeRecordRef + private::BufferSlice { /// [`Error::Decode`](crate::Error::Decode) will be returned. fn decode_record(&mut self) -> crate::Result>; - /// Converts the decoder into a streaming iterator of records of type `T`. This - /// lazily decodes the data. - fn decode_stream(self) -> StreamIterDecoder - where - Self: Sized; - /// Tries to decode all records into a `Vec`. This eagerly decodes the data. /// /// # Errors @@ -99,6 +99,18 @@ pub trait DecodeDbn: DecodeRecordRef + private::BufferSlice { } } +/// A trait alias for DBN decoders with metadata. +pub trait DecodeDbn: DecodeRecord + DecodeRecordRef + DbnMetadata {} + +/// A trait for decoders that can be converted to streaming iterators. +pub trait DecodeStream: DecodeRecord + private::BufferSlice { + /// Converts the decoder into a streaming iterator of records of type `T`. This + /// lazily decodes the data. + fn decode_stream(self) -> StreamIterDecoder + where + Self: Sized; +} + /// A decoder implementing [`DecodeDbn`] whose [`Encoding`](crate::enums::Encoding) and /// [`Compression`] are determined at runtime by peeking at the first few bytes. pub struct DynDecoder<'a, R>(DynDecoderImpl<'a, R>) @@ -250,7 +262,7 @@ where } #[allow(deprecated)] -impl<'a, R> DecodeDbn for DynDecoder<'a, R> +impl<'a, R> DbnMetadata for DynDecoder<'a, R> where R: io::BufRead, { @@ -262,6 +274,20 @@ where } } + fn metadata_mut(&mut self) -> &mut Metadata { + match &mut self.0 { + DynDecoderImpl::Dbn(decoder) => decoder.metadata_mut(), + DynDecoderImpl::ZstdDbn(decoder) => decoder.metadata_mut(), + DynDecoderImpl::LegacyDbz(decoder) => decoder.metadata_mut(), + } + } +} + +#[allow(deprecated)] +impl<'a, R> DecodeRecord for DynDecoder<'a, R> +where + R: io::BufRead, +{ fn decode_record(&mut self) -> crate::Result> { match &mut self.0 { DynDecoderImpl::Dbn(decoder) => decoder.decode_record(), @@ -269,7 +295,12 @@ where DynDecoderImpl::LegacyDbz(decoder) => decoder.decode_record(), } } +} +impl<'a, R> DecodeStream for DynDecoder<'a, R> +where + R: io::BufRead, +{ fn decode_stream(self) -> StreamIterDecoder where Self: Sized, diff --git a/rust/dbn/src/decode/dbn/sync.rs b/rust/dbn/src/decode/dbn/sync.rs index 97e75c3..0f2be11 100644 --- a/rust/dbn/src/decode/dbn/sync.rs +++ b/rust/dbn/src/decode/dbn/sync.rs @@ -11,8 +11,8 @@ use super::{DBN_PREFIX, DBN_PREFIX_LEN}; use crate::{ compat::{self, SYMBOL_CSTR_LEN_V1}, decode::{ - private::BufferSlice, DecodeDbn, DecodeRecordRef, FromLittleEndianSlice, StreamIterDecoder, - VersionUpgradePolicy, + private::BufferSlice, DbnMetadata, DecodeRecord, DecodeRecordRef, DecodeStream, + FromLittleEndianSlice, StreamIterDecoder, VersionUpgradePolicy, }, error::silence_eof_error, HasRType, MappingInterval, Metadata, Record, RecordHeader, RecordRef, SType, Schema, @@ -20,10 +20,7 @@ use crate::{ }; /// Type for decoding files and streams in Databento Binary Encoding (DBN), both metadata and records. -pub struct Decoder -where - R: io::Read, -{ +pub struct Decoder { metadata: Metadata, decoder: RecordDecoder, } @@ -170,18 +167,29 @@ where } } -impl DecodeDbn for Decoder -where - R: io::Read, -{ +impl DbnMetadata for Decoder { fn metadata(&self) -> &Metadata { &self.metadata } + fn metadata_mut(&mut self) -> &mut Metadata { + &mut self.metadata + } +} + +impl DecodeRecord for Decoder +where + R: io::Read, +{ fn decode_record(&mut self) -> crate::Result> { self.decoder.decode() } +} +impl DecodeStream for Decoder +where + R: io::Read, +{ fn decode_stream(self) -> StreamIterDecoder { StreamIterDecoder::new(self) } @@ -197,10 +205,7 @@ where } /// A DBN decoder of records -pub struct RecordDecoder -where - R: io::Read, -{ +pub struct RecordDecoder { /// For future use with reading different DBN versions. version: u8, upgrade_policy: VersionUpgradePolicy, diff --git a/rust/dbn/src/decode/dbz.rs b/rust/dbn/src/decode/dbz.rs index 515842d..9cb03a9 100644 --- a/rust/dbn/src/decode/dbz.rs +++ b/rust/dbn/src/decode/dbz.rs @@ -9,8 +9,8 @@ use std::{ }; use super::{ - private::BufferSlice, zstd::ZSTD_SKIPPABLE_MAGIC_RANGE, DecodeDbn, DecodeRecordRef, - StreamIterDecoder, VersionUpgradePolicy, + private::BufferSlice, zstd::ZSTD_SKIPPABLE_MAGIC_RANGE, DbnMetadata, DecodeRecord, + DecodeRecordRef, DecodeStream, StreamIterDecoder, VersionUpgradePolicy, }; use crate::{ compat, @@ -127,11 +127,17 @@ impl DecodeRecordRef for Decoder { } } -impl DecodeDbn for Decoder { +impl DbnMetadata for Decoder { fn metadata(&self) -> &Metadata { &self.metadata } + fn metadata_mut(&mut self) -> &mut Metadata { + &mut self.metadata + } +} + +impl DecodeRecord for Decoder { fn decode_record(&mut self) -> crate::Result> { let rec_ref = self.decode_record_ref()?; if let Some(rec_ref) = rec_ref { @@ -148,7 +154,9 @@ impl DecodeDbn for Decoder { Ok(None) } } +} +impl DecodeStream for Decoder { /// Try to decode the DBZ file into a streaming iterator. This decodes the /// data lazily. /// diff --git a/rust/dbn/src/decode/stream.rs b/rust/dbn/src/decode/stream.rs index a0bf065..991c566 100644 --- a/rust/dbn/src/decode/stream.rs +++ b/rust/dbn/src/decode/stream.rs @@ -2,16 +2,16 @@ use std::marker::PhantomData; use streaming_iterator::StreamingIterator; -use super::DecodeDbn; +use super::{DecodeRecord, DecodeStream}; use crate::record::{transmute_record_bytes, HasRType}; -/// A consuming iterator wrapping a [`DecodeDbn`]. Lazily decodes the contents of the file -/// or other input stream. +/// A consuming iterator wrapping a [`DecodeRecord`]. Lazily decodes the contents of the +/// file or other input stream. /// /// Implements [`streaming_iterator::StreamingIterator`]. pub struct StreamIterDecoder where - D: DecodeDbn, + D: DecodeRecord, T: HasRType, { /// The underlying decoder implementation. @@ -27,7 +27,7 @@ where impl StreamIterDecoder where - D: DecodeDbn, + D: DecodeRecord, T: HasRType, { pub(crate) fn new(decoder: D) -> Self { @@ -47,7 +47,7 @@ where impl StreamingIterator for StreamIterDecoder where - D: DecodeDbn, + D: DecodeStream, T: HasRType, { type Item = T; diff --git a/rust/dbn/src/encode.rs b/rust/dbn/src/encode.rs index eacd9ba..b1b0ae8 100644 --- a/rust/dbn/src/encode.rs +++ b/rust/dbn/src/encode.rs @@ -26,8 +26,9 @@ pub use self::{ }; use crate::{ - decode::DecodeDbn, rtype_method_dispatch, rtype_ts_out_method_dispatch, Compression, Encoding, - Error, HasRType, Metadata, Record, RecordRef, Result, Schema, + decode::{DbnMetadata, DecodeRecordRef}, + rtype_method_dispatch, rtype_ts_out_method_dispatch, Compression, Encoding, Error, HasRType, + Metadata, Record, RecordRef, Result, Schema, }; use self::{csv::serialize::CsvSerialize, json::serialize::JsonSerialize}; @@ -112,7 +113,7 @@ pub trait EncodeDbn: EncodeRecord + EncodeRecordRef { /// # Errors /// This function returns an error if it's unable to write to the underlying writer /// or there's a serialization error. - fn encode_decoded(&mut self, mut decoder: D) -> Result<()> { + fn encode_decoded(&mut self, mut decoder: D) -> Result<()> { let ts_out = decoder.metadata().ts_out; while let Some(record) = decoder.decode_record_ref()? { // Safety: It's safe to cast to `WithTsOut` because we're passing in the `ts_out` @@ -129,7 +130,7 @@ pub trait EncodeDbn: EncodeRecord + EncodeRecordRef { /// # Errors /// This function returns an error if it's unable to write to the underlying writer /// or there's a serialization error. - fn encode_decoded_with_limit( + fn encode_decoded_with_limit( &mut self, mut decoder: D, limit: NonZeroU64, @@ -417,7 +418,7 @@ where self.0.encode_stream(stream) } - fn encode_decoded(&mut self, decoder: D) -> Result<()> { + fn encode_decoded(&mut self, decoder: D) -> Result<()> { self.0.encode_decoded(decoder) } } @@ -490,7 +491,7 @@ macro_rules! encoder_enum_dispatch { } } - fn encode_decoded( + fn encode_decoded( &mut self, decoder: D, ) -> Result<()> { diff --git a/rust/dbn/src/encode/csv/sync.rs b/rust/dbn/src/encode/csv/sync.rs index 639c9de..4c5308b 100644 --- a/rust/dbn/src/encode/csv/sync.rs +++ b/rust/dbn/src/encode/csv/sync.rs @@ -3,7 +3,7 @@ use std::{io, num::NonZeroU64}; use streaming_iterator::StreamingIterator; use crate::{ - decode::DecodeDbn, + decode::{DbnMetadata, DecodeRecordRef}, encode::{DbnEncodable, EncodeDbn, EncodeRecord, EncodeRecordRef, EncodeRecordTextExt}, rtype_method_dispatch, rtype_ts_out_method_dispatch, schema_method_dispatch, schema_ts_out_method_dispatch, Error, RType, Record, Result, Schema, @@ -178,7 +178,7 @@ where /// # Errors /// This function returns an error if it's unable to write to the underlying writer /// or there's a serialization error. - fn encode_decoded(&mut self, mut decoder: D) -> Result<()> { + fn encode_decoded(&mut self, mut decoder: D) -> Result<()> { let ts_out = decoder.metadata().ts_out; if let Some(schema) = decoder.metadata().schema { schema_method_dispatch!(schema, self, encode_header, false)?; @@ -198,7 +198,7 @@ where } } - fn encode_decoded_with_limit( + fn encode_decoded_with_limit( &mut self, mut decoder: D, limit: NonZeroU64, diff --git a/rust/dbn/src/python/metadata.rs b/rust/dbn/src/python/metadata.rs index 066d845..ebd0b0d 100644 --- a/rust/dbn/src/python/metadata.rs +++ b/rust/dbn/src/python/metadata.rs @@ -8,7 +8,7 @@ use pyo3::{ }; use crate::{ - decode::{DecodeDbn, DynDecoder}, + decode::{DbnMetadata, DynDecoder}, encode::dbn::MetadataEncoder, enums::{SType, Schema}, MappingInterval, Metadata, SymbolMapping, VersionUpgradePolicy, From c1b03ff2bbf8bfdf7abf0a6acfd181cd81ce7c6e Mon Sep 17 00:00:00 2001 From: Carter Green Date: Wed, 13 Dec 2023 11:46:48 -0600 Subject: [PATCH 12/21] ADD: Add dbn CLI support for map_symbols --- CHANGELOG.md | 10 + c/src/text_serialization.rs | 53 +-- python/src/transcoder.rs | 17 +- rust/dbn-cli/src/encode.rs | 86 ++--- rust/dbn-cli/src/lib.rs | 9 + rust/dbn-cli/src/main.rs | 6 +- rust/dbn-cli/tests/integration_tests.rs | 39 +++ rust/dbn/src/encode.rs | 408 +----------------------- rust/dbn/src/encode/csv.rs | 2 +- rust/dbn/src/encode/csv/sync.rs | 126 +++++++- rust/dbn/src/encode/dyn_encoder.rs | 374 ++++++++++++++++++++++ rust/dbn/src/encode/dyn_writer.rs | 163 ++++++++++ rust/dbn/src/encode/json.rs | 2 +- rust/dbn/src/encode/json/sync.rs | 66 ++++ rust/dbn/src/enums.rs | 1 - rust/dbn/src/metadata.rs | 26 +- 16 files changed, 910 insertions(+), 478 deletions(-) create mode 100644 rust/dbn/src/encode/dyn_encoder.rs create mode 100644 rust/dbn/src/encode/dyn_writer.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 9965a76..3d253d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,14 @@ allows outputting saved live data to CSV - Allowed passing `--limit` option to `dbn` CLI tool with `--metadata` flag - Improved performance of decoding uncompressed DBN fragments with the `dbn` CLI tool +- Added builders to `CsvEncoder`, `DynEncoder`, and `JsonEncoder` to assist with the + growing number of customizations + - Added option to write CSV header as part of creating `CsvEncoder` to make it harder + to forget +- Added `-s`/`--map-symbols` flag to CLI to create a `symbol` field in the output with + the text symbol mapped from the instrument ID - Added `version` param to Python `Metadata` contructor choose between DBNv1 and DBNv2 +- Implemented `EncodeRecordTextExt` for `DynEncoder` - Implemented `Deserialize` and `Serialize` for all records and enums (with `serde` feature enabled). This allows serializing records with additional encodings not supported by the DBN crate @@ -15,6 +22,7 @@ - Added Python type definition for `Metadata.__init__` - Added `metadata_mut` method to decoders to get a mutable reference to the decoded metadata +- Added `encode::ZSTD_COMPRESSION_LEVEL` constant ### Breaking changes - Split `DecodeDbn` trait into `DecodeRecord` and `DbnMetadata` traits for more @@ -28,6 +36,8 @@ - Added missing Python `__eq__` and `__ne__` implementations for `BidAskPair` - Fixed Python `size_hint` return value for `InstrumentDefMsgV1` and `SymbolMappingMsgV1` +- Fixed cases where `dbn` CLI tool would write a broken pipe error to standard error + such as when piping to `head` ## 0.14.2 - 2023-11-17 ### Enhancements diff --git a/c/src/text_serialization.rs b/c/src/text_serialization.rs index b3c3c85..03cc543 100644 --- a/c/src/text_serialization.rs +++ b/c/src/text_serialization.rs @@ -75,10 +75,13 @@ pub unsafe extern "C" fn s_serialize_record_header( let mut cursor = io::Cursor::new(buffer); let res = match options.encoding { TextEncoding::Json => return 0, - TextEncoding::Csv => { - let mut encoder = csv::Encoder::new(&mut cursor, options.pretty_px, options.pretty_ts); - rtype_ts_out_dispatch!(record, options.ts_out, serialize_csv_header, &mut encoder) - } + TextEncoding::Csv => csv::Encoder::builder(&mut cursor) + .use_pretty_px(options.pretty_px) + .use_pretty_ts(options.pretty_ts) + .build() + .and_then(|mut encoder| { + rtype_ts_out_dispatch!(record, options.ts_out, serialize_csv_header, &mut encoder) + }), } // flatten .and_then(|res| res); @@ -117,10 +120,14 @@ pub unsafe extern "C" fn f_serialize_record_header( TextEncoding::Json => { return 0; } - TextEncoding::Csv => { - let mut encoder = csv::Encoder::new(&mut cfile, options.pretty_px, options.pretty_ts); - rtype_ts_out_dispatch!(record, options.ts_out, serialize_csv_header, &mut encoder) - } + TextEncoding::Csv => csv::Encoder::builder(&mut cfile) + .write_header(false) + .use_pretty_px(options.pretty_px) + .use_pretty_ts(options.pretty_ts) + .build() + .and_then(|mut encoder| { + rtype_ts_out_dispatch!(record, options.ts_out, serialize_csv_header, &mut encoder) + }), }; res.map(|_| cfile.bytes_written() as i32) .unwrap_or(SerializeError::Serialization as libc::c_int) @@ -159,12 +166,17 @@ pub unsafe extern "C" fn s_serialize_record( }; let mut cursor = io::Cursor::new(buffer); let res = match options.encoding { - TextEncoding::Json => { - json::Encoder::new(&mut cursor, false, options.pretty_px, options.pretty_ts) - .encode_record_ref_ts_out(record, options.ts_out) - } - TextEncoding::Csv => csv::Encoder::new(&mut cursor, options.pretty_px, options.pretty_ts) + TextEncoding::Json => json::Encoder::builder(&mut cursor) + .use_pretty_px(options.pretty_px) + .use_pretty_ts(options.pretty_ts) + .build() .encode_record_ref_ts_out(record, options.ts_out), + TextEncoding::Csv => csv::Encoder::builder(&mut cursor) + .write_header(false) + .use_pretty_px(options.pretty_px) + .use_pretty_ts(options.pretty_ts) + .build() + .and_then(|mut enc| enc.encode_record_ref_ts_out(record, options.ts_out)), }; write_null_and_ret(cursor, res) } @@ -197,12 +209,17 @@ pub unsafe extern "C" fn f_serialize_record( return SerializeError::NullOptions as libc::c_int; }; let res = match options.encoding { - TextEncoding::Json => { - json::Encoder::new(&mut cfile, false, options.pretty_px, options.pretty_ts) - .encode_record_ref_ts_out(record, options.ts_out) - } - TextEncoding::Csv => csv::Encoder::new(&mut cfile, options.pretty_px, options.pretty_ts) + TextEncoding::Json => json::Encoder::builder(&mut cfile) + .use_pretty_px(options.pretty_px) + .use_pretty_ts(options.pretty_ts) + .build() .encode_record_ref_ts_out(record, options.ts_out), + TextEncoding::Csv => csv::Encoder::builder(&mut cfile) + .write_header(false) + .use_pretty_px(options.pretty_px) + .use_pretty_ts(options.pretty_ts) + .build() + .and_then(|mut enc| enc.encode_record_ref_ts_out(record, options.ts_out)), }; res.map(|_| cfile.bytes_written() as i32) .unwrap_or(SerializeError::Serialization as libc::c_int) diff --git a/python/src/transcoder.rs b/python/src/transcoder.rs index c45d8a1..ac8a009 100644 --- a/python/src/transcoder.rs +++ b/python/src/transcoder.rs @@ -249,7 +249,12 @@ impl Inner { ) .map_err(to_val_err)?; - let mut encoder = CsvEncoder::new(&mut self.output, self.use_pretty_px, self.use_pretty_ts); + let mut encoder = CsvEncoder::builder(&mut self.output) + .use_pretty_px(self.use_pretty_px) + .use_pretty_ts(self.use_pretty_ts) + .write_header(false) + .build() + .map_err(to_val_err)?; loop { match decoder.decode_record_ref() { Ok(Some(rec)) => { @@ -298,12 +303,10 @@ impl Inner { ) .map_err(to_val_err)?; - let mut encoder = JsonEncoder::new( - &mut self.output, - false, - self.use_pretty_px, - self.use_pretty_ts, - ); + let mut encoder = JsonEncoder::builder(&mut self.output) + .use_pretty_px(self.use_pretty_px) + .use_pretty_ts(self.use_pretty_ts) + .build(); loop { match decoder.decode_record_ref() { Ok(Some(rec)) => { diff --git a/rust/dbn-cli/src/encode.rs b/rust/dbn-cli/src/encode.rs index 7f9f8c9..92b0741 100644 --- a/rust/dbn-cli/src/encode.rs +++ b/rust/dbn-cli/src/encode.rs @@ -4,20 +4,36 @@ use dbn::{ decode::{DbnMetadata, DecodeRecordRef}, encode::{ json, DbnEncodable, DbnRecordEncoder, DynEncoder, DynWriter, EncodeDbn, EncodeRecordRef, + EncodeRecordTextExt, }, - rtype_dispatch, Compression, Encoding, MetadataBuilder, SType, + rtype_dispatch, Compression, Encoding, MetadataBuilder, SType, SymbolIndex, }; use crate::{infer_encoding_and_compression, output_from_args, Args}; -pub fn encode_from_dbn(decoder: D, args: &Args) -> anyhow::Result<()> +pub fn silence_broken_pipe(err: anyhow::Error) -> anyhow::Result<()> { + // Handle broken pipe as a non-error. + if let Some(err) = err.downcast_ref::() { + if matches!(err, dbn::Error::Io { source, .. } if source.kind() == std::io::ErrorKind::BrokenPipe) + { + return Ok(()); + } + } + Err(err) +} + +pub fn encode_from_dbn(mut decoder: D, args: &Args) -> anyhow::Result<()> where D: DecodeRecordRef + DbnMetadata, { let writer = output_from_args(args)?; let (encoding, compression) = infer_encoding_and_compression(args)?; - let encode_res = if args.should_output_metadata { - assert!(args.json); + Ok(if args.should_output_metadata { + if encoding != Encoding::Json { + return Err(anyhow::format_err!( + "Metadata flag is only valid with JSON encoding" + )); + } json::Encoder::new( writer, args.should_pretty_print, @@ -28,24 +44,25 @@ where } else if args.fragment { encode_fragment(decoder, writer, compression) } else { - DynEncoder::new( - writer, - encoding, - compression, - decoder.metadata(), - args.should_pretty_print, - args.should_pretty_print, - args.should_pretty_print, - )? - .encode_decoded(decoder) - }; - match encode_res { - // Handle broken pipe as a non-error. - Err(dbn::Error::Io { source, .. }) if source.kind() == std::io::ErrorKind::BrokenPipe => { + let mut encoder = DynEncoder::builder(writer, encoding, compression, decoder.metadata()) + .all_pretty(args.should_pretty_print) + .with_symbol(args.map_symbols) + .build()?; + if args.map_symbols { + let symbol_map = decoder.metadata().symbol_map()?; + let ts_out = decoder.metadata().ts_out; + while let Some(rec) = decoder.decode_record_ref()? { + let sym = symbol_map.get_for_rec(&rec).map(String::as_str); + // Safety: ts_out is accurate because we get it from the metadata + unsafe { + encoder.encode_ref_ts_out_with_sym(rec, ts_out, sym)?; + } + } Ok(()) + } else { + encoder.encode_decoded(decoder) } - res => Ok(res?), - } + }?) } pub fn encode_from_frag(mut decoder: D, args: &Args) -> anyhow::Result<()> @@ -60,7 +77,7 @@ where } assert!(!args.should_output_metadata); - let mut encoder = DynEncoder::new( + let mut encoder = DynEncoder::builder( writer, encoding, compression, @@ -72,10 +89,11 @@ where .stype_in(None) .stype_out(SType::InstrumentId) .build(), - args.should_pretty_print, - args.should_pretty_print, - args.should_pretty_print, - )?; + ) + // Can't write header until we know the record type + .write_header(false) + .all_pretty(args.should_pretty_print) + .build()?; let mut has_written_header = encoding != Encoding::Csv; fn write_header( _record: &T, @@ -85,24 +103,10 @@ where } while let Some(record) = decoder.decode_record_ref()? { if !has_written_header { - match rtype_dispatch!(record, write_header, &mut encoder)? { - Err(dbn::Error::Io { source, .. }) - if source.kind() == io::ErrorKind::BrokenPipe => - { - return Ok(()) - } - res => res?, - } + rtype_dispatch!(record, write_header, &mut encoder)??; has_written_header = true; } - // Assume no ts_out for safety - match encoder.encode_record_ref(record) { - // Handle broken pipe as a non-error. - Err(dbn::Error::Io { source, .. }) if source.kind() == io::ErrorKind::BrokenPipe => { - return Ok(()); - } - res => res?, - }; + encoder.encode_record_ref(record)?; } Ok(()) } diff --git a/rust/dbn-cli/src/lib.rs b/rust/dbn-cli/src/lib.rs index 1bbe991..07681b9 100644 --- a/rust/dbn-cli/src/lib.rs +++ b/rust/dbn-cli/src/lib.rs @@ -115,6 +115,15 @@ pub struct Args { help ="Make the CSV or JSON output easier to read by converting timestamps to ISO 8601 and prices to decimals" )] pub should_pretty_print: bool, + #[clap( + short = 's', + long = "map-symbols", + action = ArgAction::SetTrue, + default_value = "false", + conflicts_with_all = ["input_fragment", "dbn", "fragment"], + help ="Use symbology mappings from the metadata to create a 'symbol' field mapping the intstrument ID to its requested symbol." + )] + pub map_symbols: bool, #[clap( short = 'l', long = "limit", diff --git a/rust/dbn-cli/src/main.rs b/rust/dbn-cli/src/main.rs index 13005ff..ff664b2 100644 --- a/rust/dbn-cli/src/main.rs +++ b/rust/dbn-cli/src/main.rs @@ -6,7 +6,7 @@ use std::{ use clap::Parser; use dbn::decode::{DbnMetadata, DbnRecordDecoder, DecodeRecordRef, DynDecoder}; use dbn_cli::{ - encode::{encode_from_dbn, encode_from_frag}, + encode::{encode_from_dbn, encode_from_frag, silence_broken_pipe}, filter::{LimitFilter, SchemaFilter}, Args, }; @@ -31,6 +31,10 @@ fn wrap( } fn main() -> anyhow::Result<()> { + main_impl().or_else(silence_broken_pipe) +} + +fn main_impl() -> anyhow::Result<()> { let args = Args::parse(); // DBN fragment if args.is_input_fragment { diff --git a/rust/dbn-cli/tests/integration_tests.rs b/rust/dbn-cli/tests/integration_tests.rs index 77b9365..e0d7b87 100644 --- a/rust/dbn-cli/tests/integration_tests.rs +++ b/rust/dbn-cli/tests/integration_tests.rs @@ -41,6 +41,8 @@ fn write_json_to_path(#[values("dbn", "dbn.zst")] extension: &str) { assert!(contents.contains('{')); assert!(contents.contains('{')); assert!(contents.ends_with('\n')); + // no map symbols + assert!(!contents.contains("\"symbol\":")); } #[test] @@ -605,6 +607,43 @@ fn writes_csv_header_for_0_records() { .stderr(is_empty()); } +#[rstest] +#[case::csv("--csv")] +#[case::json("--json")] +fn map_symbols(#[case] output_flag: &str) { + let cmd = cmd() + .args([ + &format!("{TEST_DATA_PATH}/test_data.mbo.dbn.zst"), + output_flag, + "--map-symbols", + ]) + .assert() + .success() + .stderr(is_empty()); + if output_flag == "--csv" { + cmd.stdout(contains(",symbol\n").count(1)); + } else { + cmd.stdout(contains("\"symbol\":\"").count(2)); + } +} + +#[rstest] +#[case::dbn("--dbn")] +#[case::fragment("--fragment")] +fn map_symbols_fails_for_invalid_output(#[case] output_flag: &str) { + cmd() + .args([ + &format!("{TEST_DATA_PATH}/test_data.mbo.dbn.zst"), + output_flag, + "--map-symbols", + ]) + .assert() + .failure() + .stdout(is_empty()) + .stderr(contains(format!("'{output_flag}'"))) + .stderr(contains("'--map-symbols'")); +} + #[test] fn passing_current_dbn_version_is_accepted() { cmd() diff --git a/rust/dbn/src/encode.rs b/rust/dbn/src/encode.rs index b1b0ae8..ee4e7b9 100644 --- a/rust/dbn/src/encode.rs +++ b/rust/dbn/src/encode.rs @@ -2,6 +2,8 @@ //! [`EncodeDbn`] trait. pub mod csv; pub mod dbn; +mod dyn_encoder; +mod dyn_writer; pub mod json; use std::{fmt, io, num::NonZeroU64}; @@ -9,26 +11,30 @@ use std::{fmt, io, num::NonZeroU64}; use streaming_iterator::StreamingIterator; // Re-exports -#[cfg(feature = "async")] -pub use self::dbn::{ - AsyncEncoder as AsyncDbnEncoder, AsyncMetadataEncoder as AsyncDbnMetadataEncoder, - AsyncRecordEncoder as AsyncDbnRecordEncoder, -}; -#[cfg(feature = "async")] -pub use self::json::AsyncEncoder as AsyncJsonEncoder; pub use self::{ csv::Encoder as CsvEncoder, dbn::{ Encoder as DbnEncoder, MetadataEncoder as DbnMetadataEncoder, RecordEncoder as DbnRecordEncoder, }, + dyn_encoder::{DynEncoder, DynEncoderBuilder}, + dyn_writer::DynWriter, json::Encoder as JsonEncoder, }; +#[cfg(feature = "async")] +pub use self::{ + dbn::{ + AsyncEncoder as AsyncDbnEncoder, AsyncMetadataEncoder as AsyncDbnMetadataEncoder, + AsyncRecordEncoder as AsyncDbnRecordEncoder, + }, + dyn_writer::DynAsyncWriter, + json::AsyncEncoder as AsyncJsonEncoder, +}; use crate::{ decode::{DbnMetadata, DecodeRecordRef}, - rtype_method_dispatch, rtype_ts_out_method_dispatch, Compression, Encoding, Error, HasRType, - Metadata, Record, RecordRef, Result, Schema, + rtype_method_dispatch, rtype_ts_out_method_dispatch, Error, HasRType, Record, RecordRef, + Result, }; use self::{csv::serialize::CsvSerialize, json::serialize::JsonSerialize}; @@ -193,46 +199,8 @@ pub trait EncodeRecordTextExt: EncodeRecord + EncodeRecordRef { } } -/// The default Zstandard compression level. -const ZSTD_COMPRESSION_LEVEL: i32 = 0; - -/// Type for runtime polymorphism over whether encoding uncompressed or ZStd-compressed -/// DBN records. Implements [`std::io::Write`]. -pub struct DynWriter<'a, W>(DynWriterImpl<'a, W>) -where - W: io::Write; - -enum DynWriterImpl<'a, W> -where - W: io::Write, -{ - Uncompressed(W), - ZStd(zstd::stream::AutoFinishEncoder<'a, W>), -} - -impl<'a, W> DynWriter<'a, W> -where - W: io::Write, -{ - /// Create a new instance of [`DynWriter`] which will wrap `writer` with `compression`. - /// - /// # Errors - /// This function returns an error if it fails to initialize the Zstd compression. - pub fn new(writer: W, compression: Compression) -> Result { - match compression { - Compression::None => Ok(Self(DynWriterImpl::Uncompressed(writer))), - Compression::ZStd => zstd_encoder(writer).map(|enc| Self(DynWriterImpl::ZStd(enc))), - } - } - - /// Returns a mutable reference to the underlying writer. - pub fn get_mut(&mut self) -> &mut W { - match &mut self.0 { - DynWriterImpl::Uncompressed(w) => w, - DynWriterImpl::ZStd(enc) => enc.get_mut(), - } - } -} +/// The default Zstandard compression level used. +pub const ZSTD_COMPRESSION_LEVEL: i32 = 0; fn zstd_encoder<'a, W: io::Write>(writer: W) -> Result> { let mut zstd_encoder = zstd::Encoder::new(writer, ZSTD_COMPRESSION_LEVEL) @@ -243,267 +211,6 @@ fn zstd_encoder<'a, W: io::Write>(writer: W) -> Result io::Write for DynWriter<'a, W> -where - W: io::Write, -{ - fn write(&mut self, buf: &[u8]) -> io::Result { - match &mut self.0 { - DynWriterImpl::Uncompressed(writer) => writer.write(buf), - DynWriterImpl::ZStd(writer) => writer.write(buf), - } - } - - fn flush(&mut self) -> io::Result<()> { - match &mut self.0 { - DynWriterImpl::Uncompressed(writer) => writer.flush(), - DynWriterImpl::ZStd(writer) => writer.flush(), - } - } - - fn write_vectored(&mut self, bufs: &[io::IoSlice<'_>]) -> io::Result { - match &mut self.0 { - DynWriterImpl::Uncompressed(writer) => writer.write_vectored(bufs), - DynWriterImpl::ZStd(writer) => writer.write_vectored(bufs), - } - } - - fn write_all(&mut self, buf: &[u8]) -> io::Result<()> { - match &mut self.0 { - DynWriterImpl::Uncompressed(writer) => writer.write_all(buf), - DynWriterImpl::ZStd(writer) => writer.write_all(buf), - } - } - - fn write_fmt(&mut self, fmt: std::fmt::Arguments<'_>) -> io::Result<()> { - match &mut self.0 { - DynWriterImpl::Uncompressed(writer) => writer.write_fmt(fmt), - DynWriterImpl::ZStd(writer) => writer.write_fmt(fmt), - } - } -} - -/// An encoder implementing [`EncodeDbn`] whose [`Encoding`] and [`Compression`] can be -/// set at runtime. -pub struct DynEncoder<'a, W>(DynEncoderImpl<'a, W>) -where - W: io::Write; - -// [`DynEncoder`] isn't cloned so this isn't a concern. -#[allow(clippy::large_enum_variant)] -enum DynEncoderImpl<'a, W> -where - W: io::Write, -{ - Dbn(dbn::Encoder>), - Csv(csv::Encoder>), - Json(json::Encoder>), -} - -impl<'a, W> DynEncoder<'a, W> -where - W: io::Write, -{ - /// Constructs a new instance of [`DynEncoder`]. - /// - /// Note: `should_pretty_print`, `user_pretty_px`, and `use_pretty_ts` are ignored - /// if `encoding` is `Dbn`. - /// - /// # Errors - /// This function returns an error if it fails to encode the DBN metadata or - /// it fails to initialize the Zstd compression. - pub fn new( - writer: W, - encoding: Encoding, - compression: Compression, - metadata: &Metadata, - should_pretty_print: bool, - use_pretty_px: bool, - use_pretty_ts: bool, - ) -> Result { - let writer = DynWriter::new(writer, compression)?; - match encoding { - Encoding::Dbn => { - dbn::Encoder::new(writer, metadata).map(|e| Self(DynEncoderImpl::Dbn(e))) - } - Encoding::Csv => Ok(Self(DynEncoderImpl::Csv(csv::Encoder::new( - writer, - use_pretty_px, - use_pretty_ts, - )))), - Encoding::Json => Ok(Self(DynEncoderImpl::Json(json::Encoder::new( - writer, - should_pretty_print, - use_pretty_px, - use_pretty_ts, - )))), - } - } - - /// Encodes the CSV header for the record type `R`, i.e. the names of each of the - /// fields to the output. - /// - /// If `with_symbol` is `true`, will add a header field for "symbol". - /// - /// # Errors - /// This function returns an error if there's an error writing to `writer`. - pub fn encode_header(&mut self, with_symbol: bool) -> Result<()> { - match &mut self.0 { - DynEncoderImpl::Csv(encoder) => encoder.encode_header::(with_symbol), - _ => Ok(()), - } - } - - /// Encodes the CSV header for `schema`, i.e. the names of each of the fields to - /// the output. - /// - /// If `ts_out` is `true`, will add a header field "ts_out". If `with_symbol` is - /// `true`, will add a header field "symbol". - /// - /// # Errors - /// This function returns an error if there's an error writing to `writer`. - pub fn encode_header_for_schema( - &mut self, - schema: Schema, - ts_out: bool, - with_symbol: bool, - ) -> Result<()> { - match &mut self.0 { - DynEncoderImpl::Csv(encoder) => { - encoder.encode_header_for_schema(schema, ts_out, with_symbol) - } - _ => Ok(()), - } - } -} - -impl<'a, W> EncodeRecord for DynEncoder<'a, W> -where - W: io::Write, -{ - fn encode_record(&mut self, record: &R) -> Result<()> { - self.0.encode_record(record) - } - - fn flush(&mut self) -> Result<()> { - self.0.flush() - } -} - -impl<'a, W> EncodeRecordRef for DynEncoder<'a, W> -where - W: io::Write, -{ - fn encode_record_ref(&mut self, record: RecordRef) -> Result<()> { - self.0.encode_record_ref(record) - } - - unsafe fn encode_record_ref_ts_out(&mut self, record: RecordRef, ts_out: bool) -> Result<()> { - self.0.encode_record_ref_ts_out(record, ts_out) - } -} - -impl<'a, W> EncodeDbn for DynEncoder<'a, W> -where - W: io::Write, -{ - fn encode_records(&mut self, records: &[R]) -> Result<()> { - self.0.encode_records(records) - } - - fn encode_stream( - &mut self, - stream: impl StreamingIterator, - ) -> Result<()> { - self.0.encode_stream(stream) - } - - fn encode_decoded(&mut self, decoder: D) -> Result<()> { - self.0.encode_decoded(decoder) - } -} - -impl<'a, W> EncodeRecord for DynEncoderImpl<'a, W> -where - W: io::Write, -{ - fn encode_record(&mut self, record: &R) -> Result<()> { - match self { - DynEncoderImpl::Dbn(enc) => enc.encode_record(record), - DynEncoderImpl::Csv(enc) => enc.encode_record(record), - DynEncoderImpl::Json(enc) => enc.encode_record(record), - } - } - - fn flush(&mut self) -> Result<()> { - match self { - DynEncoderImpl::Dbn(enc) => enc.flush(), - DynEncoderImpl::Csv(enc) => enc.flush(), - DynEncoderImpl::Json(enc) => enc.flush(), - } - } -} - -impl<'a, W> EncodeRecordRef for DynEncoderImpl<'a, W> -where - W: io::Write, -{ - fn encode_record_ref(&mut self, record: RecordRef) -> Result<()> { - match self { - DynEncoderImpl::Dbn(enc) => enc.encode_record_ref(record), - DynEncoderImpl::Csv(enc) => enc.encode_record_ref(record), - DynEncoderImpl::Json(enc) => enc.encode_record_ref(record), - } - } - - unsafe fn encode_record_ref_ts_out(&mut self, record: RecordRef, ts_out: bool) -> Result<()> { - match self { - DynEncoderImpl::Dbn(enc) => enc.encode_record_ref_ts_out(record, ts_out), - DynEncoderImpl::Csv(enc) => enc.encode_record_ref_ts_out(record, ts_out), - DynEncoderImpl::Json(enc) => enc.encode_record_ref_ts_out(record, ts_out), - } - } -} - -impl<'a, W> EncodeDbn for DynEncoderImpl<'a, W> -where - W: io::Write, -{ - encoder_enum_dispatch! {Dbn, Csv, Json} -} - -/// An aid the with boilerplate code of calling the same method on each enum variant's -/// inner value. -macro_rules! encoder_enum_dispatch { - ($($variant:ident),*) => { - fn encode_records(&mut self, records: &[R]) -> Result<()> { - match self { - $(Self::$variant(v) => v.encode_records(records),)* - } - } - - fn encode_stream( - &mut self, - stream: impl StreamingIterator, - ) -> Result<()> { - match self { - $(Self::$variant(v) => v.encode_stream(stream),)* - } - } - - fn encode_decoded( - &mut self, - decoder: D, - ) -> Result<()> { - match self { - $(Self::$variant(v) => v.encode_decoded(decoder),)* - } - } - }; -} - -pub(crate) use encoder_enum_dispatch; - #[cfg(test)] mod test_data { use streaming_iterator::StreamingIterator; @@ -554,84 +261,3 @@ mod test_data { } } } - -#[cfg(feature = "async")] -pub use r#async::DynWriter as DynAsyncWriter; - -#[cfg(feature = "async")] -mod r#async { - use std::{ - pin::Pin, - task::{Context, Poll}, - }; - - use async_compression::tokio::write::ZstdEncoder; - use tokio::io; - - use crate::enums::Compression; - - /// An object that allows for abstracting over compressed and uncompressed output. - pub struct DynWriter(DynWriterImpl) - where - W: io::AsyncWriteExt + Unpin; - - enum DynWriterImpl - where - W: io::AsyncWriteExt + Unpin, - { - Uncompressed(W), - ZStd(ZstdEncoder), - } - - impl DynWriter - where - W: io::AsyncWriteExt + Unpin, - { - /// Creates a new instance of [`DynWriter`] which will wrap `writer` with - /// `compression`. - pub fn new(writer: W, compression: Compression) -> Self { - Self(match compression { - Compression::None => DynWriterImpl::Uncompressed(writer), - Compression::ZStd => DynWriterImpl::ZStd(ZstdEncoder::new(writer)), - }) - } - - /// Returns a mutable reference to the underlying writer. - pub fn get_mut(&mut self) -> &mut W { - match &mut self.0 { - DynWriterImpl::Uncompressed(w) => w, - DynWriterImpl::ZStd(enc) => enc.get_mut(), - } - } - } - - impl io::AsyncWrite for DynWriter - where - W: io::AsyncWrite + io::AsyncWriteExt + Unpin, - { - fn poll_write( - mut self: Pin<&mut Self>, - cx: &mut Context<'_>, - buf: &[u8], - ) -> Poll> { - match &mut self.0 { - DynWriterImpl::Uncompressed(w) => io::AsyncWrite::poll_write(Pin::new(w), cx, buf), - DynWriterImpl::ZStd(enc) => io::AsyncWrite::poll_write(Pin::new(enc), cx, buf), - } - } - - fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - match &mut self.0 { - DynWriterImpl::Uncompressed(w) => io::AsyncWrite::poll_flush(Pin::new(w), cx), - DynWriterImpl::ZStd(enc) => io::AsyncWrite::poll_flush(Pin::new(enc), cx), - } - } - - fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - match &mut self.0 { - DynWriterImpl::Uncompressed(w) => io::AsyncWrite::poll_shutdown(Pin::new(w), cx), - DynWriterImpl::ZStd(enc) => io::AsyncWrite::poll_shutdown(Pin::new(enc), cx), - } - } - } -} diff --git a/rust/dbn/src/encode/csv.rs b/rust/dbn/src/encode/csv.rs index 692b4c7..ef243aa 100644 --- a/rust/dbn/src/encode/csv.rs +++ b/rust/dbn/src/encode/csv.rs @@ -3,4 +3,4 @@ pub(crate) mod serialize; mod sync; -pub use sync::Encoder; +pub use sync::{Encoder, EncoderBuilder}; diff --git a/rust/dbn/src/encode/csv/sync.rs b/rust/dbn/src/encode/csv/sync.rs index 4c5308b..d268497 100644 --- a/rust/dbn/src/encode/csv/sync.rs +++ b/rust/dbn/src/encode/csv/sync.rs @@ -17,14 +17,122 @@ where W: io::Write, { writer: csv::Writer, + // Prevent writing header twice + has_written_header: bool, use_pretty_px: bool, use_pretty_ts: bool, } +/// Helper for constructing a CSV [`Encoder`]. +/// +/// If writing a CSV header (`write_header`), which is enabled by default, +/// `schema` is required, otherwise no fields are required. +pub struct EncoderBuilder +where + W: io::Write, +{ + writer: W, + use_pretty_px: bool, + use_pretty_ts: bool, + write_header: bool, + schema: Option, + ts_out: bool, + with_symbol: bool, +} + +impl EncoderBuilder +where + W: io::Write, +{ + /// Creates a new CSV encoder builder. + pub fn new(writer: W) -> Self { + Self { + writer, + use_pretty_px: false, + use_pretty_ts: false, + write_header: true, + schema: None, + ts_out: false, + with_symbol: false, + } + } + + /// Sets whether the CSV encoder will serialize price fields as a decimal. Defaults + /// to `false`. + pub fn use_pretty_px(mut self, use_pretty_px: bool) -> Self { + self.use_pretty_px = use_pretty_px; + self + } + + /// Sets whether the CSV encoder will serialize timestamp fields as ISO8601 datetime + /// strings. Defaults to `false`. + pub fn use_pretty_ts(mut self, use_pretty_ts: bool) -> Self { + self.use_pretty_ts = use_pretty_ts; + self + } + + /// Sets whether the CSV encoder will write a header row when it's created. + /// Defaults to `true`. If `false`, a header row can still be written with + /// [`Encoder::encode_header()`] or [`Encoder::encode_header_for_schema()`]. + pub fn write_header(mut self, write_header: bool) -> Self { + self.write_header = write_header; + self + } + + /// Sets the schema that will be encoded. This is required if writing a header row. + /// + /// # Errors + /// This function returns an error if `schema` is `None`. It accepts to an `Option` to + /// more easily work with [`Metadata::schema`](crate::Metadata::schema). + pub fn schema(mut self, schema: Option) -> crate::Result { + if schema.is_none() { + return Err(Error::encode("can't encode a CSV with mixed schemas")); + }; + self.schema = schema; + Ok(self) + } + + /// Sets whether to add a header field "ts_out". Defaults to `false`. + pub fn ts_out(mut self, ts_out: bool) -> Self { + self.ts_out = ts_out; + self + } + + /// Sets whether to add a header field "symbol". Defaults to `false`. + pub fn with_symbol(mut self, with_symbol: bool) -> Self { + self.with_symbol = with_symbol; + self + } + + /// Creates the new encoder with the previously specified settings and if + /// `write_header` is `true`, encodes the header row. + /// + /// # Errors + /// This function returns an error if it fails to write the header row. + pub fn build(self) -> crate::Result> { + let mut encoder = Encoder::new(self.writer, self.use_pretty_px, self.use_pretty_ts); + if self.write_header { + let Some(schema) = self.schema else { + return Err(Error::BadArgument { + param_name: "schema".to_owned(), + desc: "need to specify schema in order to write header".to_owned(), + }); + }; + encoder.encode_header_for_schema(schema, self.ts_out, self.with_symbol)?; + } + Ok(encoder) + } +} + impl Encoder where W: io::Write, { + /// Creates a builder for configuring an `Encoder` object. + pub fn builder(writer: W) -> EncoderBuilder { + EncoderBuilder::new(writer) + } + /// Creates a new [`Encoder`] that will write to `writer`. If `use_pretty_px` /// is `true`, price fields will be serialized as a decimal. If `pretty_ts` is /// `true`, timestamp fields will be serialized in a ISO8601 datetime string. @@ -36,6 +144,7 @@ where writer: csv_writer, use_pretty_px, use_pretty_ts, + has_written_header: false, } } @@ -61,6 +170,7 @@ where } // end of line self.writer.write_record(None::<&[u8]>)?; + self.has_written_header = true; Ok(()) } @@ -82,7 +192,9 @@ where ts_out: bool, with_symbol: bool, ) -> Result<()> { - schema_ts_out_method_dispatch!(schema, ts_out, self, encode_header, with_symbol) + schema_ts_out_method_dispatch!(schema, ts_out, self, encode_header, with_symbol)?; + self.has_written_header = true; + Ok(()) } fn encode_record_impl(&mut self, record: &R) -> csv::Result<()> { @@ -146,7 +258,9 @@ where W: io::Write, { fn encode_records(&mut self, records: &[R]) -> Result<()> { - self.encode_header::(false)?; + if !self.has_written_header { + self.encode_header::(false)?; + } for record in records { self.encode_record(record)?; } @@ -163,7 +277,9 @@ where &mut self, mut stream: impl StreamingIterator, ) -> Result<()> { - self.encode_header::(false)?; + if !self.has_written_header { + self.encode_header::(false)?; + } while let Some(record) = stream.next() { self.encode_record(record)?; } @@ -181,7 +297,9 @@ where fn encode_decoded(&mut self, mut decoder: D) -> Result<()> { let ts_out = decoder.metadata().ts_out; if let Some(schema) = decoder.metadata().schema { - schema_method_dispatch!(schema, self, encode_header, false)?; + if !self.has_written_header { + self.encode_header_for_schema(schema, ts_out, false)?; + } let rtype = RType::from(schema); while let Some(record) = decoder.decode_record_ref()? { if record.rtype().map_or(true, |r| r != rtype) { diff --git a/rust/dbn/src/encode/dyn_encoder.rs b/rust/dbn/src/encode/dyn_encoder.rs new file mode 100644 index 0000000..af7c8b0 --- /dev/null +++ b/rust/dbn/src/encode/dyn_encoder.rs @@ -0,0 +1,374 @@ +use std::io; + +use streaming_iterator::StreamingIterator; + +use super::{ + CsvEncoder, DbnEncodable, DbnEncoder, DynWriter, EncodeDbn, EncodeRecord, EncodeRecordRef, + EncodeRecordTextExt, JsonEncoder, +}; +use crate::{ + decode::{DbnMetadata, DecodeRecordRef}, + Compression, Encoding, Metadata, RecordRef, Result, Schema, +}; + +/// An encoder whose [`Encoding`] and [`Compression`] can be set at runtime. +pub struct DynEncoder<'a, W>(DynEncoderImpl<'a, W>) +where + W: io::Write; + +// [`DynEncoder`] isn't cloned so this isn't a concern. +#[allow(clippy::large_enum_variant)] +enum DynEncoderImpl<'a, W> +where + W: io::Write, +{ + Dbn(DbnEncoder>), + Csv(CsvEncoder>), + Json(JsonEncoder>), +} + +/// Helper for constructing a [`DynEncoder`]. +pub struct DynEncoderBuilder<'m, W> +where + W: io::Write, +{ + writer: W, + encoding: Encoding, + compression: Compression, + metadata: &'m Metadata, + write_header: bool, + should_pretty_print: bool, + use_pretty_px: bool, + use_pretty_ts: bool, + with_symbol: bool, +} + +impl<'m, W> DynEncoderBuilder<'m, W> +where + W: io::Write, +{ + /// Creates a new builder. All required fields for the builder are passed to this + /// function. + pub fn new( + writer: W, + encoding: Encoding, + compression: Compression, + metadata: &'m Metadata, + ) -> Self { + Self { + writer, + encoding, + compression, + metadata, + write_header: true, + should_pretty_print: false, + use_pretty_px: false, + use_pretty_ts: false, + with_symbol: false, + } + } + + /// Sets whether the encoder will write a header row when it's created if encoding + /// CSV. Defaults to `true`. If `false`, a header row can still be written with + /// [`DynEncoder::encode_header()`] or [`DynEncoder::encode_header_for_schema()`]. + pub fn write_header(mut self, write_header: bool) -> Self { + self.write_header = write_header; + self + } + + /// Sets all three pretty options together: `should_pretty_print`, `use_pretty_px`, + /// and `use_pretty_ts`. By default all are `false`. + pub fn all_pretty(self, all_pretty: bool) -> Self { + self.should_pretty_print(all_pretty) + .use_pretty_px(all_pretty) + .use_pretty_ts(all_pretty) + } + + /// Sets whether the encoder should encode nicely-formatted JSON objects with + /// indentation if encoding JSON. Defaults to `false` where each JSON object is + /// compact with no spacing. + pub fn should_pretty_print(mut self, should_pretty_print: bool) -> Self { + self.should_pretty_print = should_pretty_print; + self + } + + /// Sets whether the encoder will serialize price fields as a decimal in CSV and + /// JSON encodings. Defaults to `false`. + pub fn use_pretty_px(mut self, use_pretty_px: bool) -> Self { + self.use_pretty_px = use_pretty_px; + self + } + + /// Sets whether the encoder will serialize timestamp fields as ISO8601 datetime + /// strings in CSV and JSON encodings. Defaults to `false`. + pub fn use_pretty_ts(mut self, use_pretty_ts: bool) -> Self { + self.use_pretty_ts = use_pretty_ts; + self + } + + /// Sets whether to add a header field "symbol" if encoding CSV. Defaults to + /// `false`. + pub fn with_symbol(mut self, with_symbol: bool) -> Self { + self.with_symbol = with_symbol; + self + } + + /// Creates the new encoder with the previously specified settings and if + /// `write_header` is `true`, encodes the header row. + /// + /// # Errors + /// This function returns an error if it fails to write the CSV header row or the + /// DBN metadata. + pub fn build<'a>(self) -> crate::Result> { + let writer = DynWriter::new(self.writer, self.compression)?; + Ok(DynEncoder(match self.encoding { + Encoding::Dbn => DynEncoderImpl::Dbn(DbnEncoder::new(writer, self.metadata)?), + Encoding::Csv => { + let builder = CsvEncoder::builder(writer) + .use_pretty_px(self.use_pretty_px) + .use_pretty_ts(self.use_pretty_ts) + .write_header(self.write_header) + .ts_out(self.metadata.ts_out) + .with_symbol(self.with_symbol); + DynEncoderImpl::Csv(if self.write_header { + builder.schema(self.metadata.schema)?.build()? + } else { + builder.build()? + }) + } + Encoding::Json => DynEncoderImpl::Json( + JsonEncoder::builder(writer) + .should_pretty_print(self.should_pretty_print) + .use_pretty_px(self.use_pretty_px) + .use_pretty_ts(self.use_pretty_ts) + .build(), + ), + })) + } +} + +impl<'a, W> DynEncoder<'a, W> +where + W: io::Write, +{ + /// Constructs a new instance of [`DynEncoder`]. + /// + /// Note: `should_pretty_print`, `use_pretty_px`, and `use_pretty_ts` are ignored + /// if `encoding` is `Dbn`. + /// + /// # Errors + /// This function returns an error if it fails to encode the DBN metadata or + /// it fails to initialize the Zstd compression. + pub fn new( + writer: W, + encoding: Encoding, + compression: Compression, + metadata: &Metadata, + should_pretty_print: bool, + use_pretty_px: bool, + use_pretty_ts: bool, + ) -> Result { + Self::builder(writer, encoding, compression, metadata) + .should_pretty_print(should_pretty_print) + .use_pretty_px(use_pretty_px) + .use_pretty_ts(use_pretty_ts) + .build() + } + + /// Creates a builder for configuring a `DynEncoder` object. + pub fn builder( + writer: W, + encoding: Encoding, + compression: Compression, + metadata: &Metadata, + ) -> DynEncoderBuilder<'_, W> { + DynEncoderBuilder::new(writer, encoding, compression, metadata) + } + + /// Encodes the CSV header for the record type `R`, i.e. the names of each of the + /// fields to the output. + /// + /// If `with_symbol` is `true`, will add a header field for "symbol". + /// + /// # Errors + /// This function returns an error if there's an error writing to `writer`. + pub fn encode_header(&mut self, with_symbol: bool) -> Result<()> { + match &mut self.0 { + DynEncoderImpl::Csv(encoder) => encoder.encode_header::(with_symbol), + _ => Ok(()), + } + } + + /// Encodes the CSV header for `schema`, i.e. the names of each of the fields to + /// the output. + /// + /// If `ts_out` is `true`, will add a header field "ts_out". If `with_symbol` is + /// `true`, will add a header field "symbol". + /// + /// # Errors + /// This function returns an error if there's an error writing to `writer`. + pub fn encode_header_for_schema( + &mut self, + schema: Schema, + ts_out: bool, + with_symbol: bool, + ) -> Result<()> { + match &mut self.0 { + DynEncoderImpl::Csv(encoder) => { + encoder.encode_header_for_schema(schema, ts_out, with_symbol) + } + _ => Ok(()), + } + } +} + +impl<'a, W> EncodeRecord for DynEncoder<'a, W> +where + W: io::Write, +{ + fn encode_record(&mut self, record: &R) -> Result<()> { + self.0.encode_record(record) + } + + fn flush(&mut self) -> Result<()> { + self.0.flush() + } +} + +impl<'a, W> EncodeRecordRef for DynEncoder<'a, W> +where + W: io::Write, +{ + fn encode_record_ref(&mut self, record: RecordRef) -> Result<()> { + self.0.encode_record_ref(record) + } + + unsafe fn encode_record_ref_ts_out(&mut self, record: RecordRef, ts_out: bool) -> Result<()> { + self.0.encode_record_ref_ts_out(record, ts_out) + } +} + +impl<'a, W> EncodeDbn for DynEncoder<'a, W> +where + W: io::Write, +{ + fn encode_records(&mut self, records: &[R]) -> Result<()> { + self.0.encode_records(records) + } + + fn encode_stream( + &mut self, + stream: impl StreamingIterator, + ) -> Result<()> { + self.0.encode_stream(stream) + } + + fn encode_decoded(&mut self, decoder: D) -> Result<()> { + self.0.encode_decoded(decoder) + } +} + +impl<'a, W> EncodeRecordTextExt for DynEncoder<'a, W> +where + W: io::Write, +{ + fn encode_record_with_sym( + &mut self, + record: &R, + symbol: Option<&str>, + ) -> Result<()> { + self.0.encode_record_with_sym(record, symbol) + } +} + +impl<'a, W> EncodeRecord for DynEncoderImpl<'a, W> +where + W: io::Write, +{ + fn encode_record(&mut self, record: &R) -> Result<()> { + match self { + DynEncoderImpl::Dbn(enc) => enc.encode_record(record), + DynEncoderImpl::Csv(enc) => enc.encode_record(record), + DynEncoderImpl::Json(enc) => enc.encode_record(record), + } + } + + fn flush(&mut self) -> Result<()> { + match self { + DynEncoderImpl::Dbn(enc) => enc.flush(), + DynEncoderImpl::Csv(enc) => enc.flush(), + DynEncoderImpl::Json(enc) => enc.flush(), + } + } +} + +impl<'a, W> EncodeRecordRef for DynEncoderImpl<'a, W> +where + W: io::Write, +{ + fn encode_record_ref(&mut self, record: RecordRef) -> Result<()> { + match self { + DynEncoderImpl::Dbn(enc) => enc.encode_record_ref(record), + DynEncoderImpl::Csv(enc) => enc.encode_record_ref(record), + DynEncoderImpl::Json(enc) => enc.encode_record_ref(record), + } + } + + unsafe fn encode_record_ref_ts_out(&mut self, record: RecordRef, ts_out: bool) -> Result<()> { + match self { + DynEncoderImpl::Dbn(enc) => enc.encode_record_ref_ts_out(record, ts_out), + DynEncoderImpl::Csv(enc) => enc.encode_record_ref_ts_out(record, ts_out), + DynEncoderImpl::Json(enc) => enc.encode_record_ref_ts_out(record, ts_out), + } + } +} + +impl<'a, W> EncodeDbn for DynEncoderImpl<'a, W> +where + W: io::Write, +{ + fn encode_records(&mut self, records: &[R]) -> Result<()> { + match self { + DynEncoderImpl::Dbn(encoder) => encoder.encode_records(records), + DynEncoderImpl::Csv(encoder) => encoder.encode_records(records), + DynEncoderImpl::Json(encoder) => encoder.encode_records(records), + } + } + + fn encode_stream( + &mut self, + stream: impl StreamingIterator, + ) -> Result<()> { + match self { + DynEncoderImpl::Dbn(encoder) => encoder.encode_stream(stream), + DynEncoderImpl::Csv(encoder) => encoder.encode_stream(stream), + DynEncoderImpl::Json(encoder) => encoder.encode_stream(stream), + } + } + + fn encode_decoded(&mut self, decoder: D) -> Result<()> { + match self { + DynEncoderImpl::Dbn(encoder) => encoder.encode_decoded(decoder), + DynEncoderImpl::Csv(encoder) => encoder.encode_decoded(decoder), + DynEncoderImpl::Json(encoder) => encoder.encode_decoded(decoder), + } + } +} + +impl<'a, W> EncodeRecordTextExt for DynEncoderImpl<'a, W> +where + W: io::Write, +{ + fn encode_record_with_sym( + &mut self, + record: &R, + symbol: Option<&str>, + ) -> Result<()> { + match self { + // Not supported for DBN so ignore `symbol` + Self::Dbn(encoder) => encoder.encode_record(record), + Self::Csv(encoder) => encoder.encode_record_with_sym(record, symbol), + Self::Json(encoder) => encoder.encode_record_with_sym(record, symbol), + } + } +} diff --git a/rust/dbn/src/encode/dyn_writer.rs b/rust/dbn/src/encode/dyn_writer.rs new file mode 100644 index 0000000..46b3571 --- /dev/null +++ b/rust/dbn/src/encode/dyn_writer.rs @@ -0,0 +1,163 @@ +use std::io; + +use super::zstd_encoder; +use crate::{Compression, Result}; + +/// Type for runtime polymorphism over whether encoding uncompressed or ZStd-compressed +/// DBN records. Implements [`std::io::Write`]. +pub struct DynWriter<'a, W>(DynWriterImpl<'a, W>) +where + W: io::Write; + +enum DynWriterImpl<'a, W> +where + W: io::Write, +{ + Uncompressed(W), + ZStd(zstd::stream::AutoFinishEncoder<'a, W>), +} + +impl<'a, W> DynWriter<'a, W> +where + W: io::Write, +{ + /// Create a new instance of [`DynWriter`] which will wrap `writer` with `compression`. + /// + /// # Errors + /// This function returns an error if it fails to initialize the Zstd compression. + pub fn new(writer: W, compression: Compression) -> Result { + match compression { + Compression::None => Ok(Self(DynWriterImpl::Uncompressed(writer))), + Compression::ZStd => zstd_encoder(writer).map(|enc| Self(DynWriterImpl::ZStd(enc))), + } + } + + /// Returns a mutable reference to the underlying writer. + pub fn get_mut(&mut self) -> &mut W { + match &mut self.0 { + DynWriterImpl::Uncompressed(w) => w, + DynWriterImpl::ZStd(enc) => enc.get_mut(), + } + } +} + +impl<'a, W> io::Write for DynWriter<'a, W> +where + W: io::Write, +{ + fn write(&mut self, buf: &[u8]) -> io::Result { + match &mut self.0 { + DynWriterImpl::Uncompressed(writer) => writer.write(buf), + DynWriterImpl::ZStd(writer) => writer.write(buf), + } + } + + fn flush(&mut self) -> io::Result<()> { + match &mut self.0 { + DynWriterImpl::Uncompressed(writer) => writer.flush(), + DynWriterImpl::ZStd(writer) => writer.flush(), + } + } + + fn write_vectored(&mut self, bufs: &[io::IoSlice<'_>]) -> io::Result { + match &mut self.0 { + DynWriterImpl::Uncompressed(writer) => writer.write_vectored(bufs), + DynWriterImpl::ZStd(writer) => writer.write_vectored(bufs), + } + } + + fn write_all(&mut self, buf: &[u8]) -> io::Result<()> { + match &mut self.0 { + DynWriterImpl::Uncompressed(writer) => writer.write_all(buf), + DynWriterImpl::ZStd(writer) => writer.write_all(buf), + } + } + + fn write_fmt(&mut self, fmt: std::fmt::Arguments<'_>) -> io::Result<()> { + match &mut self.0 { + DynWriterImpl::Uncompressed(writer) => writer.write_fmt(fmt), + DynWriterImpl::ZStd(writer) => writer.write_fmt(fmt), + } + } +} + +#[cfg(feature = "async")] +pub use r#async::DynWriter as DynAsyncWriter; + +#[cfg(feature = "async")] +mod r#async { + use std::{ + pin::Pin, + task::{Context, Poll}, + }; + + use async_compression::tokio::write::ZstdEncoder; + use tokio::io; + + use crate::enums::Compression; + + /// An object that allows for abstracting over compressed and uncompressed output. + pub struct DynWriter(DynWriterImpl) + where + W: io::AsyncWriteExt + Unpin; + + enum DynWriterImpl + where + W: io::AsyncWriteExt + Unpin, + { + Uncompressed(W), + ZStd(ZstdEncoder), + } + + impl DynWriter + where + W: io::AsyncWriteExt + Unpin, + { + /// Creates a new instance of [`DynWriter`] which will wrap `writer` with + /// `compression`. + pub fn new(writer: W, compression: Compression) -> Self { + Self(match compression { + Compression::None => DynWriterImpl::Uncompressed(writer), + Compression::ZStd => DynWriterImpl::ZStd(ZstdEncoder::new(writer)), + }) + } + + /// Returns a mutable reference to the underlying writer. + pub fn get_mut(&mut self) -> &mut W { + match &mut self.0 { + DynWriterImpl::Uncompressed(w) => w, + DynWriterImpl::ZStd(enc) => enc.get_mut(), + } + } + } + + impl io::AsyncWrite for DynWriter + where + W: io::AsyncWrite + io::AsyncWriteExt + Unpin, + { + fn poll_write( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + match &mut self.0 { + DynWriterImpl::Uncompressed(w) => io::AsyncWrite::poll_write(Pin::new(w), cx, buf), + DynWriterImpl::ZStd(enc) => io::AsyncWrite::poll_write(Pin::new(enc), cx, buf), + } + } + + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + match &mut self.0 { + DynWriterImpl::Uncompressed(w) => io::AsyncWrite::poll_flush(Pin::new(w), cx), + DynWriterImpl::ZStd(enc) => io::AsyncWrite::poll_flush(Pin::new(enc), cx), + } + } + + fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + match &mut self.0 { + DynWriterImpl::Uncompressed(w) => io::AsyncWrite::poll_shutdown(Pin::new(w), cx), + DynWriterImpl::ZStd(enc) => io::AsyncWrite::poll_shutdown(Pin::new(enc), cx), + } + } + } +} diff --git a/rust/dbn/src/encode/json.rs b/rust/dbn/src/encode/json.rs index 9f75dc0..41b558a 100644 --- a/rust/dbn/src/encode/json.rs +++ b/rust/dbn/src/encode/json.rs @@ -2,7 +2,7 @@ pub(crate) mod serialize; mod sync; -pub use sync::Encoder; +pub use sync::{Encoder, EncoderBuilder}; #[cfg(feature = "async")] mod r#async; #[cfg(feature = "async")] diff --git a/rust/dbn/src/encode/json/sync.rs b/rust/dbn/src/encode/json/sync.rs index f6bce6e..ababde8 100644 --- a/rust/dbn/src/encode/json/sync.rs +++ b/rust/dbn/src/encode/json/sync.rs @@ -17,6 +17,67 @@ where use_pretty_ts: bool, } +/// Helper for constructing a JSON [`Encoder`]. +/// +/// No fields are required. +pub struct EncoderBuilder +where + W: io::Write, +{ + writer: W, + should_pretty_print: bool, + use_pretty_px: bool, + use_pretty_ts: bool, +} + +impl EncoderBuilder +where + W: io::Write, +{ + /// Creates a new JSON encoder builder. + pub fn new(writer: W) -> Self { + Self { + writer, + should_pretty_print: false, + use_pretty_px: false, + use_pretty_ts: false, + } + } + + /// Sets whether the JSON encoder should encode nicely-formatted JSON objects + /// with indentation. Defaults to `false` where each JSON object is compact with + /// no spacing. + pub fn should_pretty_print(mut self, should_pretty_print: bool) -> Self { + self.should_pretty_print = should_pretty_print; + self + } + + /// Sets whether the JSON encoder will serialize price fields as a decimal. Defaults + /// to `false`. + pub fn use_pretty_px(mut self, use_pretty_px: bool) -> Self { + self.use_pretty_px = use_pretty_px; + self + } + + /// Sets whether the JSON encoder will serialize timestamp fields as ISO8601 + /// datetime strings. Defaults to `false`. + pub fn use_pretty_ts(mut self, use_pretty_ts: bool) -> Self { + self.use_pretty_ts = use_pretty_ts; + self + } + + /// Creates the new encoder with the previously specified settings and if + /// `write_header` is `true`, encodes the header row. + pub fn build(self) -> Encoder { + Encoder::new( + self.writer, + self.should_pretty_print, + self.use_pretty_px, + self.use_pretty_ts, + ) + } +} + impl Encoder where W: io::Write, @@ -38,6 +99,11 @@ where } } + /// Creates a builder for configuring an `Encoder` object. + pub fn builder(writer: W) -> EncoderBuilder { + EncoderBuilder::new(writer) + } + /// Encodes `metadata` into JSON. /// /// # Errors diff --git a/rust/dbn/src/enums.rs b/rust/dbn/src/enums.rs index d6594d4..bfd4cea 100644 --- a/rust/dbn/src/enums.rs +++ b/rust/dbn/src/enums.rs @@ -661,7 +661,6 @@ pub enum SecurityUpdateAction { Modify = b'M', /// Removal of an instrument definition. Delete = b'D', - // FIXME: can this be removed? #[doc(hidden)] #[deprecated = "Still present in legacy files."] Invalid = b'~', diff --git a/rust/dbn/src/metadata.rs b/rust/dbn/src/metadata.rs index 8a61486..29b6964 100644 --- a/rust/dbn/src/metadata.rs +++ b/rust/dbn/src/metadata.rs @@ -172,13 +172,13 @@ impl AsRef<[u8]> for Metadata { } impl MetadataBuilder { - /// Sets the [`version`](Metadata::version) and returns the builder. + /// Sets [`version`](Metadata::version) and returns the builder. pub fn version(mut self, version: u8) -> Self { self.version = version; self } - /// Sets the [`dataset`](Metadata::dataset) and returns the builder. + /// Sets [`dataset`](Metadata::dataset) and returns the builder. pub fn dataset(self, dataset: String) -> MetadataBuilder { MetadataBuilder { version: self.version, @@ -197,7 +197,7 @@ impl MetadataBuilder { } } - /// Sets the [`schema`](Metadata::schema) and returns the builder. + /// Sets [`schema`](Metadata::schema) and returns the builder. pub fn schema( self, schema: Option, @@ -219,7 +219,7 @@ impl MetadataBuilder { } } - /// Sets the [`start`](Metadata::start) and returns the builder. + /// Sets [`start`](Metadata::start) and returns the builder. pub fn start(self, start: u64) -> MetadataBuilder { MetadataBuilder { version: self.version, @@ -238,19 +238,19 @@ impl MetadataBuilder { } } - /// Sets the [`end`](Metadata::end) and returns the builder. + /// Sets [`end`](Metadata::end) and returns the builder. pub fn end(mut self, end: Option) -> Self { self.end = end; self } - /// Sets the [`limit`](Metadata::limit) and returns the builder. + /// Sets [`limit`](Metadata::limit) and returns the builder. pub fn limit(mut self, limit: Option) -> Self { self.limit = limit; self } - /// Sets the [`stype_in`](Metadata::stype_in) and returns the builder. + /// Sets [`stype_in`](Metadata::stype_in) and returns the builder. pub fn stype_in( self, stype_in: Option, @@ -272,7 +272,7 @@ impl MetadataBuilder { } } - /// Sets the [`stype_out`](Metadata::stype_out) and returns the builder. + /// Sets [`stype_out`](Metadata::stype_out) and returns the builder. pub fn stype_out(self, stype_out: SType) -> MetadataBuilder { MetadataBuilder { version: self.version, @@ -291,31 +291,31 @@ impl MetadataBuilder { } } - /// Sets the [`ts_out`](Metadata::ts_out) and returns the builder. + /// Sets [`ts_out`](Metadata::ts_out) and returns the builder. pub fn ts_out(mut self, ts_out: bool) -> Self { self.ts_out = ts_out; self } - /// Sets the [`symbols`](Metadata::symbols) and returns the builder. + /// Sets [`symbols`](Metadata::symbols) and returns the builder. pub fn symbols(mut self, symbols: Vec) -> Self { self.symbols = symbols; self } - /// Sets the [`partial`](Metadata::partial) and returns the builder. + /// Sets [`partial`](Metadata::partial) and returns the builder. pub fn partial(mut self, partial: Vec) -> Self { self.partial = partial; self } - /// Sets the [`not_found`](Metadata::not_found) and returns the builder. + /// Sets [`not_found`](Metadata::not_found) and returns the builder. pub fn not_found(mut self, not_found: Vec) -> Self { self.not_found = not_found; self } - /// Sets the [`mappings`](Metadata::mappings) and returns the builder. + /// Sets [`mappings`](Metadata::mappings) and returns the builder. pub fn mappings(mut self, mappings: Vec) -> Self { self.mappings = mappings; self From 3e9b4c54aec9afdf249cf4be65f14f6a030ae653 Mon Sep 17 00:00:00 2001 From: Carter Green Date: Wed, 3 Jan 2024 09:41:42 -0600 Subject: [PATCH 13/21] DEL: Remove unused re-exports --- rust/dbn/src/json_writer.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rust/dbn/src/json_writer.rs b/rust/dbn/src/json_writer.rs index 48508d0..0736430 100644 --- a/rust/dbn/src/json_writer.rs +++ b/rust/dbn/src/json_writer.rs @@ -2,7 +2,6 @@ // Re-export for version and casing consistency pub use json_writer::{ - JSONArrayWriter as JsonArrayWriter, JSONObjectWriter as JsonObjectWriter, JSONWriter, - JSONWriter as JsonWriter, JSONWriterValue as JsonWriterValue, + JSONObjectWriter as JsonObjectWriter, JSONWriter as JsonWriter, PrettyJSONWriter as PrettyJsonWriter, NULL, }; From aede39b73c9858eb95ce2fd678ef0378ec38bc9e Mon Sep 17 00:00:00 2001 From: Carter Green Date: Wed, 3 Jan 2024 11:47:51 -0600 Subject: [PATCH 14/21] FIX: Fix use of symbol_cstr_len in MetadataEncoder --- CHANGELOG.md | 2 ++ rust/dbn/src/encode/dbn/async.rs | 61 ++++++++++++++------------------ rust/dbn/src/encode/dbn/sync.rs | 57 +++++++++++++---------------- 3 files changed, 54 insertions(+), 66 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d253d3..56cdd35 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,8 @@ `SymbolMappingMsgV1` - Fixed cases where `dbn` CLI tool would write a broken pipe error to standard error such as when piping to `head` +- Fixed bug in sync and async `MetadataEncoder`s where `version` was used to determine + the encoded length of fixed-length symbols instead of the `symbol_cstr_len` field ## 0.14.2 - 2023-11-17 ### Enhancements diff --git a/rust/dbn/src/encode/dbn/async.rs b/rust/dbn/src/encode/dbn/async.rs index 431871b..310a94d 100644 --- a/rust/dbn/src/encode/dbn/async.rs +++ b/rust/dbn/src/encode/dbn/async.rs @@ -237,7 +237,7 @@ where .write_u32_le(length) .await .map_err(metadata_err)?; - self.encode_fixed_len_cstr::<{ crate::METADATA_DATASET_CSTR_LEN }>(&metadata.dataset) + self.encode_fixed_len_cstr(crate::METADATA_DATASET_CSTR_LEN, &metadata.dataset) .await?; self.writer .write_u16_le(metadata.schema.map(|s| s as u16).unwrap_or(NULL_SCHEMA)) @@ -279,13 +279,13 @@ where .map_err(metadata_err)?; // schema_definition_length self.writer.write_u32_le(0).await.map_err(metadata_err)?; - self.encode_repeated_symbol_cstr(metadata.version, &metadata.symbols) + self.encode_repeated_symbol_cstr(metadata.symbol_cstr_len, &metadata.symbols) .await?; - self.encode_repeated_symbol_cstr(metadata.version, &metadata.partial) + self.encode_repeated_symbol_cstr(metadata.symbol_cstr_len, &metadata.partial) .await?; - self.encode_repeated_symbol_cstr(metadata.version, &metadata.not_found) + self.encode_repeated_symbol_cstr(metadata.symbol_cstr_len, &metadata.not_found) .await?; - self.encode_symbol_mappings(metadata.version, &metadata.mappings) + self.encode_symbol_mappings(metadata.symbol_cstr_len, &metadata.mappings) .await?; Ok(()) @@ -331,19 +331,17 @@ where Ok(()) } - async fn encode_repeated_symbol_cstr(&mut self, version: u8, symbols: &[String]) -> Result<()> { + async fn encode_repeated_symbol_cstr( + &mut self, + symbol_cstr_len: usize, + symbols: &[String], + ) -> Result<()> { self.writer .write_u32_le(symbols.len() as u32) .await - .map_err(|e| Error::io(e, "writing cstr length"))?; + .map_err(|e| Error::io(e, "writing repeated symbols length"))?; for symbol in symbols { - if version == 1 { - self.encode_fixed_len_cstr::<{ crate::compat::SYMBOL_CSTR_LEN_V1 }>(symbol) - .await - } else { - self.encode_fixed_len_cstr::<{ crate::SYMBOL_CSTR_LEN }>(symbol) - .await - }?; + self.encode_fixed_len_cstr(symbol_cstr_len, symbol).await?; } Ok(()) @@ -351,7 +349,7 @@ where async fn encode_symbol_mappings( &mut self, - version: u8, + symbol_cstr_len: usize, symbol_mappings: &[SymbolMapping], ) -> Result<()> { // encode mappings_count @@ -359,25 +357,19 @@ where .write_u32_le(symbol_mappings.len() as u32) .await .map_err(|e| Error::io(e, "writing symbol mappings length"))?; - if version == 1 { - for symbol_mapping in symbol_mappings { - self.encode_symbol_mapping::<{ crate::compat::SYMBOL_CSTR_LEN_V1 }>(symbol_mapping) - .await?; - } - } else { - for symbol_mapping in symbol_mappings { - self.encode_symbol_mapping::<{ crate::SYMBOL_CSTR_LEN }>(symbol_mapping) - .await?; - } + for symbol_mapping in symbol_mappings { + self.encode_symbol_mapping(symbol_cstr_len, symbol_mapping) + .await?; } Ok(()) } - async fn encode_symbol_mapping( + async fn encode_symbol_mapping( &mut self, + symbol_cstr_len: usize, symbol_mapping: &SymbolMapping, ) -> Result<()> { - self.encode_fixed_len_cstr::(&symbol_mapping.raw_symbol) + self.encode_fixed_len_cstr(symbol_cstr_len, &symbol_mapping.raw_symbol) .await?; // encode interval_count self.writer @@ -391,22 +383,23 @@ where self.encode_date(interval.end_date) .await .map_err(|e| Error::io(e, "writing end date"))?; - self.encode_fixed_len_cstr::(&interval.symbol).await?; + self.encode_fixed_len_cstr(symbol_cstr_len, &interval.symbol) + .await?; } Ok(()) } - async fn encode_fixed_len_cstr(&mut self, string: &str) -> Result<()> { + async fn encode_fixed_len_cstr(&mut self, symbol_cstr_len: usize, string: &str) -> Result<()> { if !string.is_ascii() { return Err(Error::Conversion { input: string.to_owned(), desired_type: "ASCII", }); } - if string.len() > LEN { + if string.len() > symbol_cstr_len { return Err(Error::encode( format!( - "'{string}' is too long to be encoded in DBN; it cannot be longer than {LEN} characters" + "'{string}' is too long to be encoded in DBN; it cannot be longer than {symbol_cstr_len} characters" ))); } let cstr_err = |e| Error::io(e, "writing cstr"); @@ -415,7 +408,7 @@ where .await .map_err(cstr_err)?; // pad remaining space with null bytes - for _ in string.len()..LEN { + for _ in string.len()..symbol_cstr_len { self.writer.write_u8(0).await.map_err(cstr_err)?; } Ok(()) @@ -533,7 +526,7 @@ mod tests { "LNQ".to_owned(), ]; target - .encode_repeated_symbol_cstr(crate::DBN_VERSION, symbols.as_slice()) + .encode_repeated_symbol_cstr(crate::SYMBOL_CSTR_LEN, symbols.as_slice()) .await .unwrap(); assert_eq!( @@ -555,7 +548,7 @@ mod tests { let mut buffer = Vec::new(); let mut target = MetadataEncoder::new(&mut buffer); target - .encode_fixed_len_cstr::<{ crate::SYMBOL_CSTR_LEN }>("NG") + .encode_fixed_len_cstr(crate::SYMBOL_CSTR_LEN, "NG") .await .unwrap(); assert_eq!(buffer.len(), crate::SYMBOL_CSTR_LEN); diff --git a/rust/dbn/src/encode/dbn/sync.rs b/rust/dbn/src/encode/dbn/sync.rs index fb53556..aade772 100644 --- a/rust/dbn/src/encode/dbn/sync.rs +++ b/rust/dbn/src/encode/dbn/sync.rs @@ -138,7 +138,7 @@ where self.writer .write_all(length.to_le_bytes().as_slice()) .map_err(metadata_err)?; - self.encode_fixed_len_cstr::<{ crate::METADATA_DATASET_CSTR_LEN }>(&metadata.dataset)?; + self.encode_fixed_len_cstr(crate::METADATA_DATASET_CSTR_LEN, &metadata.dataset)?; self.writer .write_all( (metadata.schema.map(|s| s as u16).unwrap_or(NULL_SCHEMA)) @@ -177,10 +177,10 @@ where .write_all(0u32.to_le_bytes().as_slice()) .map_err(metadata_err)?; - self.encode_repeated_symbol_cstr(metadata.version, metadata.symbols.as_slice())?; - self.encode_repeated_symbol_cstr(metadata.version, metadata.partial.as_slice())?; - self.encode_repeated_symbol_cstr(metadata.version, metadata.not_found.as_slice())?; - self.encode_symbol_mappings(metadata.version, metadata.mappings.as_slice())?; + self.encode_repeated_symbol_cstr(metadata.symbol_cstr_len, metadata.symbols.as_slice())?; + self.encode_repeated_symbol_cstr(metadata.symbol_cstr_len, metadata.partial.as_slice())?; + self.encode_repeated_symbol_cstr(metadata.symbol_cstr_len, metadata.not_found.as_slice())?; + self.encode_symbol_mappings(metadata.symbol_cstr_len, metadata.mappings.as_slice())?; Ok(()) } @@ -244,16 +244,16 @@ where Ok(()) } - fn encode_repeated_symbol_cstr(&mut self, version: u8, symbols: &[String]) -> Result<()> { + fn encode_repeated_symbol_cstr( + &mut self, + symbol_cstr_len: usize, + symbols: &[String], + ) -> Result<()> { self.writer .write_all((symbols.len() as u32).to_le_bytes().as_slice()) - .map_err(|e| Error::io(e, "writing cstr length"))?; + .map_err(|e| Error::io(e, "writing repeated symbols length"))?; for symbol in symbols { - if version == 1 { - self.encode_fixed_len_cstr::<{ crate::compat::SYMBOL_CSTR_LEN_V1 }>(symbol)?; - } else { - self.encode_fixed_len_cstr::<{ crate::SYMBOL_CSTR_LEN }>(symbol)?; - } + self.encode_fixed_len_cstr(symbol_cstr_len, symbol)?; } Ok(()) @@ -261,32 +261,25 @@ where fn encode_symbol_mappings( &mut self, - version: u8, + symbol_cstr_len: usize, symbol_mappings: &[SymbolMapping], ) -> Result<()> { // encode mappings_count self.writer .write_all((symbol_mappings.len() as u32).to_le_bytes().as_slice()) .map_err(|e| Error::io(e, "writing symbol mappings length"))?; - if version == 1 { - for symbol_mapping in symbol_mappings { - self.encode_symbol_mapping::<{ crate::compat::SYMBOL_CSTR_LEN_V1 }>( - symbol_mapping, - )?; - } - } else { - for symbol_mapping in symbol_mappings { - self.encode_symbol_mapping::<{ crate::SYMBOL_CSTR_LEN }>(symbol_mapping)?; - } + for symbol_mapping in symbol_mappings { + self.encode_symbol_mapping(symbol_cstr_len, symbol_mapping)?; } Ok(()) } - fn encode_symbol_mapping( + fn encode_symbol_mapping( &mut self, + symbol_cstr_len: usize, symbol_mapping: &SymbolMapping, ) -> Result<()> { - self.encode_fixed_len_cstr::(&symbol_mapping.raw_symbol)?; + self.encode_fixed_len_cstr(symbol_cstr_len, &symbol_mapping.raw_symbol)?; // encode interval_count self.writer .write_all( @@ -300,28 +293,28 @@ where .map_err(|e| Error::io(e, "writing start date"))?; self.encode_date(interval.end_date) .map_err(|e| Error::io(e, "writing end date"))?; - self.encode_fixed_len_cstr::(&interval.symbol)?; + self.encode_fixed_len_cstr(symbol_cstr_len, &interval.symbol)?; } Ok(()) } - fn encode_fixed_len_cstr(&mut self, string: &str) -> Result<()> { + fn encode_fixed_len_cstr(&mut self, symbol_cstr_len: usize, string: &str) -> Result<()> { if !string.is_ascii() { return Err(Error::Conversion { input: string.to_owned(), desired_type: "ASCII", }); } - if string.len() > LEN { + if string.len() > symbol_cstr_len { return Err(Error::encode( format!( - "'{string}' is too long to be encoded in DBN; it cannot be longer than {LEN} characters" + "'{string}' is too long to be encoded in DBN; it cannot be longer than {symbol_cstr_len} characters" ))); } let cstr_err = |e| Error::io(e, "writing cstr"); self.writer.write_all(string.as_bytes()).map_err(cstr_err)?; // pad remaining space with null bytes - for _ in string.len()..LEN { + for _ in string.len()..symbol_cstr_len { self.writer.write_all(&[0]).map_err(cstr_err)?; } Ok(()) @@ -529,7 +522,7 @@ mod tests { "LNQ".to_owned(), ]; target - .encode_repeated_symbol_cstr(crate::DBN_VERSION, symbols.as_slice()) + .encode_repeated_symbol_cstr(crate::SYMBOL_CSTR_LEN, symbols.as_slice()) .unwrap(); assert_eq!( buffer.len(), @@ -550,7 +543,7 @@ mod tests { let mut buffer = Vec::new(); let mut target = MetadataEncoder::new(&mut buffer); target - .encode_fixed_len_cstr::<{ crate::SYMBOL_CSTR_LEN }>("NG") + .encode_fixed_len_cstr(crate::SYMBOL_CSTR_LEN, "NG") .unwrap(); assert_eq!(buffer.len(), crate::SYMBOL_CSTR_LEN); assert_eq!(&buffer[..2], b"NG"); From 94442423a9159a8cdaa60d4a6e7b7e13c95a6ebb Mon Sep 17 00:00:00 2001 From: Carter Green Date: Fri, 5 Jan 2024 14:20:07 -0600 Subject: [PATCH 15/21] MOD: Improve `RecordRef::get` panic message --- CHANGELOG.md | 2 ++ rust/dbn/src/record_ref.rs | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 56cdd35..e15f297 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,8 @@ - Added Python type definition for `Metadata.__init__` - Added `metadata_mut` method to decoders to get a mutable reference to the decoded metadata +- Improved panic message on `RecordRef::get` when length doesn't match expected to be + actionable - Added `encode::ZSTD_COMPRESSION_LEVEL` constant ### Breaking changes diff --git a/rust/dbn/src/record_ref.rs b/rust/dbn/src/record_ref.rs index d26db92..b0841d1 100644 --- a/rust/dbn/src/record_ref.rs +++ b/rust/dbn/src/record_ref.rs @@ -77,7 +77,9 @@ impl<'a> RecordRef<'a> { if self.has::() { assert!( self.record_size() >= mem::size_of::(), - "Malformed record. Expected length of at least {} bytes, found {} bytes", + "Malformed `{}` record: expected length of at least {} bytes, found {} bytes. \ + Confirm the DBN version in the Metadata header and the version upgrade policy", + std::any::type_name::(), mem::size_of::(), self.record_size() ); From 76654c8883384532693551faaced6935f137f9c5 Mon Sep 17 00:00:00 2001 From: Carter Green Date: Fri, 5 Jan 2024 08:39:29 -0600 Subject: [PATCH 16/21] MOD: Improve record `Debug` implementations --- CHANGELOG.md | 5 ++ rust/dbn-macros/src/dbn_attr.rs | 21 ++++++ rust/dbn-macros/src/debug.rs | 77 ++++++++++++++++++++++ rust/dbn-macros/src/has_rtype.rs | 5 +- rust/dbn-macros/src/lib.rs | 51 +++++++++++++-- rust/dbn/src/compat.rs | 19 +++++- rust/dbn/src/macros.rs | 2 +- rust/dbn/src/pretty.rs | 2 +- rust/dbn/src/record.rs | 57 +++++++++++----- rust/dbn/src/record/methods.rs | 109 +++++++++++++++++++++++++++++++ 10 files changed, 321 insertions(+), 27 deletions(-) create mode 100644 rust/dbn-macros/src/debug.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index e15f297..9b903fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ ## 0.15.0 - TBD ### Enhancements +- Improved `Debug` implementation for all record types + - Prices are formatted as decimals + - Fixed-length strings are formatted as strings + - Bit flag fields are formatted as binary + - Several fields are formatted as enums instead of their raw representations - Added `--schema` option to `dbn` CLI tool to filter a DBN to a particular schema. This allows outputting saved live data to CSV - Allowed passing `--limit` option to `dbn` CLI tool with `--metadata` flag diff --git a/rust/dbn-macros/src/dbn_attr.rs b/rust/dbn-macros/src/dbn_attr.rs index 5f41bdf..3127836 100644 --- a/rust/dbn-macros/src/dbn_attr.rs +++ b/rust/dbn-macros/src/dbn_attr.rs @@ -8,6 +8,8 @@ use syn::{parenthesized, spanned::Spanned, token, Field, FieldsNamed, Meta}; pub const C_CHAR_ATTR: &str = "c_char"; pub const FIXED_PRICE_ATTR: &str = "fixed_price"; +pub const FMT_BINARY: &str = "fmt_binary"; +pub const FMT_METHOD: &str = "fmt_method"; pub const INDEX_TS_ATTR: &str = "index_ts"; pub const SKIP_ATTR: &str = "skip"; pub const UNIX_NANOS_ATTR: &str = "unix_nanos"; @@ -62,6 +64,8 @@ pub fn find_dbn_attr_args(field: &Field) -> syn::Result> { } else if let Some(i) = meta.path.get_ident() { if i == C_CHAR_ATTR || i == FIXED_PRICE_ATTR + || i == FMT_BINARY + || i == FMT_METHOD || i == INDEX_TS_ATTR || i == SKIP_ATTR || i == UNIX_NANOS_ATTR @@ -119,6 +123,23 @@ pub fn is_hidden(field: &Field) -> bool { .any(|id| id == SKIP_ATTR) } +pub fn find_dbn_debug_attr(field: &Field) -> syn::Result> { + let mut args: Vec<_> = find_dbn_attr_args(field)? + .into_iter() + .filter(|id| { + id == C_CHAR_ATTR || id == FIXED_PRICE_ATTR || id == FMT_BINARY || id == FMT_METHOD + }) + .collect(); + match args.len() { + 0 => Ok(None), + 1 => Ok(Some(args.pop().unwrap())), + _ => Err(syn::Error::new( + field.span(), + "Passed incompatible format arguments to dbn attr", + )), + } +} + pub fn find_dbn_serialize_attr(field: &Field) -> syn::Result> { let mut args: Vec<_> = find_dbn_attr_args(field)? .into_iter() diff --git a/rust/dbn-macros/src/debug.rs b/rust/dbn-macros/src/debug.rs new file mode 100644 index 0000000..eb8c6f7 --- /dev/null +++ b/rust/dbn-macros/src/debug.rs @@ -0,0 +1,77 @@ +use proc_macro2::TokenStream; +use quote::quote; +use syn::{parse_macro_input, Field, ItemStruct}; + +use crate::{ + dbn_attr::{ + find_dbn_debug_attr, is_hidden, C_CHAR_ATTR, FIXED_PRICE_ATTR, FMT_BINARY, FMT_METHOD, + }, + utils::crate_name, +}; + +pub fn record_debug_impl(input_struct: &ItemStruct) -> TokenStream { + let record_type = &input_struct.ident; + let field_iter = input_struct + .fields + .iter() + .map(|f| format_field(f).unwrap_or_else(|e| e.into_compile_error())); + quote! { + impl ::std::fmt::Debug for #record_type { + fn fmt(&self, f: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result { + let mut debug_struct = f.debug_struct(stringify!(#record_type)); + #(#field_iter)* + debug_struct.finish() + } + } + } +} + +pub fn derive_impl(input: proc_macro::TokenStream) -> proc_macro::TokenStream { + // let DeriveInput { ident, data, .. } = parse_macro_input!(input as DeriveInput); + let input_struct = parse_macro_input!(input as ItemStruct); + let record_type = &input_struct.ident; + let field_iter = input_struct + .fields + .iter() + .map(|f| format_field(f).unwrap_or_else(|e| e.into_compile_error())); + quote! { + impl ::std::fmt::Debug for #record_type { + fn fmt(&self, f: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result { + let mut debug_struct = f.debug_struct(stringify!(#record_type)); + #(#field_iter)* + debug_struct.finish() + } + } + } + .into() +} + +fn format_field(field: &Field) -> syn::Result { + let ident = field.ident.as_ref().unwrap(); + if is_hidden(field) { + return Ok(quote!()); + } + Ok(match find_dbn_debug_attr(field)? { + Some(id) if id == C_CHAR_ATTR => { + quote! { debug_struct.field(stringify!(#ident), &(self.#ident as u8 as char)); } + } + Some(id) if id == FIXED_PRICE_ATTR => { + let crate_name = crate_name(); + quote! { debug_struct.field(stringify!(#ident), &#crate_name::pretty::Px(self.#ident)); } + } + Some(id) if id == FMT_BINARY => { + // format as `0b00101010` + quote! { debug_struct.field(stringify!(#ident), &format_args!("{:#010b}", &self.#ident)); } + } + Some(id) if id == FMT_METHOD => { + // Try to use method to format, otherwise fallback on raw value + return Ok(quote! { + match self.#ident() { + Ok(s) => debug_struct.field(stringify!(#ident), &s), + Err(_) => debug_struct.field(stringify!(#ident), &self.#ident), + }; + }); + } + _ => quote! { debug_struct.field(stringify!(#ident), &self.#ident); }, + }) +} diff --git a/rust/dbn-macros/src/has_rtype.rs b/rust/dbn-macros/src/has_rtype.rs index 4824b6a..df34473 100644 --- a/rust/dbn-macros/src/has_rtype.rs +++ b/rust/dbn-macros/src/has_rtype.rs @@ -28,6 +28,7 @@ pub fn attribute_macro_impl( let raw_index_ts = get_raw_index_ts(&input_struct).unwrap_or_else(|e| e.into_compile_error()); let rtypes = args.args.iter(); let crate_name = crate::utils::crate_name(); + let impl_debug = crate::debug::record_debug_impl(&input_struct); quote! ( #input_struct @@ -67,11 +68,13 @@ pub fn attribute_macro_impl( } } } + + #impl_debug ) .into() } -struct Args { +pub(crate) struct Args { args: Vec, span: Span, } diff --git a/rust/dbn-macros/src/lib.rs b/rust/dbn-macros/src/lib.rs index 3aa6844..13cc2c3 100644 --- a/rust/dbn-macros/src/lib.rs +++ b/rust/dbn-macros/src/lib.rs @@ -1,6 +1,7 @@ use proc_macro::TokenStream; mod dbn_attr; +mod debug; mod has_rtype; mod py_field_desc; mod serialize; @@ -15,6 +16,15 @@ pub fn derive_mock_pyo3(_item: TokenStream) -> TokenStream { TokenStream::new() } +/// Dummy derive macro to enable enable the `dbn` helper attribute for record types +/// using the `dbn_record` proc macro but neither `CsvSerialize` nor `JsonSerialize` as +/// helper attributes aren't supported for proc macros alone. See +/// . +#[proc_macro_derive(DbnAttr, attributes(dbn))] +pub fn dbn_attr(_item: TokenStream) -> TokenStream { + TokenStream::new() +} + /// Derive macro for CSV serialization. Supports the following `dbn` attributes: /// - `c_char`: serializes the field as a `char` /// - `encode_order`: overrides the position of the field in the CSV table @@ -31,7 +41,9 @@ pub fn derive_csv_serialize(input: TokenStream) -> TokenStream { serialize::derive_csv_macro_impl(input) } -/// Derive macro for JSON serialization. Supports the following `dbn` attributes: +/// Derive macro for JSON serialization. +/// +/// Supports the following `dbn` attributes: /// - `c_char`: serializes the field as a `char` /// - `fixed_price`: serializes the field as fixed-price, with the output format /// depending on `PRETTY_PX` @@ -46,8 +58,9 @@ pub fn derive_json_serialize(input: TokenStream) -> TokenStream { serialize::derive_json_macro_impl(input) } -/// Derive macro for field descriptions exposed to Python. Supports the following `dbn` -/// attributes: +/// Derive macro for field descriptions exposed to Python. +/// +/// Supports the following `dbn` attributes: /// - `c_char`: indicates the field dtype should be a single-character string rather /// than an integer /// - `encode_order`: overrides the position of the field in the ordered list @@ -59,19 +72,47 @@ pub fn derive_py_field_desc(input: TokenStream) -> TokenStream { py_field_desc::derive_impl(input) } -/// Attribute macro that acts like a derive macro for for `HasRType` and -/// `AsRef<[u8]>`. +/// Attribute macro that acts like a derive macro for for `Debug` (with customization), +/// `Record`, `RecordMut`, `HasRType`, `PartialOrd`, and `AsRef<[u8]>`. /// /// Expects 1 or more paths to `u8` constants that are the RTypes associated /// with this record. /// /// Supports the following `dbn` attributes: +/// - `c_char`: format the type as a `char` instead of as a numeric +/// - `fixed_price`: format the integer as a fixed-precision decimal +/// - `fmt_binary`: format as a binary +/// - `fmt_method`: try to format by calling the getter method with the same name as the /// - `index_ts`: indicates this field is the primary timestamp for the record +/// field. If the getter returns an error, the raw field value will be used +/// - `skip`: won't be included in the `Debug` output +/// +/// Note: attribute macros don't support helper attributes on their own. If not deriving +/// `CsvSerialize` or `JsonSerialize`, derive `DbnAttr` to use the `dbn` helper attribute +/// without a compiler error. #[proc_macro_attribute] pub fn dbn_record(attr: TokenStream, input: TokenStream) -> TokenStream { has_rtype::attribute_macro_impl(attr, input) } +/// Derive macro for Debug representations with the same extensions for DBN records +/// as `dbn_record`. +/// +/// Supports the following `dbn` attributes: +/// - `c_char`: format the type as a `char` instead of as a numeric +/// - `fixed_price`: format the integer as a fixed-precision decimal +/// - `fmt_binary`: format as a binary +/// - `fmt_method`: try to format by calling the getter method with the same name as the +/// field. If the getter returns an error, the raw field value will be used +/// - `skip`: won't be included in the `Debug` output +/// +/// Note: fields beginning with `_` will automatically be skipped, e.g. `_dummy` isn't +/// included in the `Debug` output. +#[proc_macro_derive(RecordDebug, attributes(dbn))] +pub fn derive_record_debug(input: TokenStream) -> TokenStream { + debug::derive_impl(input) +} + #[cfg(test)] mod tests { #[test] diff --git a/rust/dbn/src/compat.rs b/rust/dbn/src/compat.rs index 4eb90be..b0b0952 100644 --- a/rust/dbn/src/compat.rs +++ b/rust/dbn/src/compat.rs @@ -73,7 +73,7 @@ pub unsafe fn decode_record_ref<'a>( /// /// Note: This will be renamed to `InstrumentDefMsg` in DBN version 2. #[repr(C)] -#[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] +#[derive(Clone, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr( @@ -208,32 +208,43 @@ pub struct InstrumentDefMsgV1 { #[pyo3(get, set)] pub channel_id: u16, /// The currency used for price fields. + #[dbn(fmt_method)] #[cfg_attr(feature = "serde", serde(with = "crate::record::cstr_serde"))] pub currency: [c_char; 4], /// The currency used for settlement, if different from `currency`. + #[dbn(fmt_method)] #[cfg_attr(feature = "serde", serde(with = "crate::record::cstr_serde"))] pub settl_currency: [c_char; 4], /// The strategy type of the spread. + #[dbn(fmt_method)] #[cfg_attr(feature = "serde", serde(with = "crate::record::cstr_serde"))] pub secsubtype: [c_char; 6], /// The instrument raw symbol assigned by the publisher. - #[dbn(encode_order(2))] + #[dbn(encode_order(2), fmt_method)] pub raw_symbol: [c_char; SYMBOL_CSTR_LEN_V1], /// The security group code of the instrument. + #[dbn(fmt_method)] pub group: [c_char; 21], /// The exchange used to identify the instrument. + #[dbn(fmt_method)] pub exchange: [c_char; 5], /// The underlying asset code (product code) of the instrument. + #[dbn(fmt_method)] pub asset: [c_char; 7], /// The ISO standard instrument categorization code. + #[dbn(fmt_method)] pub cfi: [c_char; 7], /// The type of the instrument, e.g. FUT for future or future spread. + #[dbn(fmt_method)] pub security_type: [c_char; 7], /// The unit of measure for the instrument’s original contract size, e.g. USD or LBS. + #[dbn(fmt_method)] pub unit_of_measure: [c_char; 31], /// The symbol of the first underlying instrument. + #[dbn(fmt_method)] pub underlying: [c_char; 21], /// The currency of [`strike_price`](Self::strike_price). + #[dbn(fmt_method)] pub strike_price_currency: [c_char; 4], /// The classification of the instrument. #[dbn(c_char, encode_order(4))] @@ -304,7 +315,7 @@ pub struct InstrumentDefMsgV1 { /// /// Note: This will be renamed to `SymbolMappingMsg` in DBN version 2. #[repr(C)] -#[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] +#[derive(Clone, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr( @@ -320,8 +331,10 @@ pub struct SymbolMappingMsgV1 { #[pyo3(get, set)] pub hd: RecordHeader, /// The input symbol. + #[dbn(fmt_method)] pub stype_in_symbol: [c_char; SYMBOL_CSTR_LEN_V1], /// The output symbol. + #[dbn(fmt_method)] pub stype_out_symbol: [c_char; SYMBOL_CSTR_LEN_V1], // Filler for alignment. #[doc(hidden)] diff --git a/rust/dbn/src/macros.rs b/rust/dbn/src/macros.rs index fb90942..a44dafd 100644 --- a/rust/dbn/src/macros.rs +++ b/rust/dbn/src/macros.rs @@ -1,7 +1,7 @@ //! Helper macros for working with multiple RTypes, Schemas, and types of records. // Re-export -pub use dbn_macros::{dbn_record, CsvSerialize, JsonSerialize, PyFieldDesc}; +pub use dbn_macros::{dbn_record, CsvSerialize, DbnAttr, JsonSerialize, PyFieldDesc, RecordDebug}; /// Base macro for type dispatch based on rtype. /// diff --git a/rust/dbn/src/pretty.rs b/rust/dbn/src/pretty.rs index 57acb6c..5139d35 100644 --- a/rust/dbn/src/pretty.rs +++ b/rust/dbn/src/pretty.rs @@ -37,7 +37,7 @@ impl fmt::Debug for Ts { impl fmt::Debug for Px { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.0.fmt(f) + f.write_str(&fmt_px(self.0)) } } diff --git a/rust/dbn/src/record.rs b/rust/dbn/src/record.rs index 1702fbb..1f6cbf9 100644 --- a/rust/dbn/src/record.rs +++ b/rust/dbn/src/record.rs @@ -18,7 +18,7 @@ use crate::{ Action, InstrumentClass, MatchAlgorithm, SecurityUpdateAction, Side, StatType, StatUpdateAction, UserDefinedInstrument, }, - macros::{dbn_record, CsvSerialize, JsonSerialize}, + macros::{dbn_record, CsvSerialize, JsonSerialize, RecordDebug}, publishers::Publisher, Error, Result, SYMBOL_CSTR_LEN, }; @@ -33,7 +33,7 @@ pub use conv::{ /// Common data for all Databento records. Always found at the beginning of a record /// struct. #[repr(C)] -#[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] +#[derive(Clone, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr( @@ -64,7 +64,7 @@ pub struct RecordHeader { /// A market-by-order (MBO) tick message. The record of the /// [`Mbo`](crate::enums::Schema::Mbo) schema. #[repr(C)] -#[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] +#[derive(Clone, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr( @@ -93,6 +93,7 @@ pub struct MboMsg { pub size: u32, /// A combination of packet end with matching engine status. See /// [`enums::flags`](crate::enums::flags) for possible values. + #[dbn(fmt_binary)] #[pyo3(get)] pub flags: u8, /// A channel ID within the venue. @@ -121,7 +122,7 @@ pub struct MboMsg { /// A level. #[repr(C)] -#[derive(Clone, Debug, JsonSerialize, PartialEq, Eq, Hash)] +#[derive(Clone, JsonSerialize, RecordDebug, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr( @@ -150,7 +151,7 @@ pub struct BidAskPair { /// Market by price implementation with a book depth of 0. Equivalent to /// MBP-0. The record of the [`Trades`](crate::enums::Schema::Trades) schema. #[repr(C)] -#[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] +#[derive(Clone, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr( @@ -181,6 +182,7 @@ pub struct TradeMsg { pub side: c_char, /// A combination of packet end with matching engine status. See /// [`enums::flags`](crate::enums::flags) for possible values. + #[dbn(fmt_binary)] #[pyo3(get)] pub flags: u8, /// The depth of actual book change. @@ -203,7 +205,7 @@ pub struct TradeMsg { /// Market by price implementation with a known book depth of 1. The record of the /// [`Mbp1`](crate::enums::Schema::Mbp1) schema. #[repr(C)] -#[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] +#[derive(Clone, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr( @@ -235,6 +237,7 @@ pub struct Mbp1Msg { pub side: c_char, /// A combination of packet end with matching engine status. See /// [`enums::flags`](crate::enums::flags) for possible values. + #[dbn(fmt_binary)] #[pyo3(get)] pub flags: u8, /// The depth of actual book change. @@ -260,7 +263,7 @@ pub struct Mbp1Msg { /// Market by price implementation with a known book depth of 10. The record of the /// [`Mbp10`](crate::enums::Schema::Mbp10) schema. #[repr(C)] -#[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] +#[derive(Clone, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr( @@ -292,6 +295,7 @@ pub struct Mbp10Msg { pub side: c_char, /// A combination of packet end with matching engine status. See /// [`enums::flags`](crate::enums::flags) for possible values. + #[dbn(fmt_binary)] #[pyo3(get)] pub flags: u8, /// The depth of actual book change. @@ -324,7 +328,7 @@ pub type TbboMsg = Mbp1Msg; /// - [`Ohlcv1D`](crate::enums::Schema::Ohlcv1D) /// - [`OhlcvEod`](crate::enums::Schema::OhlcvEod) #[repr(C)] -#[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] +#[derive(Clone, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr( @@ -364,7 +368,7 @@ pub struct OhlcvMsg { /// [`Status`](crate::enums::Schema::Status) schema. #[doc(hidden)] #[repr(C)] -#[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] +#[derive(Clone, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr( @@ -384,6 +388,7 @@ pub struct StatusMsg { #[dbn(unix_nanos)] #[pyo3(get, set)] pub ts_recv: u64, + #[dbn(fmt_method)] #[cfg_attr(feature = "serde", serde(with = "conv::cstr_serde"))] pub group: [c_char; 21], #[pyo3(get, set)] @@ -397,7 +402,7 @@ pub struct StatusMsg { /// Definition of an instrument. The record of the /// [`Definition`](crate::enums::Schema::Definition) schema. #[repr(C)] -#[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] +#[derive(Clone, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr( @@ -533,40 +538,51 @@ pub struct InstrumentDefMsg { #[pyo3(get, set)] pub channel_id: u16, /// The currency used for price fields. + #[dbn(fmt_method)] #[cfg_attr(feature = "serde", serde(with = "conv::cstr_serde"))] pub currency: [c_char; 4], /// The currency used for settlement, if different from `currency`. + #[dbn(fmt_method)] #[cfg_attr(feature = "serde", serde(with = "conv::cstr_serde"))] pub settl_currency: [c_char; 4], /// The strategy type of the spread. + #[dbn(fmt_method)] #[cfg_attr(feature = "serde", serde(with = "conv::cstr_serde"))] pub secsubtype: [c_char; 6], /// The instrument raw symbol assigned by the publisher. - #[dbn(encode_order(2))] + #[dbn(encode_order(2), fmt_method)] #[cfg_attr(feature = "serde", serde(with = "conv::cstr_serde"))] pub raw_symbol: [c_char; SYMBOL_CSTR_LEN], /// The security group code of the instrument. + #[dbn(fmt_method)] #[cfg_attr(feature = "serde", serde(with = "conv::cstr_serde"))] pub group: [c_char; 21], /// The exchange used to identify the instrument. + #[dbn(fmt_method)] #[cfg_attr(feature = "serde", serde(with = "conv::cstr_serde"))] pub exchange: [c_char; 5], /// The underlying asset code (product code) of the instrument. + #[dbn(fmt_method)] #[cfg_attr(feature = "serde", serde(with = "conv::cstr_serde"))] pub asset: [c_char; 7], /// The ISO standard instrument categorization code. + #[dbn(fmt_method)] #[cfg_attr(feature = "serde", serde(with = "conv::cstr_serde"))] pub cfi: [c_char; 7], /// The type of the instrument, e.g. FUT for future or future spread. + #[dbn(fmt_method)] #[cfg_attr(feature = "serde", serde(with = "conv::cstr_serde"))] pub security_type: [c_char; 7], /// The unit of measure for the instrument’s original contract size, e.g. USD or LBS. + #[dbn(fmt_method)] #[cfg_attr(feature = "serde", serde(with = "conv::cstr_serde"))] pub unit_of_measure: [c_char; 31], /// The symbol of the first underlying instrument. + #[dbn(fmt_method)] #[cfg_attr(feature = "serde", serde(with = "conv::cstr_serde"))] pub underlying: [c_char; 21], /// The currency of [`strike_price`](Self::strike_price). + #[dbn(fmt_method)] #[cfg_attr(feature = "serde", serde(with = "conv::cstr_serde"))] pub strike_price_currency: [c_char; 4], /// The classification of the instrument. @@ -628,7 +644,7 @@ pub struct InstrumentDefMsg { /// An auction imbalance message. #[repr(C)] -#[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] +#[derive(Clone, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr( @@ -722,7 +738,7 @@ pub struct ImbalanceMsg { /// A statistics message. A catchall for various data disseminated by publishers. /// The [`stat_type`](Self::stat_type) indicates the statistic contained in the message. #[repr(C)] -#[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] +#[derive(Clone, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr( @@ -759,13 +775,16 @@ pub struct StatMsg { pub ts_in_delta: i32, /// The type of statistic value contained in the message. Refer to the /// [`StatType`](crate::enums::StatType) for variants. + #[dbn(fmt_method)] pub stat_type: u16, /// A channel ID within the venue. pub channel_id: u16, /// Indicates if the statistic is newly added (1) or deleted (2). (Deleted is only used with /// some stat types) + #[dbn(fmt_method)] pub update_action: u8, /// Additional flags associate with certain stat types. + #[dbn(fmt_binary)] pub stat_flags: u8, // Filler for alignment #[doc(hidden)] @@ -775,7 +794,7 @@ pub struct StatMsg { /// An error message from the Databento Live Subscription Gateway (LSG). #[repr(C)] -#[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] +#[derive(Clone, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr( @@ -791,6 +810,7 @@ pub struct ErrorMsg { #[pyo3(get, set)] pub hd: RecordHeader, /// The error message. + #[dbn(fmt_method)] #[cfg_attr(feature = "serde", serde(with = "conv::cstr_serde"))] pub err: [c_char; 64], } @@ -798,7 +818,7 @@ pub struct ErrorMsg { /// A symbol mapping message which maps a symbol of one [`SType`](crate::enums::SType) /// to another. #[repr(C)] -#[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] +#[derive(Clone, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr( @@ -815,15 +835,19 @@ pub struct SymbolMappingMsg { pub hd: RecordHeader, // TODO(carter): special serialization to string? /// The input symbology type of `stype_in_symbol`. + #[dbn(fmt_method)] #[pyo3(get, set)] pub stype_in: u8, /// The input symbol. + #[dbn(fmt_method)] #[cfg_attr(feature = "serde", serde(with = "conv::cstr_serde"))] pub stype_in_symbol: [c_char; SYMBOL_CSTR_LEN], /// The output symbology type of `stype_out_symbol`. + #[dbn(fmt_method)] #[pyo3(get, set)] pub stype_out: u8, /// The output symbol. + #[dbn(fmt_method)] #[cfg_attr(feature = "serde", serde(with = "conv::cstr_serde"))] pub stype_out_symbol: [c_char; SYMBOL_CSTR_LEN], /// The start of the mapping interval expressed as the number of nanoseconds since @@ -841,7 +865,7 @@ pub struct SymbolMappingMsg { /// A non-error message from the Databento Live Subscription Gateway (LSG). Also used /// for heartbeating. #[repr(C)] -#[derive(Clone, Debug, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] +#[derive(Clone, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr( @@ -857,6 +881,7 @@ pub struct SystemMsg { #[pyo3(get, set)] pub hd: RecordHeader, /// The message from the Databento Live Subscription Gateway (LSG). + #[dbn(fmt_method)] #[cfg_attr(feature = "serde", serde(with = "conv::cstr_serde"))] pub msg: [c_char; 64], } diff --git a/rust/dbn/src/record/methods.rs b/rust/dbn/src/record/methods.rs index 043a035..17c0fba 100644 --- a/rust/dbn/src/record/methods.rs +++ b/rust/dbn/src/record/methods.rs @@ -1,3 +1,5 @@ +use std::fmt::Debug; + use crate::{ compat::{InstrumentDefMsgV1, SymbolMappingMsgV1}, SType, @@ -64,6 +66,25 @@ impl RecordHeader { } } +impl Debug for RecordHeader { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut debug_struct = f.debug_struct("RecordHeader"); + debug_struct.field("length", &self.length); + match self.rtype() { + Ok(rtype) => debug_struct.field("rtype", &format_args!("{rtype:?}")), + Err(_) => debug_struct.field("rtype", &format_args!("{:#04X}", &self.rtype)), + }; + match self.publisher() { + Ok(p) => debug_struct.field("publisher_id", &format_args!("{p:?}")), + Err(_) => debug_struct.field("publisher_id", &self.publisher_id), + }; + debug_struct + .field("instrument_id", &self.instrument_id) + .field("ts_event", &self.ts_event) + .finish() + } +} + impl MboMsg { /// Tries to convert the raw order side to an enum. /// @@ -786,6 +807,8 @@ impl WithTsOut { #[cfg(test)] mod tests { + use crate::flags; + use super::*; #[test] @@ -796,4 +819,90 @@ mod tests { "couldn't convert 0x0E to dbn::enums::rtype::RType" ); } + + #[test] + fn debug_mbo() { + let rec = MboMsg { + hd: RecordHeader::new::( + rtype::MBO, + Publisher::OpraPillarXcbo as u16, + 678, + 1704468548242628731, + ), + flags: flags::LAST | flags::BAD_TS_RECV, + price: 4_500_500_000_000, + side: b'B' as c_char, + action: b'A' as c_char, + ..Default::default() + }; + assert_eq!( + format!("{rec:?}"), + "MboMsg { hd: RecordHeader { length: 14, rtype: Mbo, publisher_id: OpraPillarXcbo, \ + instrument_id: 678, ts_event: 1704468548242628731 }, order_id: 0, \ + price: 4500.500000000, size: 4294967295, flags: 0b10001000, channel_id: 0, \ + action: 'A', side: 'B', ts_recv: 18446744073709551615, ts_in_delta: 0, sequence: 0 }" + ); + } + + #[test] + fn debug_stats() { + let rec = StatMsg { + stat_type: StatType::OpenInterest as u16, + update_action: StatUpdateAction::New as u8, + quantity: 5, + stat_flags: 0b00000010, + ..Default::default() + }; + assert_eq!( + format!("{rec:?}"), + "StatMsg { hd: RecordHeader { length: 16, rtype: Statistics, publisher_id: 0, \ + instrument_id: 0, ts_event: 18446744073709551615 }, ts_recv: 18446744073709551615, \ + ts_ref: 18446744073709551615, price: UNDEF_PRICE, quantity: 5, sequence: 0, ts_in_delta: 0, \ + stat_type: OpenInterest, channel_id: 0, update_action: New, stat_flags: 0b00000010 }" + ); + } + + #[test] + fn debug_instrument_err() { + let rec = ErrorMsg { + err: str_to_c_chars("Missing stype_in").unwrap(), + ..Default::default() + }; + assert_eq!( + format!("{rec:?}"), + "ErrorMsg { hd: RecordHeader { length: 20, rtype: Error, publisher_id: 0, \ + instrument_id: 0, ts_event: 18446744073709551615 }, err: \"Missing stype_in\" }" + ); + } + + #[test] + fn debug_instrument_sys() { + let rec = SystemMsg::heartbeat(123); + assert_eq!( + format!("{rec:?}"), + "SystemMsg { hd: RecordHeader { length: 20, rtype: System, publisher_id: 0, \ + instrument_id: 0, ts_event: 123 }, msg: \"Heartbeat\" }" + ); + } + + #[test] + fn debug_instrument_symbol_mapping() { + let rec = SymbolMappingMsg { + hd: RecordHeader::new::( + rtype::SYMBOL_MAPPING, + 0, + 5602, + 1704466940331347283, + ), + stype_in: SType::RawSymbol as u8, + stype_in_symbol: str_to_c_chars("ESM4").unwrap(), + stype_out: SType::RawSymbol as u8, + stype_out_symbol: str_to_c_chars("ESM4").unwrap(), + ..Default::default() + }; + assert_eq!( + format!("{rec:?}"), + "SymbolMappingMsg { hd: RecordHeader { length: 44, rtype: SymbolMapping, publisher_id: 0, instrument_id: 5602, ts_event: 1704466940331347283 }, stype_in: RawSymbol, stype_in_symbol: \"ESM4\", stype_out: RawSymbol, stype_out_symbol: \"ESM4\", start_ts: 18446744073709551615, end_ts: 18446744073709551615 }" + ); + } } From 18a8d5f8b611c4d8cea7a54e7c8916832b9f3d5e Mon Sep 17 00:00:00 2001 From: Carter Green Date: Wed, 10 Jan 2024 09:13:54 -0600 Subject: [PATCH 17/21] DEL: Remove specialty `__repr__` logic --- rust/dbn/src/python/record.rs | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/rust/dbn/src/python/record.rs b/rust/dbn/src/python/record.rs index 6c28e29..fc08672 100644 --- a/rust/dbn/src/python/record.rs +++ b/rust/dbn/src/python/record.rs @@ -2037,11 +2037,7 @@ impl ErrorMsg { } fn __repr__(&self) -> String { - if let Ok(err_msg) = self.err() { - format!("ErrorMsg {{ hd: {:?}, err: '{}' }}", self.hd, err_msg) - } else { - format!("ErrorMsg {{ hd: {:?}, err: '{:?}' }}", self.hd, self.err) - } + format!("{self:?}") } #[getter] @@ -2420,11 +2416,7 @@ impl SystemMsg { } fn __repr__(&self) -> String { - if let Ok(sys_msg) = self.msg() { - format!("SystemMsg {{ hd: {:?}, msg: '{}' }}", self.hd, sys_msg) - } else { - format!("SystemMsg {{ hd: {:?}, msg: '{:?}' }}", self.hd, self.msg) - } + format!("{self:?}") } #[getter] From 39757749a20384993f482ca47fecd1de51b02479 Mon Sep 17 00:00:00 2001 From: Carter Green Date: Wed, 10 Jan 2024 14:16:47 -0600 Subject: [PATCH 18/21] MOD: Improve `Debug` for `RecordRef` --- CHANGELOG.md | 1 + rust/dbn/src/record_ref.rs | 30 +++++++++++++++++++++++++----- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b903fe..727baba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - Fixed-length strings are formatted as strings - Bit flag fields are formatted as binary - Several fields are formatted as enums instead of their raw representations +- Improved `Debug` implementation for `RecordRef` to show `RecordHeader` - Added `--schema` option to `dbn` CLI tool to filter a DBN to a particular schema. This allows outputting saved live data to CSV - Allowed passing `--limit` option to `dbn` CLI tool with `--metadata` flag diff --git a/rust/dbn/src/record_ref.rs b/rust/dbn/src/record_ref.rs index b0841d1..4202e23 100644 --- a/rust/dbn/src/record_ref.rs +++ b/rust/dbn/src/record_ref.rs @@ -1,6 +1,6 @@ //! The [`RecordRef`] struct for non-owning references to DBN records. -use std::{marker::PhantomData, mem, ptr::NonNull}; +use std::{fmt::Debug, marker::PhantomData, mem, ptr::NonNull}; use crate::{ record::{HasRType, Record, RecordHeader}, @@ -9,7 +9,7 @@ use crate::{ /// A wrapper around a non-owning immutable reference to a DBN record. This wrapper /// allows for mixing of record types and schemas, and runtime record polymorphism. -#[derive(Copy, Clone, Debug)] +#[derive(Copy, Clone)] pub struct RecordRef<'a> { ptr: NonNull, /// Associates the object with the lifetime of the memory pointed to by `ptr`. @@ -189,6 +189,17 @@ impl<'a> From> for RecordRef<'a> { } } +impl<'a> Debug for RecordRef<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RecordRef") + .field( + "ptr", + &format_args!("{:?} --> {:?}", self.ptr, self.header()), + ) + .finish() + } +} + #[cfg(test)] mod tests { use std::ffi::c_char; @@ -216,10 +227,19 @@ mod tests { #[test] fn test_header() { - let target = unsafe { RecordRef::new(SOURCE_RECORD.as_ref()) }; + let target = RecordRef::from(&SOURCE_RECORD); assert_eq!(*target.header(), SOURCE_RECORD.hd); } + #[test] + fn test_fmt_debug() { + let target = RecordRef::from(&SOURCE_RECORD); + let string = format!("{target:?}"); + dbg!(&string); + assert!(string.starts_with("RecordRef { ptr: 0x")); + assert!(string.ends_with("--> RecordHeader { length: 14, rtype: Mbo, publisher_id: GlbxMdp3Glbx, instrument_id: 1, ts_event: 0 } }")); + } + #[test] fn test_has_and_get() { let target = RecordRef::from(&SOURCE_RECORD); @@ -230,7 +250,7 @@ mod tests { assert!(!target.has::()); assert!(!target.has::()); assert!(target.has::()); - assert_eq!(*unsafe { target.get_unchecked::() }, SOURCE_RECORD); + assert_eq!(*target.get::().unwrap(), SOURCE_RECORD); } #[test] @@ -246,7 +266,7 @@ mod tests { fn test_get_too_short() { let mut src = SOURCE_RECORD; src.hd.length -= 1; - let target = unsafe { RecordRef::new(src.as_ref()) }; + let target = RecordRef::from(&src); // panic due to unexpected length target.get::(); } From b5a58765fca340fca02a1d50ecb3223f71ac423c Mon Sep 17 00:00:00 2001 From: Carter Green Date: Fri, 12 Jan 2024 16:16:13 -0600 Subject: [PATCH 19/21] ADD: Increase SystemMsg and ErrorMsg size --- CHANGELOG.md | 9 ++ python/databento_dbn.pyi | 58 ++++++++ python/src/dbn_decoder.rs | 4 +- python/src/lib.rs | 4 +- python/src/transcoder.rs | 4 +- rust/dbn/src/compat.rs | 131 +++++++++++++++--- rust/dbn/src/decode/dbn/async.rs | 2 +- rust/dbn/src/decode/dbn/sync.rs | 4 +- rust/dbn/src/encode/json/sync.rs | 4 +- rust/dbn/src/python/record.rs | 201 +++++++++++++++++++++++++++- rust/dbn/src/record.rs | 22 ++- rust/dbn/src/record/impl_default.rs | 25 +++- rust/dbn/src/record/methods.rs | 80 ++++++++++- 13 files changed, 501 insertions(+), 47 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 727baba..6a67a91 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,15 @@ - Added `encode::ZSTD_COMPRESSION_LEVEL` constant ### Breaking changes +- Increased size of `SystemMsg` and `ErrorMsg` to provide better messages from Live + gateway + - Increased length of `err` and `msg` fields for more detailed messages + - Added `is_last` field to `ErrorMsg` to indicate the last error in a chain + - Added `code` field to `SystemMsg` and `ErrorMsg`, although currently unused + - Added new `is_last` parameter to `ErrorMsg::new` + - Decoding these is backwards-compatible and records with longer messages won't be + sent during the DBN version 2 migration period + - Renamed previous records to `compat::ErrorMsgV1` and `compat::SystemMsgV1` - Split `DecodeDbn` trait into `DecodeRecord` and `DbnMetadata` traits for more flexibility. `DecodeDbn` continues to exist as a trait alias - Moved `decode_stream` out of `DecodeDbn` to its own separate trait `DecodeStream` diff --git a/python/databento_dbn.pyi b/python/databento_dbn.pyi index d12a9c3..0dff9bf 100644 --- a/python/databento_dbn.pyi +++ b/python/databento_dbn.pyi @@ -27,9 +27,11 @@ _DBNRecord = Union[ InstrumentDefMsgV1, ImbalanceMsg, ErrorMsg, + ErrorMsgV1, SymbolMappingMsg, SymbolMappingMsgV1, SystemMsg, + SystemMsgV1, StatMsg, ] @@ -3201,6 +3203,31 @@ class ErrorMsg(Record): An error message from the Databento Live Subscription Gateway (LSG). """ + @property + def err(self) -> str: + """ + The error message. + + Returns + ------- + str + + """ + @property + def is_last(self) -> int: + """ + Whether this is the last record in a chain. + + Returns + ------- + int + """ + +class ErrorMsgV1(Record): + """ + A DBN version 1 error message from the Databento Live Subscription Gateway (LSG). + """ + @property def err(self) -> str: """ @@ -3402,6 +3429,37 @@ class SystemMsg(Record): """ +class SystemMsgV1(Record): + """ + A DBN version 1 non-error message from the Databento Live Subscription Gateway + (LSG). + + Also used for heartbeating. + + """ + + @property + def msg(self) -> str: + """ + The message from the Databento Live Subscription Gateway (LSG). + + Returns + ------- + str + + """ + @property + def is_heartbeat(self) -> bool: + """ + `true` if this message is a heartbeat, used to indicate the connection + with the gateway is still open. + + Returns + ------- + bool + + """ + class DBNDecoder: """ A class for decoding DBN data to Python objects. diff --git a/python/src/dbn_decoder.rs b/python/src/dbn_decoder.rs index 0e76aff..2803231 100644 --- a/python/src/dbn_decoder.rs +++ b/python/src/dbn_decoder.rs @@ -162,7 +162,7 @@ mod tests { let metadata_pos = encoder.get_ref().len(); assert!(matches!(target.decode(), Ok(recs) if recs.len() == 1)); assert!(target.has_decoded_metadata); - let rec = ErrorMsg::new(1680708278000000000, "Python"); + let rec = ErrorMsg::new(1680708278000000000, "Python", true); encoder.encode_record(&rec).unwrap(); assert!(target.buffer.get_ref().is_empty()); let record_pos = encoder.get_ref().len(); @@ -201,7 +201,7 @@ mod tests { let metadata_pos = encoder.get_ref().len(); assert!(matches!(decoder.decode(), Ok(recs) if recs.len() == 1)); assert!(decoder.has_decoded_metadata); - let rec1 = ErrorMsg::new(1680708278000000000, "Python"); + let rec1 = ErrorMsg::new(1680708278000000000, "Python", true); let rec2 = OhlcvMsg { hd: RecordHeader::new::(rtype::OHLCV_1S, 1, 1, 1681228173000000000), open: 100, diff --git a/python/src/lib.rs b/python/src/lib.rs index 54291b0..30a1c49 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -3,7 +3,7 @@ use pyo3::{prelude::*, wrap_pyfunction, PyClass}; use dbn::{ - compat::{InstrumentDefMsgV1, SymbolMappingMsgV1}, + compat::{ErrorMsgV1, InstrumentDefMsgV1, SymbolMappingMsgV1, SystemMsgV1}, enums::{Compression, Encoding, SType, Schema}, flags, python::EnumIterator, @@ -48,9 +48,11 @@ fn databento_dbn(_py: Python<'_>, m: &PyModule) -> PyResult<()> { checked_add_class::(m)?; checked_add_class::(m)?; checked_add_class::(m)?; + checked_add_class::(m)?; checked_add_class::(m)?; checked_add_class::(m)?; checked_add_class::(m)?; + checked_add_class::(m)?; checked_add_class::(m)?; // PyClass enums checked_add_class::(m)?; diff --git a/python/src/transcoder.rs b/python/src/transcoder.rs index ac8a009..937020d 100644 --- a/python/src/transcoder.rs +++ b/python/src/transcoder.rs @@ -527,7 +527,7 @@ mod tests { .has_decoded_metadata ); let metadata_pos = encoder.get_ref().len(); - let rec = ErrorMsg::new(1680708278000000000, "This is a test"); + let rec = ErrorMsg::new(1680708278000000000, "This is a test", true); encoder.encode_record(&rec).unwrap(); assert!(target.buffer().is_empty()); let record_pos = encoder.get_ref().len(); @@ -591,7 +591,7 @@ mod tests { .downcast_unchecked::<{ Encoding::Csv as u8 }>() .has_decoded_metadata ); - let rec1 = ErrorMsg::new(1680708278000000000, "This is a test"); + let rec1 = ErrorMsg::new(1680708278000000000, "This is a test", true); let rec2 = OhlcvMsg { hd: RecordHeader::new::(rtype::OHLCV_1S, 1, 1, 1681228173000000000), open: 100, diff --git a/rust/dbn/src/compat.rs b/rust/dbn/src/compat.rs index b0b0952..f023c75 100644 --- a/rust/dbn/src/compat.rs +++ b/rust/dbn/src/compat.rs @@ -4,8 +4,8 @@ use std::os::raw::c_char; use crate::{ macros::{dbn_record, CsvSerialize, JsonSerialize}, record::{transmute_header_bytes, transmute_record_bytes}, - rtype, HasRType, InstrumentDefMsg, RecordHeader, RecordRef, SecurityUpdateAction, - SymbolMappingMsg, UserDefinedInstrument, VersionUpgradePolicy, + rtype, HasRType, RecordHeader, RecordRef, SecurityUpdateAction, UserDefinedInstrument, + VersionUpgradePolicy, }; // Dummy derive macro to get around `cfg_attr` incompatibility of several @@ -13,9 +13,9 @@ use crate::{ #[cfg(not(feature = "python"))] use dbn_macros::MockPyo3; -/// The length of symbol fields in DBN version 1 (current version). +/// The length of symbol fields in DBN version 1 (prior version being phased out). pub const SYMBOL_CSTR_LEN_V1: usize = 22; -/// The length of symbol fields in DBN version 2 (future version). +/// The length of symbol fields in DBN version 2 (current version). pub const SYMBOL_CSTR_LEN_V2: usize = 71; pub(crate) const METADATA_RESERVED_LEN_V1: usize = 47; @@ -27,8 +27,10 @@ pub const fn version_symbol_cstr_len(version: u8) -> usize { SYMBOL_CSTR_LEN_V2 } } +pub use crate::record::ErrorMsg as ErrorMsgV2; pub use crate::record::InstrumentDefMsg as InstrumentDefMsgV2; pub use crate::record::SymbolMappingMsg as SymbolMappingMsgV2; +pub use crate::record::SystemMsg as SystemMsgV2; /// Decodes bytes into a [`RecordRef`], optionally applying conversion from structs /// of a prior DBN version to the current DBN version, according to the `version` and @@ -49,19 +51,30 @@ pub unsafe fn decode_record_ref<'a>( let header = transmute_header_bytes(input).unwrap(); match header.rtype { rtype::INSTRUMENT_DEF => { - let definition = InstrumentDefMsg::from( + let definition = InstrumentDefMsgV2::from( transmute_record_bytes::(input).unwrap(), ); std::ptr::copy_nonoverlapping(&definition, compat_buffer.as_mut_ptr().cast(), 1); return RecordRef::new(compat_buffer); } rtype::SYMBOL_MAPPING => { - let definition = SymbolMappingMsg::from( + let definition = SymbolMappingMsgV2::from( transmute_record_bytes::(input).unwrap(), ); std::ptr::copy_nonoverlapping(&definition, compat_buffer.as_mut_ptr().cast(), 1); return RecordRef::new(compat_buffer); } + rtype::ERROR => { + let system = ErrorMsgV2::from(transmute_record_bytes::(input).unwrap()); + std::ptr::copy_nonoverlapping(&system, compat_buffer.as_mut_ptr().cast(), 1); + return RecordRef::new(compat_buffer); + } + rtype::SYSTEM => { + let system = + SystemMsgV2::from(transmute_record_bytes::(input).unwrap()); + std::ptr::copy_nonoverlapping(&system, compat_buffer.as_mut_ptr().cast(), 1); + return RecordRef::new(compat_buffer); + } _ => (), } } @@ -70,8 +83,6 @@ pub unsafe fn decode_record_ref<'a>( /// Definition of an instrument in DBN version 1. The record of the /// [`Definition`](crate::enums::Schema::Definition) schema. -/// -/// Note: This will be renamed to `InstrumentDefMsg` in DBN version 2. #[repr(C)] #[derive(Clone, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] @@ -310,10 +321,33 @@ pub struct InstrumentDefMsgV1 { #[doc(hidden)] pub _dummy: [u8; 3], } + +/// An error message from the Databento Live Subscription Gateway (LSG) in DBN version +/// 1. +#[repr(C)] +#[derive(Clone, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] +#[cfg_attr(feature = "trivial_copy", derive(Copy))] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + feature = "python", + pyo3::pyclass(dict, module = "databento_dbn"), + derive(crate::macros::PyFieldDesc) +)] +#[cfg_attr(not(feature = "python"), derive(MockPyo3))] // bring `pyo3` attribute into scope +#[cfg_attr(test, derive(type_layout::TypeLayout))] +#[dbn_record(rtype::ERROR)] +pub struct ErrorMsgV1 { + /// The common header. + #[pyo3(get, set)] + pub hd: RecordHeader, + /// The error message. + #[dbn(fmt_method)] + #[cfg_attr(feature = "serde", serde(with = "crate::record::cstr_serde"))] + pub err: [c_char; 64], +} + /// A symbol mapping message in DBN version 1 which maps a symbol of one /// [`SType`](crate::SType) to another. -/// -/// Note: This will be renamed to `SymbolMappingMsg` in DBN version 2. #[repr(C)] #[derive(Clone, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] #[cfg_attr(feature = "trivial_copy", derive(Copy))] @@ -351,7 +385,31 @@ pub struct SymbolMappingMsgV1 { pub end_ts: u64, } -impl From<&InstrumentDefMsgV1> for InstrumentDefMsg { +/// A non-error message from the Databento Live Subscription Gateway (LSG) in DBN +/// version 1. Also used for heartbeating. +#[repr(C)] +#[derive(Clone, CsvSerialize, JsonSerialize, PartialEq, Eq, Hash)] +#[cfg_attr(feature = "trivial_copy", derive(Copy))] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + feature = "python", + pyo3::pyclass(dict, module = "databento_dbn"), + derive(crate::macros::PyFieldDesc) +)] +#[cfg_attr(not(feature = "python"), derive(MockPyo3))] // bring `pyo3` attribute into scope +#[cfg_attr(test, derive(type_layout::TypeLayout))] +#[dbn_record(rtype::SYSTEM)] +pub struct SystemMsgV1 { + /// The common header. + #[pyo3(get, set)] + pub hd: RecordHeader, + /// The message from the Databento Live Subscription Gateway (LSG). + #[dbn(fmt_method)] + #[cfg_attr(feature = "serde", serde(with = "crate::record::cstr_serde"))] + pub msg: [c_char; 64], +} + +impl From<&InstrumentDefMsgV1> for InstrumentDefMsgV2 { fn from(old: &InstrumentDefMsgV1) -> Self { let mut res = Self { // recalculate length @@ -395,7 +453,6 @@ impl From<&InstrumentDefMsgV1> for InstrumentDefMsg { currency: old.currency, settl_currency: old.settl_currency, secsubtype: old.secsubtype, - raw_symbol: [0; SYMBOL_CSTR_LEN_V2], group: old.group, exchange: old.exchange, asset: old.asset, @@ -421,7 +478,7 @@ impl From<&InstrumentDefMsgV1> for InstrumentDefMsg { contract_multiplier_unit: old.contract_multiplier_unit, flow_schedule_type: old.flow_schedule_type, tick_rule: old.tick_rule, - _reserved: Default::default(), + ..Default::default() }; // Safety: SYMBOL_CSTR_LEN_V1 is less than SYMBOL_CSTR_LEN unsafe { @@ -435,7 +492,26 @@ impl From<&InstrumentDefMsgV1> for InstrumentDefMsg { } } -impl From<&SymbolMappingMsgV1> for SymbolMappingMsg { +impl From<&ErrorMsgV1> for ErrorMsgV2 { + fn from(old: &ErrorMsgV1) -> Self { + let mut new = Self { + hd: RecordHeader::new::( + rtype::ERROR, + old.hd.publisher_id, + old.hd.instrument_id, + old.hd.ts_event, + ), + ..Default::default() + }; + // Safety: new `err` is longer than older + unsafe { + std::ptr::copy_nonoverlapping(old.err.as_ptr(), new.err.as_mut_ptr(), new.err.len()); + } + new + } +} + +impl From<&SymbolMappingMsgV1> for SymbolMappingMsgV2 { fn from(old: &SymbolMappingMsgV1) -> Self { let mut res = Self { hd: RecordHeader::new::( @@ -444,13 +520,9 @@ impl From<&SymbolMappingMsgV1> for SymbolMappingMsg { old.hd.instrument_id, old.hd.ts_event, ), - stype_in_symbol: [0; SYMBOL_CSTR_LEN_V2], - stype_out_symbol: [0; SYMBOL_CSTR_LEN_V2], start_ts: old.start_ts, end_ts: old.end_ts, - // Invalid - stype_in: u8::MAX, - stype_out: u8::MAX, + ..Default::default() }; // Safety: SYMBOL_CSTR_LEN_V1 is less than SYMBOL_CSTR_LEN unsafe { @@ -469,6 +541,25 @@ impl From<&SymbolMappingMsgV1> for SymbolMappingMsg { } } +impl From<&SystemMsgV1> for SystemMsgV2 { + fn from(old: &SystemMsgV1) -> Self { + let mut new = Self { + hd: RecordHeader::new::( + rtype::SYSTEM, + old.hd.publisher_id, + old.hd.instrument_id, + old.hd.ts_event, + ), + ..Default::default() + }; + // Safety: new `msg` is longer than older + unsafe { + std::ptr::copy_nonoverlapping(old.msg.as_ptr(), new.msg.as_mut_ptr(), new.msg.len()); + } + new + } +} + /// A trait for symbol mapping records. pub trait SymbolMappingRec: HasRType { /// Returns the input symbol as a `&str`. @@ -542,7 +633,7 @@ mod tests { use crate::python::PyFieldDesc; assert_eq!( - InstrumentDefMsg::ordered_fields(""), + InstrumentDefMsgV1::ordered_fields(""), InstrumentDefMsgV2::ordered_fields("") ); } diff --git a/rust/dbn/src/decode/dbn/async.rs b/rust/dbn/src/decode/dbn/async.rs index 74df077..b244005 100644 --- a/rust/dbn/src/decode/dbn/async.rs +++ b/rust/dbn/src/decode/dbn/async.rs @@ -816,7 +816,7 @@ mod tests { #[tokio::test] async fn test_decode_record_length_longer_than_buffer() { - let rec = ErrorMsg::new(1680703198000000000, "Test"); + let rec = ErrorMsg::new(1680703198000000000, "Test", true); let mut target = RecordDecoder::new(&rec.as_ref()[..rec.record_size() - 1]); let res = target.decode_ref().await; dbg!(&res); diff --git a/rust/dbn/src/decode/dbn/sync.rs b/rust/dbn/src/decode/dbn/sync.rs index 0f2be11..96536db 100644 --- a/rust/dbn/src/decode/dbn/sync.rs +++ b/rust/dbn/src/decode/dbn/sync.rs @@ -970,7 +970,7 @@ mod tests { close: 125, volume: 65, }; - let error_msg: ErrorMsg = ErrorMsg::new(0, "Test failed successfully"); + let error_msg: ErrorMsg = ErrorMsg::new(0, "Test failed successfully", true); encoder.encode_record(&OHLCV_MSG).unwrap(); encoder.encode_record(&error_msg).unwrap(); @@ -1004,7 +1004,7 @@ mod tests { #[test] fn test_decode_record_length_longer_than_buffer() { - let rec = ErrorMsg::new(1680703198000000000, "Test"); + let rec = ErrorMsg::new(1680703198000000000, "Test", true); let mut target = RecordDecoder::new(&rec.as_ref()[..rec.record_size() - 1]); assert!(matches!(target.decode_ref(), Ok(None))); } diff --git a/rust/dbn/src/encode/json/sync.rs b/rust/dbn/src/encode/json/sync.rs index ababde8..ef874ef 100644 --- a/rust/dbn/src/encode/json/sync.rs +++ b/rust/dbn/src/encode/json/sync.rs @@ -673,14 +673,14 @@ mod tests { #[test] fn test_serialize_quoted_str_to_json() { let json = write_json_to_string( - vec![ErrorMsg::new(0, "\"A test")].as_slice(), + vec![ErrorMsg::new(0, "\"A test", true)].as_slice(), false, true, true, ); assert_eq!( json, - r#"{"hd":{"ts_event":null,"rtype":21,"publisher_id":0,"instrument_id":0},"err":"\"A test"} + r#"{"hd":{"ts_event":null,"rtype":21,"publisher_id":0,"instrument_id":0},"err":"\"A test","code":255,"is_last":1} "# ); } diff --git a/rust/dbn/src/python/record.rs b/rust/dbn/src/python/record.rs index fc08672..3c20f64 100644 --- a/rust/dbn/src/python/record.rs +++ b/rust/dbn/src/python/record.rs @@ -8,7 +8,7 @@ use pyo3::{ }; use crate::{ - compat::{InstrumentDefMsgV1, SymbolMappingMsgV1}, + compat::{ErrorMsgV1, InstrumentDefMsgV1, SymbolMappingMsgV1, SystemMsgV1}, record::str_to_c_chars, rtype, BidAskPair, ErrorMsg, HasRType, ImbalanceMsg, InstrumentDefMsg, MboMsg, Mbp10Msg, Mbp1Msg, OhlcvMsg, Record, RecordHeader, SType, SecurityUpdateAction, StatMsg, @@ -2019,9 +2019,105 @@ impl StatMsg { #[pymethods] impl ErrorMsg { + #[new] + fn py_new(ts_event: u64, err: &str, is_last: Option) -> PyResult { + Ok(ErrorMsg::new(ts_event, err, is_last.unwrap_or(true))) + } + + fn __bytes__(&self) -> &[u8] { + self.as_ref() + } + + fn __richcmp__(&self, other: &Self, op: CompareOp, py: Python<'_>) -> Py { + match op { + CompareOp::Eq => self.eq(other).into_py(py), + CompareOp::Ne => self.ne(other).into_py(py), + _ => py.NotImplemented(), + } + } + + fn __repr__(&self) -> String { + format!("{self:?}") + } + + #[getter] + fn rtype(&self) -> u8 { + self.hd.rtype + } + + #[getter] + fn publisher_id(&self) -> u16 { + self.hd.publisher_id + } + + #[getter] + fn instrument_id(&self) -> u32 { + self.hd.instrument_id + } + + #[getter] + fn ts_event(&self) -> u64 { + self.hd.ts_event + } + + #[getter] + #[pyo3(name = "pretty_ts_event")] + fn py_pretty_ts_event(&self, py: Python<'_>) -> PyResult { + get_utc_nanosecond_timestamp(py, self.ts_event()) + } + + #[pyo3(name = "record_size")] + fn py_record_size(&self) -> usize { + self.record_size() + } + + #[classattr] + fn size_hint() -> PyResult { + Ok(mem::size_of::()) + } + + #[getter] + #[pyo3(name = "err")] + fn py_err(&self) -> PyResult<&str> { + self.err().map_err(to_val_err) + } + + #[classattr] + #[pyo3(name = "_dtypes")] + fn py_dtypes() -> Vec<(String, String)> { + Self::field_dtypes("") + } + + #[classattr] + #[pyo3(name = "_price_fields")] + fn py_price_fields() -> Vec { + Self::price_fields("") + } + + #[classattr] + #[pyo3(name = "_timestamp_fields")] + fn py_timestamp_fields() -> Vec { + Self::timestamp_fields("") + } + + #[classattr] + #[pyo3(name = "_hidden_fields")] + fn py_hidden_fields() -> Vec { + Self::hidden_fields("") + } + + #[classattr] + #[pyo3(name = "_ordered_fields")] + fn py_ordered_fields() -> Vec { + Self::ordered_fields("") + } +} + +#[pymethods] +impl ErrorMsgV1 { #[new] fn py_new(ts_event: u64, err: &str) -> PyResult { - Ok(ErrorMsg::new(ts_event, err)) + Ok(ErrorMsgV1::new(ts_event, err)) } fn __bytes__(&self) -> &[u8] { @@ -2497,6 +2593,107 @@ impl SystemMsg { } } +#[pymethods] +impl SystemMsgV1 { + #[new] + fn py_new(ts_event: u64, msg: &str) -> PyResult { + Self::new(ts_event, msg).map_err(to_val_err) + } + + fn __bytes__(&self) -> &[u8] { + self.as_ref() + } + + fn __richcmp__(&self, other: &Self, op: CompareOp, py: Python<'_>) -> Py { + match op { + CompareOp::Eq => self.eq(other).into_py(py), + CompareOp::Ne => self.ne(other).into_py(py), + _ => py.NotImplemented(), + } + } + + fn __repr__(&self) -> String { + format!("{self:?}") + } + + #[getter] + fn rtype(&self) -> u8 { + self.hd.rtype + } + + #[getter] + fn publisher_id(&self) -> u16 { + self.hd.publisher_id + } + + #[getter] + fn instrument_id(&self) -> u32 { + self.hd.instrument_id + } + + #[getter] + fn ts_event(&self) -> u64 { + self.hd.ts_event + } + + #[getter] + #[pyo3(name = "pretty_ts_event")] + fn py_pretty_ts_event(&self, py: Python<'_>) -> PyResult { + get_utc_nanosecond_timestamp(py, self.ts_event()) + } + + #[pyo3(name = "record_size")] + fn py_record_size(&self) -> usize { + self.record_size() + } + + #[classattr] + fn size_hint() -> PyResult { + Ok(mem::size_of::()) + } + + #[getter] + #[pyo3(name = "msg")] + fn py_msg(&self) -> PyResult<&str> { + self.msg().map_err(to_val_err) + } + + #[pyo3(name = "is_heartbeat")] + fn py_is_heartbeat(&self) -> bool { + self.is_heartbeat() + } + + #[classattr] + #[pyo3(name = "_dtypes")] + fn py_dtypes() -> Vec<(String, String)> { + Self::field_dtypes("") + } + + #[classattr] + #[pyo3(name = "_price_fields")] + fn py_price_fields() -> Vec { + Self::price_fields("") + } + + #[classattr] + #[pyo3(name = "_timestamp_fields")] + fn py_timestamp_fields() -> Vec { + Self::timestamp_fields("") + } + + #[classattr] + #[pyo3(name = "_hidden_fields")] + fn py_hidden_fields() -> Vec { + Self::hidden_fields("") + } + + #[classattr] + #[pyo3(name = "_ordered_fields")] + fn py_ordered_fields() -> Vec { + Self::ordered_fields("") + } +} + impl PyFieldDesc for [BidAskPair; N] { fn field_dtypes(_field_name: &str) -> Vec<(String, String)> { let mut res = Vec::new(); diff --git a/rust/dbn/src/record.rs b/rust/dbn/src/record.rs index 1f6cbf9..eca3942 100644 --- a/rust/dbn/src/record.rs +++ b/rust/dbn/src/record.rs @@ -1,7 +1,7 @@ //! Market data types for encoding different Databento [`Schema`](crate::enums::Schema)s //! and conversion functions. -mod conv; +pub(crate) mod conv; mod impl_default; mod methods; @@ -812,7 +812,14 @@ pub struct ErrorMsg { /// The error message. #[dbn(fmt_method)] #[cfg_attr(feature = "serde", serde(with = "conv::cstr_serde"))] - pub err: [c_char; 64], + pub err: [c_char; 302], + /// The error code. Currently unused. + #[pyo3(get, set)] + pub code: u8, + /// Sometimes multiple errors are sent together. This field will be non-zero for the + /// last error. + #[pyo3(get, set)] + pub is_last: u8, } /// A symbol mapping message which maps a symbol of one [`SType`](crate::enums::SType) @@ -883,7 +890,10 @@ pub struct SystemMsg { /// The message from the Databento Live Subscription Gateway (LSG). #[dbn(fmt_method)] #[cfg_attr(feature = "serde", serde(with = "conv::cstr_serde"))] - pub msg: [c_char; 64], + pub msg: [c_char; 303], + /// Type of system message, currently unused. + #[pyo3(get, set)] + pub code: u8, } /// Used for polymorphism around types all beginning with a [`RecordHeader`] where @@ -1044,9 +1054,9 @@ mod tests { #[case::status(StatusMsg::default(), 48)] #[case::imbalance(ImbalanceMsg::default(), 112)] #[case::stat(StatMsg::default(), 64)] - #[case::error(ErrorMsg::default(), 80)] + #[case::error(ErrorMsg::default(), 320)] #[case::symbol_mapping(SymbolMappingMsg::default(), 176)] - #[case::system(SystemMsg::default(), 80)] + #[case::system(SystemMsg::default(), 320)] #[case::with_ts_out(WithTsOut::new(SystemMsg::default(), 0), mem::size_of::() + 8)] fn test_sizes(#[case] _rec: R, #[case] exp: usize) { assert_eq!(mem::size_of::(), exp); @@ -1105,6 +1115,6 @@ mod tests { #[test] fn test_record_object_safe() { - let _record: Box = Box::new(ErrorMsg::new(1, "Boxed record")); + let _record: Box = Box::new(ErrorMsg::new(1, "Boxed record", true)); } } diff --git a/rust/dbn/src/record/impl_default.rs b/rust/dbn/src/record/impl_default.rs index 96009c3..76ac962 100644 --- a/rust/dbn/src/record/impl_default.rs +++ b/rust/dbn/src/record/impl_default.rs @@ -1,5 +1,5 @@ use crate::{ - compat::{InstrumentDefMsgV1, SymbolMappingMsgV1, SYMBOL_CSTR_LEN_V1}, + compat::{ErrorMsgV1, InstrumentDefMsgV1, SymbolMappingMsgV1, SystemMsgV1, SYMBOL_CSTR_LEN_V1}, SType, Schema, UNDEF_ORDER_SIZE, UNDEF_PRICE, UNDEF_STAT_QUANTITY, UNDEF_TIMESTAMP, }; @@ -314,7 +314,7 @@ impl Default for StatMsg { } } -impl Default for ErrorMsg { +impl Default for ErrorMsgV1 { fn default() -> Self { Self { hd: RecordHeader::default::(rtype::ERROR), @@ -323,6 +323,17 @@ impl Default for ErrorMsg { } } +impl Default for ErrorMsg { + fn default() -> Self { + Self { + hd: RecordHeader::default::(rtype::ERROR), + err: [0; 302], + code: u8::MAX, + is_last: u8::MAX, + } + } +} + impl Default for SymbolMappingMsg { fn default() -> Self { Self { @@ -351,6 +362,16 @@ impl Default for SymbolMappingMsgV1 { } impl Default for SystemMsg { + fn default() -> Self { + Self { + hd: RecordHeader::default::(rtype::SYSTEM), + msg: [0; 303], + code: u8::MAX, + } + } +} + +impl Default for SystemMsgV1 { fn default() -> Self { Self { hd: RecordHeader::default::(rtype::SYSTEM), diff --git a/rust/dbn/src/record/methods.rs b/rust/dbn/src/record/methods.rs index 17c0fba..fd64117 100644 --- a/rust/dbn/src/record/methods.rs +++ b/rust/dbn/src/record/methods.rs @@ -1,7 +1,7 @@ use std::fmt::Debug; use crate::{ - compat::{InstrumentDefMsgV1, SymbolMappingMsgV1}, + compat::{ErrorMsgV1, InstrumentDefMsgV1, SymbolMappingMsgV1, SystemMsgV1}, SType, }; @@ -565,15 +565,42 @@ impl StatMsg { } } +impl ErrorMsgV1 { + /// Creates a new `ErrorMsgV1`. + /// + /// # Errors + /// This function returns an error if `msg` is too long. + pub fn new(ts_event: u64, msg: &str) -> Self { + let mut error = Self { + hd: RecordHeader::new::(rtype::ERROR, 0, 0, ts_event), + ..Default::default() + }; + // leave at least one null byte + for (i, byte) in msg.as_bytes().iter().take(error.err.len() - 1).enumerate() { + error.err[i] = *byte as c_char; + } + error + } + + /// Returns `err` as a `&str`. + /// + /// # Errors + /// This function returns an error if `err` contains invalid UTF-8. + pub fn err(&self) -> Result<&str> { + c_chars_to_str(&self.err) + } +} + impl ErrorMsg { /// Creates a new `ErrorMsg`. /// /// # Errors /// This function returns an error if `msg` is too long. - pub fn new(ts_event: u64, msg: &str) -> Self { + pub fn new(ts_event: u64, msg: &str, is_last: bool) -> Self { let mut error = Self { hd: RecordHeader::new::(rtype::ERROR, 0, 0, ts_event), - err: [0; 64], + is_last: is_last as u8, + ..Default::default() }; // leave at least one null byte for (i, byte) in msg.as_bytes().iter().take(error.err.len() - 1).enumerate() { @@ -730,6 +757,7 @@ impl SystemMsg { Ok(Self { hd: RecordHeader::new::(rtype::SYSTEM, 0, 0, ts_event), msg: str_to_c_chars(msg)?, + ..Default::default() }) } @@ -738,6 +766,7 @@ impl SystemMsg { Self { hd: RecordHeader::new::(rtype::SYSTEM, 0, 0, ts_event), msg: str_to_c_chars(Self::HEARTBEAT).unwrap(), + code: u8::MAX, } } @@ -758,6 +787,43 @@ impl SystemMsg { } } +impl SystemMsgV1 { + /// Creates a new `SystemMsgV1`. + /// + /// # Errors + /// This function returns an error if `msg` is too long. + pub fn new(ts_event: u64, msg: &str) -> Result { + Ok(Self { + hd: RecordHeader::new::(rtype::SYSTEM, 0, 0, ts_event), + msg: str_to_c_chars(msg)?, + }) + } + + /// Creates a new heartbeat `SystemMsg`. + pub fn heartbeat(ts_event: u64) -> Self { + Self { + hd: RecordHeader::new::(rtype::SYSTEM, 0, 0, ts_event), + msg: str_to_c_chars(SystemMsg::HEARTBEAT).unwrap(), + } + } + + /// Checks whether the message is a heartbeat from the gateway. + pub fn is_heartbeat(&self) -> bool { + self.msg() + .map(|msg| msg == SystemMsg::HEARTBEAT) + .unwrap_or_default() + } + + /// Returns the message from the Databento Live Subscription Gateway (LSG) as + /// a `&str`. + /// + /// # Errors + /// This function returns an error if `msg` contains invalid UTF-8. + pub fn msg(&self) -> Result<&str> { + c_chars_to_str(&self.msg) + } +} + impl Record for WithTsOut { fn header(&self) -> &RecordHeader { self.rec.header() @@ -870,8 +936,8 @@ mod tests { }; assert_eq!( format!("{rec:?}"), - "ErrorMsg { hd: RecordHeader { length: 20, rtype: Error, publisher_id: 0, \ - instrument_id: 0, ts_event: 18446744073709551615 }, err: \"Missing stype_in\" }" + "ErrorMsg { hd: RecordHeader { length: 80, rtype: Error, publisher_id: 0, \ + instrument_id: 0, ts_event: 18446744073709551615 }, err: \"Missing stype_in\", code: 255, is_last: 255 }" ); } @@ -880,8 +946,8 @@ mod tests { let rec = SystemMsg::heartbeat(123); assert_eq!( format!("{rec:?}"), - "SystemMsg { hd: RecordHeader { length: 20, rtype: System, publisher_id: 0, \ - instrument_id: 0, ts_event: 123 }, msg: \"Heartbeat\" }" + "SystemMsg { hd: RecordHeader { length: 80, rtype: System, publisher_id: 0, \ + instrument_id: 0, ts_event: 123 }, msg: \"Heartbeat\", code: 255 }" ); } From 9679d24c29b4c32953278bff45c1d4a65e23715b Mon Sep 17 00:00:00 2001 From: Carter Green Date: Fri, 12 Jan 2024 16:34:50 -0600 Subject: [PATCH 20/21] VER: Release DBN 0.15.0 --- CHANGELOG.md | 2 +- Cargo.lock | 366 ++++++++++++++++++++++--------------- c/Cargo.toml | 6 +- python/Cargo.toml | 4 +- python/pyproject.toml | 4 +- rust/dbn-cli/Cargo.toml | 8 +- rust/dbn-macros/Cargo.toml | 12 +- rust/dbn/Cargo.toml | 6 +- 8 files changed, 237 insertions(+), 171 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a67a91..9ae3de7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## 0.15.0 - TBD +## 0.15.0 - 2023-01-13 ### Enhancements - Improved `Debug` implementation for all record types - Prices are formatted as decimals diff --git a/Cargo.lock b/Cargo.lock index 1f6a1a6..ecade3d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -28,9 +28,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.4" +version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab91ebe16eb252986481c5b62f6098f3b698a45e34b5b98200cf20dd2484a44" +checksum = "4cd2405b3ac1faab2990b74d728624cd9fd115651fcecc7c2d8daf01376275ba" dependencies = [ "anstyle", "anstyle-parse", @@ -48,43 +48,43 @@ checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" [[package]] name = "anstyle-parse" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "317b9a89c1868f5ea6ff1d9539a69f45dffc21ce321ac1fd1160dfa48c8e2140" +checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.0.0" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648" dependencies = [ - "windows-sys", + "windows-sys 0.52.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.1" +version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0699d10d2f4d628a98ee7b57b289abbc98ff3bad977cb3152709d4bf2330628" +checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" dependencies = [ "anstyle", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] name = "anyhow" -version = "1.0.75" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" +checksum = "080e9890a082662b09c1ad45f567faeeb47f22b5fb23895fbe1e651e718e25ca" [[package]] name = "assert_cmd" -version = "2.0.12" +version = "2.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88903cb14723e4d4003335bb7f8a14f27691649105346a0f0957466c096adfe6" +checksum = "00ad3f3a942eee60335ab4342358c161ee296829e0d16ff42fc1d6cb07815467" dependencies = [ "anstyle", "bstr", @@ -132,9 +132,9 @@ dependencies = [ [[package]] name = "basic-toml" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f2139706359229bfa8f19142ac1155b4b80beafb7a60471ac5dd109d4a19778" +checksum = "2db21524cad41c5591204d22d75e1970a2d1f71060214ca931dc7d5afe2c14e5" dependencies = [ "serde", ] @@ -153,9 +153,9 @@ checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" [[package]] name = "bstr" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "542f33a8835a0884b006a0c3df3dadd99c0c3f296ed26c2fdc8028e01ad6230c" +checksum = "c48f0051a4b4c5e0b6d365cd04af53aeaa209e3cc15ec2cdb69e73cc87fbd0dc" dependencies = [ "memchr", "regex-automata", @@ -204,9 +204,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "clap" -version = "4.4.8" +version = "4.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2275f18819641850fa26c89acc84d465c1bf91ce57bc2748b28c420473352f64" +checksum = "58e54881c004cec7895b0068a0a954cd5d62da01aef83fa35b1e594497bf5445" dependencies = [ "clap_builder", "clap_derive", @@ -214,9 +214,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.8" +version = "4.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07cdf1b148b25c1e1f7a42225e30a0d99a615cd4637eae7365548dd4529b95bc" +checksum = "59cb82d7f531603d2fd1f507441cdd35184fa81beff7bd489570de7f773460bb" dependencies = [ "anstream", "anstyle", @@ -234,7 +234,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.48", ] [[package]] @@ -272,7 +272,7 @@ dependencies = [ [[package]] name = "databento-dbn" -version = "0.14.2" +version = "0.15.0" dependencies = [ "dbn", "pyo3", @@ -283,7 +283,7 @@ dependencies = [ [[package]] name = "dbn" -version = "0.14.2" +version = "0.15.0" dependencies = [ "async-compression", "csv", @@ -305,7 +305,7 @@ dependencies = [ [[package]] name = "dbn-c" -version = "0.14.2" +version = "0.15.0" dependencies = [ "anyhow", "cbindgen", @@ -315,7 +315,7 @@ dependencies = [ [[package]] name = "dbn-cli" -version = "0.14.2" +version = "0.15.0" dependencies = [ "anyhow", "assert_cmd", @@ -330,22 +330,22 @@ dependencies = [ [[package]] name = "dbn-macros" -version = "0.14.2" +version = "0.15.0" dependencies = [ "csv", "dbn", "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.48", "trybuild", ] [[package]] name = "deranged" -version = "0.3.9" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f32d04922c60427da6f9fef14d042d9edddef64cb9d4ce0d64d0685fbeb1fd3" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" dependencies = [ "powerfmt", "serde", @@ -377,12 +377,12 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f258a7194e7f7c2a7837a8913aeab7fd8c383457034fa20ce4dd3dcb813e8eb8" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -402,9 +402,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" dependencies = [ "futures-channel", "futures-core", @@ -417,9 +417,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", "futures-sink", @@ -427,15 +427,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" [[package]] name = "futures-executor" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" dependencies = [ "futures-core", "futures-task", @@ -444,32 +444,32 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" [[package]] name = "futures-macro" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.48", ] [[package]] name = "futures-sink" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" [[package]] name = "futures-task" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" [[package]] name = "futures-timer" @@ -479,9 +479,9 @@ checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" [[package]] name = "futures-util" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ "futures-channel", "futures-core", @@ -497,9 +497,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.28.0" +version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" +checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" [[package]] name = "glob" @@ -515,9 +515,9 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "hashbrown" -version = "0.14.2" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" [[package]] name = "heck" @@ -548,7 +548,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" dependencies = [ "equivalent", - "hashbrown 0.14.2", + "hashbrown 0.14.3", ] [[package]] @@ -568,9 +568,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" +checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" [[package]] name = "jobserver" @@ -593,15 +593,15 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.150" +version = "0.2.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" +checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7" [[package]] name = "linux-raw-sys" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "969488b55f8ac402214f3f5fd243ebb7206cf82de60d3172994707a4bcc2b829" +checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" [[package]] name = "lock_api" @@ -621,9 +621,9 @@ checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" [[package]] name = "memchr" -version = "2.6.4" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" +checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" [[package]] name = "memoffset" @@ -679,39 +679,39 @@ dependencies = [ [[package]] name = "num_enum" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683751d591e6d81200c39fb0d1032608b77724f34114db54f571ff1317b337c0" +checksum = "02339744ee7253741199f897151b38e72257d13802d4ee837285cc2990a90845" dependencies = [ "num_enum_derive", ] [[package]] name = "num_enum_derive" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c11e44798ad209ccdd91fc192f0526a369a01234f7373e1b141c96d7cee4f0e" +checksum = "681030a937600a36906c185595136d26abfebb4aa9c65701cefcaf8578bb982b" dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.48", ] [[package]] name = "object" -version = "0.32.1" +version = "0.32.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" dependencies = [ "memchr", ] [[package]] name = "once_cell" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "parking_lot" @@ -733,7 +733,7 @@ dependencies = [ "libc", "redox_syscall", "smallvec", - "windows-targets", + "windows-targets 0.48.5", ] [[package]] @@ -750,9 +750,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.27" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" +checksum = "69d3587f8a9e599cc7ec2c00e331f71c4e69a5f9a4b8a6efd5b07466b9736f9a" [[package]] name = "powerfmt" @@ -793,27 +793,27 @@ dependencies = [ [[package]] name = "proc-macro-crate" -version = "2.0.0" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e8366a6159044a37876a2b9817124296703c586a5c92e2c53751fa06d8d43e8" +checksum = "6b2685dd208a3771337d8d386a89840f0f43cd68be8dae90a5f8c2384effc9cd" dependencies = [ "toml_edit", ] [[package]] name = "proc-macro2" -version = "1.0.69" +version = "1.0.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" +checksum = "95fc56cda0b5c3325f5fbbd7ff9fda9e02bb00bb3dac51252d2f1bfa1cb8cc8c" dependencies = [ "unicode-ident", ] [[package]] name = "pyo3" -version = "0.20.0" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04e8453b658fe480c3e70c8ed4e3d3ec33eb74988bd186561b0cc66b85c3bc4b" +checksum = "9a89dc7a5850d0e983be1ec2a463a171d20990487c3cfcd68b5363f1ee3d6fe0" dependencies = [ "cfg-if", "indoc", @@ -828,9 +828,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.20.0" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a96fe70b176a89cff78f2fa7b3c930081e163d5379b4dcdf993e3ae29ca662e5" +checksum = "07426f0d8fe5a601f26293f300afd1a7b1ed5e78b2a705870c5f30893c5163be" dependencies = [ "once_cell", "target-lexicon", @@ -838,9 +838,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.20.0" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "214929900fd25e6604661ed9cf349727c8920d47deff196c4e28165a6ef2a96b" +checksum = "dbb7dec17e17766b46bca4f1a4215a85006b4c2ecde122076c562dd058da6cf1" dependencies = [ "libc", "pyo3-build-config", @@ -848,33 +848,33 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.20.0" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dac53072f717aa1bfa4db832b39de8c875b7c7af4f4a6fe93cdbf9264cf8383b" +checksum = "05f738b4e40d50b5711957f142878cfa0f28e054aa0ebdfc3fd137a843f74ed3" dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.39", + "syn 2.0.48", ] [[package]] name = "pyo3-macros-backend" -version = "0.20.0" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7774b5a8282bd4f25f803b1f0d945120be959a36c72e08e7cd031c792fdfd424" +checksum = "0fc910d4851847827daf9d6cdd4a823fbdaab5b8818325c5e97a86da79e8881f" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.48", ] [[package]] name = "quote" -version = "1.0.33" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" dependencies = [ "proc-macro2", ] @@ -919,9 +919,9 @@ checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "relative-path" -version = "1.9.0" +version = "1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c707298afce11da2efef2f600116fa93ffa7a032b5d7b628aa17711ec81383ca" +checksum = "e898588f33fdd5b9420719948f9f2a32c922a246964576f71ba7f24f80610fbc" [[package]] name = "rstest" @@ -948,7 +948,7 @@ dependencies = [ "regex", "relative-path", "rustc_version", - "syn 2.0.39", + "syn 2.0.48", "unicode-ident", ] @@ -969,15 +969,15 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.24" +version = "0.38.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ad981d6c340a49cdc40a1028d9c6084ec7e9fa33fcb839cab656a267071e234" +checksum = "0a1a81a2478639a14e68937903356dbac62cf52171148924f754bb8a8cd7a96c" dependencies = [ "bitflags 2.4.1", "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -988,9 +988,9 @@ checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" [[package]] name = "ryu" -version = "1.0.15" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" +checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" [[package]] name = "scopeguard" @@ -1000,35 +1000,35 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "semver" -version = "1.0.20" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" +checksum = "b97ed7a9823b74f99c7742f5336af7be5ecd3eeafcb1507d1fa93347b1d589b0" [[package]] name = "serde" -version = "1.0.192" +version = "1.0.195" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bca2a08484b285dcb282d0f67b26cadc0df8b19f8c12502c13d966bf9482f001" +checksum = "63261df402c67811e9ac6def069e4786148c4563f4b50fd4bf30aa370d626b02" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.192" +version = "1.0.195" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6c7207fbec9faa48073f3e3074cbe553af6ea512d7c21ba46e434e70ea9fbc1" +checksum = "46fe8f8603d81ba86327b23a2e9cdf49e1255fb94a4c5f297f6ee0547178ea2c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.48", ] [[package]] name = "serde_json" -version = "1.0.108" +version = "1.0.111" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" +checksum = "176e46fa42316f18edd598015a5166857fc835ec732f5215eac6b7bdbf0a84f4" dependencies = [ "itoa", "ryu", @@ -1081,7 +1081,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.39", + "syn 2.0.48", ] [[package]] @@ -1097,9 +1097,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.39" +version = "2.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" +checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" dependencies = [ "proc-macro2", "quote", @@ -1108,28 +1108,28 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.12.12" +version = "0.12.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c39fd04924ca3a864207c66fc2cd7d22d7c016007f9ce846cbb9326331930a" +checksum = "69758bda2e78f098e4ccb393021a0963bb3442eac05f135c30f61b7370bbafae" [[package]] name = "tempfile" -version = "3.8.1" +version = "3.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" +checksum = "01ce4141aa927a6d1bd34a041795abd0db1cccba5d5f24b009f694bdf3a1f3fa" dependencies = [ "cfg-if", "fastrand", "redox_syscall", "rustix", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] name = "termcolor" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff1bc3d3f05aff0403e8ac0d92ced918ec05b666a43f83297ccef5bea8a3d449" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" dependencies = [ "winapi-util", ] @@ -1141,7 +1141,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "21bebf2b7c9e0a515f6e0f8c51dc0f8e4696391e6f1ff30379559f8365fb0df7" dependencies = [ "rustix", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -1152,29 +1152,29 @@ checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" [[package]] name = "thiserror" -version = "1.0.50" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2" +checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.50" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" +checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.48", ] [[package]] name = "time" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4a34ab300f2dee6e562c10a046fc05e358b29f9bf92277f30c3c8d82275f6f5" +checksum = "f657ba42c3f86e7680e53c8cd3af8abbe56b5491790b46e22e19c0d57463583e" dependencies = [ "deranged", "itoa", @@ -1192,18 +1192,18 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ad70d68dba9e1f8aceda7aa6711965dfec1cac869f311a51bd08b3a2ccbce20" +checksum = "26197e33420244aeb70c3e8c78376ca46571bc4e701e4791c2cd9f57dcb3a43f" dependencies = [ "time-core", ] [[package]] name = "tokio" -version = "1.34.0" +version = "1.35.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0c014766411e834f7af5b8f4cf46257aab4036ca95e9d2c144a10f59ad6f5b9" +checksum = "c89b4efa943be685f629b149f53829423f8f5531ea21249408e8e2f8671ec104" dependencies = [ "backtrace", "bytes", @@ -1220,7 +1220,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.48", ] [[package]] @@ -1240,9 +1240,9 @@ checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" [[package]] name = "toml_edit" -version = "0.20.7" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70f427fce4d84c72b5b732388bf4a9f4531b53f74e2887e3ecb2481f68f66d81" +checksum = "d34d383cd00a163b4a5b85053df514d45bc330f6de7737edfe0a93311d1eaa03" dependencies = [ "indexmap 2.1.0", "toml_datetime", @@ -1251,9 +1251,9 @@ dependencies = [ [[package]] name = "trybuild" -version = "1.0.85" +version = "1.0.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "196a58260a906cedb9bf6d8034b6379d0c11f552416960452f267402ceeddff1" +checksum = "76de4f783e610194f6c98bfd53f9fc52bb2e0d02c947621e8a0f4ecc799b2880" dependencies = [ "basic-toml", "glob", @@ -1349,7 +1349,16 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets", + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.0", ] [[package]] @@ -1358,13 +1367,28 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +dependencies = [ + "windows_aarch64_gnullvm 0.52.0", + "windows_aarch64_msvc 0.52.0", + "windows_i686_gnu 0.52.0", + "windows_i686_msvc 0.52.0", + "windows_x86_64_gnu 0.52.0", + "windows_x86_64_gnullvm 0.52.0", + "windows_x86_64_msvc 0.52.0", ] [[package]] @@ -1373,47 +1397,89 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" + [[package]] name = "windows_i686_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" +[[package]] +name = "windows_i686_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" + [[package]] name = "windows_i686_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" +[[package]] +name = "windows_i686_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" + [[package]] name = "winnow" -version = "0.5.19" +version = "0.5.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "829846f3e3db426d4cee4510841b71a8e58aa2a76b1132579487ae430ccd9c7b" +checksum = "b7cf47b659b318dccbd69cc4797a39ae128f533dce7902a1096044d1967b9c16" dependencies = [ "memchr", ] diff --git a/c/Cargo.toml b/c/Cargo.toml index e37a6c5..e83acb2 100644 --- a/c/Cargo.toml +++ b/c/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "dbn-c" authors = ["Databento "] -version = "0.14.2" +version = "0.15.0" edition = "2021" description = "C bindings for working with Databento Binary Encoding (DBN)" license = "Apache-2.0" @@ -14,10 +14,10 @@ name = "dbn_c" crate-type = ["staticlib"] [dependencies] -anyhow = "1.0.75" +anyhow = "1.0.79" # DBN library dbn = { path = "../rust/dbn", features = [] } -libc = "0.2.150" +libc = "0.2.152" [build-dependencies] cbindgen = { version = "0.26.0", default-features = false } diff --git a/python/Cargo.toml b/python/Cargo.toml index 7cf93b7..25c080e 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "databento-dbn" authors = ["Databento "] -version = "0.14.2" +version = "0.15.0" edition = "2021" description = "Python library written in Rust for working with Databento Binary Encoding (DBN)" license = "Apache-2.0" @@ -18,7 +18,7 @@ dbn = { path = "../rust/dbn", features = ["python"] } # Python bindings for Rust pyo3 = "0.20" # Dates and datetimes -time = "0.3.30" +time = "0.3.31" [build-dependencies] pyo3-build-config = { version = "0.20" } diff --git a/python/pyproject.toml b/python/pyproject.toml index 9ac0c97..8f32d28 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "databento-dbn" -version = "0.14.2" +version = "0.15.0" description = "Python bindings for encoding and decoding Databento Binary Encoding (DBN)" authors = ["Databento "] license = "Apache-2.0" @@ -17,7 +17,7 @@ build-backend = "maturin" [project] name = "databento-dbn" -version = "0.14.2" +version = "0.15.0" authors = [ { name = "Databento", email = "support@databento.com" } ] diff --git a/rust/dbn-cli/Cargo.toml b/rust/dbn-cli/Cargo.toml index bb652de..bbf7e7a 100644 --- a/rust/dbn-cli/Cargo.toml +++ b/rust/dbn-cli/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "dbn-cli" authors = ["Databento "] -version = "0.14.2" +version = "0.15.0" edition = "2021" description = "Command-line utility for converting Databento Binary Encoding (DBN) files to text-based formats" default-run = "dbn" @@ -17,7 +17,7 @@ path = "src/main.rs" [dependencies] # Databento common DBN library -dbn = { path = "../dbn", version = "=0.14.2", default-features = false } +dbn = { path = "../dbn", version = "=0.15.0", default-features = false } # Error handling anyhow = "1.0" @@ -29,9 +29,9 @@ zstd = "0.13" [dev-dependencies] # CLI integration tests -assert_cmd = "2.0.12" +assert_cmd = "2.0.13" # assert_cmd companion predicates = "3.0.4" rstest = "0.18.2" # A library for managing temporary files and directories -tempfile = "3.8.1" +tempfile = "3.9.0" diff --git a/rust/dbn-macros/Cargo.toml b/rust/dbn-macros/Cargo.toml index 3a18840..658fddb 100644 --- a/rust/dbn-macros/Cargo.toml +++ b/rust/dbn-macros/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "dbn-macros" authors = ["Databento "] -version = "0.14.2" +version = "0.15.0" edition = "2021" description = "Proc macros for dbn crate" license = "Apache-2.0" @@ -12,14 +12,14 @@ proc-macro = true [dependencies] # Get name of current crate in macros, like $crate in macro_rules macros -proc-macro-crate = "2.0.0" -proc-macro2 = "1.0.69" +proc-macro-crate = "3.0.0" +proc-macro2 = "1.0.76" # Convert code to token streams -quote = "1.0.33" +quote = "1.0.35" # Token parsing -syn = { version = "2.0.39", features = ["full"] } +syn = { version = "2.0.48", features = ["full"] } [dev-dependencies] csv = "1" dbn = { path = "../dbn" } -trybuild = "1.0.85" +trybuild = "1.0.88" diff --git a/rust/dbn/Cargo.toml b/rust/dbn/Cargo.toml index fff12c0..bc0bb49 100644 --- a/rust/dbn/Cargo.toml +++ b/rust/dbn/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "dbn" authors = ["Databento "] -version = "0.14.2" +version = "0.15.0" edition = "2021" description = "Library for working with Databento Binary Encoding (DBN)" license = "Apache-2.0" @@ -25,10 +25,10 @@ serde = ["dep:serde", "time/parsing", "time/serde"] trivial_copy = [] [dependencies] -dbn-macros = { version = "=0.14.2", path = "../dbn-macros" } +dbn-macros = { version = "=0.15.0", path = "../dbn-macros" } # async (de)compression -async-compression = { version = "0.4.4", features = ["tokio", "zstd"], optional = true } +async-compression = { version = "0.4.5", features = ["tokio", "zstd"], optional = true } # CSV serialization csv = "1.3" # Fast integer to string conversion From 5effa2ad747cce3911d49476f187601986c88d44 Mon Sep 17 00:00:00 2001 From: Carter Green Date: Tue, 16 Jan 2024 08:51:52 -0600 Subject: [PATCH 21/21] FIX: Fix release date --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ae3de7..3942d2d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## 0.15.0 - 2023-01-13 +## 0.15.0 - 2023-01-16 ### Enhancements - Improved `Debug` implementation for all record types - Prices are formatted as decimals