VER: Release 0.13.0

CHANGELOG.md

@@ -1,5 +1,35 @@
 # Changelog
 
+## 0.11.0 - 2023-09-21
+### Enhancements
+- Added new `EncodeRecordTextExt` trait which is implemented for the CSV and JSON
+  encoders. It adds two methods for encoding a `symbol` field along side the rest of the
+  record fields, matching the behavior of `map_symbols` in the historical API
+- Added `encode_header` and `encode_header_for_schema` methods to `CsvEncoder` and
+  `DynEncoder` to give more flexibility for encoding CSV headers
+- Added `from_file` and `from_zstd_file` functions to `AsyncDbnDecoder` to match
+  synchronous decoder
+- Implemented `Copy` for `RecordRef` to make it behave more like a reference
+- Added `AsyncDbnEncoder` for simpler DBN encoding and to match sync API
+- Added `RecordEnum` and `RecordRefEnum` to more easily be able to pattern match on
+  records of different types
+- Added `ARCX.PILLAR.ARCX` publisher
+- Added `From` DBN records for `RecordRef`
+- Added re-exports to the top level of the crate for all enums and records for simpler
+  imports
+- Added `ClosePrice` and `NetChange` `StatType`s used in the `OPRA.PILLAR` dataset
+
+### Breaking changes
+- Split `encode_record_ref` into a safe method with no arguments and an unsafe method
+  with a `ts_out` parameter to reduce `unsafe` usage when not working with live data
+  that may contain `ts_out`
+
+### Bug fixes
+- Fixed `dbn` CLI not writing CSV header when using `--fragment` and `--zstd-fragment`
+  flags
+- Fixed lifetime on return value from `RecordRef::get_unchecked`
+- Fixed missing check for `stype_out` before building `Metadata` symbology maps
+
 ## 0.10.2 - 2023-09-12
 ### Bug fixes
 - Fixed query range checking in `Metadata::symbol_map_for_date`

c/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "dbn-c"
 authors = ["Databento <[email protected]>"]
-version = "0.10.2"
+version = "0.11.0"
 edition = "2021"
 description = "C bindings for working with Databento Binary Encoding (DBN)"
 license = "Apache-2.0"

c/src/text_serialization.rs

@@ -179,10 +179,10 @@ pub unsafe extern "C" fn s_serialize_record(
     let res = match options.encoding {
         TextEncoding::Json => {
             json::Encoder::new(&mut cursor, false, options.pretty_px, options.pretty_ts)
-                .encode_record_ref(record, options.ts_out)
+                .encode_record_ref_ts_out(record, options.ts_out)
         }
         TextEncoding::Csv => csv::Encoder::new(&mut cursor, options.pretty_px, options.pretty_ts)
-            .encode_record_ref(record, options.ts_out),
+            .encode_record_ref_ts_out(record, options.ts_out),
     }
     // null byte
     .and_then(|_| {
@@ -232,10 +232,10 @@ pub unsafe extern "C" fn f_serialize_record(
     let res = match options.encoding {
         TextEncoding::Json => {
             json::Encoder::new(&mut cfile, false, options.pretty_px, options.pretty_ts)
-                .encode_record_ref(record, options.ts_out)
+                .encode_record_ref_ts_out(record, options.ts_out)
         }
         TextEncoding::Csv => csv::Encoder::new(&mut cfile, options.pretty_px, options.pretty_ts)
-            .encode_record_ref(record, options.ts_out),
+            .encode_record_ref_ts_out(record, options.ts_out),
     };
     if res.is_ok() {
         cfile.bytes_written() as i32
@@ -266,5 +266,5 @@ fn serialize_csv_header<W: io::Write, R: DbnEncodable>(
     _rec: &R,
     encoder: &mut csv::Encoder<W>,
 ) -> dbn::Result<()> {
-    encoder.encode_header::<R>()
+    encoder.encode_header::<R>(false)
 }

python/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "databento-dbn"
 authors = ["Databento <[email protected]>"]
-version = "0.10.2"
+version = "0.11.0"
 edition = "2021"
 description = "Python library written in Rust for working with Databento Binary Encoding (DBN)"
 license = "Apache-2.0"

python/databento_dbn.pyi

@@ -5,14 +5,7 @@ from collections.abc import Iterable
 from collections.abc import Sequence
 from datetime import datetime
 from enum import Enum
-from typing import (
-    Any,
-    BinaryIO,
-    ClassVar,
-    SupportsBytes,
-    TextIO,
-    Union,
-)
+from typing import Any, BinaryIO, ClassVar, SupportsBytes, TextIO, Union
 
 
 FIXED_PRICE_SCALE: int

python/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "databento-dbn"
-version = "0.10.2"
+version = "0.11.0"
 description = "Python bindings for encoding and decoding Databento Binary Encoding (DBN)"
 authors = ["Databento <[email protected]>"]
 license = "Apache-2.0"

python/src/transcoder.rs

@@ -92,7 +92,8 @@ impl Transcoder {
         loop {
             match decoder.decode_record_ref() {
                 Ok(Some(rec)) => {
-                    unsafe { encoder.encode_record_ref(rec, self.ts_out) }.map_err(to_val_err)?;
+                    unsafe { encoder.encode_record_ref_ts_out(rec, self.ts_out) }
+                        .map_err(to_val_err)?;
                     // keep track of position after last _successful_ decoding to
                     // ensure buffer is left in correct state in the case where one
                     // or more successful decodings is followed by a partial one, i.e.

rust/dbn-cli/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "dbn-cli"
 authors = ["Databento <[email protected]>"]
-version = "0.10.2"
+version = "0.11.0"
 edition = "2021"
 description = "Command-line utility for converting Databento Binary Encoding (DBN) files to text-based formats"
 default-run = "dbn"
@@ -17,7 +17,7 @@ path = "src/main.rs"
 
 [dependencies]
 # Databento common DBN library
-dbn = { path = "../dbn", version = "=0.10.2", default-features = false }
+dbn = { path = "../dbn", version = "=0.11.0", default-features = false }
 
 # Error handling
 anyhow = "1.0.72"

rust/dbn-cli/src/main.rs

@@ -3,9 +3,9 @@ use std::{fs::File, io};
 use clap::Parser;
 use dbn::{
     decode::{DbnRecordDecoder, DecodeDbn, DecodeRecordRef, DynDecoder},
-    encode::{json, DynEncoder, EncodeDbn, EncodeRecordRef},
+    encode::{json, DbnEncodable, DynEncoder, EncodeDbn, EncodeRecordRef},
     enums::SType,
-    MetadataBuilder,
+    rtype_dispatch, Encoding, MetadataBuilder,
 };
 use dbn_cli::{infer_encoding_and_compression, output_from_args, Args};
 
@@ -81,9 +81,20 @@ fn write_dbn_frag<R: io::Read>(
         args.should_pretty_print,
     )?;
     let mut n = 0;
+    let mut has_written_header = encoding != Encoding::Csv;
+    fn write_header<T: DbnEncodable>(
+        _record: &T,
+        encoder: &mut DynEncoder<Box<dyn io::Write>>,
+    ) -> dbn::Result<()> {
+        encoder.encode_header::<T>(false)
+    }
     while let Some(record) = decoder.decode_record_ref()? {
+        if !has_written_header {
+            rtype_dispatch!(record, write_header, &mut encoder)??;
+            has_written_header = true;
+        }
         // Assume no ts_out for safety
-        match unsafe { encoder.encode_record_ref(record, false) } {
+        match encoder.encode_record_ref(record) {
             // Handle broken pipe as a non-error.
             Err(dbn::Error::Io { source, .. })
                 if source.kind() == std::io::ErrorKind::BrokenPipe =>

rust/dbn-cli/tests/integration_tests.rs

@@ -427,6 +427,20 @@ fn test_fragment() {
         .stderr(is_empty());
 }
 
+#[test]
+fn test_writes_csv_header_for_fragment() {
+    cmd()
+        .args([
+            &format!("{TEST_DATA_PATH}/test_data.definition.dbn.frag"),
+            "--fragment",
+            "--csv",
+        ])
+        .assert()
+        .success()
+        .stdout(contains('\n').count(3))
+        .stderr(is_empty());
+}
+
 #[test]
 fn test_zstd_fragment() {
     cmd()
@@ -441,6 +455,20 @@ fn test_zstd_fragment() {
         .stderr(is_empty());
 }
 
+#[test]
+fn test_writes_csv_header_for_zstd_fragment() {
+    cmd()
+        .args([
+            &format!("{TEST_DATA_PATH}/test_data.definition.dbn.frag.zst"),
+            "--zstd-fragment",
+            "--csv",
+        ])
+        .assert()
+        .success()
+        .stdout(contains('\n').count(3))
+        .stderr(is_empty());
+}
+
 #[test]
 fn test_limit_updates_metadata() {
     // Check metadata shows limit = 2

rust/dbn-macros/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "dbn-macros"
 authors = ["Databento <[email protected]>"]
-version = "0.10.2"
+version = "0.11.0"
 edition = "2021"
 description = "Proc macros for dbn crate"
 license = "Apache-2.0"

rust/dbn/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "dbn"
 authors = ["Databento <[email protected]>"]
-version = "0.10.2"
+version = "0.11.0"
 edition = "2021"
 description = "Library for working with Databento Binary Encoding (DBN)"
 license = "Apache-2.0"
@@ -25,7 +25,7 @@ serde = ["dep:serde", "time/parsing", "time/serde"]
 trivial_copy = []
 
 [dependencies]
-dbn-macros = { version = "=0.10.2", path = "../dbn-macros" }
+dbn-macros = { version = "=0.11.0", path = "../dbn-macros" }
 
 # async (de)compression
 async-compression = { version = "0.4.1", features = ["tokio", "zstd"], optional = true }
@@ -48,7 +48,7 @@ thiserror = "1.0"
 # date and datetime support
 time = { version = "0.3", features = ["formatting", "macros"] }
 # async traits
-tokio = { version = "1", features = ["io-util"], optional = true }
+tokio = { version = "1", features = ["fs", "io-util"], optional = true }
 # (de)compression
 zstd = "0.12"
 

rust/dbn/src/decode.rs

@@ -69,8 +69,8 @@ pub trait DecodeDbn: DecodeRecordRef + private::BufferSlice {
     /// [`Error::Decode`](crate::Error::Decode) will be returned.
     fn decode_record<T: HasRType>(&mut self) -> crate::Result<Option<&T>>;
 
-    /// Tries to convert the decoder into a streaming iterator. This lazily decodes the
-    /// data.
+    /// Converts the decoder into a streaming iterator of records of type `T`. This
+    /// lazily decodes the data.
     fn decode_stream<T: HasRType>(self) -> StreamIterDecoder<Self, T>
     where
         Self: Sized;

rust/dbn/src/decode/dbn/async.rs

@@ -1,5 +1,10 @@
+use std::path::Path;
+
 use async_compression::tokio::bufread::ZstdDecoder;
-use tokio::io::{self, BufReader};
+use tokio::{
+    fs::File,
+    io::{self, BufReader},
+};
 
 use crate::{
     decode::FromLittleEndianSlice,
@@ -87,7 +92,7 @@ where
     }
 }
 
-impl<'a, R> Decoder<ZstdDecoder<BufReader<R>>>
+impl<R> Decoder<ZstdDecoder<BufReader<R>>>
 where
     R: io::AsyncReadExt + Unpin,
 {
@@ -100,7 +105,7 @@ where
     }
 }
 
-impl<'a, R> Decoder<ZstdDecoder<R>>
+impl<R> Decoder<ZstdDecoder<R>>
 where
     R: io::AsyncBufReadExt + Unpin,
 {
@@ -113,6 +118,46 @@ where
     }
 }
 
+impl Decoder<BufReader<File>> {
+    /// Creates a new async DBN [`Decoder`] from the file at `path`.
+    ///
+    /// # Errors
+    /// This function will return an error if it is unable to read the file at `path` or
+    /// if it is unable to parse the metadata in the file.
+    pub async fn from_file(path: impl AsRef<Path>) -> crate::Result<Self> {
+        let file = File::open(path.as_ref()).await.map_err(|e| {
+            crate::Error::io(
+                e,
+                format!(
+                    "Error opening DBN file at path '{}'",
+                    path.as_ref().display()
+                ),
+            )
+        })?;
+        Self::new(BufReader::new(file)).await
+    }
+}
+
+impl Decoder<ZstdDecoder<BufReader<File>>> {
+    /// Creates a new async DBN [`Decoder`] from the Zstandard-compressed file at `path`.
+    ///
+    /// # Errors
+    /// This function will return an error if it is unable to read the file at `path` or
+    /// if it is unable to parse the metadata in the file.
+    pub async fn from_zstd_file(path: impl AsRef<Path>) -> crate::Result<Self> {
+        let file = File::open(path.as_ref()).await.map_err(|e| {
+            crate::Error::io(
+                e,
+                format!(
+                    "Error opening Zstandard-compressed DBN file at path '{}'",
+                    path.as_ref().display()
+                ),
+            )
+        })?;
+        Self::with_zstd(file).await
+    }
+}
+
 /// An async decoder for files and streams of Databento Binary Encoding (DBN) records.
 pub struct RecordDecoder<R>
 where