diff --git a/crates/sparrow-runtime/src/execute.rs b/crates/sparrow-runtime/src/execute.rs index b8a6012fe..fa78263c3 100644 --- a/crates/sparrow-runtime/src/execute.rs +++ b/crates/sparrow-runtime/src/execute.rs @@ -73,12 +73,12 @@ pub async fn execute( #[derive(Default, Debug)] pub struct ExecutionOptions { - changed_since_time: Timestamp, - final_at_time: Option, - bounded_lateness_ns: Option, - compute_snapshot_config: Option, - limits: Option, - stop_signal_rx: Option>, + pub changed_since_time: Timestamp, + pub final_at_time: Option, + pub bounded_lateness_ns: Option, + pub compute_snapshot_config: Option, + pub limits: Option, + pub stop_signal_rx: Option>, } impl ExecutionOptions { diff --git a/crates/sparrow-session/src/lib.rs b/crates/sparrow-session/src/lib.rs index 06f5c4503..bbc72adef 100644 --- a/crates/sparrow-session/src/lib.rs +++ b/crates/sparrow-session/src/lib.rs @@ -7,5 +7,5 @@ mod table; pub use error::Error; pub use execution::Execution; pub use expr::{Expr, Literal}; -pub use session::Session; +pub use session::{ExecutionOptions, Session}; pub use table::Table; diff --git a/crates/sparrow-session/src/session.rs b/crates/sparrow-session/src/session.rs index 061d5280c..b74700270 100644 --- a/crates/sparrow-session/src/session.rs +++ b/crates/sparrow-session/src/session.rs @@ -4,13 +4,13 @@ use arrow_schema::SchemaRef; use error_stack::{IntoReport, IntoReportCompat, ResultExt}; use futures::{StreamExt, TryStreamExt}; use itertools::Itertools; +use sparrow_api::kaskada::v1alpha::execute_request::Limits; use sparrow_api::kaskada::v1alpha::{ ComputeTable, FeatureSet, PerEntityBehavior, TableConfig, TableMetadata, }; use sparrow_compiler::{AstDfgRef, DataContext, Dfg, DiagnosticCollector}; use sparrow_plan::TableId; use sparrow_runtime::execute::output::Destination; -use sparrow_runtime::execute::ExecutionOptions; use sparrow_syntax::{ExprOp, LiteralValue, Located, Location, Resolved}; use uuid::Uuid; @@ -23,6 +23,12 @@ pub struct Session { dfg: Dfg, } +#[derive(Default)] +pub struct ExecutionOptions { + /// The maximum number of rows to return. + pub row_limit: Option, +} + /// Adds a table to the session. impl Session { pub fn add_literal(&mut self, literal: Literal) -> error_stack::Result { @@ -231,7 +237,11 @@ impl Session { } } - pub fn execute(&self, expr: &Expr) -> error_stack::Result { + pub fn execute( + &self, + expr: &Expr, + options: ExecutionOptions, + ) -> error_stack::Result { // TODO: Decorations? let primary_group_info = self .data_context @@ -276,7 +286,8 @@ impl Session { let destination = Destination::Channel(output_tx); let data_context = self.data_context.clone(); - let options = ExecutionOptions::default(); + + let options = options.to_sparrow_options(); // Hacky. Use the existing execution logic. This weird things with downloading checkpoints, etc. let result = rt @@ -291,39 +302,6 @@ impl Session { .boxed(); Ok(Execution::new(rt, output_rx, result)) - - // // Spawn the root task - // rt.block_on(async move { - // let (output_tx, output_rx) = tokio::sync::mpsc::channel(10); - - // let destination = Destination::Channel(output_tx); - // let data_context = self.data_context.clone(); - // let options = ExecutionOptions::default(); - - // // Hacky. Use the existing execution logic. This weird things with downloading checkpoints, etc. - // let mut results = - // sparrow_runtime::execute::execute_new(plan, destination, data_context, options) - // .await - // .change_context(Error::Execute)? - // .boxed(); - - // // Hacky. Try to get the last response so we can see if there are any errors, etc. - // let mut _last = None; - // while let Some(response) = results.try_next().await.change_context(Error::Execute)? { - // _last = Some(response); - // } - - // let batches: Vec<_> = tokio_stream::wrappers::ReceiverStream::new(output_rx) - // .collect() - // .await; - - // // Hacky: Assume we produce at least one batch. - // // New execution plans contain the schema ref which cleans this up. - // let schema = batches[0].schema(); - // let batch = arrow_select::concat::concat_batches(&schema, &batches) - // .into_report() - // .change_context(Error::Execute)?; - // Ok(batch) } pub(super) fn hacky_table_mut( @@ -349,6 +327,20 @@ static RECORD_EXTENSION_ARGUMENTS: [Located; 2] = [ Located::internal_string("base"), ]; +impl ExecutionOptions { + fn to_sparrow_options(&self) -> sparrow_runtime::execute::ExecutionOptions { + let mut options = sparrow_runtime::execute::ExecutionOptions::default(); + + if let Some(row_limit) = self.row_limit { + options.limits = Some(Limits { + preview_rows: row_limit as i64, + }); + } + + options + } +} + #[cfg(test)] mod tests { use std::sync::Arc; diff --git a/sparrow-py/.darglint b/sparrow-py/.darglint index b60eee9d8..89956bb7a 100644 --- a/sparrow-py/.darglint +++ b/sparrow-py/.darglint @@ -1,4 +1,5 @@ [darglint] strictness = short docstring_style=numpy -ignore_regex=^(test)?_(.*), \ No newline at end of file +ignore_regex=^(test)?_(.*), +message_template={path}:{line} in {obj}: {msg_id} {msg} \ No newline at end of file diff --git a/sparrow-py/noxfile.py b/sparrow-py/noxfile.py index 5ffdae6ca..bc7a6105c 100644 --- a/sparrow-py/noxfile.py +++ b/sparrow-py/noxfile.py @@ -93,7 +93,11 @@ def mypy(session: Session) -> None: args = session.posargs or ["pysrc", "pytests", "docs/conf.py"] session.install("mypy", "pytest", "pandas-stubs") install_self(session) - session.run("mypy", *args) + # Using `--install-types` should make this less picky about missing stubs. + # However, there is a possibility it slows things down, by making mypy + # run twice -- once to determine what types need to be installed, then once + # to check things with those stubs. + session.run("mypy", "--install-types", *args) if not session.posargs: session.run("mypy", f"--python-executable={sys.executable}", "noxfile.py") diff --git a/sparrow-py/pyproject.toml b/sparrow-py/pyproject.toml index c827c079b..8596cb5b7 100644 --- a/sparrow-py/pyproject.toml +++ b/sparrow-py/pyproject.toml @@ -83,7 +83,6 @@ show_error_context = true # pyproject.toml [tool.pytest.ini_options] -addopts = "-ra -q" testpaths = [ "pytests", ] diff --git a/sparrow-py/pysrc/sparrow_py/__init__.py b/sparrow-py/pysrc/sparrow_py/__init__.py index 378bbfdc0..58c509de8 100644 --- a/sparrow-py/pysrc/sparrow_py/__init__.py +++ b/sparrow-py/pysrc/sparrow_py/__init__.py @@ -3,10 +3,12 @@ from typing import List from typing import Union +from . import sources +from ._expr import Expr +from ._result import Result +from ._session import init_session from ._windows import SinceWindow from ._windows import SlidingWindow -from .expr import Expr -from .session import init_session def record(fields: Dict[str, Expr]) -> Expr: @@ -17,4 +19,12 @@ def record(fields: Dict[str, Expr]) -> Expr: return Expr.call("record", *args) -__all__ = ["Expr", "init_session", "record", "SinceWindow", "SlidingWindow"] +__all__ = [ + "Expr", + "init_session", + "record", + "SinceWindow", + "SlidingWindow", + "sources", + "Result", +] diff --git a/sparrow-py/pysrc/sparrow_py/expr.py b/sparrow-py/pysrc/sparrow_py/_expr.py similarity index 90% rename from sparrow-py/pysrc/sparrow_py/expr.py rename to sparrow-py/pysrc/sparrow_py/_expr.py index ccdc459bf..66a6c9483 100644 --- a/sparrow-py/pysrc/sparrow_py/expr.py +++ b/sparrow-py/pysrc/sparrow_py/_expr.py @@ -11,14 +11,16 @@ import pandas as pd import pyarrow as pa -import sparrow_py +import sparrow_py as kt import sparrow_py._ffi as _ffi +from ._result import Result from ._windows import SinceWindow from ._windows import SlidingWindow from ._windows import Window +#: The type of arguments to expressions. Arg = Union["Expr", int, str, float, None] @@ -268,7 +270,7 @@ def extend(self, fields: Dict[str, "Expr"]) -> "Expr": """Extend this record with the additoonal fields.""" # This argument order is weird, and we shouldn't need to make a record # in order to do the extension. - extension = sparrow_py.record(fields) + extension = kt.record(fields) return Expr.call("extend_record", extension, self) def neg(self) -> "Expr": @@ -287,9 +289,8 @@ def is_not_null(self) -> "Expr": """Return a boolean expression indicating if the expression is not null.""" return Expr.call("is_valid", self) - def collect( - self, max: Optional[int] = None, window: Optional[Window] = None - ) -> "Expr": + def collect(self, max: Optional[int], window: Optional[Window] = None) -> "Expr": + """Return an expression collecting the last `max` values in the `window`.""" return _aggregation("collect", self, window, max) def sum(self, window: Optional[Window] = None) -> "Expr": @@ -304,9 +305,29 @@ def last(self, window: Optional[Window] = None) -> "Expr": """Return the last aggregation of the expression.""" return _aggregation("last", self, window) - def run(self) -> pd.DataFrame: + def show(self, limit: int = 100) -> None: + """ + Print the first N rows of the result. + + This is intended for debugging purposes. + + Parameters + ---------- + limit : int + Maximum number of rows to print. + """ + df = self.run(row_limit=limit) + try: + import ipython # type: ignore + + ipython.display(df) + except ImportError: + print(df) + + def run(self, row_limit: Optional[int] = None) -> pd.DataFrame: """Run the expression.""" - batches = self._ffi_expr.execute().collect_pyarrow() + options = _ffi.ExecutionOptions(row_limit=row_limit) + batches = self._ffi_expr.execute(options).collect_pyarrow() schema = batches[0].schema table = pa.Table.from_batches(batches, schema=schema) return table.to_pandas() @@ -330,13 +351,18 @@ def _aggregation( input : Expr The input to the expression. window : Optional[Window] - The window to use for the aggregation. + The window to use for the aggregation. *args : Optional[Arg] Additional arguments to provide before `input` and the flattened window. Returns ------- The resulting expression. + + Raises + ------ + UnimplementedError + If the window is not a known type. """ # Note: things would be easier if we had a more normal order, which @@ -345,9 +371,9 @@ def _aggregation( # us to add the *args like so. if window is None: return Expr.call(op, *args, input, None, None) - elif isinstance(window, SinceWindow): + elif isinstance(window, kt.SinceWindow): return Expr.call(op, *args, input, window._predicate, None) elif isinstance(window, SlidingWindow): return Expr.call(op, *args, input, window._predicate, window._duration) else: - raise ValueError(f"Unknown window type {window!r}") + raise UnimplementedError(f"Unknown window type {window!r}") diff --git a/sparrow-py/pysrc/sparrow_py/_ffi.pyi b/sparrow-py/pysrc/sparrow_py/_ffi.pyi index 3becf24e1..184542a11 100644 --- a/sparrow-py/pysrc/sparrow_py/_ffi.pyi +++ b/sparrow-py/pysrc/sparrow_py/_ffi.pyi @@ -1,6 +1,6 @@ +from typing import List from typing import Optional from typing import Sequence -from typing import List import pyarrow as pa from sparrow_py.udf import Udf @@ -8,6 +8,9 @@ from sparrow_py.udf import Udf class Session: def __init__(self) -> None: ... +class ExecutionOptions(object): + def __init__(self, row_limit: Optional[int] = None) -> None: ... + row_limit: Optional[int] class Execution(object): def collect_pyarrow(self) -> List[pa.RecordBatch]: ... @@ -23,7 +26,7 @@ class Expr: def data_type_string(self) -> str: ... def equivalent(self, other: Expr) -> bool: ... def session(self) -> Session: ... - def execute(self) -> Execution: ... + def execute(self, options: Optional[ExecutionOptions] = None) -> Execution: ... def call_udf(udf: Udf, result_type: pa.DataType, *args: pa.Array) -> pa.Array: ... @@ -40,4 +43,4 @@ class Table(Expr): ) -> None: ... @property def name(self) -> str: ... - def add_pyarrow(self, data: pa.RecordBatch) -> None: ... \ No newline at end of file + def add_pyarrow(self, data: pa.RecordBatch) -> None: ... diff --git a/sparrow-py/pysrc/sparrow_py/_result.py b/sparrow-py/pysrc/sparrow_py/_result.py new file mode 100644 index 000000000..c597fa329 --- /dev/null +++ b/sparrow-py/pysrc/sparrow_py/_result.py @@ -0,0 +1,2 @@ +class Result(object): + """Result of running a timestream query.""" diff --git a/sparrow-py/pysrc/sparrow_py/session.py b/sparrow-py/pysrc/sparrow_py/_session.py similarity index 100% rename from sparrow-py/pysrc/sparrow_py/session.py rename to sparrow-py/pysrc/sparrow_py/_session.py diff --git a/sparrow-py/pysrc/sparrow_py/_windows.py b/sparrow-py/pysrc/sparrow_py/_windows.py index 5fd88c8e8..14437016c 100644 --- a/sparrow-py/pysrc/sparrow_py/_windows.py +++ b/sparrow-py/pysrc/sparrow_py/_windows.py @@ -11,7 +11,7 @@ def __init__(self) -> None: class SinceWindow(Window): """Window since the last time a predicate was true.""" - def __init__(self, predicate: "sparrow_py.expr.Expr") -> None: + def __init__(self, predicate: "sparrow_py.Expr") -> None: super().__init__() self._predicate = predicate @@ -19,7 +19,7 @@ def __init__(self, predicate: "sparrow_py.expr.Expr") -> None: class SlidingWindow(Window): """Sliding windows where the width is a multiple of some condition.""" - def __init__(self, duration: int, predicate: "sparrow_py.expr.Expr") -> None: + def __init__(self, duration: int, predicate: "sparrow_py.Expr") -> None: super().__init__() self._duration = duration self._predicate = predicate diff --git a/sparrow-py/pysrc/sparrow_py/sources/source.py b/sparrow-py/pysrc/sparrow_py/sources/source.py index 2e472720d..c43425a9e 100644 --- a/sparrow-py/pysrc/sparrow_py/sources/source.py +++ b/sparrow-py/pysrc/sparrow_py/sources/source.py @@ -2,15 +2,16 @@ from typing import Optional import pyarrow as pa +import sparrow_py as kt import sparrow_py._ffi as _ffi -from sparrow_py.expr import Expr -from sparrow_py.session import _get_session + +from .._session import _get_session _TABLE_NUM: int = 0 -class Source(Expr): +class Source(kt.Expr): """A source expression.""" # TODO: Clean-up naming on the FFI side. diff --git a/sparrow-py/pytests/aggregation_test.py b/sparrow-py/pytests/aggregation_test.py index 8b9f341e1..e47a7f94f 100644 --- a/sparrow-py/pytests/aggregation_test.py +++ b/sparrow-py/pytests/aggregation_test.py @@ -1,13 +1,10 @@ """Tests for the Kaskada query builder.""" import pytest -from sparrow_py import SinceWindow -from sparrow_py import SlidingWindow -from sparrow_py import record -from sparrow_py.sources import CsvSource +import sparrow_py as kt @pytest.fixture(scope="module") -def source() -> CsvSource: +def source() -> kt.sources.CsvSource: """Create an empty table for testing.""" content = "\n".join( [ @@ -20,14 +17,14 @@ def source() -> CsvSource: "1996-12-19T16:40:02-08:00,A,,", ] ) - return CsvSource("time", "key", content) + return kt.sources.CsvSource("time", "key", content) def test_sum_unwindowed(source, golden) -> None: """Test we can create a record.""" m = source["m"] n = source["n"] - golden(record({"m": m, "sum_m": m.sum(), "n": n, "sum_n": n.sum()})) + golden(kt.record({"m": m, "sum_m": m.sum(), "n": n, "sum_n": n.sum()})) def test_sum_windowed(source, golden) -> None: @@ -35,12 +32,12 @@ def test_sum_windowed(source, golden) -> None: m = source["m"] n = source["n"] golden( - record( + kt.record( { "m": m, - "sum_m": m.sum(window=SinceWindow(m > 20)), + "sum_m": m.sum(window=kt.SinceWindow(m > 20)), "n": n, - "sum_n": n.sum(window=SlidingWindow(2, m > 10)), + "sum_n": n.sum(window=kt.SlidingWindow(2, m > 10)), } ) ) diff --git a/sparrow-py/pytests/collect_test.py b/sparrow-py/pytests/collect_test.py index 470d1a8a0..7d91356d5 100644 --- a/sparrow-py/pytests/collect_test.py +++ b/sparrow-py/pytests/collect_test.py @@ -1,13 +1,10 @@ """Tests for the collect function.""" import pytest -from sparrow_py import SinceWindow -from sparrow_py import SlidingWindow -from sparrow_py import record -from sparrow_py.sources import CsvSource +import sparrow_py as kt @pytest.fixture(scope="module") -def source() -> CsvSource: +def source() -> kt.sources.CsvSource: """Create an empty table for testing.""" content = "\n".join( [ @@ -20,7 +17,7 @@ def source() -> CsvSource: "1996-12-19T16:40:02-08:00,A,,", ] ) - return CsvSource("time", "key", content) + return kt.sources.CsvSource("time", "key", content) def test_collect_basic(source, golden) -> None: @@ -28,7 +25,7 @@ def test_collect_basic(source, golden) -> None: m = source["m"] n = source["n"] golden( - record( + kt.record( { "m": m, "collect_m": m.collect(max=None), @@ -44,7 +41,7 @@ def test_collect_with_max(source, golden) -> None: m = source["m"] n = source["n"] golden( - record( + kt.record( { "m": m, "collect_m_max_2": m.collect(max=2), @@ -58,4 +55,4 @@ def test_collect_with_max(source, golden) -> None: def test_collect_since_window(source, golden) -> None: """Test we can collect values to a list in a since window""" m = source["m"] - golden(record({"m": m, "since_m": m.sum(window=SinceWindow(m > 10))})) + golden(kt.record({"m": m, "since_m": m.sum(window=kt.SinceWindow(m > 10))})) diff --git a/sparrow-py/pytests/conftest.py b/sparrow-py/pytests/conftest.py index 3c549a5f6..0416e5299 100644 --- a/sparrow-py/pytests/conftest.py +++ b/sparrow-py/pytests/conftest.py @@ -39,6 +39,11 @@ def handler( format : str, optional The format to store the golden file in. Defaults to "json". + + Raises + ------ + ValueError + If the `format` is not recognized. """ nonlocal output @@ -62,7 +67,7 @@ def handler( elif format == "parquet": df.to_parquet(filename) elif format == "json": - df.to_json(filename, orient="records", lines=True) + df.to_json(filename, orient="records", lines=True, date_unit="ns") else: raise ValueError(f"Unknown format {format}") else: @@ -83,10 +88,17 @@ def handler( correct = pd.read_parquet(filename) elif format == "json": correct = pd.read_json( - filename, orient="records", lines=True, dtype=df.dtypes.to_dict() + filename, + orient="records", + lines=True, + dtype=df.dtypes.to_dict(), + date_unit="ns", ) else: raise ValueError(f"Unknown format {format}") + print(df.iloc[:, 0]) + print(df.dtypes) + print(correct.iloc[:, 0]) pd.testing.assert_frame_equal(df, correct) return handler diff --git a/sparrow-py/pytests/expr_test.py b/sparrow-py/pytests/expr_test.py index f01f980ee..59d8d6f5e 100644 --- a/sparrow-py/pytests/expr_test.py +++ b/sparrow-py/pytests/expr_test.py @@ -3,11 +3,11 @@ import pyarrow as pa import pytest -from sparrow_py.sources import Source +import sparrow_py as kt @pytest.fixture(scope="module") -def source1() -> Source: +def source1() -> kt.sources.Source: """Create a table for testing.""" schema = pa.schema( [ @@ -17,7 +17,7 @@ def source1() -> Source: pa.field("y", pa.int32()), ] ) - return Source("time", "key", schema) + return kt.sources.Source("time", "key", schema) def test_field_ref(source1) -> None: @@ -89,3 +89,35 @@ def test_expr_arithmetic_types(source1) -> None: if sys.version_info >= (3, 11): assert "Arg[0]: Expr of type bool" in e.value.__notes__ assert "Arg[1]: Expr of type int32" in e.value.__notes__ + + +def test_expr_show(source1, capsys) -> None: + """Test the output of showing a dataframe.""" + content = "\n".join( + [ + "time,key,m,n", + "1996-12-19T16:39:57-08:00,A,5,10", + "1996-12-19T16:39:58-08:00,B,24,3", + "1996-12-19T16:39:59-08:00,A,17,6", + "1996-12-19T16:40:00-08:00,A,,9", + "1996-12-19T16:40:01-08:00,A,12,", + "1996-12-19T16:40:02-08:00,A,,", + ] + ) + source = kt.sources.CsvSource("time", "key", content) + + source.show(limit=4) + + output = capsys.readouterr().out + assert output == "\n".join( + [ + " _time _subsort _key_hash ... key m n", + "0 1996-12-20 00:39:57 0 12960666915911099378 ... A 5.0 10", + "1 1996-12-20 00:39:58 1 2867199309159137213 ... B 24.0 3", + "2 1996-12-20 00:39:59 2 12960666915911099378 ... A 17.0 6", + "3 1996-12-20 00:40:00 3 12960666915911099378 ... A NaN 9", + "", + "[4 rows x 8 columns]", + "", + ] + ) diff --git a/sparrow-py/pytests/golden/aggregation_test/test_sum_unwindowed.json b/sparrow-py/pytests/golden/aggregation_test/test_sum_unwindowed.json index 64c3cdf4d..dc4c460e1 100644 --- a/sparrow-py/pytests/golden/aggregation_test/test_sum_unwindowed.json +++ b/sparrow-py/pytests/golden/aggregation_test/test_sum_unwindowed.json @@ -1,6 +1,6 @@ -{"_time":851042397000,"_subsort":0,"_key_hash":12960666915911099378,"_key":"A","m":5.0,"sum_m":5,"n":10.0,"sum_n":10} -{"_time":851042398000,"_subsort":1,"_key_hash":2867199309159137213,"_key":"B","m":24.0,"sum_m":24,"n":3.0,"sum_n":3} -{"_time":851042399000,"_subsort":2,"_key_hash":12960666915911099378,"_key":"A","m":17.0,"sum_m":22,"n":6.0,"sum_n":16} -{"_time":851042400000,"_subsort":3,"_key_hash":12960666915911099378,"_key":"A","m":null,"sum_m":22,"n":9.0,"sum_n":25} -{"_time":851042401000,"_subsort":4,"_key_hash":12960666915911099378,"_key":"A","m":12.0,"sum_m":34,"n":null,"sum_n":25} -{"_time":851042402000,"_subsort":5,"_key_hash":12960666915911099378,"_key":"A","m":null,"sum_m":34,"n":null,"sum_n":25} +{"_time":851042397000000000,"_subsort":0,"_key_hash":12960666915911099378,"_key":"A","m":5.0,"sum_m":5,"n":10.0,"sum_n":10} +{"_time":851042398000000000,"_subsort":1,"_key_hash":2867199309159137213,"_key":"B","m":24.0,"sum_m":24,"n":3.0,"sum_n":3} +{"_time":851042399000000000,"_subsort":2,"_key_hash":12960666915911099378,"_key":"A","m":17.0,"sum_m":22,"n":6.0,"sum_n":16} +{"_time":851042400000000000,"_subsort":3,"_key_hash":12960666915911099378,"_key":"A","m":null,"sum_m":22,"n":9.0,"sum_n":25} +{"_time":851042401000000000,"_subsort":4,"_key_hash":12960666915911099378,"_key":"A","m":12.0,"sum_m":34,"n":null,"sum_n":25} +{"_time":851042402000000000,"_subsort":5,"_key_hash":12960666915911099378,"_key":"A","m":null,"sum_m":34,"n":null,"sum_n":25} diff --git a/sparrow-py/pytests/golden/aggregation_test/test_sum_windowed.json b/sparrow-py/pytests/golden/aggregation_test/test_sum_windowed.json index 17bfd1c88..628d72c49 100644 --- a/sparrow-py/pytests/golden/aggregation_test/test_sum_windowed.json +++ b/sparrow-py/pytests/golden/aggregation_test/test_sum_windowed.json @@ -1,6 +1,6 @@ -{"_time":851042397000,"_subsort":0,"_key_hash":12960666915911099378,"_key":"A","m":5.0,"sum_m":5,"n":10.0,"sum_n":10} -{"_time":851042398000,"_subsort":1,"_key_hash":2867199309159137213,"_key":"B","m":24.0,"sum_m":24,"n":3.0,"sum_n":3} -{"_time":851042399000,"_subsort":2,"_key_hash":12960666915911099378,"_key":"A","m":17.0,"sum_m":22,"n":6.0,"sum_n":16} -{"_time":851042400000,"_subsort":3,"_key_hash":12960666915911099378,"_key":"A","m":null,"sum_m":22,"n":9.0,"sum_n":25} -{"_time":851042401000,"_subsort":4,"_key_hash":12960666915911099378,"_key":"A","m":12.0,"sum_m":34,"n":null,"sum_n":25} -{"_time":851042402000,"_subsort":5,"_key_hash":12960666915911099378,"_key":"A","m":null,"sum_m":34,"n":null,"sum_n":9} +{"_time":851042397000000000,"_subsort":0,"_key_hash":12960666915911099378,"_key":"A","m":5.0,"sum_m":5,"n":10.0,"sum_n":10} +{"_time":851042398000000000,"_subsort":1,"_key_hash":2867199309159137213,"_key":"B","m":24.0,"sum_m":24,"n":3.0,"sum_n":3} +{"_time":851042399000000000,"_subsort":2,"_key_hash":12960666915911099378,"_key":"A","m":17.0,"sum_m":22,"n":6.0,"sum_n":16} +{"_time":851042400000000000,"_subsort":3,"_key_hash":12960666915911099378,"_key":"A","m":null,"sum_m":22,"n":9.0,"sum_n":25} +{"_time":851042401000000000,"_subsort":4,"_key_hash":12960666915911099378,"_key":"A","m":12.0,"sum_m":34,"n":null,"sum_n":25} +{"_time":851042402000000000,"_subsort":5,"_key_hash":12960666915911099378,"_key":"A","m":null,"sum_m":34,"n":null,"sum_n":9} diff --git a/sparrow-py/pytests/golden/collect_test/test_collect_basic.json b/sparrow-py/pytests/golden/collect_test/test_collect_basic.json index baa0e1e7c..f5b3a477c 100644 --- a/sparrow-py/pytests/golden/collect_test/test_collect_basic.json +++ b/sparrow-py/pytests/golden/collect_test/test_collect_basic.json @@ -1,6 +1,6 @@ -{"_time":851042397000,"_subsort":0,"_key_hash":12960666915911099378,"_key":"A","m":5.0,"collect_m":[5.0],"n":10.0,"collect_n":[10.0]} -{"_time":851042398000,"_subsort":1,"_key_hash":2867199309159137213,"_key":"B","m":24.0,"collect_m":[24.0],"n":3.0,"collect_n":[3.0]} -{"_time":851042399000,"_subsort":2,"_key_hash":12960666915911099378,"_key":"A","m":17.0,"collect_m":[5.0,17.0],"n":6.0,"collect_n":[10.0,6.0]} -{"_time":851042400000,"_subsort":3,"_key_hash":12960666915911099378,"_key":"A","m":null,"collect_m":[5.0,17.0,null],"n":9.0,"collect_n":[10.0,6.0,9.0]} -{"_time":851042401000,"_subsort":4,"_key_hash":12960666915911099378,"_key":"A","m":12.0,"collect_m":[5.0,17.0,null,12.0],"n":null,"collect_n":[10.0,6.0,9.0,null]} -{"_time":851042402000,"_subsort":5,"_key_hash":12960666915911099378,"_key":"A","m":null,"collect_m":[5.0,17.0,null,12.0,null],"n":null,"collect_n":[10.0,6.0,9.0,null,null]} +{"_time":851042397000000000,"_subsort":0,"_key_hash":12960666915911099378,"_key":"A","m":5.0,"collect_m":[5.0],"n":10.0,"collect_n":[10.0]} +{"_time":851042398000000000,"_subsort":1,"_key_hash":2867199309159137213,"_key":"B","m":24.0,"collect_m":[24.0],"n":3.0,"collect_n":[3.0]} +{"_time":851042399000000000,"_subsort":2,"_key_hash":12960666915911099378,"_key":"A","m":17.0,"collect_m":[5.0,17.0],"n":6.0,"collect_n":[10.0,6.0]} +{"_time":851042400000000000,"_subsort":3,"_key_hash":12960666915911099378,"_key":"A","m":null,"collect_m":[5.0,17.0,null],"n":9.0,"collect_n":[10.0,6.0,9.0]} +{"_time":851042401000000000,"_subsort":4,"_key_hash":12960666915911099378,"_key":"A","m":12.0,"collect_m":[5.0,17.0,null,12.0],"n":null,"collect_n":[10.0,6.0,9.0,null]} +{"_time":851042402000000000,"_subsort":5,"_key_hash":12960666915911099378,"_key":"A","m":null,"collect_m":[5.0,17.0,null,12.0,null],"n":null,"collect_n":[10.0,6.0,9.0,null,null]} diff --git a/sparrow-py/pytests/golden/collect_test/test_collect_since_window.json b/sparrow-py/pytests/golden/collect_test/test_collect_since_window.json index b1ce71f47..19c9cb6b8 100644 --- a/sparrow-py/pytests/golden/collect_test/test_collect_since_window.json +++ b/sparrow-py/pytests/golden/collect_test/test_collect_since_window.json @@ -1,6 +1,6 @@ -{"_time":851042397000,"_subsort":0,"_key_hash":12960666915911099378,"_key":"A","m":5.0,"since_m":5.0} -{"_time":851042398000,"_subsort":1,"_key_hash":2867199309159137213,"_key":"B","m":24.0,"since_m":24.0} -{"_time":851042399000,"_subsort":2,"_key_hash":12960666915911099378,"_key":"A","m":17.0,"since_m":22.0} -{"_time":851042400000,"_subsort":3,"_key_hash":12960666915911099378,"_key":"A","m":null,"since_m":null} -{"_time":851042401000,"_subsort":4,"_key_hash":12960666915911099378,"_key":"A","m":12.0,"since_m":12.0} -{"_time":851042402000,"_subsort":5,"_key_hash":12960666915911099378,"_key":"A","m":null,"since_m":null} +{"_time":851042397000000000,"_subsort":0,"_key_hash":12960666915911099378,"_key":"A","m":5.0,"since_m":5.0} +{"_time":851042398000000000,"_subsort":1,"_key_hash":2867199309159137213,"_key":"B","m":24.0,"since_m":24.0} +{"_time":851042399000000000,"_subsort":2,"_key_hash":12960666915911099378,"_key":"A","m":17.0,"since_m":22.0} +{"_time":851042400000000000,"_subsort":3,"_key_hash":12960666915911099378,"_key":"A","m":null,"since_m":null} +{"_time":851042401000000000,"_subsort":4,"_key_hash":12960666915911099378,"_key":"A","m":12.0,"since_m":12.0} +{"_time":851042402000000000,"_subsort":5,"_key_hash":12960666915911099378,"_key":"A","m":null,"since_m":null} diff --git a/sparrow-py/pytests/golden/collect_test/test_collect_with_max.json b/sparrow-py/pytests/golden/collect_test/test_collect_with_max.json index 1006a8011..e58ec209a 100644 --- a/sparrow-py/pytests/golden/collect_test/test_collect_with_max.json +++ b/sparrow-py/pytests/golden/collect_test/test_collect_with_max.json @@ -1,6 +1,6 @@ -{"_time":851042397000,"_subsort":0,"_key_hash":12960666915911099378,"_key":"A","m":5.0,"collect_m_max_2":[5.0],"n":10.0,"collect_n_max_2":[10.0]} -{"_time":851042398000,"_subsort":1,"_key_hash":2867199309159137213,"_key":"B","m":24.0,"collect_m_max_2":[24.0],"n":3.0,"collect_n_max_2":[3.0]} -{"_time":851042399000,"_subsort":2,"_key_hash":12960666915911099378,"_key":"A","m":17.0,"collect_m_max_2":[5.0,17.0],"n":6.0,"collect_n_max_2":[10.0,6.0]} -{"_time":851042400000,"_subsort":3,"_key_hash":12960666915911099378,"_key":"A","m":null,"collect_m_max_2":[17.0,null],"n":9.0,"collect_n_max_2":[6.0,9.0]} -{"_time":851042401000,"_subsort":4,"_key_hash":12960666915911099378,"_key":"A","m":12.0,"collect_m_max_2":[null,12.0],"n":null,"collect_n_max_2":[9.0,null]} -{"_time":851042402000,"_subsort":5,"_key_hash":12960666915911099378,"_key":"A","m":null,"collect_m_max_2":[12.0,null],"n":null,"collect_n_max_2":[null,null]} +{"_time":851042397000000000,"_subsort":0,"_key_hash":12960666915911099378,"_key":"A","m":5.0,"collect_m_max_2":[5.0],"n":10.0,"collect_n_max_2":[10.0]} +{"_time":851042398000000000,"_subsort":1,"_key_hash":2867199309159137213,"_key":"B","m":24.0,"collect_m_max_2":[24.0],"n":3.0,"collect_n_max_2":[3.0]} +{"_time":851042399000000000,"_subsort":2,"_key_hash":12960666915911099378,"_key":"A","m":17.0,"collect_m_max_2":[5.0,17.0],"n":6.0,"collect_n_max_2":[10.0,6.0]} +{"_time":851042400000000000,"_subsort":3,"_key_hash":12960666915911099378,"_key":"A","m":null,"collect_m_max_2":[17.0,null],"n":9.0,"collect_n_max_2":[6.0,9.0]} +{"_time":851042401000000000,"_subsort":4,"_key_hash":12960666915911099378,"_key":"A","m":12.0,"collect_m_max_2":[null,12.0],"n":null,"collect_n_max_2":[9.0,null]} +{"_time":851042402000000000,"_subsort":5,"_key_hash":12960666915911099378,"_key":"A","m":null,"collect_m_max_2":[12.0,null],"n":null,"collect_n_max_2":[null,null]} diff --git a/sparrow-py/pytests/golden/math_test/test_math_int64.json b/sparrow-py/pytests/golden/math_test/test_math_int64.json index 7d0599587..6132e15fa 100644 --- a/sparrow-py/pytests/golden/math_test/test_math_int64.json +++ b/sparrow-py/pytests/golden/math_test/test_math_int64.json @@ -1,6 +1,6 @@ -{"_time":851042397000,"_subsort":0,"_key_hash":12960666915911099378,"_key":"A","m":5.0,"n":10.0,"add":15.0,"sub":-5.0} -{"_time":851042398000,"_subsort":1,"_key_hash":2867199309159137213,"_key":"B","m":24.0,"n":3.0,"add":27.0,"sub":21.0} -{"_time":851042399000,"_subsort":2,"_key_hash":12960666915911099378,"_key":"A","m":17.0,"n":6.0,"add":23.0,"sub":11.0} -{"_time":851042400000,"_subsort":3,"_key_hash":12960666915911099378,"_key":"A","m":null,"n":9.0,"add":null,"sub":null} -{"_time":851042401000,"_subsort":4,"_key_hash":12960666915911099378,"_key":"A","m":12.0,"n":null,"add":null,"sub":null} -{"_time":851042402000,"_subsort":5,"_key_hash":12960666915911099378,"_key":"A","m":null,"n":null,"add":null,"sub":null} +{"_time":851042397000000000,"_subsort":0,"_key_hash":12960666915911099378,"_key":"A","m":5.0,"n":10.0,"add":15.0,"sub":-5.0} +{"_time":851042398000000000,"_subsort":1,"_key_hash":2867199309159137213,"_key":"B","m":24.0,"n":3.0,"add":27.0,"sub":21.0} +{"_time":851042399000000000,"_subsort":2,"_key_hash":12960666915911099378,"_key":"A","m":17.0,"n":6.0,"add":23.0,"sub":11.0} +{"_time":851042400000000000,"_subsort":3,"_key_hash":12960666915911099378,"_key":"A","m":null,"n":9.0,"add":null,"sub":null} +{"_time":851042401000000000,"_subsort":4,"_key_hash":12960666915911099378,"_key":"A","m":12.0,"n":null,"add":null,"sub":null} +{"_time":851042402000000000,"_subsort":5,"_key_hash":12960666915911099378,"_key":"A","m":null,"n":null,"add":null,"sub":null} diff --git a/sparrow-py/pytests/golden/record_test/test_extend_record.json b/sparrow-py/pytests/golden/record_test/test_extend_record.json index 77351f7a6..785a1b18f 100644 --- a/sparrow-py/pytests/golden/record_test/test_extend_record.json +++ b/sparrow-py/pytests/golden/record_test/test_extend_record.json @@ -1,6 +1,6 @@ -{"_time":851042397000,"_subsort":0,"_key_hash":12960666915911099378,"_key":"A","add":15.0,"time":"1996-12-19T16:39:57-08:00","key":"A","m":5.0,"n":10.0} -{"_time":851042398000,"_subsort":1,"_key_hash":2867199309159137213,"_key":"B","add":27.0,"time":"1996-12-19T16:39:58-08:00","key":"B","m":24.0,"n":3.0} -{"_time":851042399000,"_subsort":2,"_key_hash":12960666915911099378,"_key":"A","add":23.0,"time":"1996-12-19T16:39:59-08:00","key":"A","m":17.0,"n":6.0} -{"_time":851042400000,"_subsort":3,"_key_hash":12960666915911099378,"_key":"A","add":null,"time":"1996-12-19T16:40:00-08:00","key":"A","m":null,"n":9.0} -{"_time":851042401000,"_subsort":4,"_key_hash":12960666915911099378,"_key":"A","add":null,"time":"1996-12-19T16:40:01-08:00","key":"A","m":12.0,"n":null} -{"_time":851042402000,"_subsort":5,"_key_hash":12960666915911099378,"_key":"A","add":null,"time":"1996-12-19T16:40:02-08:00","key":"A","m":null,"n":null} +{"_time":851042397000000000,"_subsort":0,"_key_hash":12960666915911099378,"_key":"A","add":15.0,"time":"1996-12-19T16:39:57-08:00","key":"A","m":5.0,"n":10.0} +{"_time":851042398000000000,"_subsort":1,"_key_hash":2867199309159137213,"_key":"B","add":27.0,"time":"1996-12-19T16:39:58-08:00","key":"B","m":24.0,"n":3.0} +{"_time":851042399000000000,"_subsort":2,"_key_hash":12960666915911099378,"_key":"A","add":23.0,"time":"1996-12-19T16:39:59-08:00","key":"A","m":17.0,"n":6.0} +{"_time":851042400000000000,"_subsort":3,"_key_hash":12960666915911099378,"_key":"A","add":null,"time":"1996-12-19T16:40:00-08:00","key":"A","m":null,"n":9.0} +{"_time":851042401000000000,"_subsort":4,"_key_hash":12960666915911099378,"_key":"A","add":null,"time":"1996-12-19T16:40:01-08:00","key":"A","m":12.0,"n":null} +{"_time":851042402000000000,"_subsort":5,"_key_hash":12960666915911099378,"_key":"A","add":null,"time":"1996-12-19T16:40:02-08:00","key":"A","m":null,"n":null} diff --git a/sparrow-py/pytests/golden/record_test/test_record.json b/sparrow-py/pytests/golden/record_test/test_record.json index 56953ae2d..a796427b6 100644 --- a/sparrow-py/pytests/golden/record_test/test_record.json +++ b/sparrow-py/pytests/golden/record_test/test_record.json @@ -1,6 +1,6 @@ -{"_time":851042397000,"_subsort":0,"_key_hash":12960666915911099378,"_key":"A","m":5.0,"n":10.0} -{"_time":851042398000,"_subsort":1,"_key_hash":2867199309159137213,"_key":"B","m":24.0,"n":3.0} -{"_time":851042399000,"_subsort":2,"_key_hash":12960666915911099378,"_key":"A","m":17.0,"n":6.0} -{"_time":851042400000,"_subsort":3,"_key_hash":12960666915911099378,"_key":"A","m":null,"n":9.0} -{"_time":851042401000,"_subsort":4,"_key_hash":12960666915911099378,"_key":"A","m":12.0,"n":null} -{"_time":851042402000,"_subsort":5,"_key_hash":12960666915911099378,"_key":"A","m":null,"n":null} +{"_time":851042397000000000,"_subsort":0,"_key_hash":12960666915911099378,"_key":"A","m":5.0,"n":10.0} +{"_time":851042398000000000,"_subsort":1,"_key_hash":2867199309159137213,"_key":"B","m":24.0,"n":3.0} +{"_time":851042399000000000,"_subsort":2,"_key_hash":12960666915911099378,"_key":"A","m":17.0,"n":6.0} +{"_time":851042400000000000,"_subsort":3,"_key_hash":12960666915911099378,"_key":"A","m":null,"n":9.0} +{"_time":851042401000000000,"_subsort":4,"_key_hash":12960666915911099378,"_key":"A","m":12.0,"n":null} +{"_time":851042402000000000,"_subsort":5,"_key_hash":12960666915911099378,"_key":"A","m":null,"n":null} diff --git a/sparrow-py/pytests/golden/record_test/test_select_record.json b/sparrow-py/pytests/golden/record_test/test_select_record.json index 9a16276ab..e55310483 100644 --- a/sparrow-py/pytests/golden/record_test/test_select_record.json +++ b/sparrow-py/pytests/golden/record_test/test_select_record.json @@ -1,6 +1,6 @@ -{"_time":851042397000,"_subsort":0,"_key_hash":12960666915911099378,"_key":"A","time":"1996-12-19T16:39:57-08:00","key":"A","m":5.0} -{"_time":851042398000,"_subsort":1,"_key_hash":2867199309159137213,"_key":"B","time":"1996-12-19T16:39:58-08:00","key":"B","m":24.0} -{"_time":851042399000,"_subsort":2,"_key_hash":12960666915911099378,"_key":"A","time":"1996-12-19T16:39:59-08:00","key":"A","m":17.0} -{"_time":851042400000,"_subsort":3,"_key_hash":12960666915911099378,"_key":"A","time":"1996-12-19T16:40:00-08:00","key":"A","m":null} -{"_time":851042401000,"_subsort":4,"_key_hash":12960666915911099378,"_key":"A","time":"1996-12-19T16:40:01-08:00","key":"A","m":12.0} -{"_time":851042402000,"_subsort":5,"_key_hash":12960666915911099378,"_key":"A","time":"1996-12-19T16:40:02-08:00","key":"A","m":null} +{"_time":851042397000000000,"_subsort":0,"_key_hash":12960666915911099378,"_key":"A","time":"1996-12-19T16:39:57-08:00","key":"A","m":5.0} +{"_time":851042398000000000,"_subsort":1,"_key_hash":2867199309159137213,"_key":"B","time":"1996-12-19T16:39:58-08:00","key":"B","m":24.0} +{"_time":851042399000000000,"_subsort":2,"_key_hash":12960666915911099378,"_key":"A","time":"1996-12-19T16:39:59-08:00","key":"A","m":17.0} +{"_time":851042400000000000,"_subsort":3,"_key_hash":12960666915911099378,"_key":"A","time":"1996-12-19T16:40:00-08:00","key":"A","m":null} +{"_time":851042401000000000,"_subsort":4,"_key_hash":12960666915911099378,"_key":"A","time":"1996-12-19T16:40:01-08:00","key":"A","m":12.0} +{"_time":851042402000000000,"_subsort":5,"_key_hash":12960666915911099378,"_key":"A","time":"1996-12-19T16:40:02-08:00","key":"A","m":null} diff --git a/sparrow-py/pytests/golden/source_test/test_add_dataframe.csv b/sparrow-py/pytests/golden/source_test/test_add_dataframe.csv deleted file mode 100644 index bd092c86b..000000000 --- a/sparrow-py/pytests/golden/source_test/test_add_dataframe.csv +++ /dev/null @@ -1,101 +0,0 @@ -_time,_subsort,_key_hash,_key,time,key -1970-01-12 14:53:20,48,17062639839782733832,4,1004000000000000,4 -1970-01-13 03:06:40,77,435660512367246687,7,1048000000000000,7 -1970-01-13 08:23:20,50,16461383214845928621,5,1067000000000000,5 -1970-01-13 16:10:00,34,5496774745203840792,3,1095000000000000,3 -1970-01-13 18:23:20,86,6794973171266502674,8,1103000000000000,8 -1970-01-14 11:53:20,24,2694864431690786590,2,1166000000000000,2 -1970-01-15 03:43:20,78,435660512367246687,7,1223000000000000,7 -1970-01-15 13:10:00,5,11832085162654999889,0,1257000000000000,0 -1970-01-16 03:03:20,83,6794973171266502674,8,1307000000000000,8 -1970-01-16 23:20:00,47,17062639839782733832,4,1380000000000000,4 -1970-01-17 06:33:20,2,11832085162654999889,0,1406000000000000,0 -1970-01-17 17:23:20,43,17062639839782733832,4,1445000000000000,4 -1970-01-18 05:03:20,56,16461383214845928621,5,1487000000000000,5 -1970-01-18 05:53:20,67,1360592371395427998,6,1490000000000000,6 -1970-01-18 13:23:20,88,6794973171266502674,8,1517000000000000,8 -1970-01-18 17:50:00,10,18433805721903975440,1,1533000000000000,1 -1970-01-19 00:46:40,90,15653042715643359010,9,1558000000000000,9 -1970-01-19 08:00:00,27,2694864431690786590,2,1584000000000000,2 -1970-01-19 08:16:40,79,435660512367246687,7,1585000000000000,7 -1970-01-20 04:00:00,45,17062639839782733832,4,1656000000000000,4 -1970-01-20 10:06:40,7,11832085162654999889,0,1678000000000000,0 -1970-01-20 22:03:20,40,17062639839782733832,4,1721000000000000,4 -1970-01-21 05:50:00,23,2694864431690786590,2,1749000000000000,2 -1970-01-21 18:36:40,57,16461383214845928621,5,1795000000000000,5 -1970-01-22 02:23:20,33,5496774745203840792,3,1823000000000000,3 -1970-01-22 04:36:40,17,18433805721903975440,1,1831000000000000,1 -1970-01-22 07:06:40,89,6794973171266502674,8,1840000000000000,8 -1970-01-22 21:50:00,38,5496774745203840792,3,1893000000000000,3 -1970-01-23 02:00:00,11,18433805721903975440,1,1908000000000000,1 -1970-01-23 08:40:00,21,2694864431690786590,2,1932000000000000,2 -1970-01-23 08:56:40,84,6794973171266502674,8,1933000000000000,8 -1970-01-23 09:13:20,62,1360592371395427998,6,1934000000000000,6 -1970-01-23 13:06:40,63,1360592371395427998,6,1948000000000000,6 -1970-01-23 23:23:20,13,18433805721903975440,1,1985000000000000,1 -1970-01-24 12:26:40,44,17062639839782733832,4,2032000000000000,4 -1970-01-24 17:43:20,52,16461383214845928621,5,2051000000000000,5 -1970-01-25 20:40:00,74,435660512367246687,7,2148000000000000,7 -1970-01-26 03:03:20,46,17062639839782733832,4,2171000000000000,4 -1970-01-26 08:53:20,53,16461383214845928621,5,2192000000000000,5 -1970-01-26 10:00:00,58,16461383214845928621,5,2196000000000000,5 -1970-01-26 10:00:00,85,6794973171266502674,8,2196000000000000,8 -1970-01-26 10:16:40,96,15653042715643359010,9,2197000000000000,9 -1970-01-26 11:56:40,68,1360592371395427998,6,2203000000000000,6 -1970-01-27 11:00:00,49,17062639839782733832,4,2286000000000000,4 -1970-01-27 17:06:40,60,1360592371395427998,6,2308000000000000,6 -1970-01-28 03:56:40,76,435660512367246687,7,2347000000000000,7 -1970-01-28 06:10:00,93,15653042715643359010,9,2355000000000000,9 -1970-01-28 13:06:40,95,15653042715643359010,9,2380000000000000,9 -1970-01-29 00:46:40,94,15653042715643359010,9,2422000000000000,9 -1970-01-29 07:10:00,4,11832085162654999889,0,2445000000000000,0 -1970-01-29 12:10:00,64,1360592371395427998,6,2463000000000000,6 -1970-01-29 19:56:40,14,18433805721903975440,1,2491000000000000,1 -1970-01-29 22:26:40,71,435660512367246687,7,2500000000000000,7 -1970-01-29 22:43:20,20,2694864431690786590,2,2501000000000000,2 -1970-01-30 03:10:00,87,6794973171266502674,8,2517000000000000,8 -1970-01-30 09:33:20,69,1360592371395427998,6,2540000000000000,6 -1970-01-30 15:23:20,59,16461383214845928621,5,2561000000000000,5 -1970-01-31 05:33:20,3,11832085162654999889,0,2612000000000000,0 -1970-02-01 06:33:20,31,5496774745203840792,3,2702000000000000,3 -1970-02-01 21:50:00,0,11832085162654999889,0,2757000000000000,0 -1970-02-02 07:16:40,9,11832085162654999889,0,2791000000000000,0 -1970-02-02 21:26:40,80,6794973171266502674,8,2842000000000000,8 -1970-02-03 06:53:20,22,2694864431690786590,2,2876000000000000,2 -1970-02-03 06:53:20,70,435660512367246687,7,2876000000000000,7 -1970-02-03 10:46:40,28,2694864431690786590,2,2890000000000000,2 -1970-02-03 18:00:00,6,11832085162654999889,0,2916000000000000,0 -1970-02-03 18:33:20,36,5496774745203840792,3,2918000000000000,3 -1970-02-03 21:36:40,75,435660512367246687,7,2929000000000000,7 -1970-02-04 05:56:40,98,15653042715643359010,9,2959000000000000,9 -1970-02-04 13:26:40,72,435660512367246687,7,2986000000000000,7 -1970-02-04 14:16:40,66,1360592371395427998,6,2989000000000000,6 -1970-02-04 16:13:20,26,2694864431690786590,2,2996000000000000,2 -1970-02-04 17:53:20,16,18433805721903975440,1,3002000000000000,1 -1970-02-05 10:33:20,92,15653042715643359010,9,3062000000000000,9 -1970-02-06 03:46:40,54,16461383214845928621,5,3124000000000000,5 -1970-02-06 22:06:40,8,11832085162654999889,0,3190000000000000,0 -1970-02-06 23:13:20,35,5496774745203840792,3,3194000000000000,3 -1970-02-07 09:13:20,39,5496774745203840792,3,3230000000000000,3 -1970-02-07 13:23:20,55,16461383214845928621,5,3245000000000000,5 -1970-02-09 19:50:00,73,435660512367246687,7,3441000000000000,7 -1970-02-10 22:30:00,65,1360592371395427998,6,3537000000000000,6 -1970-02-11 13:13:20,91,15653042715643359010,9,3590000000000000,9 -1970-02-11 17:23:20,51,16461383214845928621,5,3605000000000000,5 -1970-02-12 11:26:40,81,6794973171266502674,8,3670000000000000,8 -1970-02-12 18:56:40,61,1360592371395427998,6,3697000000000000,6 -1970-02-13 03:33:20,19,18433805721903975440,1,3728000000000000,1 -1970-02-13 07:43:20,1,11832085162654999889,0,3743000000000000,0 -1970-02-13 09:23:20,12,18433805721903975440,1,3749000000000000,1 -1970-02-13 17:43:20,25,2694864431690786590,2,3779000000000000,2 -1970-02-13 18:33:20,15,18433805721903975440,1,3782000000000000,1 -1970-02-13 20:30:00,18,18433805721903975440,1,3789000000000000,1 -1970-02-14 02:36:40,99,15653042715643359010,9,3811000000000000,9 -1970-02-14 08:26:40,32,5496774745203840792,3,3832000000000000,3 -1970-02-14 12:20:00,97,15653042715643359010,9,3846000000000000,9 -1970-02-14 16:30:00,41,17062639839782733832,4,3861000000000000,4 -1970-02-14 23:10:00,42,17062639839782733832,4,3885000000000000,4 -1970-02-15 05:50:00,30,5496774745203840792,3,3909000000000000,3 -1970-02-15 08:03:20,82,6794973171266502674,8,3917000000000000,8 -1970-02-15 10:16:40,29,2694864431690786590,2,3925000000000000,2 -1970-02-15 13:03:20,37,5496774745203840792,3,3935000000000000,3 diff --git a/sparrow-py/pytests/golden/source_test/test_add_dataframe.json b/sparrow-py/pytests/golden/source_test/test_add_dataframe.json new file mode 100644 index 000000000..bbe522cb4 --- /dev/null +++ b/sparrow-py/pytests/golden/source_test/test_add_dataframe.json @@ -0,0 +1,100 @@ +{"_time":1004000000000000,"_subsort":48,"_key_hash":17062639839782733832,"_key":4,"time":1004000000000000,"key":4} +{"_time":1048000000000000,"_subsort":77,"_key_hash":435660512367246687,"_key":7,"time":1048000000000000,"key":7} +{"_time":1067000000000000,"_subsort":50,"_key_hash":16461383214845928621,"_key":5,"time":1067000000000000,"key":5} +{"_time":1095000000000000,"_subsort":34,"_key_hash":5496774745203840792,"_key":3,"time":1095000000000000,"key":3} +{"_time":1103000000000000,"_subsort":86,"_key_hash":6794973171266502674,"_key":8,"time":1103000000000000,"key":8} +{"_time":1166000000000000,"_subsort":24,"_key_hash":2694864431690786590,"_key":2,"time":1166000000000000,"key":2} +{"_time":1223000000000000,"_subsort":78,"_key_hash":435660512367246687,"_key":7,"time":1223000000000000,"key":7} +{"_time":1257000000000000,"_subsort":5,"_key_hash":11832085162654999889,"_key":0,"time":1257000000000000,"key":0} +{"_time":1307000000000000,"_subsort":83,"_key_hash":6794973171266502674,"_key":8,"time":1307000000000000,"key":8} +{"_time":1380000000000000,"_subsort":47,"_key_hash":17062639839782733832,"_key":4,"time":1380000000000000,"key":4} +{"_time":1406000000000000,"_subsort":2,"_key_hash":11832085162654999889,"_key":0,"time":1406000000000000,"key":0} +{"_time":1445000000000000,"_subsort":43,"_key_hash":17062639839782733832,"_key":4,"time":1445000000000000,"key":4} +{"_time":1487000000000000,"_subsort":56,"_key_hash":16461383214845928621,"_key":5,"time":1487000000000000,"key":5} +{"_time":1490000000000000,"_subsort":67,"_key_hash":1360592371395427998,"_key":6,"time":1490000000000000,"key":6} +{"_time":1517000000000000,"_subsort":88,"_key_hash":6794973171266502674,"_key":8,"time":1517000000000000,"key":8} +{"_time":1533000000000000,"_subsort":10,"_key_hash":18433805721903975440,"_key":1,"time":1533000000000000,"key":1} +{"_time":1558000000000000,"_subsort":90,"_key_hash":15653042715643359010,"_key":9,"time":1558000000000000,"key":9} +{"_time":1584000000000000,"_subsort":27,"_key_hash":2694864431690786590,"_key":2,"time":1584000000000000,"key":2} +{"_time":1585000000000000,"_subsort":79,"_key_hash":435660512367246687,"_key":7,"time":1585000000000000,"key":7} +{"_time":1656000000000000,"_subsort":45,"_key_hash":17062639839782733832,"_key":4,"time":1656000000000000,"key":4} +{"_time":1678000000000000,"_subsort":7,"_key_hash":11832085162654999889,"_key":0,"time":1678000000000000,"key":0} +{"_time":1721000000000000,"_subsort":40,"_key_hash":17062639839782733832,"_key":4,"time":1721000000000000,"key":4} +{"_time":1749000000000000,"_subsort":23,"_key_hash":2694864431690786590,"_key":2,"time":1749000000000000,"key":2} +{"_time":1795000000000000,"_subsort":57,"_key_hash":16461383214845928621,"_key":5,"time":1795000000000000,"key":5} +{"_time":1823000000000000,"_subsort":33,"_key_hash":5496774745203840792,"_key":3,"time":1823000000000000,"key":3} +{"_time":1831000000000000,"_subsort":17,"_key_hash":18433805721903975440,"_key":1,"time":1831000000000000,"key":1} +{"_time":1840000000000000,"_subsort":89,"_key_hash":6794973171266502674,"_key":8,"time":1840000000000000,"key":8} +{"_time":1893000000000000,"_subsort":38,"_key_hash":5496774745203840792,"_key":3,"time":1893000000000000,"key":3} +{"_time":1908000000000000,"_subsort":11,"_key_hash":18433805721903975440,"_key":1,"time":1908000000000000,"key":1} +{"_time":1932000000000000,"_subsort":21,"_key_hash":2694864431690786590,"_key":2,"time":1932000000000000,"key":2} +{"_time":1933000000000000,"_subsort":84,"_key_hash":6794973171266502674,"_key":8,"time":1933000000000000,"key":8} +{"_time":1934000000000000,"_subsort":62,"_key_hash":1360592371395427998,"_key":6,"time":1934000000000000,"key":6} +{"_time":1948000000000000,"_subsort":63,"_key_hash":1360592371395427998,"_key":6,"time":1948000000000000,"key":6} +{"_time":1985000000000000,"_subsort":13,"_key_hash":18433805721903975440,"_key":1,"time":1985000000000000,"key":1} +{"_time":2032000000000000,"_subsort":44,"_key_hash":17062639839782733832,"_key":4,"time":2032000000000000,"key":4} +{"_time":2051000000000000,"_subsort":52,"_key_hash":16461383214845928621,"_key":5,"time":2051000000000000,"key":5} +{"_time":2148000000000000,"_subsort":74,"_key_hash":435660512367246687,"_key":7,"time":2148000000000000,"key":7} +{"_time":2171000000000000,"_subsort":46,"_key_hash":17062639839782733832,"_key":4,"time":2171000000000000,"key":4} +{"_time":2192000000000000,"_subsort":53,"_key_hash":16461383214845928621,"_key":5,"time":2192000000000000,"key":5} +{"_time":2196000000000000,"_subsort":58,"_key_hash":16461383214845928621,"_key":5,"time":2196000000000000,"key":5} +{"_time":2196000000000000,"_subsort":85,"_key_hash":6794973171266502674,"_key":8,"time":2196000000000000,"key":8} +{"_time":2197000000000000,"_subsort":96,"_key_hash":15653042715643359010,"_key":9,"time":2197000000000000,"key":9} +{"_time":2203000000000000,"_subsort":68,"_key_hash":1360592371395427998,"_key":6,"time":2203000000000000,"key":6} +{"_time":2286000000000000,"_subsort":49,"_key_hash":17062639839782733832,"_key":4,"time":2286000000000000,"key":4} +{"_time":2308000000000000,"_subsort":60,"_key_hash":1360592371395427998,"_key":6,"time":2308000000000000,"key":6} +{"_time":2347000000000000,"_subsort":76,"_key_hash":435660512367246687,"_key":7,"time":2347000000000000,"key":7} +{"_time":2355000000000000,"_subsort":93,"_key_hash":15653042715643359010,"_key":9,"time":2355000000000000,"key":9} +{"_time":2380000000000000,"_subsort":95,"_key_hash":15653042715643359010,"_key":9,"time":2380000000000000,"key":9} +{"_time":2422000000000000,"_subsort":94,"_key_hash":15653042715643359010,"_key":9,"time":2422000000000000,"key":9} +{"_time":2445000000000000,"_subsort":4,"_key_hash":11832085162654999889,"_key":0,"time":2445000000000000,"key":0} +{"_time":2463000000000000,"_subsort":64,"_key_hash":1360592371395427998,"_key":6,"time":2463000000000000,"key":6} +{"_time":2491000000000000,"_subsort":14,"_key_hash":18433805721903975440,"_key":1,"time":2491000000000000,"key":1} +{"_time":2500000000000000,"_subsort":71,"_key_hash":435660512367246687,"_key":7,"time":2500000000000000,"key":7} +{"_time":2501000000000000,"_subsort":20,"_key_hash":2694864431690786590,"_key":2,"time":2501000000000000,"key":2} +{"_time":2517000000000000,"_subsort":87,"_key_hash":6794973171266502674,"_key":8,"time":2517000000000000,"key":8} +{"_time":2540000000000000,"_subsort":69,"_key_hash":1360592371395427998,"_key":6,"time":2540000000000000,"key":6} +{"_time":2561000000000000,"_subsort":59,"_key_hash":16461383214845928621,"_key":5,"time":2561000000000000,"key":5} +{"_time":2612000000000000,"_subsort":3,"_key_hash":11832085162654999889,"_key":0,"time":2612000000000000,"key":0} +{"_time":2702000000000000,"_subsort":31,"_key_hash":5496774745203840792,"_key":3,"time":2702000000000000,"key":3} +{"_time":2757000000000000,"_subsort":0,"_key_hash":11832085162654999889,"_key":0,"time":2757000000000000,"key":0} +{"_time":2791000000000000,"_subsort":9,"_key_hash":11832085162654999889,"_key":0,"time":2791000000000000,"key":0} +{"_time":2842000000000000,"_subsort":80,"_key_hash":6794973171266502674,"_key":8,"time":2842000000000000,"key":8} +{"_time":2876000000000000,"_subsort":22,"_key_hash":2694864431690786590,"_key":2,"time":2876000000000000,"key":2} +{"_time":2876000000000000,"_subsort":70,"_key_hash":435660512367246687,"_key":7,"time":2876000000000000,"key":7} +{"_time":2890000000000000,"_subsort":28,"_key_hash":2694864431690786590,"_key":2,"time":2890000000000000,"key":2} +{"_time":2916000000000000,"_subsort":6,"_key_hash":11832085162654999889,"_key":0,"time":2916000000000000,"key":0} +{"_time":2918000000000000,"_subsort":36,"_key_hash":5496774745203840792,"_key":3,"time":2918000000000000,"key":3} +{"_time":2929000000000000,"_subsort":75,"_key_hash":435660512367246687,"_key":7,"time":2929000000000000,"key":7} +{"_time":2959000000000000,"_subsort":98,"_key_hash":15653042715643359010,"_key":9,"time":2959000000000000,"key":9} +{"_time":2986000000000000,"_subsort":72,"_key_hash":435660512367246687,"_key":7,"time":2986000000000000,"key":7} +{"_time":2989000000000000,"_subsort":66,"_key_hash":1360592371395427998,"_key":6,"time":2989000000000000,"key":6} +{"_time":2996000000000000,"_subsort":26,"_key_hash":2694864431690786590,"_key":2,"time":2996000000000000,"key":2} +{"_time":3002000000000000,"_subsort":16,"_key_hash":18433805721903975440,"_key":1,"time":3002000000000000,"key":1} +{"_time":3062000000000000,"_subsort":92,"_key_hash":15653042715643359010,"_key":9,"time":3062000000000000,"key":9} +{"_time":3124000000000000,"_subsort":54,"_key_hash":16461383214845928621,"_key":5,"time":3124000000000000,"key":5} +{"_time":3190000000000000,"_subsort":8,"_key_hash":11832085162654999889,"_key":0,"time":3190000000000000,"key":0} +{"_time":3194000000000000,"_subsort":35,"_key_hash":5496774745203840792,"_key":3,"time":3194000000000000,"key":3} +{"_time":3230000000000000,"_subsort":39,"_key_hash":5496774745203840792,"_key":3,"time":3230000000000000,"key":3} +{"_time":3245000000000000,"_subsort":55,"_key_hash":16461383214845928621,"_key":5,"time":3245000000000000,"key":5} +{"_time":3441000000000000,"_subsort":73,"_key_hash":435660512367246687,"_key":7,"time":3441000000000000,"key":7} +{"_time":3537000000000000,"_subsort":65,"_key_hash":1360592371395427998,"_key":6,"time":3537000000000000,"key":6} +{"_time":3590000000000000,"_subsort":91,"_key_hash":15653042715643359010,"_key":9,"time":3590000000000000,"key":9} +{"_time":3605000000000000,"_subsort":51,"_key_hash":16461383214845928621,"_key":5,"time":3605000000000000,"key":5} +{"_time":3670000000000000,"_subsort":81,"_key_hash":6794973171266502674,"_key":8,"time":3670000000000000,"key":8} +{"_time":3697000000000000,"_subsort":61,"_key_hash":1360592371395427998,"_key":6,"time":3697000000000000,"key":6} +{"_time":3728000000000000,"_subsort":19,"_key_hash":18433805721903975440,"_key":1,"time":3728000000000000,"key":1} +{"_time":3743000000000000,"_subsort":1,"_key_hash":11832085162654999889,"_key":0,"time":3743000000000000,"key":0} +{"_time":3749000000000000,"_subsort":12,"_key_hash":18433805721903975440,"_key":1,"time":3749000000000000,"key":1} +{"_time":3779000000000000,"_subsort":25,"_key_hash":2694864431690786590,"_key":2,"time":3779000000000000,"key":2} +{"_time":3782000000000000,"_subsort":15,"_key_hash":18433805721903975440,"_key":1,"time":3782000000000000,"key":1} +{"_time":3789000000000000,"_subsort":18,"_key_hash":18433805721903975440,"_key":1,"time":3789000000000000,"key":1} +{"_time":3811000000000000,"_subsort":99,"_key_hash":15653042715643359010,"_key":9,"time":3811000000000000,"key":9} +{"_time":3832000000000000,"_subsort":32,"_key_hash":5496774745203840792,"_key":3,"time":3832000000000000,"key":3} +{"_time":3846000000000000,"_subsort":97,"_key_hash":15653042715643359010,"_key":9,"time":3846000000000000,"key":9} +{"_time":3861000000000000,"_subsort":41,"_key_hash":17062639839782733832,"_key":4,"time":3861000000000000,"key":4} +{"_time":3885000000000000,"_subsort":42,"_key_hash":17062639839782733832,"_key":4,"time":3885000000000000,"key":4} +{"_time":3909000000000000,"_subsort":30,"_key_hash":5496774745203840792,"_key":3,"time":3909000000000000,"key":3} +{"_time":3917000000000000,"_subsort":82,"_key_hash":6794973171266502674,"_key":8,"time":3917000000000000,"key":8} +{"_time":3925000000000000,"_subsort":29,"_key_hash":2694864431690786590,"_key":2,"time":3925000000000000,"key":2} +{"_time":3935000000000000,"_subsort":37,"_key_hash":5496774745203840792,"_key":3,"time":3935000000000000,"key":3} diff --git a/sparrow-py/pytests/golden/source_test/test_add_dataframe_1.csv b/sparrow-py/pytests/golden/source_test/test_add_dataframe_1.csv deleted file mode 100644 index bacd28a05..000000000 --- a/sparrow-py/pytests/golden/source_test/test_add_dataframe_1.csv +++ /dev/null @@ -1,201 +0,0 @@ -_time,_subsort,_key_hash,_key,time,key -1970-01-12 14:53:20,48,17062639839782733832,4,1004000000000000,4 -1970-01-13 03:06:40,77,435660512367246687,7,1048000000000000,7 -1970-01-13 08:23:20,50,16461383214845928621,5,1067000000000000,5 -1970-01-13 16:10:00,34,5496774745203840792,3,1095000000000000,3 -1970-01-13 18:23:20,86,6794973171266502674,8,1103000000000000,8 -1970-01-14 11:53:20,24,2694864431690786590,2,1166000000000000,2 -1970-01-15 03:43:20,78,435660512367246687,7,1223000000000000,7 -1970-01-15 13:10:00,5,11832085162654999889,0,1257000000000000,0 -1970-01-16 03:03:20,83,6794973171266502674,8,1307000000000000,8 -1970-01-16 23:20:00,47,17062639839782733832,4,1380000000000000,4 -1970-01-17 06:33:20,2,11832085162654999889,0,1406000000000000,0 -1970-01-17 17:23:20,43,17062639839782733832,4,1445000000000000,4 -1970-01-18 05:03:20,56,16461383214845928621,5,1487000000000000,5 -1970-01-18 05:53:20,67,1360592371395427998,6,1490000000000000,6 -1970-01-18 13:23:20,88,6794973171266502674,8,1517000000000000,8 -1970-01-18 17:50:00,10,18433805721903975440,1,1533000000000000,1 -1970-01-19 00:46:40,90,15653042715643359010,9,1558000000000000,9 -1970-01-19 08:00:00,27,2694864431690786590,2,1584000000000000,2 -1970-01-19 08:16:40,79,435660512367246687,7,1585000000000000,7 -1970-01-20 04:00:00,45,17062639839782733832,4,1656000000000000,4 -1970-01-20 10:06:40,7,11832085162654999889,0,1678000000000000,0 -1970-01-20 22:03:20,40,17062639839782733832,4,1721000000000000,4 -1970-01-21 05:50:00,23,2694864431690786590,2,1749000000000000,2 -1970-01-21 18:36:40,57,16461383214845928621,5,1795000000000000,5 -1970-01-22 02:23:20,33,5496774745203840792,3,1823000000000000,3 -1970-01-22 04:36:40,17,18433805721903975440,1,1831000000000000,1 -1970-01-22 07:06:40,89,6794973171266502674,8,1840000000000000,8 -1970-01-22 21:50:00,38,5496774745203840792,3,1893000000000000,3 -1970-01-23 02:00:00,11,18433805721903975440,1,1908000000000000,1 -1970-01-23 08:40:00,21,2694864431690786590,2,1932000000000000,2 -1970-01-23 08:56:40,84,6794973171266502674,8,1933000000000000,8 -1970-01-23 09:13:20,62,1360592371395427998,6,1934000000000000,6 -1970-01-23 13:06:40,63,1360592371395427998,6,1948000000000000,6 -1970-01-23 23:23:20,13,18433805721903975440,1,1985000000000000,1 -1970-01-24 12:26:40,44,17062639839782733832,4,2032000000000000,4 -1970-01-24 17:43:20,52,16461383214845928621,5,2051000000000000,5 -1970-01-25 20:40:00,74,435660512367246687,7,2148000000000000,7 -1970-01-26 03:03:20,46,17062639839782733832,4,2171000000000000,4 -1970-01-26 08:53:20,53,16461383214845928621,5,2192000000000000,5 -1970-01-26 10:00:00,58,16461383214845928621,5,2196000000000000,5 -1970-01-26 10:00:00,85,6794973171266502674,8,2196000000000000,8 -1970-01-26 10:16:40,96,15653042715643359010,9,2197000000000000,9 -1970-01-26 11:56:40,68,1360592371395427998,6,2203000000000000,6 -1970-01-27 11:00:00,49,17062639839782733832,4,2286000000000000,4 -1970-01-27 17:06:40,60,1360592371395427998,6,2308000000000000,6 -1970-01-28 03:56:40,76,435660512367246687,7,2347000000000000,7 -1970-01-28 06:10:00,93,15653042715643359010,9,2355000000000000,9 -1970-01-28 13:06:40,95,15653042715643359010,9,2380000000000000,9 -1970-01-29 00:46:40,94,15653042715643359010,9,2422000000000000,9 -1970-01-29 07:10:00,4,11832085162654999889,0,2445000000000000,0 -1970-01-29 12:10:00,64,1360592371395427998,6,2463000000000000,6 -1970-01-29 19:56:40,14,18433805721903975440,1,2491000000000000,1 -1970-01-29 22:26:40,71,435660512367246687,7,2500000000000000,7 -1970-01-29 22:43:20,20,2694864431690786590,2,2501000000000000,2 -1970-01-30 03:10:00,87,6794973171266502674,8,2517000000000000,8 -1970-01-30 09:33:20,69,1360592371395427998,6,2540000000000000,6 -1970-01-30 15:23:20,59,16461383214845928621,5,2561000000000000,5 -1970-01-31 05:33:20,3,11832085162654999889,0,2612000000000000,0 -1970-02-01 06:33:20,31,5496774745203840792,3,2702000000000000,3 -1970-02-01 21:50:00,0,11832085162654999889,0,2757000000000000,0 -1970-02-02 07:16:40,9,11832085162654999889,0,2791000000000000,0 -1970-02-02 21:26:40,80,6794973171266502674,8,2842000000000000,8 -1970-02-03 06:53:20,22,2694864431690786590,2,2876000000000000,2 -1970-02-03 06:53:20,70,435660512367246687,7,2876000000000000,7 -1970-02-03 10:46:40,28,2694864431690786590,2,2890000000000000,2 -1970-02-03 18:00:00,6,11832085162654999889,0,2916000000000000,0 -1970-02-03 18:33:20,36,5496774745203840792,3,2918000000000000,3 -1970-02-03 21:36:40,75,435660512367246687,7,2929000000000000,7 -1970-02-04 05:56:40,98,15653042715643359010,9,2959000000000000,9 -1970-02-04 13:26:40,72,435660512367246687,7,2986000000000000,7 -1970-02-04 14:16:40,66,1360592371395427998,6,2989000000000000,6 -1970-02-04 16:13:20,26,2694864431690786590,2,2996000000000000,2 -1970-02-04 17:53:20,16,18433805721903975440,1,3002000000000000,1 -1970-02-05 00:16:40,119,18433805721903975440,1,3025000000000000,1 -1970-02-05 10:33:20,92,15653042715643359010,9,3062000000000000,9 -1970-02-05 11:40:00,179,435660512367246687,7,3066000000000000,7 -1970-02-05 15:50:00,181,6794973171266502674,8,3081000000000000,8 -1970-02-05 21:06:40,135,5496774745203840792,3,3100000000000000,3 -1970-02-06 03:46:40,54,16461383214845928621,5,3124000000000000,5 -1970-02-06 19:03:20,140,17062639839782733832,4,3179000000000000,4 -1970-02-06 22:06:40,8,11832085162654999889,0,3190000000000000,0 -1970-02-06 23:13:20,35,5496774745203840792,3,3194000000000000,3 -1970-02-06 23:30:00,139,5496774745203840792,3,3195000000000000,3 -1970-02-07 04:30:00,161,1360592371395427998,6,3213000000000000,6 -1970-02-07 09:13:20,39,5496774745203840792,3,3230000000000000,3 -1970-02-07 13:23:20,55,16461383214845928621,5,3245000000000000,5 -1970-02-08 04:56:40,105,11832085162654999889,0,3301000000000000,0 -1970-02-08 05:46:40,150,16461383214845928621,5,3304000000000000,5 -1970-02-08 18:16:40,171,435660512367246687,7,3349000000000000,7 -1970-02-09 09:16:40,149,17062639839782733832,4,3403000000000000,4 -1970-02-09 09:50:00,125,2694864431690786590,2,3405000000000000,2 -1970-02-09 13:43:20,157,16461383214845928621,5,3419000000000000,5 -1970-02-09 14:50:00,144,17062639839782733832,4,3423000000000000,4 -1970-02-09 19:50:00,73,435660512367246687,7,3441000000000000,7 -1970-02-10 22:30:00,65,1360592371395427998,6,3537000000000000,6 -1970-02-11 00:43:20,151,16461383214845928621,5,3545000000000000,5 -1970-02-11 09:20:00,148,17062639839782733832,4,3576000000000000,4 -1970-02-11 12:56:40,184,6794973171266502674,8,3589000000000000,8 -1970-02-11 13:13:20,91,15653042715643359010,9,3590000000000000,9 -1970-02-11 17:23:20,51,16461383214845928621,5,3605000000000000,5 -1970-02-12 01:26:40,163,1360592371395427998,6,3634000000000000,6 -1970-02-12 05:53:20,162,1360592371395427998,6,3650000000000000,6 -1970-02-12 11:26:40,81,6794973171266502674,8,3670000000000000,8 -1970-02-12 13:40:00,124,2694864431690786590,2,3678000000000000,2 -1970-02-12 16:10:00,193,15653042715643359010,9,3687000000000000,9 -1970-02-12 17:00:00,170,435660512367246687,7,3690000000000000,7 -1970-02-12 18:56:40,61,1360592371395427998,6,3697000000000000,6 -1970-02-13 03:33:20,19,18433805721903975440,1,3728000000000000,1 -1970-02-13 07:43:20,1,11832085162654999889,0,3743000000000000,0 -1970-02-13 09:23:20,12,18433805721903975440,1,3749000000000000,1 -1970-02-13 17:43:20,25,2694864431690786590,2,3779000000000000,2 -1970-02-13 18:33:20,15,18433805721903975440,1,3782000000000000,1 -1970-02-13 20:30:00,18,18433805721903975440,1,3789000000000000,1 -1970-02-14 02:36:40,99,15653042715643359010,9,3811000000000000,9 -1970-02-14 04:50:00,146,17062639839782733832,4,3819000000000000,4 -1970-02-14 05:56:40,143,17062639839782733832,4,3823000000000000,4 -1970-02-14 06:30:00,100,11832085162654999889,0,3825000000000000,0 -1970-02-14 06:46:40,142,17062639839782733832,4,3826000000000000,4 -1970-02-14 08:26:40,32,5496774745203840792,3,3832000000000000,3 -1970-02-14 12:20:00,97,15653042715643359010,9,3846000000000000,9 -1970-02-14 15:23:20,154,16461383214845928621,5,3857000000000000,5 -1970-02-14 16:30:00,41,17062639839782733832,4,3861000000000000,4 -1970-02-14 16:30:00,159,16461383214845928621,5,3861000000000000,5 -1970-02-14 23:10:00,42,17062639839782733832,4,3885000000000000,4 -1970-02-14 23:10:00,176,435660512367246687,7,3885000000000000,7 -1970-02-15 05:50:00,30,5496774745203840792,3,3909000000000000,3 -1970-02-15 08:03:20,82,6794973171266502674,8,3917000000000000,8 -1970-02-15 08:53:20,175,435660512367246687,7,3920000000000000,7 -1970-02-15 09:26:40,107,11832085162654999889,0,3922000000000000,0 -1970-02-15 10:16:40,29,2694864431690786590,2,3925000000000000,2 -1970-02-15 13:03:20,37,5496774745203840792,3,3935000000000000,3 -1970-02-15 17:46:40,191,15653042715643359010,9,3952000000000000,9 -1970-02-15 23:20:00,182,6794973171266502674,8,3972000000000000,8 -1970-02-16 04:20:00,110,18433805721903975440,1,3990000000000000,1 -1970-02-17 05:53:20,195,15653042715643359010,9,4082000000000000,9 -1970-02-17 09:46:40,153,16461383214845928621,5,4096000000000000,5 -1970-02-17 13:56:40,177,435660512367246687,7,4111000000000000,7 -1970-02-17 23:06:40,173,435660512367246687,7,4144000000000000,7 -1970-02-18 20:30:00,136,5496774745203840792,3,4221000000000000,3 -1970-02-18 22:26:40,113,18433805721903975440,1,4228000000000000,1 -1970-02-19 00:06:40,111,18433805721903975440,1,4234000000000000,1 -1970-02-20 01:23:20,118,18433805721903975440,1,4325000000000000,1 -1970-02-20 20:50:00,156,16461383214845928621,5,4395000000000000,5 -1970-02-21 00:10:00,145,17062639839782733832,4,4407000000000000,4 -1970-02-22 01:43:20,167,1360592371395427998,6,4499000000000000,6 -1970-02-22 12:33:20,131,5496774745203840792,3,4538000000000000,3 -1970-02-23 12:26:40,132,5496774745203840792,3,4624000000000000,3 -1970-02-23 15:13:20,103,11832085162654999889,0,4634000000000000,0 -1970-02-23 23:33:20,198,15653042715643359010,9,4664000000000000,9 -1970-02-24 01:30:00,127,2694864431690786590,2,4671000000000000,2 -1970-02-24 09:00:00,116,18433805721903975440,1,4698000000000000,1 -1970-02-25 12:13:20,190,15653042715643359010,9,4796000000000000,9 -1970-02-26 21:50:00,130,5496774745203840792,3,4917000000000000,3 -1970-02-27 08:56:40,192,15653042715643359010,9,4957000000000000,9 -1970-02-27 18:23:20,126,2694864431690786590,2,4991000000000000,2 -1970-02-27 21:10:00,172,435660512367246687,7,5001000000000000,7 -1970-03-01 07:03:20,197,15653042715643359010,9,5123000000000000,9 -1970-03-01 07:53:20,133,5496774745203840792,3,5126000000000000,3 -1970-03-01 11:13:20,189,6794973171266502674,8,5138000000000000,8 -1970-03-01 20:56:40,188,6794973171266502674,8,5173000000000000,8 -1970-03-02 17:46:40,152,16461383214845928621,5,5248000000000000,5 -1970-03-03 02:56:40,155,16461383214845928621,5,5281000000000000,5 -1970-03-03 07:56:40,120,2694864431690786590,2,5299000000000000,2 -1970-03-04 17:33:20,122,2694864431690786590,2,5420000000000000,2 -1970-03-04 17:50:00,168,1360592371395427998,6,5421000000000000,6 -1970-03-05 16:20:00,187,6794973171266502674,8,5502000000000000,8 -1970-03-05 17:43:20,178,435660512367246687,7,5507000000000000,7 -1970-03-05 23:16:40,129,2694864431690786590,2,5527000000000000,2 -1970-03-06 02:53:20,180,6794973171266502674,8,5540000000000000,8 -1970-03-06 15:56:40,160,1360592371395427998,6,5587000000000000,6 -1970-03-10 02:10:00,199,15653042715643359010,9,5883000000000000,9 -1970-03-10 04:56:40,123,2694864431690786590,2,5893000000000000,2 -1970-03-10 18:33:20,174,435660512367246687,7,5942000000000000,7 -1970-03-11 04:00:00,158,16461383214845928621,5,5976000000000000,5 -1970-03-12 02:46:40,165,1360592371395427998,6,6058000000000000,6 -1970-03-12 08:20:00,183,6794973171266502674,8,6078000000000000,8 -1970-03-12 12:30:00,104,11832085162654999889,0,6093000000000000,0 -1970-03-12 21:23:20,185,6794973171266502674,8,6125000000000000,8 -1970-03-13 00:43:20,114,18433805721903975440,1,6137000000000000,1 -1970-03-13 01:00:00,164,1360592371395427998,6,6138000000000000,6 -1970-03-13 07:40:00,186,6794973171266502674,8,6162000000000000,8 -1970-03-13 21:16:40,194,15653042715643359010,9,6211000000000000,9 -1970-03-14 16:43:20,117,18433805721903975440,1,6281000000000000,1 -1970-03-15 19:40:00,109,11832085162654999889,0,6378000000000000,0 -1970-03-16 09:16:40,196,15653042715643359010,9,6427000000000000,9 -1970-03-17 09:10:00,169,1360592371395427998,6,6513000000000000,6 -1970-03-17 12:13:20,141,17062639839782733832,4,6524000000000000,4 -1970-03-17 20:16:40,106,11832085162654999889,0,6553000000000000,0 -1970-03-18 00:43:20,102,11832085162654999889,0,6569000000000000,0 -1970-03-18 10:26:40,121,2694864431690786590,2,6604000000000000,2 -1970-03-18 13:13:20,147,17062639839782733832,4,6614000000000000,4 -1970-03-19 00:36:40,112,18433805721903975440,1,6655000000000000,1 -1970-03-19 04:30:00,138,5496774745203840792,3,6669000000000000,3 -1970-03-19 11:10:00,101,11832085162654999889,0,6693000000000000,0 -1970-03-20 06:36:40,134,5496774745203840792,3,6763000000000000,3 -1970-03-20 12:26:40,166,1360592371395427998,6,6784000000000000,6 -1970-03-20 18:33:20,115,18433805721903975440,1,6806000000000000,1 -1970-03-20 21:53:20,108,11832085162654999889,0,6818000000000000,0 -1970-03-21 05:40:00,137,5496774745203840792,3,6846000000000000,3 -1970-03-21 17:03:20,128,2694864431690786590,2,6887000000000000,2 diff --git a/sparrow-py/pytests/golden/source_test/test_add_dataframe_1.json b/sparrow-py/pytests/golden/source_test/test_add_dataframe_1.json new file mode 100644 index 000000000..07a55d00e --- /dev/null +++ b/sparrow-py/pytests/golden/source_test/test_add_dataframe_1.json @@ -0,0 +1,200 @@ +{"_time":1004000000000000,"_subsort":48,"_key_hash":17062639839782733832,"_key":4,"time":1004000000000000,"key":4} +{"_time":1048000000000000,"_subsort":77,"_key_hash":435660512367246687,"_key":7,"time":1048000000000000,"key":7} +{"_time":1067000000000000,"_subsort":50,"_key_hash":16461383214845928621,"_key":5,"time":1067000000000000,"key":5} +{"_time":1095000000000000,"_subsort":34,"_key_hash":5496774745203840792,"_key":3,"time":1095000000000000,"key":3} +{"_time":1103000000000000,"_subsort":86,"_key_hash":6794973171266502674,"_key":8,"time":1103000000000000,"key":8} +{"_time":1166000000000000,"_subsort":24,"_key_hash":2694864431690786590,"_key":2,"time":1166000000000000,"key":2} +{"_time":1223000000000000,"_subsort":78,"_key_hash":435660512367246687,"_key":7,"time":1223000000000000,"key":7} +{"_time":1257000000000000,"_subsort":5,"_key_hash":11832085162654999889,"_key":0,"time":1257000000000000,"key":0} +{"_time":1307000000000000,"_subsort":83,"_key_hash":6794973171266502674,"_key":8,"time":1307000000000000,"key":8} +{"_time":1380000000000000,"_subsort":47,"_key_hash":17062639839782733832,"_key":4,"time":1380000000000000,"key":4} +{"_time":1406000000000000,"_subsort":2,"_key_hash":11832085162654999889,"_key":0,"time":1406000000000000,"key":0} +{"_time":1445000000000000,"_subsort":43,"_key_hash":17062639839782733832,"_key":4,"time":1445000000000000,"key":4} +{"_time":1487000000000000,"_subsort":56,"_key_hash":16461383214845928621,"_key":5,"time":1487000000000000,"key":5} +{"_time":1490000000000000,"_subsort":67,"_key_hash":1360592371395427998,"_key":6,"time":1490000000000000,"key":6} +{"_time":1517000000000000,"_subsort":88,"_key_hash":6794973171266502674,"_key":8,"time":1517000000000000,"key":8} +{"_time":1533000000000000,"_subsort":10,"_key_hash":18433805721903975440,"_key":1,"time":1533000000000000,"key":1} +{"_time":1558000000000000,"_subsort":90,"_key_hash":15653042715643359010,"_key":9,"time":1558000000000000,"key":9} +{"_time":1584000000000000,"_subsort":27,"_key_hash":2694864431690786590,"_key":2,"time":1584000000000000,"key":2} +{"_time":1585000000000000,"_subsort":79,"_key_hash":435660512367246687,"_key":7,"time":1585000000000000,"key":7} +{"_time":1656000000000000,"_subsort":45,"_key_hash":17062639839782733832,"_key":4,"time":1656000000000000,"key":4} +{"_time":1678000000000000,"_subsort":7,"_key_hash":11832085162654999889,"_key":0,"time":1678000000000000,"key":0} +{"_time":1721000000000000,"_subsort":40,"_key_hash":17062639839782733832,"_key":4,"time":1721000000000000,"key":4} +{"_time":1749000000000000,"_subsort":23,"_key_hash":2694864431690786590,"_key":2,"time":1749000000000000,"key":2} +{"_time":1795000000000000,"_subsort":57,"_key_hash":16461383214845928621,"_key":5,"time":1795000000000000,"key":5} +{"_time":1823000000000000,"_subsort":33,"_key_hash":5496774745203840792,"_key":3,"time":1823000000000000,"key":3} +{"_time":1831000000000000,"_subsort":17,"_key_hash":18433805721903975440,"_key":1,"time":1831000000000000,"key":1} +{"_time":1840000000000000,"_subsort":89,"_key_hash":6794973171266502674,"_key":8,"time":1840000000000000,"key":8} +{"_time":1893000000000000,"_subsort":38,"_key_hash":5496774745203840792,"_key":3,"time":1893000000000000,"key":3} +{"_time":1908000000000000,"_subsort":11,"_key_hash":18433805721903975440,"_key":1,"time":1908000000000000,"key":1} +{"_time":1932000000000000,"_subsort":21,"_key_hash":2694864431690786590,"_key":2,"time":1932000000000000,"key":2} +{"_time":1933000000000000,"_subsort":84,"_key_hash":6794973171266502674,"_key":8,"time":1933000000000000,"key":8} +{"_time":1934000000000000,"_subsort":62,"_key_hash":1360592371395427998,"_key":6,"time":1934000000000000,"key":6} +{"_time":1948000000000000,"_subsort":63,"_key_hash":1360592371395427998,"_key":6,"time":1948000000000000,"key":6} +{"_time":1985000000000000,"_subsort":13,"_key_hash":18433805721903975440,"_key":1,"time":1985000000000000,"key":1} +{"_time":2032000000000000,"_subsort":44,"_key_hash":17062639839782733832,"_key":4,"time":2032000000000000,"key":4} +{"_time":2051000000000000,"_subsort":52,"_key_hash":16461383214845928621,"_key":5,"time":2051000000000000,"key":5} +{"_time":2148000000000000,"_subsort":74,"_key_hash":435660512367246687,"_key":7,"time":2148000000000000,"key":7} +{"_time":2171000000000000,"_subsort":46,"_key_hash":17062639839782733832,"_key":4,"time":2171000000000000,"key":4} +{"_time":2192000000000000,"_subsort":53,"_key_hash":16461383214845928621,"_key":5,"time":2192000000000000,"key":5} +{"_time":2196000000000000,"_subsort":58,"_key_hash":16461383214845928621,"_key":5,"time":2196000000000000,"key":5} +{"_time":2196000000000000,"_subsort":85,"_key_hash":6794973171266502674,"_key":8,"time":2196000000000000,"key":8} +{"_time":2197000000000000,"_subsort":96,"_key_hash":15653042715643359010,"_key":9,"time":2197000000000000,"key":9} +{"_time":2203000000000000,"_subsort":68,"_key_hash":1360592371395427998,"_key":6,"time":2203000000000000,"key":6} +{"_time":2286000000000000,"_subsort":49,"_key_hash":17062639839782733832,"_key":4,"time":2286000000000000,"key":4} +{"_time":2308000000000000,"_subsort":60,"_key_hash":1360592371395427998,"_key":6,"time":2308000000000000,"key":6} +{"_time":2347000000000000,"_subsort":76,"_key_hash":435660512367246687,"_key":7,"time":2347000000000000,"key":7} +{"_time":2355000000000000,"_subsort":93,"_key_hash":15653042715643359010,"_key":9,"time":2355000000000000,"key":9} +{"_time":2380000000000000,"_subsort":95,"_key_hash":15653042715643359010,"_key":9,"time":2380000000000000,"key":9} +{"_time":2422000000000000,"_subsort":94,"_key_hash":15653042715643359010,"_key":9,"time":2422000000000000,"key":9} +{"_time":2445000000000000,"_subsort":4,"_key_hash":11832085162654999889,"_key":0,"time":2445000000000000,"key":0} +{"_time":2463000000000000,"_subsort":64,"_key_hash":1360592371395427998,"_key":6,"time":2463000000000000,"key":6} +{"_time":2491000000000000,"_subsort":14,"_key_hash":18433805721903975440,"_key":1,"time":2491000000000000,"key":1} +{"_time":2500000000000000,"_subsort":71,"_key_hash":435660512367246687,"_key":7,"time":2500000000000000,"key":7} +{"_time":2501000000000000,"_subsort":20,"_key_hash":2694864431690786590,"_key":2,"time":2501000000000000,"key":2} +{"_time":2517000000000000,"_subsort":87,"_key_hash":6794973171266502674,"_key":8,"time":2517000000000000,"key":8} +{"_time":2540000000000000,"_subsort":69,"_key_hash":1360592371395427998,"_key":6,"time":2540000000000000,"key":6} +{"_time":2561000000000000,"_subsort":59,"_key_hash":16461383214845928621,"_key":5,"time":2561000000000000,"key":5} +{"_time":2612000000000000,"_subsort":3,"_key_hash":11832085162654999889,"_key":0,"time":2612000000000000,"key":0} +{"_time":2702000000000000,"_subsort":31,"_key_hash":5496774745203840792,"_key":3,"time":2702000000000000,"key":3} +{"_time":2757000000000000,"_subsort":0,"_key_hash":11832085162654999889,"_key":0,"time":2757000000000000,"key":0} +{"_time":2791000000000000,"_subsort":9,"_key_hash":11832085162654999889,"_key":0,"time":2791000000000000,"key":0} +{"_time":2842000000000000,"_subsort":80,"_key_hash":6794973171266502674,"_key":8,"time":2842000000000000,"key":8} +{"_time":2876000000000000,"_subsort":22,"_key_hash":2694864431690786590,"_key":2,"time":2876000000000000,"key":2} +{"_time":2876000000000000,"_subsort":70,"_key_hash":435660512367246687,"_key":7,"time":2876000000000000,"key":7} +{"_time":2890000000000000,"_subsort":28,"_key_hash":2694864431690786590,"_key":2,"time":2890000000000000,"key":2} +{"_time":2916000000000000,"_subsort":6,"_key_hash":11832085162654999889,"_key":0,"time":2916000000000000,"key":0} +{"_time":2918000000000000,"_subsort":36,"_key_hash":5496774745203840792,"_key":3,"time":2918000000000000,"key":3} +{"_time":2929000000000000,"_subsort":75,"_key_hash":435660512367246687,"_key":7,"time":2929000000000000,"key":7} +{"_time":2959000000000000,"_subsort":98,"_key_hash":15653042715643359010,"_key":9,"time":2959000000000000,"key":9} +{"_time":2986000000000000,"_subsort":72,"_key_hash":435660512367246687,"_key":7,"time":2986000000000000,"key":7} +{"_time":2989000000000000,"_subsort":66,"_key_hash":1360592371395427998,"_key":6,"time":2989000000000000,"key":6} +{"_time":2996000000000000,"_subsort":26,"_key_hash":2694864431690786590,"_key":2,"time":2996000000000000,"key":2} +{"_time":3002000000000000,"_subsort":16,"_key_hash":18433805721903975440,"_key":1,"time":3002000000000000,"key":1} +{"_time":3025000000000000,"_subsort":119,"_key_hash":18433805721903975440,"_key":1,"time":3025000000000000,"key":1} +{"_time":3062000000000000,"_subsort":92,"_key_hash":15653042715643359010,"_key":9,"time":3062000000000000,"key":9} +{"_time":3066000000000000,"_subsort":179,"_key_hash":435660512367246687,"_key":7,"time":3066000000000000,"key":7} +{"_time":3081000000000000,"_subsort":181,"_key_hash":6794973171266502674,"_key":8,"time":3081000000000000,"key":8} +{"_time":3100000000000000,"_subsort":135,"_key_hash":5496774745203840792,"_key":3,"time":3100000000000000,"key":3} +{"_time":3124000000000000,"_subsort":54,"_key_hash":16461383214845928621,"_key":5,"time":3124000000000000,"key":5} +{"_time":3179000000000000,"_subsort":140,"_key_hash":17062639839782733832,"_key":4,"time":3179000000000000,"key":4} +{"_time":3190000000000000,"_subsort":8,"_key_hash":11832085162654999889,"_key":0,"time":3190000000000000,"key":0} +{"_time":3194000000000000,"_subsort":35,"_key_hash":5496774745203840792,"_key":3,"time":3194000000000000,"key":3} +{"_time":3195000000000000,"_subsort":139,"_key_hash":5496774745203840792,"_key":3,"time":3195000000000000,"key":3} +{"_time":3213000000000000,"_subsort":161,"_key_hash":1360592371395427998,"_key":6,"time":3213000000000000,"key":6} +{"_time":3230000000000000,"_subsort":39,"_key_hash":5496774745203840792,"_key":3,"time":3230000000000000,"key":3} +{"_time":3245000000000000,"_subsort":55,"_key_hash":16461383214845928621,"_key":5,"time":3245000000000000,"key":5} +{"_time":3301000000000000,"_subsort":105,"_key_hash":11832085162654999889,"_key":0,"time":3301000000000000,"key":0} +{"_time":3304000000000000,"_subsort":150,"_key_hash":16461383214845928621,"_key":5,"time":3304000000000000,"key":5} +{"_time":3349000000000000,"_subsort":171,"_key_hash":435660512367246687,"_key":7,"time":3349000000000000,"key":7} +{"_time":3403000000000000,"_subsort":149,"_key_hash":17062639839782733832,"_key":4,"time":3403000000000000,"key":4} +{"_time":3405000000000000,"_subsort":125,"_key_hash":2694864431690786590,"_key":2,"time":3405000000000000,"key":2} +{"_time":3419000000000000,"_subsort":157,"_key_hash":16461383214845928621,"_key":5,"time":3419000000000000,"key":5} +{"_time":3423000000000000,"_subsort":144,"_key_hash":17062639839782733832,"_key":4,"time":3423000000000000,"key":4} +{"_time":3441000000000000,"_subsort":73,"_key_hash":435660512367246687,"_key":7,"time":3441000000000000,"key":7} +{"_time":3537000000000000,"_subsort":65,"_key_hash":1360592371395427998,"_key":6,"time":3537000000000000,"key":6} +{"_time":3545000000000000,"_subsort":151,"_key_hash":16461383214845928621,"_key":5,"time":3545000000000000,"key":5} +{"_time":3576000000000000,"_subsort":148,"_key_hash":17062639839782733832,"_key":4,"time":3576000000000000,"key":4} +{"_time":3589000000000000,"_subsort":184,"_key_hash":6794973171266502674,"_key":8,"time":3589000000000000,"key":8} +{"_time":3590000000000000,"_subsort":91,"_key_hash":15653042715643359010,"_key":9,"time":3590000000000000,"key":9} +{"_time":3605000000000000,"_subsort":51,"_key_hash":16461383214845928621,"_key":5,"time":3605000000000000,"key":5} +{"_time":3634000000000000,"_subsort":163,"_key_hash":1360592371395427998,"_key":6,"time":3634000000000000,"key":6} +{"_time":3650000000000000,"_subsort":162,"_key_hash":1360592371395427998,"_key":6,"time":3650000000000000,"key":6} +{"_time":3670000000000000,"_subsort":81,"_key_hash":6794973171266502674,"_key":8,"time":3670000000000000,"key":8} +{"_time":3678000000000000,"_subsort":124,"_key_hash":2694864431690786590,"_key":2,"time":3678000000000000,"key":2} +{"_time":3687000000000000,"_subsort":193,"_key_hash":15653042715643359010,"_key":9,"time":3687000000000000,"key":9} +{"_time":3690000000000000,"_subsort":170,"_key_hash":435660512367246687,"_key":7,"time":3690000000000000,"key":7} +{"_time":3697000000000000,"_subsort":61,"_key_hash":1360592371395427998,"_key":6,"time":3697000000000000,"key":6} +{"_time":3728000000000000,"_subsort":19,"_key_hash":18433805721903975440,"_key":1,"time":3728000000000000,"key":1} +{"_time":3743000000000000,"_subsort":1,"_key_hash":11832085162654999889,"_key":0,"time":3743000000000000,"key":0} +{"_time":3749000000000000,"_subsort":12,"_key_hash":18433805721903975440,"_key":1,"time":3749000000000000,"key":1} +{"_time":3779000000000000,"_subsort":25,"_key_hash":2694864431690786590,"_key":2,"time":3779000000000000,"key":2} +{"_time":3782000000000000,"_subsort":15,"_key_hash":18433805721903975440,"_key":1,"time":3782000000000000,"key":1} +{"_time":3789000000000000,"_subsort":18,"_key_hash":18433805721903975440,"_key":1,"time":3789000000000000,"key":1} +{"_time":3811000000000000,"_subsort":99,"_key_hash":15653042715643359010,"_key":9,"time":3811000000000000,"key":9} +{"_time":3819000000000000,"_subsort":146,"_key_hash":17062639839782733832,"_key":4,"time":3819000000000000,"key":4} +{"_time":3823000000000000,"_subsort":143,"_key_hash":17062639839782733832,"_key":4,"time":3823000000000000,"key":4} +{"_time":3825000000000000,"_subsort":100,"_key_hash":11832085162654999889,"_key":0,"time":3825000000000000,"key":0} +{"_time":3826000000000000,"_subsort":142,"_key_hash":17062639839782733832,"_key":4,"time":3826000000000000,"key":4} +{"_time":3832000000000000,"_subsort":32,"_key_hash":5496774745203840792,"_key":3,"time":3832000000000000,"key":3} +{"_time":3846000000000000,"_subsort":97,"_key_hash":15653042715643359010,"_key":9,"time":3846000000000000,"key":9} +{"_time":3857000000000000,"_subsort":154,"_key_hash":16461383214845928621,"_key":5,"time":3857000000000000,"key":5} +{"_time":3861000000000000,"_subsort":41,"_key_hash":17062639839782733832,"_key":4,"time":3861000000000000,"key":4} +{"_time":3861000000000000,"_subsort":159,"_key_hash":16461383214845928621,"_key":5,"time":3861000000000000,"key":5} +{"_time":3885000000000000,"_subsort":42,"_key_hash":17062639839782733832,"_key":4,"time":3885000000000000,"key":4} +{"_time":3885000000000000,"_subsort":176,"_key_hash":435660512367246687,"_key":7,"time":3885000000000000,"key":7} +{"_time":3909000000000000,"_subsort":30,"_key_hash":5496774745203840792,"_key":3,"time":3909000000000000,"key":3} +{"_time":3917000000000000,"_subsort":82,"_key_hash":6794973171266502674,"_key":8,"time":3917000000000000,"key":8} +{"_time":3920000000000000,"_subsort":175,"_key_hash":435660512367246687,"_key":7,"time":3920000000000000,"key":7} +{"_time":3922000000000000,"_subsort":107,"_key_hash":11832085162654999889,"_key":0,"time":3922000000000000,"key":0} +{"_time":3925000000000000,"_subsort":29,"_key_hash":2694864431690786590,"_key":2,"time":3925000000000000,"key":2} +{"_time":3935000000000000,"_subsort":37,"_key_hash":5496774745203840792,"_key":3,"time":3935000000000000,"key":3} +{"_time":3952000000000000,"_subsort":191,"_key_hash":15653042715643359010,"_key":9,"time":3952000000000000,"key":9} +{"_time":3972000000000000,"_subsort":182,"_key_hash":6794973171266502674,"_key":8,"time":3972000000000000,"key":8} +{"_time":3990000000000000,"_subsort":110,"_key_hash":18433805721903975440,"_key":1,"time":3990000000000000,"key":1} +{"_time":4082000000000000,"_subsort":195,"_key_hash":15653042715643359010,"_key":9,"time":4082000000000000,"key":9} +{"_time":4096000000000000,"_subsort":153,"_key_hash":16461383214845928621,"_key":5,"time":4096000000000000,"key":5} +{"_time":4111000000000000,"_subsort":177,"_key_hash":435660512367246687,"_key":7,"time":4111000000000000,"key":7} +{"_time":4144000000000000,"_subsort":173,"_key_hash":435660512367246687,"_key":7,"time":4144000000000000,"key":7} +{"_time":4221000000000000,"_subsort":136,"_key_hash":5496774745203840792,"_key":3,"time":4221000000000000,"key":3} +{"_time":4228000000000000,"_subsort":113,"_key_hash":18433805721903975440,"_key":1,"time":4228000000000000,"key":1} +{"_time":4234000000000000,"_subsort":111,"_key_hash":18433805721903975440,"_key":1,"time":4234000000000000,"key":1} +{"_time":4325000000000000,"_subsort":118,"_key_hash":18433805721903975440,"_key":1,"time":4325000000000000,"key":1} +{"_time":4395000000000000,"_subsort":156,"_key_hash":16461383214845928621,"_key":5,"time":4395000000000000,"key":5} +{"_time":4407000000000000,"_subsort":145,"_key_hash":17062639839782733832,"_key":4,"time":4407000000000000,"key":4} +{"_time":4499000000000000,"_subsort":167,"_key_hash":1360592371395427998,"_key":6,"time":4499000000000000,"key":6} +{"_time":4538000000000000,"_subsort":131,"_key_hash":5496774745203840792,"_key":3,"time":4538000000000000,"key":3} +{"_time":4624000000000000,"_subsort":132,"_key_hash":5496774745203840792,"_key":3,"time":4624000000000000,"key":3} +{"_time":4634000000000000,"_subsort":103,"_key_hash":11832085162654999889,"_key":0,"time":4634000000000000,"key":0} +{"_time":4664000000000000,"_subsort":198,"_key_hash":15653042715643359010,"_key":9,"time":4664000000000000,"key":9} +{"_time":4671000000000000,"_subsort":127,"_key_hash":2694864431690786590,"_key":2,"time":4671000000000000,"key":2} +{"_time":4698000000000000,"_subsort":116,"_key_hash":18433805721903975440,"_key":1,"time":4698000000000000,"key":1} +{"_time":4796000000000000,"_subsort":190,"_key_hash":15653042715643359010,"_key":9,"time":4796000000000000,"key":9} +{"_time":4917000000000000,"_subsort":130,"_key_hash":5496774745203840792,"_key":3,"time":4917000000000000,"key":3} +{"_time":4957000000000000,"_subsort":192,"_key_hash":15653042715643359010,"_key":9,"time":4957000000000000,"key":9} +{"_time":4991000000000000,"_subsort":126,"_key_hash":2694864431690786590,"_key":2,"time":4991000000000000,"key":2} +{"_time":5001000000000000,"_subsort":172,"_key_hash":435660512367246687,"_key":7,"time":5001000000000000,"key":7} +{"_time":5123000000000000,"_subsort":197,"_key_hash":15653042715643359010,"_key":9,"time":5123000000000000,"key":9} +{"_time":5126000000000000,"_subsort":133,"_key_hash":5496774745203840792,"_key":3,"time":5126000000000000,"key":3} +{"_time":5138000000000000,"_subsort":189,"_key_hash":6794973171266502674,"_key":8,"time":5138000000000000,"key":8} +{"_time":5173000000000000,"_subsort":188,"_key_hash":6794973171266502674,"_key":8,"time":5173000000000000,"key":8} +{"_time":5248000000000000,"_subsort":152,"_key_hash":16461383214845928621,"_key":5,"time":5248000000000000,"key":5} +{"_time":5281000000000000,"_subsort":155,"_key_hash":16461383214845928621,"_key":5,"time":5281000000000000,"key":5} +{"_time":5299000000000000,"_subsort":120,"_key_hash":2694864431690786590,"_key":2,"time":5299000000000000,"key":2} +{"_time":5420000000000000,"_subsort":122,"_key_hash":2694864431690786590,"_key":2,"time":5420000000000000,"key":2} +{"_time":5421000000000000,"_subsort":168,"_key_hash":1360592371395427998,"_key":6,"time":5421000000000000,"key":6} +{"_time":5502000000000000,"_subsort":187,"_key_hash":6794973171266502674,"_key":8,"time":5502000000000000,"key":8} +{"_time":5507000000000000,"_subsort":178,"_key_hash":435660512367246687,"_key":7,"time":5507000000000000,"key":7} +{"_time":5527000000000000,"_subsort":129,"_key_hash":2694864431690786590,"_key":2,"time":5527000000000000,"key":2} +{"_time":5540000000000000,"_subsort":180,"_key_hash":6794973171266502674,"_key":8,"time":5540000000000000,"key":8} +{"_time":5587000000000000,"_subsort":160,"_key_hash":1360592371395427998,"_key":6,"time":5587000000000000,"key":6} +{"_time":5883000000000000,"_subsort":199,"_key_hash":15653042715643359010,"_key":9,"time":5883000000000000,"key":9} +{"_time":5893000000000000,"_subsort":123,"_key_hash":2694864431690786590,"_key":2,"time":5893000000000000,"key":2} +{"_time":5942000000000000,"_subsort":174,"_key_hash":435660512367246687,"_key":7,"time":5942000000000000,"key":7} +{"_time":5976000000000000,"_subsort":158,"_key_hash":16461383214845928621,"_key":5,"time":5976000000000000,"key":5} +{"_time":6058000000000000,"_subsort":165,"_key_hash":1360592371395427998,"_key":6,"time":6058000000000000,"key":6} +{"_time":6078000000000000,"_subsort":183,"_key_hash":6794973171266502674,"_key":8,"time":6078000000000000,"key":8} +{"_time":6093000000000000,"_subsort":104,"_key_hash":11832085162654999889,"_key":0,"time":6093000000000000,"key":0} +{"_time":6125000000000000,"_subsort":185,"_key_hash":6794973171266502674,"_key":8,"time":6125000000000000,"key":8} +{"_time":6137000000000000,"_subsort":114,"_key_hash":18433805721903975440,"_key":1,"time":6137000000000000,"key":1} +{"_time":6138000000000000,"_subsort":164,"_key_hash":1360592371395427998,"_key":6,"time":6138000000000000,"key":6} +{"_time":6162000000000000,"_subsort":186,"_key_hash":6794973171266502674,"_key":8,"time":6162000000000000,"key":8} +{"_time":6211000000000000,"_subsort":194,"_key_hash":15653042715643359010,"_key":9,"time":6211000000000000,"key":9} +{"_time":6281000000000000,"_subsort":117,"_key_hash":18433805721903975440,"_key":1,"time":6281000000000000,"key":1} +{"_time":6378000000000000,"_subsort":109,"_key_hash":11832085162654999889,"_key":0,"time":6378000000000000,"key":0} +{"_time":6427000000000000,"_subsort":196,"_key_hash":15653042715643359010,"_key":9,"time":6427000000000000,"key":9} +{"_time":6513000000000000,"_subsort":169,"_key_hash":1360592371395427998,"_key":6,"time":6513000000000000,"key":6} +{"_time":6524000000000000,"_subsort":141,"_key_hash":17062639839782733832,"_key":4,"time":6524000000000000,"key":4} +{"_time":6553000000000000,"_subsort":106,"_key_hash":11832085162654999889,"_key":0,"time":6553000000000000,"key":0} +{"_time":6569000000000000,"_subsort":102,"_key_hash":11832085162654999889,"_key":0,"time":6569000000000000,"key":0} +{"_time":6604000000000000,"_subsort":121,"_key_hash":2694864431690786590,"_key":2,"time":6604000000000000,"key":2} +{"_time":6614000000000000,"_subsort":147,"_key_hash":17062639839782733832,"_key":4,"time":6614000000000000,"key":4} +{"_time":6655000000000000,"_subsort":112,"_key_hash":18433805721903975440,"_key":1,"time":6655000000000000,"key":1} +{"_time":6669000000000000,"_subsort":138,"_key_hash":5496774745203840792,"_key":3,"time":6669000000000000,"key":3} +{"_time":6693000000000000,"_subsort":101,"_key_hash":11832085162654999889,"_key":0,"time":6693000000000000,"key":0} +{"_time":6763000000000000,"_subsort":134,"_key_hash":5496774745203840792,"_key":3,"time":6763000000000000,"key":3} +{"_time":6784000000000000,"_subsort":166,"_key_hash":1360592371395427998,"_key":6,"time":6784000000000000,"key":6} +{"_time":6806000000000000,"_subsort":115,"_key_hash":18433805721903975440,"_key":1,"time":6806000000000000,"key":1} +{"_time":6818000000000000,"_subsort":108,"_key_hash":11832085162654999889,"_key":0,"time":6818000000000000,"key":0} +{"_time":6846000000000000,"_subsort":137,"_key_hash":5496774745203840792,"_key":3,"time":6846000000000000,"key":3} +{"_time":6887000000000000,"_subsort":128,"_key_hash":2694864431690786590,"_key":2,"time":6887000000000000,"key":2} diff --git a/sparrow-py/pytests/math_test.py b/sparrow-py/pytests/math_test.py index 36a14c7af..450f85bd6 100644 --- a/sparrow-py/pytests/math_test.py +++ b/sparrow-py/pytests/math_test.py @@ -1,11 +1,10 @@ """Tests for the Kaskada query builder.""" import pytest -import sparrow_py as s -from sparrow_py.sources import CsvSource +import sparrow_py as kt @pytest.fixture -def source_int64() -> CsvSource: +def source_int64() -> kt.sources.CsvSource: """Create an empty table for testing.""" content = "\n".join( [ @@ -18,7 +17,7 @@ def source_int64() -> CsvSource: "1996-12-19T16:40:02-08:00,A,,", ] ) - return CsvSource("time", "key", content) + return kt.sources.CsvSource("time", "key", content) def test_math_int64(golden, source_int64) -> None: @@ -26,7 +25,7 @@ def test_math_int64(golden, source_int64) -> None: m = source_int64["m"] n = source_int64["n"] golden( - s.record( + kt.record( { "m": m, "n": n, diff --git a/sparrow-py/pytests/record_test.py b/sparrow-py/pytests/record_test.py index 04b6fffa5..28ff17eb7 100644 --- a/sparrow-py/pytests/record_test.py +++ b/sparrow-py/pytests/record_test.py @@ -1,11 +1,10 @@ """Tests for the Kaskada query builder.""" import pytest -from sparrow_py import record -from sparrow_py.sources import CsvSource +import sparrow_py as kt @pytest.fixture -def source() -> CsvSource: +def source() -> kt.sources.CsvSource: """Create an empty table for testing.""" content = "\n".join( [ @@ -18,7 +17,7 @@ def source() -> CsvSource: "1996-12-19T16:40:02-08:00,A,,", ] ) - return CsvSource("time", "key", content) + return kt.sources.CsvSource("time", "key", content) def test_record(source, golden) -> None: @@ -27,7 +26,7 @@ def test_record(source, golden) -> None: n = source["n"] golden( - record( + kt.record( { "m": m, "n": n, diff --git a/sparrow-py/pytests/source_test.py b/sparrow-py/pytests/source_test.py index b0aadabf2..8d07f3e84 100644 --- a/sparrow-py/pytests/source_test.py +++ b/sparrow-py/pytests/source_test.py @@ -4,8 +4,7 @@ import pandas as pd import pyarrow as pa import pytest -from sparrow_py.sources import ArrowSource -from sparrow_py.sources import Source +import sparrow_py as kt def test_table_valid() -> None: @@ -17,7 +16,7 @@ def test_table_valid() -> None: ] ) - Source("time", "key", schema) + kt.sources.Source("time", "key", schema) def test_table_invalid_names() -> None: @@ -33,19 +32,19 @@ def test_table_invalid_names() -> None: # Currently, this doesn't propagate the suggestions from # existing column names from Sparrow. # TODO: Do that. - Source("non_existant_time", "key", schema) + kt.sources.Source("non_existant_time", "key", schema) with pytest.raises(KeyError): # Currently, this doesn't propagate the suggestions from # existing column names from Sparrow. # TODO: Do that. - Source("time", "non_existant_key", schema) + kt.sources.Source("time", "non_existant_key", schema) with pytest.raises(KeyError): # Currently, this doesn't propagate the suggestions from # existing column names from Sparrow. # TODO: Do that. - Source( + kt.sources.Source( "time", "key", subsort_column_name="non_existant_subsort", @@ -69,7 +68,8 @@ def test_add_dataframe(golden) -> None: ) dataset1 = pd.DataFrame(records) - table = ArrowSource("time", "key", dataset1) + table = kt.sources.ArrowSource("time", "key", dataset1) + table.show() golden(table) records.clear() @@ -84,4 +84,4 @@ def test_add_dataframe(golden) -> None: ) dataset2 = pd.DataFrame(records) table.add(dataset2) - golden(table) \ No newline at end of file + golden(table) diff --git a/sparrow-py/src/execution.rs b/sparrow-py/src/execution.rs index 405282f5f..91b5c3fb1 100644 --- a/sparrow-py/src/execution.rs +++ b/sparrow-py/src/execution.rs @@ -4,6 +4,13 @@ use sparrow_session::Execution as RustExecution; use crate::error::{Error, ErrorContext}; +#[pyclass] +#[derive(Default)] +pub(crate) struct ExecutionOptions { + #[pyo3(get, set)] + pub(crate) row_limit: Option, +} + /// Kaskada execution object. #[pyclass] pub(crate) struct Execution(Option); @@ -29,3 +36,19 @@ impl Execution { Ok(results) } } + +#[pymethods] +impl ExecutionOptions { + #[new] + fn new(row_limit: Option) -> Self { + Self { row_limit } + } +} + +impl ExecutionOptions { + pub(crate) fn to_rust_options(&self) -> sparrow_session::ExecutionOptions { + sparrow_session::ExecutionOptions { + row_limit: self.row_limit, + } + } +} diff --git a/sparrow-py/src/expr.rs b/sparrow-py/src/expr.rs index a5dcab343..9bbe9cff0 100644 --- a/sparrow-py/src/expr.rs +++ b/sparrow-py/src/expr.rs @@ -1,5 +1,5 @@ use crate::error::Error; -use crate::execution::Execution; +use crate::execution::{Execution, ExecutionOptions}; use crate::session::Session; use arrow::pyarrow::ToPyArrow; use pyo3::exceptions::{PyRuntimeError, PyValueError}; @@ -73,9 +73,12 @@ impl Expr { self.session.clone() } - fn execute(&self) -> Result { + fn execute(&self, options: Option<&ExecutionOptions>) -> Result { let session = self.session.rust_session()?; - let execution = session.execute(&self.rust_expr)?; + let options = options + .map(ExecutionOptions::to_rust_options) + .unwrap_or_default(); + let execution = session.execute(&self.rust_expr, options)?; Ok(Execution::new(execution)) } diff --git a/sparrow-py/src/lib.rs b/sparrow-py/src/lib.rs index 1495b6326..97334b3e1 100644 --- a/sparrow-py/src/lib.rs +++ b/sparrow-py/src/lib.rs @@ -18,6 +18,7 @@ fn ffi(_py: Python<'_>, m: &PyModule) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; Ok(()) }