-
Notifications
You must be signed in to change notification settings - Fork 15
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: sparrow py collect function (#595)
support `collect` function in sparrow-py --------- Co-authored-by: Ben Chambers <[email protected]>
- Loading branch information
1 parent
5be52a0
commit ed026ae
Showing
22 changed files
with
191 additions
and
57 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,61 @@ | ||
"""Tests for the collect function.""" | ||
import pytest | ||
from sparrow_py import SinceWindow | ||
from sparrow_py import SlidingWindow | ||
from sparrow_py import record | ||
from sparrow_py.sources import CsvSource | ||
|
||
|
||
@pytest.fixture(scope="module") | ||
def source() -> CsvSource: | ||
"""Create an empty table for testing.""" | ||
content = "\n".join( | ||
[ | ||
"time,key,m,n", | ||
"1996-12-19T16:39:57-08:00,A,5,10", | ||
"1996-12-19T16:39:58-08:00,B,24,3", | ||
"1996-12-19T16:39:59-08:00,A,17,6", | ||
"1996-12-19T16:40:00-08:00,A,,9", | ||
"1996-12-19T16:40:01-08:00,A,12,", | ||
"1996-12-19T16:40:02-08:00,A,,", | ||
] | ||
) | ||
return CsvSource("time", "key", content) | ||
|
||
|
||
def test_collect_basic(source, golden) -> None: | ||
"""Test we can collect values to a list""" | ||
m = source["m"] | ||
n = source["n"] | ||
golden( | ||
record( | ||
{ | ||
"m": m, | ||
"collect_m": m.collect(max=None), | ||
"n": n, | ||
"collect_n": n.collect(max=None), | ||
} | ||
) | ||
) | ||
|
||
|
||
def test_collect_with_max(source, golden) -> None: | ||
"""Test we can collect values to a list with a max""" | ||
m = source["m"] | ||
n = source["n"] | ||
golden( | ||
record( | ||
{ | ||
"m": m, | ||
"collect_m_max_2": m.collect(max=2), | ||
"n": n, | ||
"collect_n_max_2": n.collect(max=2), | ||
} | ||
) | ||
) | ||
|
||
|
||
def test_collect_since_window(source, golden) -> None: | ||
"""Test we can collect values to a list in a since window""" | ||
m = source["m"] | ||
golden(record({"m": m, "since_m": m.sum(window=SinceWindow(m > 10))})) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
7 changes: 0 additions & 7 deletions
7
sparrow-py/pytests/golden/aggregation_test/test_sum_unwindowed.csv
This file was deleted.
Oops, something went wrong.
6 changes: 6 additions & 0 deletions
6
sparrow-py/pytests/golden/aggregation_test/test_sum_unwindowed.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,6 @@ | ||
{"_time":851042397000,"_subsort":0,"_key_hash":12960666915911099378,"_key":"A","m":5.0,"sum_m":5,"n":10.0,"sum_n":10} | ||
{"_time":851042398000,"_subsort":1,"_key_hash":2867199309159137213,"_key":"B","m":24.0,"sum_m":24,"n":3.0,"sum_n":3} | ||
{"_time":851042399000,"_subsort":2,"_key_hash":12960666915911099378,"_key":"A","m":17.0,"sum_m":22,"n":6.0,"sum_n":16} | ||
{"_time":851042400000,"_subsort":3,"_key_hash":12960666915911099378,"_key":"A","m":null,"sum_m":22,"n":9.0,"sum_n":25} | ||
{"_time":851042401000,"_subsort":4,"_key_hash":12960666915911099378,"_key":"A","m":12.0,"sum_m":34,"n":null,"sum_n":25} | ||
{"_time":851042402000,"_subsort":5,"_key_hash":12960666915911099378,"_key":"A","m":null,"sum_m":34,"n":null,"sum_n":25} |
7 changes: 0 additions & 7 deletions
7
sparrow-py/pytests/golden/aggregation_test/test_sum_windowed.csv
This file was deleted.
Oops, something went wrong.
6 changes: 6 additions & 0 deletions
6
sparrow-py/pytests/golden/aggregation_test/test_sum_windowed.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,6 @@ | ||
{"_time":851042397000,"_subsort":0,"_key_hash":12960666915911099378,"_key":"A","m":5.0,"sum_m":5,"n":10.0,"sum_n":10} | ||
{"_time":851042398000,"_subsort":1,"_key_hash":2867199309159137213,"_key":"B","m":24.0,"sum_m":24,"n":3.0,"sum_n":3} | ||
{"_time":851042399000,"_subsort":2,"_key_hash":12960666915911099378,"_key":"A","m":17.0,"sum_m":22,"n":6.0,"sum_n":16} | ||
{"_time":851042400000,"_subsort":3,"_key_hash":12960666915911099378,"_key":"A","m":null,"sum_m":22,"n":9.0,"sum_n":25} | ||
{"_time":851042401000,"_subsort":4,"_key_hash":12960666915911099378,"_key":"A","m":12.0,"sum_m":34,"n":null,"sum_n":25} | ||
{"_time":851042402000,"_subsort":5,"_key_hash":12960666915911099378,"_key":"A","m":null,"sum_m":34,"n":null,"sum_n":9} |
6 changes: 6 additions & 0 deletions
6
sparrow-py/pytests/golden/collect_test/test_collect_basic.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,6 @@ | ||
{"_time":851042397000,"_subsort":0,"_key_hash":12960666915911099378,"_key":"A","m":5.0,"collect_m":[5.0],"n":10.0,"collect_n":[10.0]} | ||
{"_time":851042398000,"_subsort":1,"_key_hash":2867199309159137213,"_key":"B","m":24.0,"collect_m":[24.0],"n":3.0,"collect_n":[3.0]} | ||
{"_time":851042399000,"_subsort":2,"_key_hash":12960666915911099378,"_key":"A","m":17.0,"collect_m":[5.0,17.0],"n":6.0,"collect_n":[10.0,6.0]} | ||
{"_time":851042400000,"_subsort":3,"_key_hash":12960666915911099378,"_key":"A","m":null,"collect_m":[5.0,17.0,null],"n":9.0,"collect_n":[10.0,6.0,9.0]} | ||
{"_time":851042401000,"_subsort":4,"_key_hash":12960666915911099378,"_key":"A","m":12.0,"collect_m":[5.0,17.0,null,12.0],"n":null,"collect_n":[10.0,6.0,9.0,null]} | ||
{"_time":851042402000,"_subsort":5,"_key_hash":12960666915911099378,"_key":"A","m":null,"collect_m":[5.0,17.0,null,12.0,null],"n":null,"collect_n":[10.0,6.0,9.0,null,null]} |
6 changes: 6 additions & 0 deletions
6
sparrow-py/pytests/golden/collect_test/test_collect_since_window.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,6 @@ | ||
{"_time":851042397000,"_subsort":0,"_key_hash":12960666915911099378,"_key":"A","m":5.0,"since_m":5.0} | ||
{"_time":851042398000,"_subsort":1,"_key_hash":2867199309159137213,"_key":"B","m":24.0,"since_m":24.0} | ||
{"_time":851042399000,"_subsort":2,"_key_hash":12960666915911099378,"_key":"A","m":17.0,"since_m":22.0} | ||
{"_time":851042400000,"_subsort":3,"_key_hash":12960666915911099378,"_key":"A","m":null,"since_m":null} | ||
{"_time":851042401000,"_subsort":4,"_key_hash":12960666915911099378,"_key":"A","m":12.0,"since_m":12.0} | ||
{"_time":851042402000,"_subsort":5,"_key_hash":12960666915911099378,"_key":"A","m":null,"since_m":null} |
6 changes: 6 additions & 0 deletions
6
sparrow-py/pytests/golden/collect_test/test_collect_with_max.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,6 @@ | ||
{"_time":851042397000,"_subsort":0,"_key_hash":12960666915911099378,"_key":"A","m":5.0,"collect_m_max_2":[5.0],"n":10.0,"collect_n_max_2":[10.0]} | ||
{"_time":851042398000,"_subsort":1,"_key_hash":2867199309159137213,"_key":"B","m":24.0,"collect_m_max_2":[24.0],"n":3.0,"collect_n_max_2":[3.0]} | ||
{"_time":851042399000,"_subsort":2,"_key_hash":12960666915911099378,"_key":"A","m":17.0,"collect_m_max_2":[5.0,17.0],"n":6.0,"collect_n_max_2":[10.0,6.0]} | ||
{"_time":851042400000,"_subsort":3,"_key_hash":12960666915911099378,"_key":"A","m":null,"collect_m_max_2":[17.0,null],"n":9.0,"collect_n_max_2":[6.0,9.0]} | ||
{"_time":851042401000,"_subsort":4,"_key_hash":12960666915911099378,"_key":"A","m":12.0,"collect_m_max_2":[null,12.0],"n":null,"collect_n_max_2":[9.0,null]} | ||
{"_time":851042402000,"_subsort":5,"_key_hash":12960666915911099378,"_key":"A","m":null,"collect_m_max_2":[12.0,null],"n":null,"collect_n_max_2":[null,null]} |
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,6 @@ | ||
{"_time":851042397000,"_subsort":0,"_key_hash":12960666915911099378,"_key":"A","m":5.0,"n":10.0,"add":15.0,"sub":-5.0} | ||
{"_time":851042398000,"_subsort":1,"_key_hash":2867199309159137213,"_key":"B","m":24.0,"n":3.0,"add":27.0,"sub":21.0} | ||
{"_time":851042399000,"_subsort":2,"_key_hash":12960666915911099378,"_key":"A","m":17.0,"n":6.0,"add":23.0,"sub":11.0} | ||
{"_time":851042400000,"_subsort":3,"_key_hash":12960666915911099378,"_key":"A","m":null,"n":9.0,"add":null,"sub":null} | ||
{"_time":851042401000,"_subsort":4,"_key_hash":12960666915911099378,"_key":"A","m":12.0,"n":null,"add":null,"sub":null} | ||
{"_time":851042402000,"_subsort":5,"_key_hash":12960666915911099378,"_key":"A","m":null,"n":null,"add":null,"sub":null} |
This file was deleted.
Oops, something went wrong.
6 changes: 6 additions & 0 deletions
6
sparrow-py/pytests/golden/record_test/test_extend_record.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,6 @@ | ||
{"_time":851042397000,"_subsort":0,"_key_hash":12960666915911099378,"_key":"A","add":15.0,"time":"1996-12-19T16:39:57-08:00","key":"A","m":5.0,"n":10.0} | ||
{"_time":851042398000,"_subsort":1,"_key_hash":2867199309159137213,"_key":"B","add":27.0,"time":"1996-12-19T16:39:58-08:00","key":"B","m":24.0,"n":3.0} | ||
{"_time":851042399000,"_subsort":2,"_key_hash":12960666915911099378,"_key":"A","add":23.0,"time":"1996-12-19T16:39:59-08:00","key":"A","m":17.0,"n":6.0} | ||
{"_time":851042400000,"_subsort":3,"_key_hash":12960666915911099378,"_key":"A","add":null,"time":"1996-12-19T16:40:00-08:00","key":"A","m":null,"n":9.0} | ||
{"_time":851042401000,"_subsort":4,"_key_hash":12960666915911099378,"_key":"A","add":null,"time":"1996-12-19T16:40:01-08:00","key":"A","m":12.0,"n":null} | ||
{"_time":851042402000,"_subsort":5,"_key_hash":12960666915911099378,"_key":"A","add":null,"time":"1996-12-19T16:40:02-08:00","key":"A","m":null,"n":null} |
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,6 @@ | ||
{"_time":851042397000,"_subsort":0,"_key_hash":12960666915911099378,"_key":"A","m":5.0,"n":10.0} | ||
{"_time":851042398000,"_subsort":1,"_key_hash":2867199309159137213,"_key":"B","m":24.0,"n":3.0} | ||
{"_time":851042399000,"_subsort":2,"_key_hash":12960666915911099378,"_key":"A","m":17.0,"n":6.0} | ||
{"_time":851042400000,"_subsort":3,"_key_hash":12960666915911099378,"_key":"A","m":null,"n":9.0} | ||
{"_time":851042401000,"_subsort":4,"_key_hash":12960666915911099378,"_key":"A","m":12.0,"n":null} | ||
{"_time":851042402000,"_subsort":5,"_key_hash":12960666915911099378,"_key":"A","m":null,"n":null} |
This file was deleted.
Oops, something went wrong.
6 changes: 6 additions & 0 deletions
6
sparrow-py/pytests/golden/record_test/test_select_record.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,6 @@ | ||
{"_time":851042397000,"_subsort":0,"_key_hash":12960666915911099378,"_key":"A","time":"1996-12-19T16:39:57-08:00","key":"A","m":5.0} | ||
{"_time":851042398000,"_subsort":1,"_key_hash":2867199309159137213,"_key":"B","time":"1996-12-19T16:39:58-08:00","key":"B","m":24.0} | ||
{"_time":851042399000,"_subsort":2,"_key_hash":12960666915911099378,"_key":"A","time":"1996-12-19T16:39:59-08:00","key":"A","m":17.0} | ||
{"_time":851042400000,"_subsort":3,"_key_hash":12960666915911099378,"_key":"A","time":"1996-12-19T16:40:00-08:00","key":"A","m":null} | ||
{"_time":851042401000,"_subsort":4,"_key_hash":12960666915911099378,"_key":"A","time":"1996-12-19T16:40:01-08:00","key":"A","m":12.0} | ||
{"_time":851042402000,"_subsort":5,"_key_hash":12960666915911099378,"_key":"A","time":"1996-12-19T16:40:02-08:00","key":"A","m":null} |