Skip to content

Commit 6e1295c

Browse files
cgarciaecgarciae
authored
Fix dependencies + imports based on Python version
1 parent f3ba9a8 commit 6e1295c

39 files changed

+353
-530
lines changed

‎poetry.lock

+217-367
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pypeln/__init__.py

+1-3
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,9 @@
33
from . import sync
44
from . import thread
55
from . import process
6+
from . import task
67
from .utils import Element
78
from .utils import BaseStage
89

9-
if sys.version_info >= (3, 7):
10-
from . import task
11-
1210

1311
__version__ = "0.3.0"

‎pypeln/process/api/each.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
from ..worker import ProcessFn, Worker, ApplyProcess
1111

1212

13-
class EachFn(tp.Protocol):
13+
class EachFn(pypeln_utils.Protocol):
1414
def __call__(self, A, **kwargs):
1515
...
1616

‎pypeln/process/api/filter.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
from ..worker import ProcessFn, Worker, ApplyProcess
1111

1212

13-
class FilterFn(tp.Protocol):
13+
class FilterFn(pypeln_utils.Protocol):
1414
def __call__(self, A, **kwargs) -> bool:
1515
...
1616

‎pypeln/process/api/flat_map.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
from ..worker import ProcessFn, Worker, ApplyProcess
1111

1212

13-
class FlatMapFn(tp.Protocol):
13+
class FlatMapFn(pypeln_utils.Protocol):
1414
def __call__(self, A, **kwargs) -> tp.Iterable[B]:
1515
...
1616

‎pypeln/process/api/from_iterable.py

-6
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,6 @@
88
from copy import copy
99

1010

11-
@tp.runtime_checkable
12-
class GeneratorFn(tp.Protocol):
13-
def __call__(self) -> tp.Union[tp.Iterable]:
14-
...
15-
16-
1711
class FromIterable(tp.NamedTuple):
1812
iterable: tp.Iterable
1913

‎pypeln/process/api/map.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
from ..worker import ProcessFn, Worker, ApplyProcess
1111

1212

13-
class MapFn(tp.Protocol):
13+
class MapFn(pypeln_utils.Protocol):
1414
def __call__(self, A, **kwargs) -> B:
1515
...
1616

‎pypeln/process/worker.py

+2-8
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@
77
import threading
88
import time
99
import typing as tp
10-
from typing import Protocol
1110

1211
import stopit
1312

@@ -21,16 +20,11 @@
2120
T = tp.TypeVar("T")
2221

2322

24-
class ProcessFn(tp.Protocol):
23+
class ProcessFn(pypeln_utils.Protocol):
2524
def __call__(self, worker: "Worker", **kwargs):
2625
...
2726

2827

29-
class ApplyFn(tp.Protocol):
30-
def __call__(self, worker: "Worker", elem: tp.Any, **kwargs):
31-
...
32-
33-
3428
class StageParams(tp.NamedTuple):
3529
input_queue: IterableQueue
3630
output_queues: OutputQueues
@@ -180,7 +174,7 @@ def measure_task_time(self):
180174
return self.MeasureTaskTime(self)
181175

182176

183-
class Applicable(tp.Protocol):
177+
class Applicable(pypeln_utils.Protocol):
184178
def apply(self, worker: "Worker", elem: tp.Any, **kwargs):
185179
...
186180

‎pypeln/sync/api/each.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
from dataclasses import dataclass
99

1010

11-
class EachFn(tp.Protocol):
11+
class EachFn(pypeln_utils.Protocol):
1212
def __call__(self, elem: A, **kwargs):
1313
...
1414

‎pypeln/sync/api/filter.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
from .to_stage import to_stage
77

88

9-
class FilterFn(tp.Protocol):
9+
class FilterFn(pypeln_utils.Protocol):
1010
def __call__(self, A, **kwargs) -> bool:
1111
...
1212

‎pypeln/sync/api/flat_map.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
from .to_stage import to_stage
77

88

9-
class FlatMapFn(tp.Protocol):
9+
class FlatMapFn(pypeln_utils.Protocol):
1010
def __call__(self, A, **kwargs) -> tp.Iterable[B]:
1111
...
1212

‎pypeln/sync/api/map.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
from .to_stage import to_stage
99

1010

11-
class MapFn(tp.Protocol):
11+
class MapFn(pypeln_utils.Protocol):
1212
def __call__(self, A, **kwargs) -> B:
1313
...
1414

‎pypeln/sync/stage.py

+2-7
Original file line numberDiff line numberDiff line change
@@ -19,16 +19,11 @@ class WorkerInfo(tp.NamedTuple):
1919
index: int
2020

2121

22-
class ProcessFn(tp.Protocol):
22+
class ProcessFn(pypeln_utils.Protocol):
2323
def __call__(self, worker: "Stage", **kwargs) -> tp.Iterable:
2424
...
2525

2626

27-
class ApplyFn(tp.Protocol):
28-
def __call__(self, worker: "Stage", elem: tp.Any, **kwargs) -> tp.Iterable:
29-
...
30-
31-
3227
@dataclass
3328
class Stage(pypeln_utils.BaseStage[T], tp.Iterable[T]):
3429
process_fn: ProcessFn
@@ -105,7 +100,7 @@ def to_iterable(self, maxsize, return_index) -> tp.Iterable:
105100
yield elem.value
106101

107102

108-
class Applicable(tp.Protocol):
103+
class Applicable(pypeln_utils.Protocol):
109104
def apply(self, worker: Stage, elem: tp.Any, **kwargs) -> tp.Iterable:
110105
...
111106

‎pypeln/task/__init__.py

+2
Original file line numberDiff line numberDiff line change
@@ -78,3 +78,5 @@ async def main()
7878
from .supervisor import Supervisor
7979
from .utils import Namespace, run_coroutine_in_loop, run_function_in_loop
8080
from .worker import StageParams, TaskPool, Worker, WorkerInfo, start_workers
81+
from . import utils
82+

‎pypeln/task/api/concat_task_test.py

+4-6
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import sys
12
import time
23
import typing as tp
34
from unittest import TestCase
@@ -6,12 +7,12 @@
67
from hypothesis import strategies as st
78

89
import pypeln as pl
9-
from pypeln.task.utils import run_test_async
1010

1111
MAX_EXAMPLES = 10
1212
T = tp.TypeVar("T")
1313

1414

15+
1516
@hp.given(nums=st.lists(st.integers()))
1617
@hp.settings(max_examples=MAX_EXAMPLES)
1718
def test_concat_basic(nums: tp.List[int]):
@@ -28,10 +29,9 @@ def test_concat_basic(nums: tp.List[int]):
2829

2930
assert sorted(nums_pl) == sorted(nums_py)
3031

31-
3232
@hp.given(nums=st.lists(st.integers()))
3333
@hp.settings(max_examples=MAX_EXAMPLES)
34-
@run_test_async
34+
@pl.task.utils.run_test_async
3535
async def test_concat_basic_2(nums: tp.List[int]):
3636

3737
nums_py = list(map(lambda x: x + 1, nums))
@@ -46,7 +46,6 @@ async def test_concat_basic_2(nums: tp.List[int]):
4646

4747
assert sorted(nums_pl) == sorted(nums_py)
4848

49-
5049
# @hp.given(nums=st.lists(st.integers()))
5150
# @hp.settings(max_examples=MAX_EXAMPLES)
5251
def test_concat_multiple(nums: tp.List[int] = [1, 2, 3]):
@@ -62,8 +61,7 @@ def test_concat_multiple(nums: tp.List[int] = [1, 2, 3]):
6261
# assert sorted(nums_py1) == sorted(list(nums_pl1))
6362
assert sorted(nums_py2) == sorted(list(nums_pl2))
6463

65-
66-
@run_test_async
64+
@pl.task.utils.run_test_async
6765
async def test_concat_multiple_2(nums: tp.List[int] = [1, 2, 3]):
6866

6967
nums_py = [x + 1 for x in nums]

‎pypeln/task/api/each.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
from ..worker import ProcessFn, Worker, ApplyProcess
1111

1212

13-
class EachFn(tp.Protocol):
13+
class EachFn(pypeln_utils.Protocol):
1414
def __call__(self, A, **kwargs) -> tp.Union[None, tp.Awaitable[None]]:
1515
...
1616

‎pypeln/task/api/each_task_test.py

+7-5
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,18 @@
1+
import sys
2+
import time
13
import typing as tp
24
from unittest import TestCase
35

46
import hypothesis as hp
57
from hypothesis import strategies as st
8+
69
import pypeln as pl
7-
from pypeln.task.utils import run_test_async
8-
import time
910

1011
MAX_EXAMPLES = 10
1112
T = tp.TypeVar("T")
1213

1314

15+
1416
class TestEach(TestCase):
1517
@hp.given(nums=st.lists(st.integers()))
1618
@hp.settings(max_examples=MAX_EXAMPLES)
@@ -52,7 +54,7 @@ def test_each_run(self, nums: tp.List[int]):
5254

5355
@hp.given(nums=st.lists(st.integers()))
5456
@hp.settings(max_examples=MAX_EXAMPLES)
55-
@run_test_async
57+
@pl.task.utils.run_test_async
5658
async def test_each_list_2(self, nums: tp.List[int]):
5759

5860
nums_pl = pl.task.each(lambda x: x, nums)
@@ -72,7 +74,7 @@ async def test_each_list_2(self, nums: tp.List[int]):
7274

7375
@hp.given(nums=st.lists(st.integers()))
7476
@hp.settings(max_examples=MAX_EXAMPLES)
75-
@run_test_async
77+
@pl.task.utils.run_test_async
7678
async def test_each_list_3(self, nums: tp.List[int]):
7779

7880
nums_pl = await pl.task.each(lambda x: x, nums)
@@ -81,7 +83,7 @@ async def test_each_list_3(self, nums: tp.List[int]):
8183

8284
@hp.given(nums=st.lists(st.integers()))
8385
@hp.settings(max_examples=MAX_EXAMPLES)
84-
@run_test_async
86+
@pl.task.utils.run_test_async
8587
async def test_each_list_4(self, nums: tp.List[int]):
8688

8789
nums_pl = await (pl.task.each(lambda x: x, nums))

‎pypeln/task/api/filter.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
from ..worker import ProcessFn, Worker, ApplyProcess
1111

1212

13-
class FilterFn(tp.Protocol):
13+
class FilterFn(pypeln_utils.Protocol):
1414
def __call__(self, A, **kwargs) -> tp.Union[bool, tp.Awaitable[bool]]:
1515
...
1616

‎pypeln/task/api/filter_task_test.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import sys
12
import typing as tp
23
import unittest
34

@@ -6,7 +7,6 @@
67
from hypothesis import strategies as st
78

89
import pypeln as pl
9-
from pypeln.task.utils import run_test_async
1010

1111
MAX_EXAMPLES = 10
1212
T = tp.TypeVar("T")
@@ -86,7 +86,7 @@ async def gt1(x):
8686

8787
@hp.given(nums=st.lists(st.integers()))
8888
@hp.settings(max_examples=MAX_EXAMPLES)
89-
@run_test_async
89+
@pl.task.utils.run_test_async
9090
async def test_flat_map_square_filter_workers_pipe_3(nums: tp.List[int]):
9191
def _generator(x):
9292
yield x

‎pypeln/task/api/flat_map.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
from ..worker import ProcessFn, Worker, ApplyProcess
1111

1212

13-
class FlatMapFn(tp.Protocol):
13+
class FlatMapFn(pypeln_utils.Protocol):
1414
def __call__(self, A, **kwargs) -> tp.Union[tp.Iterable[B], tp.AsyncIterable[B]]:
1515
...
1616

0 commit comments

Comments
 (0)