From 1bce23d72e91c41a39ecc0364be305caaec3410d Mon Sep 17 00:00:00 2001 From: zjhsdtc Date: Sun, 23 Feb 2020 09:08:52 +0800 Subject: [PATCH 1/7] feat(client): implement create command --- README.md | 29 +++++++++++++++++++++++ pymarketstore/__init__.py | 2 +- pymarketstore/client.py | 48 +++++++++++++++++++++++++++++++++++++++ tests/test_client.py | 26 +++++++++++++++++++++ 4 files changed, 104 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 76f0f2c..ccfed3c 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,35 @@ Construct a client object with endpoint. ## Query +`pymkts.Client#create(tbk, datashapes, schema='Symbol/Timeframe/AttributeGroup', row_type="fixed")` + +You can create a new time bucket and build the datashapes using `pymkts.DataShapes`. + +```python +from pymarketstore import Client, DataShape, DataShapes + +cli = Client() + +o = DataShape(name='Open', typ='float64') +h = DataShape(name='High', typ='float64') +l = DataShape(name='Low', typ='float64') +c = DataShape(name='Close', typ='float64') +v = DataShape(name='Volume', typ='int64') +e = DataShape(name='Epoch', typ='int64') + +shapes = DataShapes() +shapes.add(o) +shapes.add(h) +shapes.add(l) +shapes.add(c) +shapes.add(v) +shapes.add(e) + +cli.create('TSLA/15Min/OHLCV', shapes) +``` + +## Query + `pymkts.Client#query(symbols, timeframe, attrgroup, start=None, end=None, limit=None, limit_from_start=False)` You can build parameters using `pymkts.Params`. diff --git a/pymarketstore/__init__.py b/pymarketstore/__init__.py index 5c00e20..1af9213 100644 --- a/pymarketstore/__init__.py +++ b/pymarketstore/__init__.py @@ -1,4 +1,4 @@ -from .client import Client, Params # noqa +from .client import Client, Params, DataShape, DataShapes # noqa # alias Param = Params # noqa diff --git a/pymarketstore/client.py b/pymarketstore/client.py index d11b9b6..d1a6687 100644 --- a/pymarketstore/client.py +++ b/pymarketstore/client.py @@ -72,6 +72,29 @@ def __repr__(self): return 'Params({})'.format(content) +class DataShape(object): + def __init__(self, name, typ): + self.name = name + self.typ = typ + + +class DataShapes(object): + def __init__(self): + self.shapes = dict() + + def add(self, shape): + """ + :type shape: DataShape + """ + self.shapes.setdefault(shape.typ, set()) + self.shapes[shape.typ].add(shape.name) + + def __str__(self): + rtn_strs = [] + return ':'.join(['%s/%s' % (','.join(sorted(self.shapes[typ])), typ) + for typ in sorted(self.shapes)]) + + class Client(object): def __init__(self, endpoint='http://localhost:5993/rpc'): @@ -89,6 +112,23 @@ def _request(self, method, **query): logger.exception(exc) raise + def create(self, + tbk, + datashapes, + schema='Symbol/Timeframe/AttributeGroup', + row_type="fixed"): + """ + :type datashapes: DataShapes + """ + tbk = '%s:%s' % (tbk, schema) + request = { + 'Key': tbk, + 'DataShapes': str(datashapes), + 'RowType': row_type + } + requests = {'Requests': [request]} + return self._request('DataService.Create', **requests) + def query(self, params): if not isiterable(params): params = [params] @@ -177,3 +217,11 @@ def stream(self): def __repr__(self): return 'Client("{}")'.format(self.endpoint) + + +if __name__ == '__main__': + param = Params('BTC', '1Min', 'OHLCV', limit=10) + cli = Client() + reply = cli.query(param) + print(reply) + diff --git a/tests/test_client.py b/tests/test_client.py index c2eb3a4..331116b 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -1,4 +1,5 @@ import pymarketstore as pymkts +from pymarketstore import DataShape, DataShapes from pymarketstore import jsonrpc import numpy as np try: @@ -21,6 +22,31 @@ def test_client_init(): assert isinstance(c.rpc, pymkts.client.MsgpackRpcClient) +@patch('pymarketstore.client.MsgpackRpcClient') +def test_create(MsgpackRpcClient): + cli = pymkts.Client() + + o = DataShape(name='Open', typ='float64') + h = DataShape(name='High', typ='float64') + l = DataShape(name='Low', typ='float64') + c = DataShape(name='Close', typ='float64') + v = DataShape(name='Volume', typ='int64') + e = DataShape(name='Epoch', typ='int64') + + shapes = DataShapes() + shapes.add(o) + shapes.add(h) + shapes.add(l) + shapes.add(c) + shapes.add(v) + shapes.add(e) + + assert str(shapes) == 'Close,High,Low,Open/float64:Epoch,Volume/int64' + + cli.create('TSLA/15Min/OHLCV', shapes) + assert MsgpackRpcClient().call.called == 1 + + @patch('pymarketstore.client.MsgpackRpcClient') def test_query(MsgpackRpcClient): c = pymkts.Client() From eb2d9e3803cce030450e76912cae3d42a3b93b71 Mon Sep 17 00:00:00 2001 From: zjhsdtc Date: Sun, 23 Feb 2020 11:55:53 +0800 Subject: [PATCH 2/7] improvement(client/query): add `columns` `is_sqlstatement` `sqlstatement` params to query cmd --- pymarketstore/client.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/pymarketstore/client.py b/pymarketstore/client.py index d1a6687..a49ddfd 100644 --- a/pymarketstore/client.py +++ b/pymarketstore/client.py @@ -41,13 +41,19 @@ def get_timestamp(value): class Params(object): - def __init__(self, symbols, timeframe, attrgroup, - start=None, end=None, - limit=None, limit_from_start=None): - if not isiterable(symbols): + def __init__(self, symbols=None, timeframe=None, attrgroup=None, + start=None, end=None, limit=None, limit_from_start=None, + columns=None, is_sqlstatement=None, sql_statement=None): + if symbols and not isiterable(symbols): symbols = [symbols] - self.tbk = ','.join(symbols) + "/" + timeframe + "/" + attrgroup + if symbols and timeframe and attrgroup: + self.tbk = ','.join(symbols) + "/" + timeframe + "/" + attrgroup + else: + self.tbk = None self.key_category = None # server default + self.is_sqlstatement = is_sqlstatement + self.sql_statement = sql_statement + self.columns = columns self.start = get_timestamp(start) self.end = get_timestamp(end) self.limit = limit @@ -185,6 +191,12 @@ def build_query(self, params): req['limit_from_start'] = bool(param.limit_from_start) if param.functions is not None: req['functions'] = param.functions + if param.columns is not None: + req['columns'] = param.columns + if param.is_sqlstatement is not None: + req['is_sqlstatement'] = param.is_sqlstatement + if param.sql_statement is not None: + req['sql_statement'] = param.sql_statement reqs.append(req) return { 'requests': reqs, From c4a50c95d4741007a183b664c674763aa8374012 Mon Sep 17 00:00:00 2001 From: zjhsdtc Date: Sun, 23 Feb 2020 19:42:48 +0800 Subject: [PATCH 3/7] fix(client/codec): remove deprecated encoding flag after `msgpack-python` 1.0, there is no `encoding` keyword anymore, https://github.com/msgpack/msgpack-python#major-breaking-changes-in-msgpack-10 --- pymarketstore/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pymarketstore/client.py b/pymarketstore/client.py index a49ddfd..d2814b7 100644 --- a/pymarketstore/client.py +++ b/pymarketstore/client.py @@ -112,7 +112,7 @@ def _request(self, method, **query): try: resp = self.rpc.call(method, **query) resp.raise_for_status() - rpc_reply = self.rpc.codec.loads(resp.content, encoding='utf-8') + rpc_reply = self.rpc.codec.loads(resp.content) return self.rpc.response(rpc_reply) except requests.exceptions.HTTPError as exc: logger.exception(exc) @@ -167,7 +167,7 @@ def write(self, recarray, tbk, isvariablelength=False): except requests.exceptions.ConnectionError: raise requests.exceptions.ConnectionError( "Could not contact server") - reply_obj = self.rpc.codec.loads(reply.content, encoding='utf-8') + reply_obj = self.rpc.codec.loads(reply.content) resp = self.rpc.response(reply_obj) return resp From 231bb3d1a4d142d7cf8d556a2577bb47190ffcb8 Mon Sep 17 00:00:00 2001 From: zjhsdtc Date: Sun, 23 Feb 2020 19:44:09 +0800 Subject: [PATCH 4/7] improvement(client/init): add option `codec` init parameter for `Client` --- pymarketstore/client.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pymarketstore/client.py b/pymarketstore/client.py index d2814b7..00da93d 100644 --- a/pymarketstore/client.py +++ b/pymarketstore/client.py @@ -103,9 +103,10 @@ def __str__(self): class Client(object): - def __init__(self, endpoint='http://localhost:5993/rpc'): + def __init__(self, endpoint='http://localhost:5993/rpc', codec='msgpack'): self.endpoint = endpoint - rpc_client = get_rpc_client('msgpack') + self.codec = codec + rpc_client = get_rpc_client(codec) self.rpc = rpc_client(self.endpoint) def _request(self, method, **query): @@ -143,6 +144,10 @@ def query(self, params): return QueryReply(reply) def write(self, recarray, tbk, isvariablelength=False): + if self.codec != 'msgpack': + print("Write action only support for msgpack codec") + return + data = {} data['types'] = [ recarray.dtype[name].str.replace('<', '') From a09045f525b3302c191be4fbcbda03eb97f15ea0 Mon Sep 17 00:00:00 2001 From: zjhsdtc Date: Sun, 23 Feb 2020 22:56:14 +0800 Subject: [PATCH 5/7] feat(client): add sql query --- README.md | 6 ++++++ pymarketstore/client.py | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/README.md b/README.md index ccfed3c..318dfc3 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,12 @@ You can build parameters using `pymkts.Params`. Pass one or multiple instances of `Params` to `Client.query()`. It will return `QueryReply` object which holds internal numpy array data returned from the server. +## Sql + +`pymkts.Client#sql(statement)` + +You can query with raw SQL statements. + ## Write `pymkts.Client#write(data, tbk)` diff --git a/pymarketstore/client.py b/pymarketstore/client.py index 00da93d..a365700 100644 --- a/pymarketstore/client.py +++ b/pymarketstore/client.py @@ -143,6 +143,10 @@ def query(self, params): reply = self._request('DataService.Query', **query) return QueryReply(reply) + def sql(self, statement): + params = Params(is_sqlstatement=True, sql_statement=statement) + return self.query(params) + def write(self, recarray, tbk, isvariablelength=False): if self.codec != 'msgpack': print("Write action only support for msgpack codec") From 75463a44c120e259d279aac26efb9a8213ae4729 Mon Sep 17 00:00:00 2001 From: zjhsdtc Date: Sun, 23 Feb 2020 22:59:31 +0800 Subject: [PATCH 6/7] chore(examples): add create and write example --- examples/create_write.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 examples/create_write.py diff --git a/examples/create_write.py b/examples/create_write.py new file mode 100644 index 0000000..f736841 --- /dev/null +++ b/examples/create_write.py @@ -0,0 +1,27 @@ +import time +import numpy as np +from pymarketstore import Client, DataShape, DataShapes + +cli = Client() + +o = DataShape(name='Open', typ='float64') +h = DataShape(name='High', typ='float64') +l = DataShape(name='Low', typ='float64') +c = DataShape(name='Close', typ='float64') +v = DataShape(name='Volume', typ='int64') +e = DataShape(name='Epoch', typ='int64') + +shapes = DataShapes() +shapes.add(o) +shapes.add(h) +shapes.add(l) +shapes.add(c) +shapes.add(v) +shapes.add(e) + +cli.create('TSLA/15Min/OHLCV', shapes) + +data = np.array([(int(time.time()), 0, 1, 1, 1, 1)], + dtype=[('Epoch', 'i8'), ('Open', 'f8'), ('Close', 'f8'), + ('High', 'f8'), ('Low', 'f8'), ('Volume', 'i8')]) +cli.write(data, 'TSLA/15Min/OHLCV') From 57f9a9f813e742ad74dcb4eb04b291411ce9a97d Mon Sep 17 00:00:00 2001 From: zjhsdtc Date: Sun, 23 Feb 2020 23:38:23 +0800 Subject: [PATCH 7/7] ci(travis): add travis ci --- .travis.yml | 15 +++++++++++++++ README.md | 4 +++- 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..6e63d51 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,15 @@ +os: + - linux + +language: python +cache: pip + +python: + - '2.7.13' + - '3.6.1' + +install: + pip install -r requirements.txt + +script: + python setup.py test diff --git a/README.md b/README.md index 318dfc3..f06b734 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,9 @@ # pymarketstore Python driver for MarketStore -Build Status: ![build status](https://circleci.com/gh/alpacahq/pymarketstore/tree/master.png?971fa5b1079e8af0568db6caf772132c54f04dc2) +Build Status: +![circleci.com](https://circleci.com/gh/alpacahq/pymarketstore/tree/master.png?971fa5b1079e8af0568db6caf772132c54f04dc2) +[![travis-ci.org](https://travis-ci.org/alpacahq/pymarketstore.svg)](https://travis-ci.org/alpacahq/pymarketstore) Pymarketstore can query and write financial timeseries data from [MarketStore](https://github.com/alpacahq/marketstore)