From 777252f703bba98133501b122600169cc0959ab7 Mon Sep 17 00:00:00 2001 From: George Vauter Date: Tue, 27 Dec 2016 21:45:25 -0500 Subject: [PATCH 1/4] adding basic paginator --- cymon/cymon.py | 53 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/cymon/cymon.py b/cymon/cymon.py index a204687..9e0cd8a 100644 --- a/cymon/cymon.py +++ b/cymon/cymon.py @@ -2,6 +2,7 @@ import requests from urllib import quote_plus + class Cymon(object): def __init__(self, auth_token=None, endpoint='https://cymon.io/api/nexus/v1'): @@ -24,6 +25,25 @@ def post(self, method, params, headers=None): r.raise_for_status() return r + def get_paginator(self, method): + """ + Returns a Paginator class to use for handling API pagination. + """ + method = method.lower() + if self._can_paginate(method): + return Paginator(self, method) + else: + raise NoPaginatorError('Cannot paginate {} method'.format(method)) + + def _can_paginate(self, method): + """ + Basic check to raise exception when method cannot paginate. + """ + if method in ['ip_events', 'ip_events', 'ip_urls', 'ip_blacklist']: + return True + else: + return False + def ip_lookup(self, ip_addr): r = self.get('/ip/' + ip_addr) return json.loads(r.text) @@ -36,7 +56,7 @@ def ip_domains(self, ip_addr): r = self.get('/ip/' + ip_addr + '/domains') return json.loads(r.text) - def ip_urls(self, ip_addr): + def ip_urls(self, ip_addr): r = self.get('/ip/' + ip_addr + '/urls') return json.loads(r.text) @@ -57,3 +77,34 @@ def domain_blacklist(self, tag, days=1, limit=10, offset=0): ''' supported tags: malware, botnet, spam, phishing, dnsbl, blacklist ''' r = self.get('/blacklist/domain/' + tag + '/?days=%d&limit=%d&offset=%d' %(days,limit,offset)) return json.loads(r.text) + + +class Paginator(object): + """ + This class uses generators to provide an iterable object for performing + recusive API calls when a result has been paginated. + """ + def __init__(self, cymon, method): + self.cymon = cymon + self.method = method + + def paginate(self, *args, **kwargs): + """ + Use Cymon client object to make recursive API calls when + result is paginated. + """ + method_to_call = getattr(self.cymon, self.method) + result = method_to_call(*args, **kwargs) + if result['next'] is not None: + has_next = True + yield result['results'] # intial API call to start recursion + while has_next: + resp = requests.get(result['next']) + result = json.loads(resp.text) + if result['next'] is None: + has_next = False + yield result['results'] + + +class NoPaginatorError(Exception): + pass From 9199924808f23ee00057d80105efc32c3e1a44cf Mon Sep 17 00:00:00 2001 From: George Vauter Date: Tue, 27 Dec 2016 22:32:46 -0500 Subject: [PATCH 2/4] bump limit to 100 unless specified by caller --- cymon/cymon.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cymon/cymon.py b/cymon/cymon.py index 9e0cd8a..9ea95c4 100644 --- a/cymon/cymon.py +++ b/cymon/cymon.py @@ -39,7 +39,7 @@ def _can_paginate(self, method): """ Basic check to raise exception when method cannot paginate. """ - if method in ['ip_events', 'ip_events', 'ip_urls', 'ip_blacklist']: + if method in ['ip_blacklist', 'domain_blacklist']: return True else: return False @@ -93,11 +93,14 @@ def paginate(self, *args, **kwargs): Use Cymon client object to make recursive API calls when result is paginated. """ + has_next = False method_to_call = getattr(self.cymon, self.method) - result = method_to_call(*args, **kwargs) + result = method_to_call(limit=100, *args, **kwargs) if result['next'] is not None: + print result['next'] has_next = True yield result['results'] # intial API call to start recursion + while has_next: resp = requests.get(result['next']) result = json.loads(resp.text) From 41c8b72f7c4d2a9de75a1e4283f403f658e17841 Mon Sep 17 00:00:00 2001 From: George Vauter Date: Wed, 28 Dec 2016 08:47:34 -0500 Subject: [PATCH 3/4] adding paginator info to readme --- README.md | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 61a05bb..8bae0fb 100644 --- a/README.md +++ b/README.md @@ -66,4 +66,24 @@ api.ip_domains('185.27.134.165') + ip_blacklist() + domain_lookup() + domain_blacklist() -+ url_lookup() \ No newline at end of file ++ url_lookup() + +## Paginators + +Paginators can be used to abstract the process of iterating over truncated API results. The get_paginator() method takes the method name (see Available methods above) and returns a Paginator object. + +The paginate() method of the Paginator takes in any arguments accepted by the Cymon method and returns an iterator. + +Example: + +``` +from cymon import Cymon + +cymon = Cymon() +paginator = cymon.get_paginator('ip_blacklist') + +pages = paginator.paginate('botnet', days=2) +for page in pages: + for item in page: + print item['addr'] +``` From da441544e2d36768689ed22e042f4848dc3741d5 Mon Sep 17 00:00:00 2001 From: George Vauter Date: Tue, 3 Jan 2017 08:25:57 -0500 Subject: [PATCH 4/4] clean up print --- cymon/cymon.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cymon/cymon.py b/cymon/cymon.py index 9ea95c4..499dd8e 100644 --- a/cymon/cymon.py +++ b/cymon/cymon.py @@ -97,7 +97,6 @@ def paginate(self, *args, **kwargs): method_to_call = getattr(self.cymon, self.method) result = method_to_call(limit=100, *args, **kwargs) if result['next'] is not None: - print result['next'] has_next = True yield result['results'] # intial API call to start recursion