From a4abdc8a412752b169a74748daea5fc476de26ab Mon Sep 17 00:00:00 2001 From: Yuning Miao Date: Wed, 6 Nov 2019 15:29:05 -0600 Subject: [PATCH 01/11] fixing "can't compare offset-naive and offset-aware datetimes" error on pub_date variable by removing timezone information --- plugins/feeds/public/cybercrimetracker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/feeds/public/cybercrimetracker.py b/plugins/feeds/public/cybercrimetracker.py index 67014767d..22a8eff83 100644 --- a/plugins/feeds/public/cybercrimetracker.py +++ b/plugins/feeds/public/cybercrimetracker.py @@ -25,9 +25,9 @@ def update(self): for item in self.update_xml( 'item', ["title", "link", "pubDate", "description"]): - pub_date = parser.parse(item['pubDate']) + pub_date = parser.parse(item['pubDate']).timestamp() if self.last_run is not None: - if since_last_run > pub_date: + if since_last_run > pub_date.replace(tzinfo=None): continue self.analyze(item, pub_date) From 8b9e440f82e9ba52f1abe118c41ee7290a223b63 Mon Sep 17 00:00:00 2001 From: "yuning.miao@gmail.com" <19910412YuningM> Date: Wed, 6 Nov 2019 17:22:03 -0600 Subject: [PATCH 02/11] Fixing "can't compare offset-naive and offset-aware datetimes" issue of pub_date variable by removing tzinfo --- plugins/feeds/public/cybercrimetracker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/feeds/public/cybercrimetracker.py b/plugins/feeds/public/cybercrimetracker.py index 67014767d..46dd375e7 100644 --- a/plugins/feeds/public/cybercrimetracker.py +++ b/plugins/feeds/public/cybercrimetracker.py @@ -27,7 +27,7 @@ def update(self): pub_date = parser.parse(item['pubDate']) if self.last_run is not None: - if since_last_run > pub_date: + if since_last_run > pub_date.replace(tzinfo=None): continue self.analyze(item, pub_date) From 0841071518beaacca99ee8b3161ce126639c0a24 Mon Sep 17 00:00:00 2001 From: Yuning Miao Date: Wed, 6 Nov 2019 15:29:05 -0600 Subject: [PATCH 03/11] fixing "can't compare offset-naive and offset-aware datetimes" error on pub_date variable by removing timezone information --- plugins/feeds/public/cybercrimetracker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/feeds/public/cybercrimetracker.py b/plugins/feeds/public/cybercrimetracker.py index 46dd375e7..22a8eff83 100644 --- a/plugins/feeds/public/cybercrimetracker.py +++ b/plugins/feeds/public/cybercrimetracker.py @@ -25,7 +25,7 @@ def update(self): for item in self.update_xml( 'item', ["title", "link", "pubDate", "description"]): - pub_date = parser.parse(item['pubDate']) + pub_date = parser.parse(item['pubDate']).timestamp() if self.last_run is not None: if since_last_run > pub_date.replace(tzinfo=None): continue From 5364ee686923e370b625638505e83f32cbc2f795 Mon Sep 17 00:00:00 2001 From: "yuning.miao@gmail.com" <19910412YuningM> Date: Wed, 6 Nov 2019 17:22:03 -0600 Subject: [PATCH 04/11] Fixing "can't compare offset-naive and offset-aware datetimes" issue of pub_date variable by removing tzinfo --- plugins/feeds/public/cybercrimetracker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/feeds/public/cybercrimetracker.py b/plugins/feeds/public/cybercrimetracker.py index 67014767d..46dd375e7 100644 --- a/plugins/feeds/public/cybercrimetracker.py +++ b/plugins/feeds/public/cybercrimetracker.py @@ -27,7 +27,7 @@ def update(self): pub_date = parser.parse(item['pubDate']) if self.last_run is not None: - if since_last_run > pub_date: + if since_last_run > pub_date.replace(tzinfo=None): continue self.analyze(item, pub_date) From cd07ebc7d1add03e1a7e0bba5085a10a9406b832 Mon Sep 17 00:00:00 2001 From: Yuning Miao Date: Wed, 6 Nov 2019 15:29:05 -0600 Subject: [PATCH 05/11] fixing "can't compare offset-naive and offset-aware datetimes" error on pub_date variable by removing timezone information --- plugins/feeds/public/cybercrimetracker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/feeds/public/cybercrimetracker.py b/plugins/feeds/public/cybercrimetracker.py index 46dd375e7..22a8eff83 100644 --- a/plugins/feeds/public/cybercrimetracker.py +++ b/plugins/feeds/public/cybercrimetracker.py @@ -25,7 +25,7 @@ def update(self): for item in self.update_xml( 'item', ["title", "link", "pubDate", "description"]): - pub_date = parser.parse(item['pubDate']) + pub_date = parser.parse(item['pubDate']).timestamp() if self.last_run is not None: if since_last_run > pub_date.replace(tzinfo=None): continue From bdacd945d91370daf055ebc2fa5cab69f921cc2a Mon Sep 17 00:00:00 2001 From: "yuning.miao@gmail.com" <19910412YuningM> Date: Wed, 6 Nov 2019 17:22:03 -0600 Subject: [PATCH 06/11] Fixing "can't compare offset-naive and offset-aware datetimes" issue of pub_date variable by removing tzinfo --- plugins/feeds/public/cybercrimetracker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/feeds/public/cybercrimetracker.py b/plugins/feeds/public/cybercrimetracker.py index 67014767d..46dd375e7 100644 --- a/plugins/feeds/public/cybercrimetracker.py +++ b/plugins/feeds/public/cybercrimetracker.py @@ -27,7 +27,7 @@ def update(self): pub_date = parser.parse(item['pubDate']) if self.last_run is not None: - if since_last_run > pub_date: + if since_last_run > pub_date.replace(tzinfo=None): continue self.analyze(item, pub_date) From 920681b226ece7d9c566a0b2a0a9d002a24c294e Mon Sep 17 00:00:00 2001 From: Yuning Miao Date: Wed, 6 Nov 2019 15:29:05 -0600 Subject: [PATCH 07/11] fixing "can't compare offset-naive and offset-aware datetimes" error on pub_date variable by removing timezone information --- plugins/feeds/public/cybercrimetracker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/feeds/public/cybercrimetracker.py b/plugins/feeds/public/cybercrimetracker.py index 46dd375e7..22a8eff83 100644 --- a/plugins/feeds/public/cybercrimetracker.py +++ b/plugins/feeds/public/cybercrimetracker.py @@ -25,7 +25,7 @@ def update(self): for item in self.update_xml( 'item', ["title", "link", "pubDate", "description"]): - pub_date = parser.parse(item['pubDate']) + pub_date = parser.parse(item['pubDate']).timestamp() if self.last_run is not None: if since_last_run > pub_date.replace(tzinfo=None): continue From eddb57a2626cd2c5bd77ddb0b6340c5f096905d9 Mon Sep 17 00:00:00 2001 From: Yuning Miao Date: Wed, 6 Nov 2019 18:47:18 -0600 Subject: [PATCH 08/11] bugfix --- plugins/feeds/public/cybercrimetracker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/feeds/public/cybercrimetracker.py b/plugins/feeds/public/cybercrimetracker.py index 22a8eff83..46dd375e7 100644 --- a/plugins/feeds/public/cybercrimetracker.py +++ b/plugins/feeds/public/cybercrimetracker.py @@ -25,7 +25,7 @@ def update(self): for item in self.update_xml( 'item', ["title", "link", "pubDate", "description"]): - pub_date = parser.parse(item['pubDate']).timestamp() + pub_date = parser.parse(item['pubDate']) if self.last_run is not None: if since_last_run > pub_date.replace(tzinfo=None): continue From 9653fc62e19e6098dec0255b8552e377b20cb372 Mon Sep 17 00:00:00 2001 From: Yuning Miao Date: Thu, 7 Nov 2019 17:03:40 -0600 Subject: [PATCH 09/11] bugfix to handle date in the following format: 04/June/2019 --- plugins/feeds/public/malwaremustdiecncs.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/plugins/feeds/public/malwaremustdiecncs.py b/plugins/feeds/public/malwaremustdiecncs.py index 955c8d767..7a1083b5a 100644 --- a/plugins/feeds/public/malwaremustdiecncs.py +++ b/plugins/feeds/public/malwaremustdiecncs.py @@ -20,9 +20,16 @@ def update(self): for item in self.update_xml('item', ["title", "description"]): if self.last_run is not None: - if datetime.strptime(item["description"], "%d/%b/%Y") < since_last_run: - continue - + try: + if datetime.strptime(item["description"], "%d/%b/%Y") < since_last_run: + continue + except ValueError: + pass + try: + if datetime.strptime(item["description"], "%d/%B/%Y") < since_last_run: + continue + except all: + raise ValueError("couldn't parse date") self.analyze(item["title"]) def analyze(self, cnc): From 1d91e73c1cb4e2805238ef037dcd034a95859611 Mon Sep 17 00:00:00 2001 From: Yuning Miao Date: Fri, 8 Nov 2019 09:08:52 -0600 Subject: [PATCH 10/11] change exception type --- plugins/feeds/public/malwaremustdiecncs.py | 12 ++++++++++-- yeti.conf.sample | 4 ++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/plugins/feeds/public/malwaremustdiecncs.py b/plugins/feeds/public/malwaremustdiecncs.py index 955c8d767..b08be0913 100644 --- a/plugins/feeds/public/malwaremustdiecncs.py +++ b/plugins/feeds/public/malwaremustdiecncs.py @@ -20,8 +20,16 @@ def update(self): for item in self.update_xml('item', ["title", "description"]): if self.last_run is not None: - if datetime.strptime(item["description"], "%d/%b/%Y") < since_last_run: - continue + try: + if datetime.strptime(item["description"], "%d/%b/%Y") < since_last_run: + continue + except ValueError: + pass + try: + if datetime.strptime(item["description"], "%d/%B/%Y") < since_last_run: + continue + except ValueError: + raise ValueError("Couldn't parse date") self.analyze(item["title"]) diff --git a/yeti.conf.sample b/yeti.conf.sample index 09aa8b97b..a41b80494 100644 --- a/yeti.conf.sample +++ b/yeti.conf.sample @@ -82,5 +82,5 @@ # token = [otx] -#otx_key = YourOTXKey -#number_page = 1 \ No newline at end of file +# otx_key = YourOTXKey +# number_page = 1 \ No newline at end of file From 4cc52fc51c76fd44124d885c6eefa40ce3b2dc9c Mon Sep 17 00:00:00 2001 From: Yuning Miao Date: Fri, 8 Nov 2019 09:13:05 -0600 Subject: [PATCH 11/11] change exception type --- plugins/feeds/public/malwaremustdiecncs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/feeds/public/malwaremustdiecncs.py b/plugins/feeds/public/malwaremustdiecncs.py index 7a1083b5a..ca8969f9d 100644 --- a/plugins/feeds/public/malwaremustdiecncs.py +++ b/plugins/feeds/public/malwaremustdiecncs.py @@ -28,7 +28,7 @@ def update(self): try: if datetime.strptime(item["description"], "%d/%B/%Y") < since_last_run: continue - except all: + except ValueError: raise ValueError("couldn't parse date") self.analyze(item["title"])