From 1a159c70fbd9a92d4f420781ae903b9fe6cc2b15 Mon Sep 17 00:00:00 2001 From: waza Date: Wed, 14 Oct 2015 19:36:17 +0700 Subject: [PATCH 1/4] WIP QE FIX --- tests/test_commands.py | 1 + tests/test_quoteengine.py | 87 +++++++++++++++++++++++++++++++++++++++ tululbot/commands.py | 2 +- tululbot/utils.py | 47 +++++++++++++++++++-- 4 files changed, 133 insertions(+), 4 deletions(-) create mode 100644 tests/test_quoteengine.py diff --git a/tests/test_commands.py b/tests/test_commands.py index 6bafb4f..5710c11 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -124,6 +124,7 @@ def test_with_multiword_term(self, mock_commands): def test_quote(): with patch('tululbot.commands.quote_engine') as mock_engine: + mock_engine.refresh_cache_if_applicable.return_value = False mock_engine.retrieve_random.return_value = 'some random quote' rv = quote() assert rv == 'some random quote' diff --git a/tests/test_quoteengine.py b/tests/test_quoteengine.py new file mode 100644 index 0000000..7382f39 --- /dev/null +++ b/tests/test_quoteengine.py @@ -0,0 +1,87 @@ +import json +from unittest.mock import patch + +import pytest + +from tululbot.utils import QuoteEngine + +generic_quote_1 = ( + '--- ' + 'quotes: ' + ' - q_no: 1' + ' quote: "Kenapa kalo mau sahur tiba2 ngantuk? Karena Anda belum punya cinta yang menemani Anda sahur."' + ' author: Anang Ferdi' + ' author_bio: Dokter cinta veteran' + ' tags: ' + ' - cinta' + ' - sahur' + ' - ramadhan' +) + +generic_quote_2 = ( + '---' + 'quotes:' + ' - q_no: 3' + ' quote: "Cinta itu bisa dibagi dengan 0 ga sih?"' + ' author: Adrian Nuradiansyah' + ' author_bio: Awesome Akhi' + ' tags: ' + ' - cinta' + ' - geek' + ' - matematika ' +) + +@pytest.fixture +def qe(): + return QuoteEngine() + +def test_refresh_time(qe): + with patch('tululbot.tils.datetime.datetime') as mock_dt: + with patch('tululbot.utils.requests') as mock_requests: + class FakeResponse: + pass + + response1 = FakeResponse() + response1.status_code = 200 + response1.text = ( + '{' + ' "commit": {' + ' "sha": "beefbeef"' + ' }' + '}' + ) + response1.ok = True + + response2 = FakeResponse() + response2.status_code = 200 + response2.text = ( + '' + '
' + '

Snowden is former CIA employee.

' + '
' + '' + ) + + mock_requests.get.side_effect = { + qe._git_branch_check_url: response1 + } + + qe.refresh_interval = 5 + + mock_dt.now.return_value = datetime(2009, 1, 6, 15, 8, 30) + qe._last_refresh = datetime(2000, 1, 6, 15, 8, 24) + + result = qe.refresh_url_if_applicable + assert result == True + result = qe.refresh_url_if_applicable + assert result == False + result = qe.refresh_url_if_applicable + assert result == False + + qe._last_refresh = datetime(2000, 1, 6, 15, 8, 24) + + result = qe.refresh_url_if_applicable + assert result == True + result = qe.refresh_url_if_applicable + assert result == False + diff --git a/tululbot/commands.py b/tululbot/commands.py index aacd376..30d8651 100644 --- a/tululbot/commands.py +++ b/tululbot/commands.py @@ -24,7 +24,6 @@ dispatcher = CommandDispatcher() quote_engine = QuoteEngine() -quote_engine.refresh_cache() @dispatcher.command(r'^/leli (?P.+)$') @@ -95,6 +94,7 @@ def search_on_google(): @dispatcher.command(r'^/quote$') def quote(): + quote_engine.refresh_cache_if_applicable() return quote_engine.retrieve_random() diff --git a/tululbot/utils.py b/tululbot/utils.py index 57bcfc8..192c94f 100644 --- a/tululbot/utils.py +++ b/tululbot/utils.py @@ -1,3 +1,4 @@ +import datetime import random import requests @@ -7,12 +8,24 @@ class QuoteEngine: def __init__(self): - self._quote_url = 'https://cdn.rawgit.com/tulul/tulul-quotes/master/quote.yaml' # noqa # Note: rawgit does not have 100% uptime, but at # least they're not throttling us. + self._quote_url = 'https://cdn.rawgit.com/tulul/tulul-quotes/master/quote.yaml' # noqa + + # Why not using rawgit's development endpoint? Because + # they can't promise 100% uptime on the development endpoint. + # Meanwhile, production endpoint's uptime is better because + # it is served by MaxCDN + self._git_branch_check_url = 'https://api.github.com/repos/tulul/tulul-quotes/branches/master' # noqa self._cache = [] + # URI refresh per interval + self._refresh_interval = 5 * 60 + # Dummy date. Must be old enough (just to trigger) + # the uri must be refreshed on the first call + self._last_refresh = datetime(2009, 1, 6, 15, 8, 24, 78915) + def retrieve_random(self): cache = self._cache return self.format_quote(random.choice(cache)) @@ -20,9 +33,37 @@ def retrieve_random(self): def format_quote(self, q): return '{q[quote]} - {q[author]}, {q[author_bio]}'.format(q=q) - def refresh_cache(self): + def refresh_cache_if_applicable(self): + if (!self.refresh_url_if_applicable()): + return False + body = requests.get(self._quote_url).text - # What if previosuly we have the cache, but this time + # What if previously we have the cache, but this time # when we try to get new cache, the network occurs error? # We will think about "don't refresh if error" later. self._cache = yaml.load(body)['quotes'] + + return True + + def refresh_url_if_applicable(self): + now = datetime.datetime.now() + delta = now - self._last_refresh + + if (delta.seconds < self._refresh_interval): + return False + + req = requests.get(self._git_branch_check_url) + + # Don't care broken request + if (!x.ok): + return False + + json = req.get_json() + sha = json['commit']['sha'] + self._quote_url = 'https://cdn.rawgit.com/tulul/tulul-quotes/%s/quote.yaml'.format(sha) + + self._refresh_interval = now + + return True + + From 0d6dadf35cd08dc0e8de173aea08d1169252325e Mon Sep 17 00:00:00 2001 From: waza Date: Mon, 26 Oct 2015 21:13:43 +0700 Subject: [PATCH 2/4] Temporarily fix build. Not all aspect tested yet. --- tests/test_quoteengine.py | 45 ++++++++++++++++----------------------- tululbot/utils.py | 23 +++++++++++++------- 2 files changed, 33 insertions(+), 35 deletions(-) diff --git a/tests/test_quoteengine.py b/tests/test_quoteengine.py index 7382f39..56bfc0b 100644 --- a/tests/test_quoteengine.py +++ b/tests/test_quoteengine.py @@ -1,3 +1,4 @@ +from datetime import datetime import json from unittest.mock import patch @@ -36,52 +37,42 @@ def qe(): return QuoteEngine() def test_refresh_time(qe): - with patch('tululbot.tils.datetime.datetime') as mock_dt: + with patch('tululbot.utils.TimeHelper') as mock_dt: with patch('tululbot.utils.requests') as mock_requests: class FakeResponse: + def get_json(self): + return self.json + pass response1 = FakeResponse() response1.status_code = 200 - response1.text = ( - '{' - ' "commit": {' - ' "sha": "beefbeef"' - ' }' - '}' - ) - response1.ok = True + response1.json = { + "commit": { + "sha": "beefbeef" + } + } - response2 = FakeResponse() - response2.status_code = 200 - response2.text = ( - '' - '
' - '

Snowden is former CIA employee.

' - '
' - '' - ) + response1.ok = True - mock_requests.get.side_effect = { - qe._git_branch_check_url: response1 - } + mock_requests.get.return_value = response1 - qe.refresh_interval = 5 + qe._refresh_interval = 5 mock_dt.now.return_value = datetime(2009, 1, 6, 15, 8, 30) qe._last_refresh = datetime(2000, 1, 6, 15, 8, 24) - result = qe.refresh_url_if_applicable + result = qe.refresh_url_if_applicable() assert result == True - result = qe.refresh_url_if_applicable + result = qe.refresh_url_if_applicable() assert result == False - result = qe.refresh_url_if_applicable + result = qe.refresh_url_if_applicable() assert result == False qe._last_refresh = datetime(2000, 1, 6, 15, 8, 24) - result = qe.refresh_url_if_applicable + result = qe.refresh_url_if_applicable() assert result == True - result = qe.refresh_url_if_applicable + result = qe.refresh_url_if_applicable() assert result == False diff --git a/tululbot/utils.py b/tululbot/utils.py index 192c94f..061e146 100644 --- a/tululbot/utils.py +++ b/tululbot/utils.py @@ -1,10 +1,17 @@ -import datetime +from datetime import datetime import random import requests import yaml +class TimeHelper: + + @classmethod + def now(): + return datetime.datetime.now() + + class QuoteEngine: def __init__(self): @@ -20,7 +27,7 @@ def __init__(self): self._cache = [] - # URI refresh per interval + # URI refresh per interval in seconds self._refresh_interval = 5 * 60 # Dummy date. Must be old enough (just to trigger) # the uri must be refreshed on the first call @@ -34,7 +41,7 @@ def format_quote(self, q): return '{q[quote]} - {q[author]}, {q[author_bio]}'.format(q=q) def refresh_cache_if_applicable(self): - if (!self.refresh_url_if_applicable()): + if (self.refresh_url_if_applicable() == False): return False body = requests.get(self._quote_url).text @@ -46,23 +53,23 @@ def refresh_cache_if_applicable(self): return True def refresh_url_if_applicable(self): - now = datetime.datetime.now() + now = TimeHelper.now() delta = now - self._last_refresh if (delta.seconds < self._refresh_interval): return False - req = requests.get(self._git_branch_check_url) + res = requests.get(self._git_branch_check_url) # Don't care broken request - if (!x.ok): + if (res.ok == False): return False - json = req.get_json() + json = res.get_json() sha = json['commit']['sha'] self._quote_url = 'https://cdn.rawgit.com/tulul/tulul-quotes/%s/quote.yaml'.format(sha) - self._refresh_interval = now + self._last_refresh = now return True From dde4de033bd77b3da10091050c100d51cc34b002 Mon Sep 17 00:00:00 2001 From: waza Date: Mon, 26 Oct 2015 21:19:58 +0700 Subject: [PATCH 3/4] Fix lint --- tests/test_quoteengine.py | 20 ++++++++++---------- tululbot/utils.py | 6 ++---- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/tests/test_quoteengine.py b/tests/test_quoteengine.py index 56bfc0b..76da24c 100644 --- a/tests/test_quoteengine.py +++ b/tests/test_quoteengine.py @@ -1,5 +1,4 @@ from datetime import datetime -import json from unittest.mock import patch import pytest @@ -10,7 +9,7 @@ '--- ' 'quotes: ' ' - q_no: 1' - ' quote: "Kenapa kalo mau sahur tiba2 ngantuk? Karena Anda belum punya cinta yang menemani Anda sahur."' + ' quote: "Kenapa kalo mau sahur tiba2 ngantuk? Karena Anda belum punya cinta yang menemani Anda sahur."' # noqa ' author: Anang Ferdi' ' author_bio: Dokter cinta veteran' ' tags: ' @@ -32,10 +31,12 @@ ' - matematika ' ) + @pytest.fixture def qe(): return QuoteEngine() + def test_refresh_time(qe): with patch('tululbot.utils.TimeHelper') as mock_dt: with patch('tululbot.utils.requests') as mock_requests: @@ -49,9 +50,9 @@ def get_json(self): response1.status_code = 200 response1.json = { "commit": { - "sha": "beefbeef" - } + "sha": "beefbeef" } + } response1.ok = True @@ -63,16 +64,15 @@ def get_json(self): qe._last_refresh = datetime(2000, 1, 6, 15, 8, 24) result = qe.refresh_url_if_applicable() - assert result == True + assert result is True result = qe.refresh_url_if_applicable() - assert result == False + assert result is False result = qe.refresh_url_if_applicable() - assert result == False + assert result is False qe._last_refresh = datetime(2000, 1, 6, 15, 8, 24) result = qe.refresh_url_if_applicable() - assert result == True + assert result is True result = qe.refresh_url_if_applicable() - assert result == False - + assert result is False diff --git a/tululbot/utils.py b/tululbot/utils.py index 061e146..5f09a35 100644 --- a/tululbot/utils.py +++ b/tululbot/utils.py @@ -41,7 +41,7 @@ def format_quote(self, q): return '{q[quote]} - {q[author]}, {q[author_bio]}'.format(q=q) def refresh_cache_if_applicable(self): - if (self.refresh_url_if_applicable() == False): + if (not self.refresh_url_if_applicable()): return False body = requests.get(self._quote_url).text @@ -62,7 +62,7 @@ def refresh_url_if_applicable(self): res = requests.get(self._git_branch_check_url) # Don't care broken request - if (res.ok == False): + if (not res.ok): return False json = res.get_json() @@ -72,5 +72,3 @@ def refresh_url_if_applicable(self): self._last_refresh = now return True - - From b5ac901d71c88519703058fc64452bb951490412 Mon Sep 17 00:00:00 2001 From: waza Date: Sat, 31 Oct 2015 17:42:32 +0700 Subject: [PATCH 4/4] Test to cover all case. Finalize everything. --- tests/test_quoteengine.py | 108 +++++++++++++++++++++++++++----------- tululbot/utils.py | 2 +- 2 files changed, 78 insertions(+), 32 deletions(-) diff --git a/tests/test_quoteengine.py b/tests/test_quoteengine.py index 76da24c..a933816 100644 --- a/tests/test_quoteengine.py +++ b/tests/test_quoteengine.py @@ -1,35 +1,17 @@ from datetime import datetime from unittest.mock import patch +from unittest.mock import MagicMock import pytest from tululbot.utils import QuoteEngine -generic_quote_1 = ( - '--- ' - 'quotes: ' - ' - q_no: 1' - ' quote: "Kenapa kalo mau sahur tiba2 ngantuk? Karena Anda belum punya cinta yang menemani Anda sahur."' # noqa - ' author: Anang Ferdi' - ' author_bio: Dokter cinta veteran' - ' tags: ' - ' - cinta' - ' - sahur' - ' - ramadhan' -) - -generic_quote_2 = ( - '---' - 'quotes:' - ' - q_no: 3' - ' quote: "Cinta itu bisa dibagi dengan 0 ga sih?"' - ' author: Adrian Nuradiansyah' - ' author_bio: Awesome Akhi' - ' tags: ' - ' - cinta' - ' - geek' - ' - matematika ' -) + +class FakeResponse: + def get_json(self): + return self.json + + pass @pytest.fixture @@ -37,15 +19,13 @@ def qe(): return QuoteEngine() +def cdn_with_branch(branch): + return 'https://cdn.rawgit.com/tulul/tulul-quotes/{}/quote.yaml'.format(branch) # noqa + + def test_refresh_time(qe): with patch('tululbot.utils.TimeHelper') as mock_dt: with patch('tululbot.utils.requests') as mock_requests: - class FakeResponse: - def get_json(self): - return self.json - - pass - response1 = FakeResponse() response1.status_code = 200 response1.json = { @@ -63,6 +43,8 @@ def get_json(self): mock_dt.now.return_value = datetime(2009, 1, 6, 15, 8, 30) qe._last_refresh = datetime(2000, 1, 6, 15, 8, 24) + assert qe._quote_url == cdn_with_branch("master") + result = qe.refresh_url_if_applicable() assert result is True result = qe.refresh_url_if_applicable() @@ -70,9 +52,73 @@ def get_json(self): result = qe.refresh_url_if_applicable() assert result is False + assert qe._quote_url == cdn_with_branch("beefbeef") + qe._last_refresh = datetime(2000, 1, 6, 15, 8, 24) result = qe.refresh_url_if_applicable() assert result is True result = qe.refresh_url_if_applicable() assert result is False + + +def test_refresh_response(qe): + with patch('tululbot.utils.requests') as mock_requests: + + # Test: stay empty if no need to refresh + qe.refresh_url_if_applicable = MagicMock() + qe.refresh_url_if_applicable.return_value = False + + qe.refresh_cache_if_applicable() + + assert qe._cache == [] + + # Test: should refresh + qe.refresh_url_if_applicable.return_value = True + mock_requests.get.return_value = generic_quote_1 + qe.refresh_cache_if_applicable() + result = qe.retrieve_random() + assert result == 'Yuzu kok lucu banget sih - Waza, Pejuang' + + # Test: Remote content changed but no need to refresh + qe.refresh_url_if_applicable.return_value = False + mock_requests.get.return_value = generic_quote_2 + qe.refresh_cache_if_applicable() + result = qe.retrieve_random() + assert result == 'Yuzu kok lucu banget sih - Waza, Pejuang' + + # Test: Remote content changed and it does refresh + qe.refresh_url_if_applicable.return_value = True + mock_requests.get.return_value = generic_quote_2 + qe.refresh_cache_if_applicable() + result = qe.retrieve_random() + assert result == 'Miki Hoshii is the best girl - Waza, Rocket Builder' + + +generic_quote_1 = FakeResponse() +generic_quote_1.text = ''' +--- +quotes: \n + - q_no: 1 + quote: "Yuzu kok lucu banget sih" + author: Waza + author_bio: Pejuang + tags: + - cinta + - sahur + - ramadhan +''' + +generic_quote_2 = FakeResponse() +generic_quote_2.text = ''' +--- +quotes: + - q_no: 3 + quote: "Miki Hoshii is the best girl" + author: Waza + author_bio: Rocket Builder + tags: + - cinta + - geek + - matematika +''' diff --git a/tululbot/utils.py b/tululbot/utils.py index 5f09a35..4f15347 100644 --- a/tululbot/utils.py +++ b/tululbot/utils.py @@ -67,7 +67,7 @@ def refresh_url_if_applicable(self): json = res.get_json() sha = json['commit']['sha'] - self._quote_url = 'https://cdn.rawgit.com/tulul/tulul-quotes/%s/quote.yaml'.format(sha) + self._quote_url = 'https://cdn.rawgit.com/tulul/tulul-quotes/{}/quote.yaml'.format(sha) self._last_refresh = now