From dce490d861a49de9990a7d9736bc5e77ce099ec4 Mon Sep 17 00:00:00 2001 From: John Cook Date: Thu, 19 Apr 2012 00:33:20 -0400 Subject: [PATCH 1/7] Added optional support for urllib2 instead of httplib2 --- .gitignore | 2 +- embedly/client.py | 34 +++++++++++--- embedly/tests.py | 111 ++++++++++++++++++++++++++-------------------- 3 files changed, 92 insertions(+), 55 deletions(-) diff --git a/.gitignore b/.gitignore index c18dd39..c6789fc 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,4 @@ build/ dist/ .virtualenv .noseids - +.idea diff --git a/embedly/client.py b/embedly/client.py index a12c08f..8b3bba0 100644 --- a/embedly/client.py +++ b/embedly/client.py @@ -6,6 +6,7 @@ """ import re import urllib +import urllib2 import httplib2 try: @@ -22,7 +23,7 @@ class Embedly(object): Client """ - def __init__(self, key=None, user_agent=USER_AGENT): + def __init__(self, key=None, user_agent=USER_AGENT, use_urllib2=False): """ Initialize the Embedly client @@ -30,15 +31,39 @@ def __init__(self, key=None, user_agent=USER_AGENT): :type user_agent: str :param key: Embedly Pro key :type key: str + :param use_urllib2: Option to use urllib2 instead of httplib2 + :type use_urllib2: bool :returns: None """ self.user_agent = user_agent self.key = key + self.use_urllib2 = use_urllib2 self.services = [] self._regex = None + def _make_request(self, url, headers={}): + """ + Makes HTTP requests using httplib2 or urllib2 + """ + if self.use_urllib2: + try: + request = urllib2.Request(url, headers=headers) + response = urllib2.urlopen(request) + resp = response.headers.dict + if "status" not in resp: + resp["status"] = str(response.code) + content = response.read() + except urllib2.HTTPError, e: + resp = {"status" : str(e.getcode())} + content = e.read() + else: + http = httplib2.Http() + resp, content = http.request(url, headers=headers) + + return resp, content + def get_services(self): """ get_services makes call to services end point of api.embed.ly to fetch @@ -49,9 +74,8 @@ def get_services(self): url = 'http://api.embed.ly/1/services/python' - http = httplib2.Http() headers = {'User-Agent' : self.user_agent} - resp, content = http.request(url, headers=headers) + resp, content = self._make_request(url, headers) if resp['status'] == '200': resp_data = json.loads(content) @@ -118,11 +142,9 @@ def _get(self, version, method, url_or_urls, **kwargs): url = 'http://api.embed.ly/%s/%s?%s' % (version, method, query) - http = httplib2.Http() - headers = {'User-Agent' : self.user_agent} - resp, content = http.request(url, headers=headers) + resp, content = self._make_request(url, headers=headers) if resp['status'] == '200': data = json.loads(content) diff --git a/embedly/tests.py b/embedly/tests.py index f6bd93a..ca98a93 100644 --- a/embedly/tests.py +++ b/embedly/tests.py @@ -74,66 +74,81 @@ def test_model(self): self.assert_(obj.object.type is None) def test_provider(self): - http = Embedly(self.key) + def do_test(http): + obj = http.oembed('http://www.scribd.com/doc/13994900/Easter') + self.assert_(obj.provider_url == 'http://www.scribd.com/') - obj = http.oembed('http://www.scribd.com/doc/13994900/Easter') - self.assert_(obj.provider_url == 'http://www.scribd.com/') + obj = http.oembed('http://www.scribd.com/doc/28452730/Easter-Cards') + self.assert_(obj.provider_url == 'http://www.scribd.com/') - obj = http.oembed('http://www.scribd.com/doc/28452730/Easter-Cards') - self.assert_(obj.provider_url == 'http://www.scribd.com/') + obj = http.oembed('http://www.youtube.com/watch?v=Zk7dDekYej0') + self.assert_(obj.provider_url == 'http://www.youtube.com/') - obj = http.oembed('http://www.youtube.com/watch?v=Zk7dDekYej0') - self.assert_(obj.provider_url == 'http://www.youtube.com/') + obj = http.oembed('http://yfrog.com/h22eu4j') + self.assert_(obj.provider_url == 'http://yfrog.com') - obj = http.oembed('http://yfrog.com/h22eu4j') - self.assert_(obj.provider_url == 'http://yfrog.com') + http = Embedly(self.key, use_urllib2=False) + do_test(http) + http = Embedly(self.key, use_urllib2=True) + do_test(http) def test_providers(self): - http = Embedly(self.key) - - objs = http.oembed(['http://www.scribd.com/doc/13994900/Easter', + def do_test(http): + objs = http.oembed(['http://www.scribd.com/doc/13994900/Easter', 'http://www.scribd.com/doc/28452730/Easter-Cards']) - self.assert_(objs[0].provider_url == 'http://www.scribd.com/') - self.assert_(objs[1].provider_url == 'http://www.scribd.com/') + self.assert_(objs[0].provider_url == 'http://www.scribd.com/') + self.assert_(objs[1].provider_url == 'http://www.scribd.com/') - objs = http.oembed(['http://www.youtube.com/watch?v=Zk7dDekYej0', - 'http://yfrog.com/h22eu4']) - self.assert_(objs[0].provider_url == 'http://www.youtube.com/') - self.assert_(objs[1].provider_url == 'http://yfrog.com') + objs = http.oembed(['http://www.youtube.com/watch?v=Zk7dDekYej0', + 'http://yfrog.com/h22eu4']) + self.assert_(objs[0].provider_url == 'http://www.youtube.com/') + self.assert_(objs[1].provider_url == 'http://yfrog.com') - def test_error(self): - http = Embedly(self.key) + http = Embedly(self.key, use_urllib2=False) + do_test(http) + http = Embedly(self.key, use_urllib2=True) + do_test(http) - obj = http.oembed('http://www.embedly.com/this/is/a/bad/url') - self.assert_(obj.error is True, obj.dict) - obj = http.oembed('http://blog.embed.ly/lsbsdlfldsf/asdfkljlas/klajsdlfkasdf') - self.assert_(obj.error is True, obj.dict) - obj = http.oembed('http://twitpic/nothing/to/see/here') - self.assert_(obj.error is True, obj.dict) + def test_error(self): + def do_test(http): + obj = http.oembed('http://www.embedly.com/this/is/a/bad/url') + self.assert_(obj.error is True, obj.dict) + obj = http.oembed('http://blog.embed.ly/lsbsdlfldsf/asdfkljlas/klajsdlfkasdf') + self.assert_(obj.error is True, obj.dict) + obj = http.oembed('http://twitpic/nothing/to/see/here') + self.assert_(obj.error is True, obj.dict) + + http = Embedly(self.key, use_urllib2=False) + do_test(http) + http = Embedly(self.key, use_urllib2=True) + do_test(http) def test_multi_errors(self): - http = Embedly(self.key) - - objs = http.oembed(['http://www.embedly.com/this/is/a/bad/url', - 'http://blog.embed.ly/alsd/slsdlf/asdlfj']) - self.assert_(objs[0].type == 'error', objs[0].dict) - self.assert_(objs[1].type == 'error', objs[1].dict) - - objs = http.oembed(['http://blog.embed.ly/lsbsdlfldsf/asdf/kl', - 'http://twitpic.com/nothing/to/see/here']) - self.assert_(objs[0].type == 'error',objs[0].dict) - self.assert_(objs[1].type == 'error',objs[1].dict) - - objs = http.oembed(['http://blog.embed.ly/lsbsdlfldsf/asdf/kl', - 'http://yfrog.com/h22eu4j']) - self.assert_(objs[0].type == 'error',objs[0].dict) - self.assert_(objs[1].type == 'photo',objs[1].dict) - - objs = http.oembed(['http://yfrog.com/h22eu4j', - 'http://www.scribd.com/asdf/asdf/asdfasdf']) - self.assert_(objs[0].type == 'photo',objs[0].dict) - self.assert_(objs[1].type == 'error',objs[1].dict) - + def do_test(http): + objs = http.oembed(['http://www.embedly.com/this/is/a/bad/url', + 'http://blog.embed.ly/alsd/slsdlf/asdlfj']) + self.assert_(objs[0].type == 'error', objs[0].dict) + self.assert_(objs[1].type == 'error', objs[1].dict) + + objs = http.oembed(['http://blog.embed.ly/lsbsdlfldsf/asdf/kl', + 'http://twitpic.com/nothing/to/see/here']) + self.assert_(objs[0].type == 'error',objs[0].dict) + self.assert_(objs[1].type == 'error',objs[1].dict) + + objs = http.oembed(['http://blog.embed.ly/lsbsdlfldsf/asdf/kl', + 'http://yfrog.com/h22eu4j']) + self.assert_(objs[0].type == 'error',objs[0].dict) + self.assert_(objs[1].type == 'photo',objs[1].dict) + + objs = http.oembed(['http://yfrog.com/h22eu4j', + 'http://www.scribd.com/asdf/asdf/asdfasdf']) + self.assert_(objs[0].type == 'photo',objs[0].dict) + self.assert_(objs[1].type == 'error',objs[1].dict) + + http = Embedly(self.key, use_urllib2=False) + do_test(http) + http = Embedly(self.key, use_urllib2=True) + do_test(http) def test_too_many_urls(self): http = Embedly(self.key) From fd4b4450963a18366dcc259bb6a04a3adc5ed488 Mon Sep 17 00:00:00 2001 From: John Cook Date: Sat, 21 Apr 2012 11:51:16 -0400 Subject: [PATCH 2/7] Custom version number so we can pull from github during buildout --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 8b7912f..eb54834 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ setup( name = 'Embedly', - version = '0.4.3', + version = 'urllib2', author = 'Embed.ly, Inc.', author_email = 'support@embed.ly', description = 'Python Library for Embedly', @@ -43,4 +43,4 @@ 'Programming Language :: Python :: 2.7', ), **extra -) \ No newline at end of file +) From 51ab386f84eb1a9bde4a07d950b91ab4e8f4a2bb Mon Sep 17 00:00:00 2001 From: John Cook Date: Wed, 25 Apr 2012 19:29:07 -0400 Subject: [PATCH 3/7] More flexible HTTP client --- embedly/client.py | 27 ++++---------------------- embedly/httpclients.py | 43 ++++++++++++++++++++++++++++++++++++++++++ embedly/tests.py | 17 +++++++++-------- 3 files changed, 56 insertions(+), 31 deletions(-) create mode 100644 embedly/httpclients.py diff --git a/embedly/client.py b/embedly/client.py index 8b3bba0..8d4e0ff 100644 --- a/embedly/client.py +++ b/embedly/client.py @@ -6,8 +6,7 @@ """ import re import urllib -import urllib2 -import httplib2 +from embedly.httpclients import Httplib2Client try: import json @@ -23,7 +22,7 @@ class Embedly(object): Client """ - def __init__(self, key=None, user_agent=USER_AGENT, use_urllib2=False): + def __init__(self, key=None, user_agent=USER_AGENT, http_client=Httplib2Client()): """ Initialize the Embedly client @@ -38,31 +37,13 @@ def __init__(self, key=None, user_agent=USER_AGENT, use_urllib2=False): """ self.user_agent = user_agent self.key = key - self.use_urllib2 = use_urllib2 + self.http_client = http_client self.services = [] self._regex = None def _make_request(self, url, headers={}): - """ - Makes HTTP requests using httplib2 or urllib2 - """ - if self.use_urllib2: - try: - request = urllib2.Request(url, headers=headers) - response = urllib2.urlopen(request) - resp = response.headers.dict - if "status" not in resp: - resp["status"] = str(response.code) - content = response.read() - except urllib2.HTTPError, e: - resp = {"status" : str(e.getcode())} - content = e.read() - else: - http = httplib2.Http() - resp, content = http.request(url, headers=headers) - - return resp, content + return self.http_client.request(url, headers) def get_services(self): """ diff --git a/embedly/httpclients.py b/embedly/httpclients.py new file mode 100644 index 0000000..2e71f27 --- /dev/null +++ b/embedly/httpclients.py @@ -0,0 +1,43 @@ +""" +Httpclients +====== + +An http client has a request() method that accepts the following arguments: + url: the URL being requested + headers: a dictionary of headers to use with the request + +request() returns a dictionary of response headers and the body of the response + +Here are 2 HTTP clients, feel free to use your own +""" +import urllib2 +import httplib2 + +class Httplib2Client(object): + def request(self, url, headers=None): + """ + Makes HTTP requests using httplib2 + """ + http = httplib2.Http() + resp, content = http.request(url, headers=headers) + + return resp, content + + +class Urllib2Client(object): + def request(self, url, headers=None): + """ + Makes HTTP requests using urllib2 + """ + try: + request = urllib2.Request(url, headers=(headers or {})) + response = urllib2.urlopen(request) + resp = response.headers.dict + if "status" not in resp: + resp["status"] = str(response.code) + content = response.read() + except urllib2.HTTPError, e: + resp = {"status" : str(e.getcode())} + content = e.read() + + return resp, content diff --git a/embedly/tests.py b/embedly/tests.py index ca98a93..9309121 100644 --- a/embedly/tests.py +++ b/embedly/tests.py @@ -3,6 +3,7 @@ import unittest from embedly.client import Embedly +from embedly.httpclients import Httplib2Client, Urllib2Client from embedly.models import Url class EmbedlyTestCase(unittest.TestCase): @@ -87,9 +88,9 @@ def do_test(http): obj = http.oembed('http://yfrog.com/h22eu4j') self.assert_(obj.provider_url == 'http://yfrog.com') - http = Embedly(self.key, use_urllib2=False) + http = Embedly(self.key, http_client=Httplib2Client()) do_test(http) - http = Embedly(self.key, use_urllib2=True) + http = Embedly(self.key, http_client=Urllib2Client()) do_test(http) def test_providers(self): @@ -104,9 +105,9 @@ def do_test(http): self.assert_(objs[0].provider_url == 'http://www.youtube.com/') self.assert_(objs[1].provider_url == 'http://yfrog.com') - http = Embedly(self.key, use_urllib2=False) + http = Embedly(self.key, http_client=Httplib2Client()) do_test(http) - http = Embedly(self.key, use_urllib2=True) + http = Embedly(self.key, http_client=Urllib2Client()) do_test(http) def test_error(self): @@ -118,9 +119,9 @@ def do_test(http): obj = http.oembed('http://twitpic/nothing/to/see/here') self.assert_(obj.error is True, obj.dict) - http = Embedly(self.key, use_urllib2=False) + http = Embedly(self.key, http_client=Httplib2Client()) do_test(http) - http = Embedly(self.key, use_urllib2=True) + http = Embedly(self.key, http_client=Urllib2Client()) do_test(http) def test_multi_errors(self): @@ -145,9 +146,9 @@ def do_test(http): self.assert_(objs[0].type == 'photo',objs[0].dict) self.assert_(objs[1].type == 'error',objs[1].dict) - http = Embedly(self.key, use_urllib2=False) + http = Embedly(self.key, http_client=Httplib2Client()) do_test(http) - http = Embedly(self.key, use_urllib2=True) + http = Embedly(self.key, http_client=Urllib2Client()) do_test(http) def test_too_many_urls(self): From c62f917385f02f14761aec6056c44ee650ba06cb Mon Sep 17 00:00:00 2001 From: John Cook Date: Wed, 25 Apr 2012 19:38:11 -0400 Subject: [PATCH 4/7] Reverting version number change --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index eb54834..789e641 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ setup( name = 'Embedly', - version = 'urllib2', + version = '0.4.3', author = 'Embed.ly, Inc.', author_email = 'support@embed.ly', description = 'Python Library for Embedly', From 845bd8f1d6d4f194ef9ffe85270d2fe836603c21 Mon Sep 17 00:00:00 2001 From: John Cook Date: Wed, 25 Apr 2012 19:48:11 -0400 Subject: [PATCH 5/7] Setting version back to urllib2 for Batterii builds --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 789e641..eb54834 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ setup( name = 'Embedly', - version = '0.4.3', + version = 'urllib2', author = 'Embed.ly, Inc.', author_email = 'support@embed.ly', description = 'Python Library for Embedly', From 556de3a48d305bc36c791bbb9b1347e1d166bc80 Mon Sep 17 00:00:00 2001 From: John Cook Date: Wed, 25 Apr 2012 19:54:19 -0400 Subject: [PATCH 6/7] OK I am a GitHub pull request dumb-dumb - back to version 0.4.3 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index eb54834..789e641 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ setup( name = 'Embedly', - version = 'urllib2', + version = '0.4.3', author = 'Embed.ly, Inc.', author_email = 'support@embed.ly', description = 'Python Library for Embedly', From 0469e7ea36b5ff4dddf092f53a15262a1792c92e Mon Sep 17 00:00:00 2001 From: John Cook Date: Thu, 26 Apr 2012 22:05:03 -0400 Subject: [PATCH 7/7] Added timeout option --- embedly/httpclients.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/embedly/httpclients.py b/embedly/httpclients.py index 2e71f27..c3ed56a 100644 --- a/embedly/httpclients.py +++ b/embedly/httpclients.py @@ -14,24 +14,30 @@ import httplib2 class Httplib2Client(object): + def __init__(self, timeout=30): + self.timeout = timeout + def request(self, url, headers=None): """ Makes HTTP requests using httplib2 """ - http = httplib2.Http() + http = httplib2.Http(timeout=self.timeout) resp, content = http.request(url, headers=headers) return resp, content class Urllib2Client(object): + def __init__(self, timeout=30): + self.timeout = timeout + def request(self, url, headers=None): """ Makes HTTP requests using urllib2 """ try: request = urllib2.Request(url, headers=(headers or {})) - response = urllib2.urlopen(request) + response = urllib2.urlopen(request, timeout=self.timeout) resp = response.headers.dict if "status" not in resp: resp["status"] = str(response.code)