v1.7.1 http timeout support

zpoint · Apr 17, 2019 · 537a599 · 537a599
1 parent a5f62cd
commit 537a599
Show file tree

Hide file tree

Showing 3 changed files with 6 additions and 3 deletions.
diff --git a/idataapi_transform/DataProcess/Config/ConfigUtil/GetterConfig.py b/idataapi_transform/DataProcess/Config/ConfigUtil/GetterConfig.py
@@ -13,6 +13,7 @@
 except Exception as e:
     pass
 
+from aiohttp.client import sentinel
 from .BaseConfig import BaseGetterConfig
 
 from ..ESConfig import get_es_client
@@ -27,7 +28,7 @@ def __init__(self, source, per_limit=DefaultVal.per_limit, max_limit=DefaultVal.
                  done_if=None, trim_to_max_limit=DefaultVal.trim_to_max_limit,
                  exclude_filtered_to_max_limit=DefaultVal.exclude_filtered_to_max_limit, post_body=None,
                  persistent_writer=None, persistent_to_disk_if_give_up=True, debug_mode=False, keep_other_fields=False,
-                 keep_fields=("dataType", "appCode"), http_headers=None, **kwargs):
+                 keep_fields=("dataType", "appCode"), http_headers=None, http_timeout=None, **kwargs):
         """
         will request until no more next_page to get, or get "max_limit" items
 
@@ -61,6 +62,7 @@ def __init__(self, source, per_limit=DefaultVal.per_limit, max_limit=DefaultVal.
         :param debug_mode: whether log every http request url
         :param keep_other_fields: keep field in "keep_fields" in each json_object
         :param http_headers: http_headers, dict object
+        :param http_timeout: in seconds
         :param args:
         :param kwargs:
 
@@ -104,6 +106,7 @@ def __init__(self, source, per_limit=DefaultVal.per_limit, max_limit=DefaultVal.
         self.keep_other_fields = keep_other_fields
         self.keep_fields = keep_fields
         self.http_headers = http_headers
+        self.http_timeout = http_timeout if http_timeout is not None else sentinel
 
 
 class RCSVConfig(BaseGetterConfig):

diff --git a/idataapi_transform/DataProcess/DataGetter/APIGetter.py b/idataapi_transform/DataProcess/DataGetter/APIGetter.py
@@ -152,7 +152,7 @@ async def __anext__(self):
                 if self.config.debug_mode:
                     log_str = "HTTP method: %s, url: %s" % (self.method, self.base_url)
                     logging.info(log_str)
-                resp = await self.config.session._request(self.method, self.base_url, headers=self.headers, data=self.config.post_body)
+                resp = await self.config.session._request(self.method, self.base_url, headers=self.headers, data=self.config.post_body, timeout=self.config.http_timeout)
                 text = await resp.text()
                 # print(text)
                 result = json.loads(text)

diff --git a/idataapi_transform/__init__.py b/idataapi_transform/__init__.py
@@ -31,4 +31,4 @@ def set_log_path(log_path, max_log_file_bytes):
         main_config_box.config_log(log_path, max_log_file_bytes)
 
 
-__version__ = '1.7.0'
+__version__ = '1.7.1'
Original file line number	Diff line number	Diff line change
Expand Up		@@ -31,4 +31,4 @@ def set_log_path(log_path, max_log_file_bytes):
		main_config_box.config_log(log_path, max_log_file_bytes)


		__version__ = '1.7.0'
		__version__ = '1.7.1'