reduced execution timeouts and bugfix

fcavallarin · Jul 24, 2017 · dcc0078 · dcc0078
1 parent a288d09
commit dcc0078
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 5 deletions.
diff --git a/core/constants.py b/core/constants.py
@@ -29,6 +29,7 @@
 REQTYPE_JSONP = "jsonp"
 REQTYPE_FORM = "form"
 REQTYPE_REDIRECT = "redirect"
+REQTYPE_IMAGE = "image"
 REQTYPE_UNKNOWN = "unknown"
 
 

diff --git a/core/crawl/crawler.py b/core/crawl/crawler.py
@@ -67,7 +67,7 @@ def __init__(self, argv):
 			"http_auth": None,
 			"use_urllib_onerror": True,
 			"group_qs": False,
-			"process_timeout": 300, # when lots of element(~25000) are added dynamically it can take some time..
+			"process_timeout": 180, # when lots of element(~25000) are added dynamically it can take some time..
 			"set_referer": True,
 			"scope": CRAWLSCOPE_DOMAIN,
 			"mode": CRAWLMODE_AGGRESSIVE,

diff --git a/core/crawl/crawler_thread.py b/core/crawl/crawler_thread.py
@@ -152,9 +152,10 @@ def send_probe(self, request,  errors):
 
 			if jsn == None:
 				errors.append(ERROR_PROBEKILLED)
-				time.sleep(self.process_retries_interval) # ... ???
-				retries -= 1
-				continue
+				# time.sleep(self.process_retries_interval) # ... ???
+				# retries -= 1
+				# continue
+				break
 
 
 			# try to decode json also after an exception .. sometimes phantom crashes BUT returns a valid json ..
@@ -241,7 +242,7 @@ def crawl(self):
 			adjust_requests(requests)
 
 			Shared.main_condition.acquire()
-			res = CrawlResult(request, requests, errors, probe.page_hash)
+			res = CrawlResult(request, requests, errors, probe.page_hash if probe else "")
 			Shared.crawl_results.append(res)
 			Shared.main_condition.notify()
 			Shared.main_condition.release()