add backoff function

scrapers/in/apiclient.py

@@ -90,15 +90,14 @@ def get_session_no(self, session):
         headers["User-Agent"] = self.user_agent
         url = urljoin(self.root, f"/{session}")
 
-        while not session_no:
-            resp = requests.get(url, headers=headers)
-            resp = resp.json()
-            if "message" in resp:
-                self.scraper.warning(resp["message"])
-                continue
-            session_no_regex = re.search(r"Session\s+(\d+).+", resp["name"])
-            if session_no_regex:
-                session_no = session_no_regex.group(1)
+        resp = requests.get(url, headers=headers).json()
+        if "message" in resp:
+            raise Exception(resp["message"])
+        session_no_regex = re.search(r"Session\s+(\d+).+", resp["name"])
+        if session_no_regex:
+            session_no = session_no_regex.group(1)
+        else:
+            raise Exception("Invalid session")
 
         return session_no
 

scrapers/in/events.py

@@ -7,7 +7,7 @@
 import pytz
 from openstates.scrape import Scraper, Event
 from .apiclient import ApiClient
-from .utils import add_space
+from .utils import add_space, backoff
 from openstates.exceptions import EmptyScrape
 
 
@@ -21,7 +21,7 @@ def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
     def scrape(self):
-        session_no = self.apiclient.get_session_no(self.session)
+        session_no = backoff(self.apiclient.get_session_no, self.session)
         response = self.apiclient.get("meetings", session=self.session)
 
         meetings = response["meetings"]

scrapers/in/utils.py

@@ -1,4 +1,9 @@
 import requests
+import logging
+import time
+
+logging.getLogger("IN").setLevel(logging.WARNING)
+log = logging.getLogger("openstates")
 
 
 def get_with_increasing_timeout(scraper, link, fail=False, kwargs={}):
@@ -57,3 +62,24 @@ def add_space(text):
     new_string = f"{alpha} {number}"
 
     return new_string
+
+
+def backoff(function, *args, **kwargs):
+    retries = 5
+
+    def _():
+        time.sleep(1)
+        return function(*args, **kwargs)
+
+    for attempt in range(retries):
+        try:
+            return _()
+        except Exception as e:
+            backoff = (attempt + 1) * 15
+            log.warning(
+                "[attempt %s]: %s. Backing off for %s seconds."
+                % (attempt, str(e), backoff)
+            )
+            time.sleep(backoff)
+
+    raise Exception("INDIANA API returns still None. Please confirm API status.")