Merge pull request #1051 from City-Bureau/chore/fix-cron-task

[c] Fix cron tasks

.github/workflows/archive.yml

@@ -31,18 +31,18 @@ jobs:
           sudo openvpn --config /etc/openvpn/ovpn.conf --daemon
           sleep 120
 
-      - name: Set up Python 3.7
+      - name: Set up Python 3.9
         uses: actions/setup-python@v1
         with:
-          python-version: 3.7
+          python-version: 3.9
 
       - name: Install Pipenv
         uses: dschep/install-pipenv-action@v1
 
       - name: Install dependencies
         run: pipenv sync
         env:
-          PIPENV_DEFAULT_PYTHON_VERSION: 3.7
+          PIPENV_DEFAULT_PYTHON_VERSION: 3.9
 
       - name: Run scrapers
         run: |

.github/workflows/ci.yml

@@ -36,6 +36,9 @@ jobs:
       - name: Check imports with isort
         run: pipenv run isort . --check-only
 
+      - name: Check style with black
+        run: pipenv run black . --check
+
       - name: Lint with flake8
         run: pipenv run flake8 .
 

.github/workflows/cron.yml

@@ -36,18 +36,18 @@ jobs:
           sudo openvpn --config /etc/openvpn/ovpn.conf --daemon
           sleep 120
 
-      - name: Set up Python 3.7
+      - name: Set up Python 3.9
         uses: actions/setup-python@v1
         with:
-          python-version: 3.7
+          python-version: 3.9
 
       - name: Install Pipenv
         uses: dschep/install-pipenv-action@v1
 
       - name: Install dependencies
         run: pipenv sync
         env:
-          PIPENV_DEFAULT_PYTHON_VERSION: 3.7
+          PIPENV_DEFAULT_PYTHON_VERSION: 3.9
 
       - name: Run scrapers
         run: |

Pipfile

@@ -8,7 +8,7 @@ python-dateutil = "*"
 pytz = "*"
 requests = "*"
 scrapy = "*"
-scrapy-sentry = "*"
+scrapy-sentry = {ref = "v1", git = "https://github.com/City-Bureau/scrapy-sentry.git"}
 scrapy-wayback-middleware = "*"
 city-scrapers-core = {ref = "main", git = "https://github.com/City-Bureau/city-scrapers-core.git", extras = ["azure"]}
 pypiwin32 = {version = "*",sys_platform = "== 'win32'"}

city_scrapers/spiders/chi_citycouncil.py

@@ -1,4 +1,3 @@
-import requests
 from city_scrapers_core.constants import NOT_CLASSIFIED
 from city_scrapers_core.items import Meeting
 from city_scrapers_core.spiders import CityScrapersSpider
@@ -9,19 +8,12 @@ class ChiCitycouncilSpider(CityScrapersSpider):
     name = "chi_citycouncil"
     agency = "Chicago City Council"
     timezone = "America/Chicago"
-    start_urls = ["https://chicityclerkelms.chicago.gov/Meetings/"]
+    start_urls = ["https://api.chicityclerkelms.chicago.gov/meeting"]
 
     def parse(self, response):
-
-        # The API endpoint
-        url = "https://api.chicityclerkelms.chicago.gov/meeting"  # noqa
-
-        # A GET request to the API
-        response = requests.get(url)
         response_json = response.json()
 
         for item in response_json["data"]:
-
             meeting = Meeting(
                 title=self._parse_title(item),
                 description=self._parse_description(item),

city_scrapers/spiders/chi_midway_noise.py

@@ -86,7 +86,7 @@ def parse(self, response):
         yield from meeting_list
 
     def _parse_title(self, item):
-        if type(item) == Selector:
+        if isinstance(item, Selector):
             item = item.get()
         text = self._clean_bad_chars(item)
         desc = ""
@@ -113,7 +113,7 @@ def _parse_start(self, item):
 
     def _parse_date(self, item):
         """Parse the meeting date."""
-        if type(item) == Selector:
+        if isinstance(item, Selector):
             # Scheduled meetings have only text; past meetings have <td> tags.
             if "<td>" in item.get():
                 item = item.xpath(".//td/text()").get()
@@ -135,7 +135,7 @@ def _parse_date(self, item):
     def _parse_links(self, item, response):
         """Parse or generate links."""
         documents = []
-        if type(item) == Selector:
+        if isinstance(item, Selector):
             relative_urls = item.xpath(".//a/@href").extract()
             for relative_url in relative_urls:
                 documents.append(self._build_link_dict(response.urljoin(relative_url)))