From b6bc27fb6b53ab8d78184b3aa7b636d22efc880a Mon Sep 17 00:00:00 2001 From: Re-st Date: Mon, 27 Nov 2023 13:50:22 +0900 Subject: [PATCH] =?UTF-8?q?[scrap]=20Temp.=20=EB=8B=A8=EC=B2=B4=EC=9E=A5?= =?UTF-8?q?=20=EC=8A=A4=ED=81=AC=EB=9E=A9=20=EC=86=90=EB=B3=B4=EA=B8=B0.?= =?UTF-8?q?=20=EC=88=AB=EC=9E=90=20id=EA=B0=80=20=ED=95=84=EC=9A=94.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scrap/group_head.py | 24 +++++++++++++++++++----- scrap/utils/runner.py | 28 ++++++++++++++++++---------- 2 files changed, 37 insertions(+), 15 deletions(-) diff --git a/scrap/group_head.py b/scrap/group_head.py index de06666..5fac463 100644 --- a/scrap/group_head.py +++ b/scrap/group_head.py @@ -3,11 +3,12 @@ """ from time import sleep -from db.types import Councilor +from db.types import CouncilType, Councilor +from scrap.utils.types import ScrapResult from scrap.utils.requests import get_selenium, By -def scrap_group_heads( +def scrap_group_leaders( url="https://laiis.go.kr/lips/mlo/lcl/groupHeadList.do", ) -> tuple[list[tuple[str, Councilor]], list[tuple[str, Councilor]]]: """내고장알리미를 이용해 광역/기초단체장 인적사항 스크랩 @@ -63,9 +64,22 @@ def scrap_group_heads( ) browser.get(url) - - return (metro_heads, local_heads) + results = dict() + for (area, councilor) in metro_heads: + results[area] = ScrapResult( + council_id=area, + council_type=CouncilType.METRO_LEADER, + councilors=councilor, + ) + for (local_area_name, councilor) in local_heads: + print(local_area_name) + results[local_area_name] = ScrapResult( + council_id=local_area_name, + council_type=CouncilType.LOCAL_LEADER, + councilors=councilor, + ) + return results if __name__ == "__main__": - print(scrap_group_heads()) + print(scrap_group_leaders()) diff --git a/scrap/utils/runner.py b/scrap/utils/runner.py index dd121b2..2a4daac 100644 --- a/scrap/utils/runner.py +++ b/scrap/utils/runner.py @@ -30,6 +30,7 @@ from scrap.local_councils import * from scrap.metropolitan_council import * from scrap.national_council import * +from scrap.group_head import * from requests.exceptions import Timeout @@ -190,12 +191,19 @@ def run(self) -> Dict[str, ScrapResult]: return result -class HeadsScraper(BaseScraper): +class LeadersScraper(BaseScraper): def __init__(self, kwargs: Dict[str, str] = {}): super().__init__(kwargs) def run(self) -> Dict[str, ScrapResult]: - raise NotImplementedError("단체장 스크랩은 아직 구현되지 않았습니다.") + result = dict() + + try: + results = scrap_group_leaders() + except Exception as e: + self.handle_errors("단체장", e) + + return results class ScraperFactory: @@ -208,8 +216,8 @@ def create_scraper(self) -> BaseScraper: return LocalCouncilScraper(self.kwargs) elif self.where == "metro": return MetroCouncilScraper(self.kwargs) - elif self.where == "heads": - return HeadsScraper(self.kwargs) + elif self.where == "leaders": + return LeadersScraper(self.kwargs) elif self.where == "national": return NationalCouncilScraper(self.kwargs) else: @@ -251,16 +259,16 @@ def parse_cids(cids_str: Optional[str], where: str) -> Optional[List[int]]: return range(1, 227) elif where == "national": return None - elif where == "heads": - raise NotImplementedError("단체장 스크랩은 몇부터 몇까지죠?") + elif where == "leaders": + return None if __name__ == "__main__": parser = argparse.ArgumentParser(description="지방의회 / 광역의회 / 국회 / 단체장 스크랩 스크립트 실행") - parser.add_argument( - "where", - help="스크랩할 의회 종류 (지방의회: 'local', 광역의회: 'metro', 국회: 'national', 단체장: 'heads')", - choices=["local", "metro", "national", "heads"], + parser.add_argument( "-w", + "--where", + help="스크랩할 의회 종류 (지방의회: 'local', 광역의회: 'metro', 국회: 'national', 단체장: 'leaders')", + choices=["local", "metro", "national", "leaders"], default="local", ) parser.add_argument(