Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/feat-scraping-advance' into feat…
Browse files Browse the repository at this point in the history
…-scraping-advance
  • Loading branch information
keonly committed Nov 22, 2023
2 parents f2a204e + 6ab2c44 commit e856a95
Show file tree
Hide file tree
Showing 3 changed files with 25 additions and 8 deletions.
30 changes: 23 additions & 7 deletions scrap/utils/export.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,19 +5,35 @@
from scrap.utils.types import ScrapResult, ScrapBasicArgument


def export_results_to_json(results: dict[int, ScrapResult], output_path: str, current_time: str):
def export_results_to_json(
results: dict[int, ScrapResult], output_path: str, current_time: str
):
os.makedirs(output_path, exist_ok=True)
results = {k: [asdict(councilor) for councilor in v.councilors] for k, v in results.items()}
results = {
k: [asdict(councilor) for councilor in v.councilors] for k, v in results.items()
}

with open(os.path.join(output_path, f"scraping_result_{current_time}.json"), "w", encoding="utf-8") as f:
with open(
os.path.join(output_path, f"scraping_result_{current_time}.json"),
"w",
encoding="utf-8",
) as f:
json.dump(results, f, ensure_ascii=False, indent=4)


def export_results_to_txt(results: dict[int, ScrapResult], output_path: str, current_time: str):
def export_results_to_txt(
results: dict[int, ScrapResult], output_path: str, current_time: str
):
os.makedirs(output_path, exist_ok=True)
results = {k: [asdict(councilor) for councilor in v.councilors] for k, v in results.items()}
results = {
k: [asdict(councilor) for councilor in v.councilors] for k, v in results.items()
}

with open(os.path.join(output_path, f"scraping_result_{current_time}.txt"), "w", encoding="utf-8") as f:
with open(
os.path.join(output_path, f"scraping_result_{current_time}.txt"),
"w",
encoding="utf-8",
) as f:
for cid, councilors in results.items():
councilors = "\n".join([c.to_txt() for c in councilors])
f.write(f"| {cid} | {councilors}\n")
f.write(f"| {cid} | {councilors}\n")
2 changes: 1 addition & 1 deletion scrap/utils/runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ def run_single_council(self, n: int) -> ScrapResult:
else:
if council_args is None:
raise ValueError(f"{n}번 의회에 대한 ScrapBasicArgument가 없습니다.")

if self.is_selenium_basic(n):
result = sel_scrap_basic(council_url, n, council_args)
else:
Expand Down
1 change: 1 addition & 0 deletions scrap/utils/spreadsheet.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
from scrap.local_councils.daegu import *
from scrap.local_councils.incheon import *
from scrap.local_councils.gwangju import *

# from scrap.local_councils.daejeon import *
from scrap.local_councils.ulsan import *
from scrap.local_councils.gyeonggi import *
Expand Down

0 comments on commit e856a95

Please sign in to comment.