diff --git a/API/MongoDB.py b/API/MongoDB.py deleted file mode 100644 index d58ca18..0000000 --- a/API/MongoDB.py +++ /dev/null @@ -1,32 +0,0 @@ -from dataclasses import dataclass - - -@dataclass -class Councilor: - sdName: str - sggName: str - wiwName: str - name: str - gender: str - birthday: str - age: int - jobId: int - job: str - eduId: int - edu: str - - @classmethod - def from_dict(cls, data: dict): - return cls( - sdName=data.get("sdName"), - sggName=data.get("sggName"), - wiwName=data.get("wiwName"), - name=data.get("name"), - gender=data.get("gender"), - birthday=data.get("birthday"), - age=data.get("age"), - jobId=data.get("jobId"), - job=data.get("job"), - eduId=data.get("eduId"), - edu=data.get("edu"), - ) diff --git a/API/candidate.py b/API/candidate.py index 7e612fb..7ddda3b 100644 --- a/API/candidate.py +++ b/API/candidate.py @@ -3,8 +3,6 @@ import os, requests, sys import xml.etree.ElementTree as ET from typing import List -from .MongoDB import Councilor -import argparse from configurations.secrets import OpenDataPortalSecrets from .utils import save_to_excel, getLocalMetroMap @@ -41,10 +39,11 @@ def fetch_data( data_list = [] for item in root.findall(".//item"): data_entry = {child.tag: child.text for child in item} + for column in drop_columns: data_entry.pop(column) - data_list.append(Councilor.from_dict(data_entry)) + data_list.append(data_entry) return data_list diff --git a/API/utils.py b/API/utils.py index c7994b5..e6d214b 100644 --- a/API/utils.py +++ b/API/utils.py @@ -6,6 +6,7 @@ from . import BASE_DIR, SG_TYPECODE, SG_TYPECODE_TYPE from configurations.secrets import MongoDBSecrets from db.client import client +from API.MongoDB import Councilor def save_to_excel(data: List[dict], sgTypecode: str, is_elected: bool) -> None: @@ -31,23 +32,25 @@ def get_local_district_id(sd_name: str, wiw_name: str) -> Optional[int]: def save_to_mongo(data: List[dict], sgTypecode: str) -> None: - db = client[str(MongoDBSecrets.database_name)] - main_collection = db[str(SG_TYPECODE_TYPE[sgTypecode])] + db = client["council"] + main_collection = db["local_councilor"] + + local_metro_map = getLocalMetroMap() # TODO: Support other types of councils if sgTypecode == "6": for entry in data: - district_id = get_local_district_id(entry["sdName"], entry["wiwName"]) - entry.pop("sdName") - entry.pop("wiwName") + district_id = local_metro_map[(entry["sdName"], entry["wiwName"])][ + "local_id" + ] if district_id: main_collection.update_one( { - "council_id": district_id, - "council_type": SG_TYPECODE_TYPE[sgTypecode], + "name": entry["name"], + "localId": district_id, }, - {"$push": {"councilors": entry}}, + {"$push": Councilor.from_dict(entry)}, upsert=True, ) else: