From 4e6455b3e370dc4952fb65be937d8dad8384bf10 Mon Sep 17 00:00:00 2001
From: venu-sambarapu-DS <venu.sambarapu@factly.in>
Date: Mon, 3 Jun 2024 19:14:12 +0530
Subject: [PATCH] Added code to read standard datasets only from google sheet

---
 app/api/api_v1/routers/dictionary.py | 27 +++++++++++++++++++++------
 1 file changed, 21 insertions(+), 6 deletions(-)

diff --git a/app/api/api_v1/routers/dictionary.py b/app/api/api_v1/routers/dictionary.py
index c76030d..c59bcad 100644
--- a/app/api/api_v1/routers/dictionary.py
+++ b/app/api/api_v1/routers/dictionary.py
@@ -1,6 +1,8 @@
 import pandas as pd
 from fastapi import APIRouter, HTTPException, status
 from fastapi.encoders import jsonable_encoder
+import io
+import requests
 from fastapi.responses import JSONResponse
 
 from app.core.config import CORE_FOLDER, Settings
@@ -12,13 +14,26 @@
 dictionary_router = router = APIRouter()
 
 
+g_sheet_session = requests.Session()
+g_sheet_response = g_sheet_session.get("https://docs.google.com/spreadsheets/d/1NEsFJGr5IHsrIakGgeNFUvz5zpLOadh_vDH7Apqmv9E/gviz/tq?tqx=out:csv&sheet=master_dictionaries")
+g_sheet_bytes_data = g_sheet_response.content
+data = pd.read_csv(io.StringIO(g_sheet_bytes_data.decode('utf-8')))
+print("reading data from google sheet@@@@")
+# data.rename(
+#     columns={
+#         "country_standard_name": "country",
+#         "unique_standard_airline_name": "airline",
+#         "standard_disease_name": "disease",
+#         "psu_companies": "psu",
+#     }
+# )
+# print(data.columns.tolist())
+
+
 @router.get("/", summary="Get all Saved Entities csv file name")
 async def get_entity_names():
     # List down all the csv files present in the config folder
-    return [
-        csv_file.name.replace(".csv", "")
-        for csv_file in CORE_FOLDER.glob("**/*.csv")
-    ]
+    return data.columns.tolist()
 
 
 @router.get(
@@ -27,9 +42,9 @@ async def get_entity_names():
     response_class=JSONResponse,
 )
 async def get_entity_data(entity: str):
-    entity_df = pd.read_csv(CORE_FOLDER / f"{entity}.csv")
+    entity_df = data[[entity]].dropna()
     # to avoid json conversion error
-    entity_df = entity_df.fillna("")
+    # entity_df = entity_df.fillna("")
 
     # convert to json
     json_compatible_item_data = jsonable_encoder(