Merge pull request #30 from rciam/develop

Develop

Dockerfile

@@ -1,5 +1,5 @@
 ARG PYTHON_IMAGE_REPO=python
-FROM FROM ${PYTHON_IMAGE_REPO}:3.11.5-bookworm
+FROM ${PYTHON_IMAGE_REPO}:3.11.5-bookworm
 RUN curl -sL https://deb.nodesource.com/setup_18.x | sed "s/exec_cmd 'apt-get update'/exec_cmd 'apt-get --allow-releaseinfo-change update'/" | bash -
 RUN echo "deb https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list
 RUN curl https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -

app/database.py

@@ -1,16 +1,38 @@
 from app.utils import configParser
-from sqlmodel import create_engine, Session
+from sqlalchemy import create_engine
+from sqlmodel import Session
+from sqlalchemy.exc import OperationalError
 
 
-def get_session():
-    # Initialize
-    config_file = 'config.global.py'
-    db_params = configParser.getConfig('database_parameters', config_file)
+class Database:
+    def __init__(self):
 
-    url = db_params['database_url']
-    pool_size = int(db_params.get('pool_size', 25))
-    max_overflow = int(db_params.get('max_overflow', 5))
-    engine = create_engine(url, pool_size=pool_size, max_overflow=max_overflow)
+        config_file = 'config.global.py'
+        db_params = configParser.getConfig('database_parameters', config_file)
 
-    with Session(engine) as session:
-        yield session
+        url = db_params['database_url']
+        pool_size = int(db_params.get('pool_size', 25))
+        max_overflow = int(db_params.get('max_overflow', 5))
+
+        self.engine = create_engine(url, pool_size=pool_size, max_overflow=max_overflow)
+
+    def check_database_connection(self):
+        try: 
+          # Attempt to connect to the database by checking the connection
+            with self.engine.connect():
+                print("Database connection successful!")
+                return True
+        except OperationalError as e:
+            print(f"Database connection failed: {e}")
+            return False 
+
+    def get_session(self):
+        with Session(self.engine) as session:
+            yield session
+
+    def create_session(self):
+        return Session(self.engine)
+
+
+# Creating an instance of Database
+db = Database()

app/ingester/communityIngester.py

@@ -1,5 +1,5 @@
 from app.logger import log
-from ..database import get_session
+
 from sqlalchemy.exc import NoResultFound
 from .utilsIngester import utilsIngester
 
@@ -77,9 +77,8 @@ def ingestCommunityDataPerTenenv(cls, tenenvId, session):
                         format(communityMappedItems))
 
     @classmethod
-    def ingestCommunityData(cls):
-        session_generator = get_session()
-        session = next(session_generator)
+    def ingestCommunityData(cls, session):
+
         tenenvIds = session.exec("""SELECT id FROM tenenv_info""").all()
         # for each tenenv on database try to ingest CommunityData
         # from statistics_raw table

app/ingester/ingestData.py

@@ -2,12 +2,15 @@
 from .usersIngester import UserDataIngester
 from .membeshipIngester import MembershipDataIngester
 from .loginsIngester import LoginDataIngester
+from ..database import db
 
+session = db.create_session()
 # Ingest Communities
-CommunityDataIngester.ingestCommunityData()
+CommunityDataIngester.ingestCommunityData(session)
 # Ingest Users
-UserDataIngester.ingestUserData()
+UserDataIngester.ingestUserData(session)
 # Ingest Memberships
-MembershipDataIngester.ingestMembershipData()
+MembershipDataIngester.ingestMembershipData(session)
 # Ingest Logins
-LoginDataIngester.ingestLoginData()
+LoginDataIngester.ingestLoginData(session)
+session.close()

app/ingester/loginsIngester.py

@@ -1,5 +1,4 @@
 from app.logger import log
-from ..database import get_session
 from app.utils.ipDatabase import geoip2Database
 from sqlalchemy.exc import NoResultFound
 from .utilsIngester import utilsIngester
@@ -222,9 +221,7 @@ def ingestLoginDataPerTenenv(cls, tenenvId, session):
             {0} new logins ingested""".format(loginMappedItems))
 
     @classmethod
-    def ingestLoginData(cls):
-        session_generator = get_session()
-        session = next(session_generator)
+    def ingestLoginData(cls, session):
         tenenvIds = session.exec("""SELECT id FROM tenenv_info""").all()
         for tenenvId in tenenvIds:
             LoginDataIngester.ingestLoginDataPerTenenv(tenenvId[0], session)

app/ingester/membeshipIngester.py

@@ -1,5 +1,4 @@
 from app.logger import log
-from ..database import get_session
 from sqlalchemy.exc import NoResultFound
 from .utilsIngester import utilsIngester
 
@@ -69,9 +68,7 @@ def ingestMembershipDataPerTenenv(cls, tenenvId, session):
                         format(membershipMappedItems))
 
     @classmethod
-    def ingestMembershipData(cls):
-        session_generator = get_session()
-        session = next(session_generator)
+    def ingestMembershipData(cls, session):
         tenenvIds = session.exec("""SELECT id FROM tenenv_info""").all()
         # for each tenenv on database try to ingest UserData
         # from statistics_raw table

app/ingester/usersIngester.py

@@ -1,5 +1,4 @@
 from app.logger import log
-from ..database import get_session
 from .utilsIngester import utilsIngester
 
 
@@ -48,9 +47,7 @@ def ingestUserDataPerTenenv(cls, tenenvId, session):
                 {0} users ingested or updated""".format(userMappedItems))
 
     @classmethod
-    def ingestUserData(cls):
-        session_generator = get_session()
-        session = next(session_generator)
+    def ingestUserData(cls, session):
         tenenvIds = session.exec("""SELECT id FROM tenenv_info""").all()
         # for each tenenv on database try to ingest UserData
         # from statistics_raw table

app/main.py

@@ -11,7 +11,6 @@
 from sqlalchemy import func
 from sqlalchemy.orm import selectinload
 
-from app.database import get_session
 from app.models.community_info_model import *
 from app.models.community_model import *
 from app.models.member_model import *

app/routers/ams.py

@@ -5,7 +5,7 @@
 from sqlmodel import Field, Session, SQLModel, create_engine, select
 from typing import Union
 from app.utils import configParser, globalMethods
-from app.database import get_session
+from app.database import db
 from app.utils.globalMethods import AuthNZCheck
 from fastapi.responses import PlainTextResponse
 from app.logger import log
@@ -44,7 +44,7 @@ async def verify_authorization_header(Authorization: Optional[str] = Header(None
 
 @router.post("/ams_stats")
 async def get_ams_stats(*,
-    session: Session = Depends(get_session),
+    session: Session = Depends(db.get_session),
     request: Request,
     response: Response,
     body = Body(..., example={"name": "Item Name"}),

app/routers/authenticate.py

@@ -27,7 +27,7 @@ def initializeAuthOb():
     oauth = OAuth()
 
     oauth.register(
-        g.tenant + '.' + g.environment + '.rciam',
+        g.tenant + '_' + g.environment + '_rciam',
         client_id=oidc_config['client_id'],
         client_secret=oidc_config['client_secret'],
         server_metadata_url=oidc_config['issuer'] + "/.well-known/openid-configuration",
@@ -46,7 +46,7 @@ async def login_endpoint(
         request: Request,
         oauth_ob= Depends(initializeAuthOb),
         server_config= Depends(getServerConfig)):
-    rciam = oauth_ob.create_client(g.tenant + '.' + g.environment + '.rciam')
+    rciam = oauth_ob.create_client(g.tenant + '_' + g.environment + '_rciam')
     redirect_uri = server_config['protocol'] + "://" + server_config['host'] + server_config['api_path'] + "/auth"
     return await rciam.authorize_redirect(request, redirect_uri)
 
@@ -71,7 +71,7 @@ async def authorize_rciam(
     response = RedirectResponse(url=urllib.parse.unquote(login_start_url))
     response.delete_cookie("login_start")
 
-    rciam = oauth_ob.create_client(g.tenant + '.' + g.environment + '.rciam')
+    rciam = oauth_ob.create_client(g.tenant + '_' + g.environment + '_rciam')
     try:
         token = await rciam.authorize_access_token(request)
     except OAuthError as error:
@@ -148,7 +148,7 @@ async def logout(
         oauth_ob= Depends(initializeAuthOb),
         server_config=Depends(getServerConfig)
 ):
-    rciam = oauth_ob.create_client(g.tenant + '.' + g.environment + '.rciam')
+    rciam = oauth_ob.create_client(g.tenant + '_' + g.environment + '_rciam')
     metadata = await rciam.load_server_metadata()
     # todo: Fix this after we complete the multitenacy
     redirect_uri = server_config['protocol'] + "://" + server_config['client'] +"/metrics"

app/routers/communities.py

@@ -2,7 +2,7 @@
 from sqlmodel import Field, Session, SQLModel, create_engine, select
 from typing import Union
 
-from app.database import get_session
+from app.database import db
 from app.models.community_info_model import *
 from app.models.community_model import *
 from app.models.member_model import MembersReadWithCommunityInfo
@@ -20,7 +20,7 @@
 @router.get("/members/", response_model=List[MembersReadWithCommunityInfo])
 async def read_members(
         *,
-        session: Session = Depends(get_session),
+        session: Session = Depends(db.get_session),
         offset: int = 0,
         # community_id: Union[None, int] = None
 ):
@@ -33,7 +33,7 @@ async def read_members(
 @router.get("/min_date_communities")
 async def read_min_date_communities(
         *,
-        session: Session = Depends(get_session),
+        session: Session = Depends(db.get_session),
         offset: int = 0,
         tenenv_id: int,
 ):
@@ -46,7 +46,7 @@ async def read_min_date_communities(
 @router.get("/members_bystatus")
 async def read_members_bystatus(
         *,
-        session: Session = Depends(get_session),
+        session: Session = Depends(db.get_session),
         offset: int = 0,
         community_id: Union[None, int] = None,
         tenenv_id: int,
@@ -67,7 +67,7 @@ async def read_members_bystatus(
 @router.get("/communities_groupby/{group_by}")
 async def read_communities(
         *,
-        session: Session = Depends(get_session),
+        session: Session = Depends(db.get_session),
         offset: int = 0,
         group_by: str,
         tenenv_id: int,
@@ -110,7 +110,7 @@ async def read_communities(
 @router.get("/communities")
 async def read_community(
         *,
-        session: Session = Depends(get_session),
+        session: Session = Depends(db.get_session),
         community_id: Union[None, int] = None,
         tenenv_id: int):
     sql_subquery = ''
@@ -130,7 +130,7 @@ async def read_community(
 @router.get("/communities_info", response_model=List[Community_InfoRead])
 async def read_communities_info(
         *,
-        session: Session = Depends(get_session),
+        session: Session = Depends(db.get_session),
         offset: int = 0
 ):
     communities = session.exec(select(Community_Info).offset(offset)).all()

app/routers/countries.py

@@ -1,5 +1,5 @@
 from fastapi import APIRouter, Depends, HTTPException, Query
-from app.database import get_session
+from app.database import db
 from sqlmodel import Field, Session, SQLModel, create_engine, select
 from typing import Union
 
@@ -20,7 +20,7 @@
 @router.get("/countries/", response_model=List[Country_CodesRead])
 async def read_countries(
         *,
-        session: Session = Depends(get_session),
+        session: Session = Depends(db.get_session),
         offset: int = 0
 ):
     countries = session.exec(select(Country_Codes).offset(offset)).all()
@@ -30,7 +30,7 @@ async def read_countries(
 @router.get("/country_stats/", response_model=List[Statistics_Country_HashedwithInfo])
 async def read_country_stats(
         *,
-        session: Session = Depends(get_session),
+        session: Session = Depends(db.get_session),
         offset: int = 0
 ):
     stats = session.exec(
@@ -41,7 +41,7 @@ async def read_country_stats(
 @router.get("/country_stats_by_vo/{community_id}")
 async def read_country_stats_by_vo(
         *,
-        session: Session = Depends(get_session),
+        session: Session = Depends(db.get_session),
         offset: int = 0,
         community_id: Union[None, int] = None
 ):

app/routers/dashboard.py

@@ -3,7 +3,7 @@
 from typing import Union
 from xmlrpc.client import boolean
 
-from app.database import get_session
+from app.database import  db
 from app.utils.globalMethods import AuthNZCheck
 
 router = APIRouter(
@@ -14,7 +14,7 @@
 @router.get("/tenenv/{tenant_name}/{environment_name}")
 async def read_tenenv_byname(
         *,
-        session: Session = Depends(get_session),
+        session: Session = Depends(db.get_session),
         offset: int = 0,
         tenant_name: str,
         environment_name: str
@@ -34,7 +34,7 @@ async def read_tenenv_byname(
 @router.get("/environment_byname/{environment_name}")
 async def read_environment_byname(
         *,
-        session: Session = Depends(get_session),
+        session: Session = Depends(db.get_session),
         offset: int = 0,
         environment_name: str
 ):
@@ -50,7 +50,7 @@ async def read_environment_byname(
 @router.get("/idps")
 async def read_idps(
         *,
-        session: Session = Depends(get_session),
+        session: Session = Depends(db.get_session),
         tenenv_id: int,
         idpId: int = None
 ):
@@ -69,7 +69,7 @@ async def read_idps(
 @router.get("/sps")
 async def read_sps(
         *,
-        session: Session = Depends(get_session),
+        session: Session = Depends(db.get_session),
         tenenv_id: int,
         spId: int = None
 ):