From b24ff7cd5a80d5aaf172f55af7aaaf9a85f80a72 Mon Sep 17 00:00:00 2001 From: Ioannis Igoumenos Date: Mon, 23 Oct 2023 10:27:14 +0300 Subject: [PATCH 1/3] distinct names for oauth object and client instances --- app/routers/authenticate.py | 8 ++++---- app/utils/globalMethods.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/routers/authenticate.py b/app/routers/authenticate.py index 1f3f072..e2a99be 100644 --- a/app/routers/authenticate.py +++ b/app/routers/authenticate.py @@ -27,7 +27,7 @@ def initializeAuthOb(): oauth = OAuth() oauth.register( - g.tenant + '.' + g.environment + '.rciam', + g.tenant + '_' + g.environment + '_rciam_ob', client_id=oidc_config['client_id'], client_secret=oidc_config['client_secret'], server_metadata_url=oidc_config['issuer'] + "/.well-known/openid-configuration", @@ -46,7 +46,7 @@ async def login_endpoint( request: Request, oauth_ob= Depends(initializeAuthOb), server_config= Depends(getServerConfig)): - rciam = oauth_ob.create_client(g.tenant + '.' + g.environment + '.rciam') + rciam = oauth_ob.create_client(g.tenant + '_' + g.environment + '_rciam_client') redirect_uri = server_config['protocol'] + "://" + server_config['host'] + server_config['api_path'] + "/auth" return await rciam.authorize_redirect(request, redirect_uri) @@ -71,7 +71,7 @@ async def authorize_rciam( response = RedirectResponse(url=urllib.parse.unquote(login_start_url)) response.delete_cookie("login_start") - rciam = oauth_ob.create_client(g.tenant + '.' + g.environment + '.rciam') + rciam = oauth_ob.create_client(g.tenant + '_' + g.environment + '_rciam_client') try: token = await rciam.authorize_access_token(request) except OAuthError as error: @@ -148,7 +148,7 @@ async def logout( oauth_ob= Depends(initializeAuthOb), server_config=Depends(getServerConfig) ): - rciam = oauth_ob.create_client(g.tenant + '.' + g.environment + '.rciam') + rciam = oauth_ob.create_client(g.tenant + '_' + g.environment + '_rciam_client') metadata = await rciam.load_server_metadata() # todo: Fix this after we complete the multitenacy redirect_uri = server_config['protocol'] + "://" + server_config['client'] +"/metrics" diff --git a/app/utils/globalMethods.py b/app/utils/globalMethods.py index d0b234f..e35c7ea 100644 --- a/app/utils/globalMethods.py +++ b/app/utils/globalMethods.py @@ -31,7 +31,7 @@ async def __call__(self, request: Request, response: Response): self.logger.debug("""{0}.{1}: Config File Name: {2}""".format(g.tenant, g.environment, config_file)) self.oauth.register( - g.tenant + '.' + g.environment + '.rciam', + g.tenant + '_' + g.environment + '_rciam_ob', client_id=oidc_config['client_id'], client_secret=oidc_config['client_secret'], server_metadata_url=oidc_config['issuer'] + "/.well-known/openid-configuration", @@ -42,7 +42,7 @@ async def __call__(self, request: Request, response: Response): # permissions calculation access_token = request.headers.get('x-access-token') - rciam = self.oauth.create_client(g.tenant + '.' + g.environment + '.rciam') + rciam = self.oauth.create_client(g.tenant + '_' + g.environment + '_rciam_client') metadata = await rciam.load_server_metadata() headers = {'Authorization': f'Bearer {access_token}'} From c345cdf4d2c8077b146df0d6ad15698f56ba2929 Mon Sep 17 00:00:00 2001 From: Nick Mastoris Date: Mon, 23 Oct 2023 10:34:52 +0300 Subject: [PATCH 2/3] fix creation of numerous db_connections problem --- Dockerfile | 2 +- app/database.py | 44 +++++++++++++++++++++++-------- app/ingester/communityIngester.py | 7 +++-- app/ingester/ingestData.py | 11 +++++--- app/ingester/loginsIngester.py | 5 +--- app/ingester/membeshipIngester.py | 5 +--- app/ingester/usersIngester.py | 5 +--- app/main.py | 1 - app/routers/ams.py | 4 +-- app/routers/communities.py | 14 +++++----- app/routers/countries.py | 8 +++--- app/routers/dashboard.py | 10 +++---- app/routers/logins.py | 14 +++++----- app/routers/users.py | 12 ++++----- 14 files changed, 78 insertions(+), 64 deletions(-) diff --git a/Dockerfile b/Dockerfile index 00256e5..e053d29 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ ARG PYTHON_IMAGE_REPO=python -FROM FROM ${PYTHON_IMAGE_REPO}:3.11.5-bookworm +FROM ${PYTHON_IMAGE_REPO}:3.11.5-bookworm RUN curl -sL https://deb.nodesource.com/setup_18.x | sed "s/exec_cmd 'apt-get update'/exec_cmd 'apt-get --allow-releaseinfo-change update'/" | bash - RUN echo "deb https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list RUN curl https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - diff --git a/app/database.py b/app/database.py index c547c36..5e97b80 100644 --- a/app/database.py +++ b/app/database.py @@ -1,16 +1,38 @@ from app.utils import configParser -from sqlmodel import create_engine, Session +from sqlalchemy import create_engine +from sqlmodel import Session +from sqlalchemy.exc import OperationalError -def get_session(): - # Initialize - config_file = 'config.global.py' - db_params = configParser.getConfig('database_parameters', config_file) +class Database: + def __init__(self): - url = db_params['database_url'] - pool_size = int(db_params.get('pool_size', 25)) - max_overflow = int(db_params.get('max_overflow', 5)) - engine = create_engine(url, pool_size=pool_size, max_overflow=max_overflow) + config_file = 'config.global.py' + db_params = configParser.getConfig('database_parameters', config_file) - with Session(engine) as session: - yield session + url = db_params['database_url'] + pool_size = int(db_params.get('pool_size', 25)) + max_overflow = int(db_params.get('max_overflow', 5)) + + self.engine = create_engine(url, pool_size=pool_size, max_overflow=max_overflow) + + def check_database_connection(self): + try: + # Attempt to connect to the database by checking the connection + with self.engine.connect(): + print("Database connection successful!") + return True + except OperationalError as e: + print(f"Database connection failed: {e}") + return False + + def get_session(self): + with Session(self.engine) as session: + yield session + + def create_session(self): + return Session(self.engine) + + +# Creating an instance of Database +db = Database() diff --git a/app/ingester/communityIngester.py b/app/ingester/communityIngester.py index 915590f..1fa8ab6 100644 --- a/app/ingester/communityIngester.py +++ b/app/ingester/communityIngester.py @@ -1,5 +1,5 @@ from app.logger import log -from ..database import get_session + from sqlalchemy.exc import NoResultFound from .utilsIngester import utilsIngester @@ -77,9 +77,8 @@ def ingestCommunityDataPerTenenv(cls, tenenvId, session): format(communityMappedItems)) @classmethod - def ingestCommunityData(cls): - session_generator = get_session() - session = next(session_generator) + def ingestCommunityData(cls, session): + tenenvIds = session.exec("""SELECT id FROM tenenv_info""").all() # for each tenenv on database try to ingest CommunityData # from statistics_raw table diff --git a/app/ingester/ingestData.py b/app/ingester/ingestData.py index a6a6bd5..a605a2d 100644 --- a/app/ingester/ingestData.py +++ b/app/ingester/ingestData.py @@ -2,12 +2,15 @@ from .usersIngester import UserDataIngester from .membeshipIngester import MembershipDataIngester from .loginsIngester import LoginDataIngester +from ..database import db +session = db.create_session() # Ingest Communities -CommunityDataIngester.ingestCommunityData() +CommunityDataIngester.ingestCommunityData(session) # Ingest Users -UserDataIngester.ingestUserData() +UserDataIngester.ingestUserData(session) # Ingest Memberships -MembershipDataIngester.ingestMembershipData() +MembershipDataIngester.ingestMembershipData(session) # Ingest Logins -LoginDataIngester.ingestLoginData() +LoginDataIngester.ingestLoginData(session) +session.close() diff --git a/app/ingester/loginsIngester.py b/app/ingester/loginsIngester.py index fe46f87..2556ccf 100644 --- a/app/ingester/loginsIngester.py +++ b/app/ingester/loginsIngester.py @@ -1,5 +1,4 @@ from app.logger import log -from ..database import get_session from app.utils.ipDatabase import geoip2Database from sqlalchemy.exc import NoResultFound from .utilsIngester import utilsIngester @@ -222,9 +221,7 @@ def ingestLoginDataPerTenenv(cls, tenenvId, session): {0} new logins ingested""".format(loginMappedItems)) @classmethod - def ingestLoginData(cls): - session_generator = get_session() - session = next(session_generator) + def ingestLoginData(cls, session): tenenvIds = session.exec("""SELECT id FROM tenenv_info""").all() for tenenvId in tenenvIds: LoginDataIngester.ingestLoginDataPerTenenv(tenenvId[0], session) diff --git a/app/ingester/membeshipIngester.py b/app/ingester/membeshipIngester.py index 5e6fa34..0396bae 100644 --- a/app/ingester/membeshipIngester.py +++ b/app/ingester/membeshipIngester.py @@ -1,5 +1,4 @@ from app.logger import log -from ..database import get_session from sqlalchemy.exc import NoResultFound from .utilsIngester import utilsIngester @@ -69,9 +68,7 @@ def ingestMembershipDataPerTenenv(cls, tenenvId, session): format(membershipMappedItems)) @classmethod - def ingestMembershipData(cls): - session_generator = get_session() - session = next(session_generator) + def ingestMembershipData(cls, session): tenenvIds = session.exec("""SELECT id FROM tenenv_info""").all() # for each tenenv on database try to ingest UserData # from statistics_raw table diff --git a/app/ingester/usersIngester.py b/app/ingester/usersIngester.py index 4ea9281..213272c 100644 --- a/app/ingester/usersIngester.py +++ b/app/ingester/usersIngester.py @@ -1,5 +1,4 @@ from app.logger import log -from ..database import get_session from .utilsIngester import utilsIngester @@ -48,9 +47,7 @@ def ingestUserDataPerTenenv(cls, tenenvId, session): {0} users ingested or updated""".format(userMappedItems)) @classmethod - def ingestUserData(cls): - session_generator = get_session() - session = next(session_generator) + def ingestUserData(cls, session): tenenvIds = session.exec("""SELECT id FROM tenenv_info""").all() # for each tenenv on database try to ingest UserData # from statistics_raw table diff --git a/app/main.py b/app/main.py index 84a65c9..64371a3 100644 --- a/app/main.py +++ b/app/main.py @@ -11,7 +11,6 @@ from sqlalchemy import func from sqlalchemy.orm import selectinload -from app.database import get_session from app.models.community_info_model import * from app.models.community_model import * from app.models.member_model import * diff --git a/app/routers/ams.py b/app/routers/ams.py index 30c716a..c3d7ec3 100644 --- a/app/routers/ams.py +++ b/app/routers/ams.py @@ -5,7 +5,7 @@ from sqlmodel import Field, Session, SQLModel, create_engine, select from typing import Union from app.utils import configParser, globalMethods -from app.database import get_session +from app.database import db from app.utils.globalMethods import AuthNZCheck from fastapi.responses import PlainTextResponse from app.logger import log @@ -44,7 +44,7 @@ async def verify_authorization_header(Authorization: Optional[str] = Header(None @router.post("/ams_stats") async def get_ams_stats(*, - session: Session = Depends(get_session), + session: Session = Depends(db.get_session), request: Request, response: Response, body = Body(..., example={"name": "Item Name"}), diff --git a/app/routers/communities.py b/app/routers/communities.py index 27c6fac..f006bbb 100644 --- a/app/routers/communities.py +++ b/app/routers/communities.py @@ -2,7 +2,7 @@ from sqlmodel import Field, Session, SQLModel, create_engine, select from typing import Union -from app.database import get_session +from app.database import db from app.models.community_info_model import * from app.models.community_model import * from app.models.member_model import MembersReadWithCommunityInfo @@ -20,7 +20,7 @@ @router.get("/members/", response_model=List[MembersReadWithCommunityInfo]) async def read_members( *, - session: Session = Depends(get_session), + session: Session = Depends(db.get_session), offset: int = 0, # community_id: Union[None, int] = None ): @@ -33,7 +33,7 @@ async def read_members( @router.get("/min_date_communities") async def read_min_date_communities( *, - session: Session = Depends(get_session), + session: Session = Depends(db.get_session), offset: int = 0, tenenv_id: int, ): @@ -46,7 +46,7 @@ async def read_min_date_communities( @router.get("/members_bystatus") async def read_members_bystatus( *, - session: Session = Depends(get_session), + session: Session = Depends(db.get_session), offset: int = 0, community_id: Union[None, int] = None, tenenv_id: int, @@ -67,7 +67,7 @@ async def read_members_bystatus( @router.get("/communities_groupby/{group_by}") async def read_communities( *, - session: Session = Depends(get_session), + session: Session = Depends(db.get_session), offset: int = 0, group_by: str, tenenv_id: int, @@ -110,7 +110,7 @@ async def read_communities( @router.get("/communities") async def read_community( *, - session: Session = Depends(get_session), + session: Session = Depends(db.get_session), community_id: Union[None, int] = None, tenenv_id: int): sql_subquery = '' @@ -130,7 +130,7 @@ async def read_community( @router.get("/communities_info", response_model=List[Community_InfoRead]) async def read_communities_info( *, - session: Session = Depends(get_session), + session: Session = Depends(db.get_session), offset: int = 0 ): communities = session.exec(select(Community_Info).offset(offset)).all() diff --git a/app/routers/countries.py b/app/routers/countries.py index c744f11..f5de351 100644 --- a/app/routers/countries.py +++ b/app/routers/countries.py @@ -1,5 +1,5 @@ from fastapi import APIRouter, Depends, HTTPException, Query -from app.database import get_session +from app.database import db from sqlmodel import Field, Session, SQLModel, create_engine, select from typing import Union @@ -20,7 +20,7 @@ @router.get("/countries/", response_model=List[Country_CodesRead]) async def read_countries( *, - session: Session = Depends(get_session), + session: Session = Depends(db.get_session), offset: int = 0 ): countries = session.exec(select(Country_Codes).offset(offset)).all() @@ -30,7 +30,7 @@ async def read_countries( @router.get("/country_stats/", response_model=List[Statistics_Country_HashedwithInfo]) async def read_country_stats( *, - session: Session = Depends(get_session), + session: Session = Depends(db.get_session), offset: int = 0 ): stats = session.exec( @@ -41,7 +41,7 @@ async def read_country_stats( @router.get("/country_stats_by_vo/{community_id}") async def read_country_stats_by_vo( *, - session: Session = Depends(get_session), + session: Session = Depends(db.get_session), offset: int = 0, community_id: Union[None, int] = None ): diff --git a/app/routers/dashboard.py b/app/routers/dashboard.py index 2e6a46d..227d153 100644 --- a/app/routers/dashboard.py +++ b/app/routers/dashboard.py @@ -3,7 +3,7 @@ from typing import Union from xmlrpc.client import boolean -from app.database import get_session +from app.database import db from app.utils.globalMethods import AuthNZCheck router = APIRouter( @@ -14,7 +14,7 @@ @router.get("/tenenv/{tenant_name}/{environment_name}") async def read_tenenv_byname( *, - session: Session = Depends(get_session), + session: Session = Depends(db.get_session), offset: int = 0, tenant_name: str, environment_name: str @@ -34,7 +34,7 @@ async def read_tenenv_byname( @router.get("/environment_byname/{environment_name}") async def read_environment_byname( *, - session: Session = Depends(get_session), + session: Session = Depends(db.get_session), offset: int = 0, environment_name: str ): @@ -50,7 +50,7 @@ async def read_environment_byname( @router.get("/idps") async def read_idps( *, - session: Session = Depends(get_session), + session: Session = Depends(db.get_session), tenenv_id: int, idpId: int = None ): @@ -69,7 +69,7 @@ async def read_idps( @router.get("/sps") async def read_sps( *, - session: Session = Depends(get_session), + session: Session = Depends(db.get_session), tenenv_id: int, spId: int = None ): diff --git a/app/routers/logins.py b/app/routers/logins.py index 8d151fa..088093f 100644 --- a/app/routers/logins.py +++ b/app/routers/logins.py @@ -7,7 +7,7 @@ from typing import Union from xmlrpc.client import boolean -from app.database import get_session +from app.database import db from app.utils.globalMethods import AuthNZCheck @@ -22,7 +22,7 @@ async def read_min_date_logins( *, request: Request, - session: Session = Depends(get_session), + session: Session = Depends(db.get_session), tenenv_id: int, unique_logins: Union[boolean, None] = False, ): @@ -39,7 +39,7 @@ async def read_min_date_logins( async def read_logins_per_idp( *, request: Request, - session: Session = Depends(get_session), + session: Session = Depends(db.get_session), offset: int = 0, sp: str = None, # type: ignore startDate: str = None, # type: ignore @@ -100,7 +100,7 @@ async def read_logins_per_idp( @router.get("/logins_per_sp") async def read_logins_per_sp( *, - session: Session = Depends(get_session), + session: Session = Depends(db.get_session), request: Request, offset: int = 0, idp: str = None, @@ -163,7 +163,7 @@ async def read_logins_per_sp( @router.get("/logins_per_country") async def read_logins_per_country( *, - session: Session = Depends(get_session), + session: Session = Depends(db.get_session), offset: int = 0, group_by: Union[str, None] = None, startDate: str = None, @@ -258,7 +258,7 @@ async def read_logins_per_country( @router.get("/logins_countby") async def read_logins_countby( *, - session: Session = Depends(get_session), + session: Session = Depends(db.get_session), offset: int = 0, interval: Union[str, None] = None, count_interval: int = None, @@ -305,7 +305,7 @@ async def read_logins_countby( @router.get("/logins_groupby/{group_by}") async def read_logins_groupby( *, - session: Session = Depends(get_session), + session: Session = Depends(db.get_session), request: Request, offset: int = 0, group_by: str, diff --git a/app/routers/users.py b/app/routers/users.py index 2daa268..94e4cde 100644 --- a/app/routers/users.py +++ b/app/routers/users.py @@ -2,7 +2,7 @@ from sqlmodel import Field, Session, SQLModel, create_engine, select from typing import Union -from app.database import get_session +from app.database import db from app.utils.globalMethods import AuthNZCheck @@ -17,7 +17,7 @@ @router.get("/min_date_registered_users") async def read_min_date_registered_users( *, - session: Session = Depends(get_session), + session: Session = Depends(db.get_session), tenenv_id: int ): min_date = session.exec(""" @@ -29,7 +29,7 @@ async def read_min_date_registered_users( @router.get("/registered_users_country") async def read_users_country( *, - session: Session = Depends(get_session), + session: Session = Depends(db.get_session), offset: int = 0, startDate: str = None, endDate: str = None, @@ -72,7 +72,7 @@ async def read_users_country( @router.get("/registered_users_country_group_by/{group_by}") async def read_users_country_groupby( *, - session: Session = Depends(get_session), + session: Session = Depends(db.get_session), offset: int = 0, group_by: str, startDate: str = None, @@ -121,7 +121,7 @@ async def read_users_country_groupby( @router.get("/registered_users_groupby/{group_by}") async def read_users_groupby( *, - session: Session = Depends(get_session), + session: Session = Depends(db.get_session), offset: int = 0, group_by: str, interval: Union[str, None] = None, @@ -154,7 +154,7 @@ async def read_users_groupby( @router.get("/registered_users_countby") async def read_users_countby( *, - session: Session = Depends(get_session), + session: Session = Depends(db.get_session), offset: int = 0, interval: Union[str, None] = None, count_interval: int = None, From b93e144baebdf9c94781dc53207618458a9da04e Mon Sep 17 00:00:00 2001 From: Ioannis Igoumenos Date: Mon, 23 Oct 2023 10:42:26 +0300 Subject: [PATCH 3/3] oauth object and client have to share the same instance name --- app/routers/authenticate.py | 8 ++++---- app/utils/globalMethods.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/routers/authenticate.py b/app/routers/authenticate.py index e2a99be..a897406 100644 --- a/app/routers/authenticate.py +++ b/app/routers/authenticate.py @@ -27,7 +27,7 @@ def initializeAuthOb(): oauth = OAuth() oauth.register( - g.tenant + '_' + g.environment + '_rciam_ob', + g.tenant + '_' + g.environment + '_rciam', client_id=oidc_config['client_id'], client_secret=oidc_config['client_secret'], server_metadata_url=oidc_config['issuer'] + "/.well-known/openid-configuration", @@ -46,7 +46,7 @@ async def login_endpoint( request: Request, oauth_ob= Depends(initializeAuthOb), server_config= Depends(getServerConfig)): - rciam = oauth_ob.create_client(g.tenant + '_' + g.environment + '_rciam_client') + rciam = oauth_ob.create_client(g.tenant + '_' + g.environment + '_rciam') redirect_uri = server_config['protocol'] + "://" + server_config['host'] + server_config['api_path'] + "/auth" return await rciam.authorize_redirect(request, redirect_uri) @@ -71,7 +71,7 @@ async def authorize_rciam( response = RedirectResponse(url=urllib.parse.unquote(login_start_url)) response.delete_cookie("login_start") - rciam = oauth_ob.create_client(g.tenant + '_' + g.environment + '_rciam_client') + rciam = oauth_ob.create_client(g.tenant + '_' + g.environment + '_rciam') try: token = await rciam.authorize_access_token(request) except OAuthError as error: @@ -148,7 +148,7 @@ async def logout( oauth_ob= Depends(initializeAuthOb), server_config=Depends(getServerConfig) ): - rciam = oauth_ob.create_client(g.tenant + '_' + g.environment + '_rciam_client') + rciam = oauth_ob.create_client(g.tenant + '_' + g.environment + '_rciam') metadata = await rciam.load_server_metadata() # todo: Fix this after we complete the multitenacy redirect_uri = server_config['protocol'] + "://" + server_config['client'] +"/metrics" diff --git a/app/utils/globalMethods.py b/app/utils/globalMethods.py index e35c7ea..02f2579 100644 --- a/app/utils/globalMethods.py +++ b/app/utils/globalMethods.py @@ -31,7 +31,7 @@ async def __call__(self, request: Request, response: Response): self.logger.debug("""{0}.{1}: Config File Name: {2}""".format(g.tenant, g.environment, config_file)) self.oauth.register( - g.tenant + '_' + g.environment + '_rciam_ob', + g.tenant + '_' + g.environment + '_rciam', client_id=oidc_config['client_id'], client_secret=oidc_config['client_secret'], server_metadata_url=oidc_config['issuer'] + "/.well-known/openid-configuration", @@ -42,7 +42,7 @@ async def __call__(self, request: Request, response: Response): # permissions calculation access_token = request.headers.get('x-access-token') - rciam = self.oauth.create_client(g.tenant + '_' + g.environment + '_rciam_client') + rciam = self.oauth.create_client(g.tenant + '_' + g.environment + '_rciam') metadata = await rciam.load_server_metadata() headers = {'Authorization': f'Bearer {access_token}'}