Develop

CHANGELOG.md

@@ -1,4 +1,10 @@
 ### Changelog
 
+
+## 2023-11-23
+
+### Added
+- Add EOSC Core Infrastructure Proxy demo instance at Metrics-Dev
+
 ## Prerelease logging
 - Development RC phase

app/ingester/communityIngester.py

@@ -78,7 +78,7 @@ def ingestCommunityDataPerTenenv(cls, tenenvId, session):
 
     @classmethod
     def ingestCommunityData(cls, session):
-       
+
         tenenvIds = session.exec("""SELECT id FROM tenenv_info""").all()
         # for each tenenv on database try to ingest CommunityData
         # from statistics_raw table

app/ingester/loginsIngester.py

@@ -192,7 +192,7 @@ def ingestLoginDataPerTenenv(cls, tenenv, session):
 
         for login in loginsNotMapped:
 
-            if (login[0]['voPersonId'] in hashed_user_ids):
+            if ('voPersonId' in login[0] and login[0]['voPersonId'] in hashed_user_ids):
                 cls.logger.info("""Ignore this user with
                     hash {0} as he is at the blacklist""". format(login[0]['voPersonId']))
                 continue

app/routers/users.py

@@ -35,11 +35,11 @@ async def read_users_country(
         endDate: str = None,
         tenenv_id: int
 ):
-    interval_subquery = ""
+    interval_subquery = "WHERE tenenv_id = {0}".format(tenenv_id)
     if startDate and endDate:
         interval_subquery = """
-                WHERE users.created BETWEEN '{0}' AND '{1}'
-            """.format(startDate, endDate)
+                WHERE users.created BETWEEN '{0}' AND '{1}' AND tenenv_id = {2}
+            """.format(startDate, endDate, tenenv_id)
     users_countries = session.exec(
         """WITH users_countries AS (
         SELECT statistics_country_hashed.hasheduserid as userid, country, countrycode, count(*) as sum_count
@@ -80,11 +80,11 @@ async def read_users_country_groupby(
         tenenv_id: int
 ):
     if group_by:
-        interval_subquery = ""
+        interval_subquery = "WHERE tenenv_id = {0}".format(tenenv_id)
         if startDate and endDate:
             interval_subquery = """
-                WHERE users.created BETWEEN '{0}' AND '{1}'
-            """.format(startDate, endDate)
+                WHERE users.created BETWEEN '{0}' AND '{1}' AND tenenv_id={2}
+            """.format(startDate, endDate, tenenv_id)
         users = session.exec(
             """WITH users_countries AS (
         SELECT statistics_country_hashed.hasheduserid as userid, country, countrycode, count(*) as sum_count
@@ -114,6 +114,7 @@ async def read_users_country_groupby(
                 GROUP BY range_date, country,countrycode
                 ORDER BY range_date, country
             ) user_country_group_by
+
         GROUP BY range_date""".format(group_by, interval_subquery, tenenv_id)).all()
         return users
 

metrics-migrate-stats/.gitignore

@@ -0,0 +1,6 @@
+log/*
+__pycache__
+config.py
+.venv/*
+env
+databases/GeoLite2-Country.mmdb

metrics-migrate-stats/CHANGELOG.md

@@ -0,0 +1,7 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+

metrics-migrate-stats/Controller/countryStatisticsController.py

@@ -0,0 +1,23 @@
+from datetime import date, timedelta
+from Model.ipStatistics import ipStatistics
+from Model.countryStatistics import countryStatistics
+from datetime import datetime, timedelta
+class countryStatisticsController:
+  @classmethod
+  def getDataNotMapped(self):
+    dateFrom = countryStatistics.getLastDate()
+
+    # we dont have any country statistics saved
+    if dateFrom[0][0] == None:
+      result = ipStatistics.getAllIpStatistics()
+    else:
+      dayAfter = dateFrom[0][0] + timedelta(days=1)
+      dayFrom = dayAfter.strftime('%Y-%m-%d 00:00:00')
+
+      yesterday = date.today() - timedelta(days=1)
+      dateTo = yesterday.strftime('%Y-%m-%d 23:59:59')
+
+      result = ipStatistics.getIpStatisticsByDate(dayFrom, dateTo)
+    return result
+
+

metrics-migrate-stats/Controller/countryStatisticsHashedUserIdsController.py

@@ -0,0 +1,23 @@
+from datetime import date, timedelta
+from Model.ipStatistics import ipStatistics
+from Model.countryStatisticsHashedUserId import countryStatisticsHashedUserId
+from datetime import datetime, timedelta
+class countryStatisticsHashedUserIdsController:
+  @classmethod
+  def getDataNotMapped(self):
+    dateFrom = countryStatisticsHashedUserId.getLastDate()
+
+    # we dont have any country statistics saved
+    if dateFrom[0][0] == None:
+      result = ipStatistics.getAllIpStatistics()
+    else:
+      dayAfter = dateFrom[0][0] + timedelta(days=1)
+      dayFrom = dayAfter.strftime('%Y-%m-%d 00:00:00')
+
+      yesterday = date.today() - timedelta(days=1)
+      dateTo = yesterday.strftime('%Y-%m-%d 23:59:59')
+
+      result = ipStatistics.getIpStatisticsByDate(dayFrom, dateTo)
+    return result
+
+

metrics-migrate-stats/Controller/identityProvidersMapController.py

@@ -0,0 +1,13 @@
+from datetime import date, timedelta
+from Model.identityProvidersMapFromProxy import identityProvidersMapFromProxy
+from Model.identityProvidersMap import identityProvidersMap
+from datetime import datetime, timedelta
+class identityProvidersMapController:
+
+  @classmethod
+  def getAllData(self):
+
+    result = identityProvidersMapFromProxy.getAllIdps()
+    return result
+
+

metrics-migrate-stats/Controller/serviceProvidersMapController.py

@@ -0,0 +1,13 @@
+from datetime import date, timedelta
+from Model.serviceProvidersMapFromProxy import serviceProvidersMapFromProxy
+from Model.serviceProvidersMap import serviceProvidersMap
+from datetime import datetime, timedelta
+class serviceProvidersMapController:
+
+  @classmethod
+  def getAllData(self):
+
+    result = serviceProvidersMapFromProxy.getAllSps()
+    return result
+
+

metrics-migrate-stats/Controller/statisticsCountryHashedController.py

@@ -0,0 +1,111 @@
+from datetime import date, timedelta
+from multiprocessing.spawn import prepare
+from Model.countries import countries
+from Model.identityProvidersMap import identityProvidersMap
+from Model.ipStatistics import ipStatistics
+from Model.statisticsDetail import statisticsDetail
+from Model.statistics import statistics
+from Model.serviceProvidersMap import serviceProvidersMap
+from Model.statisticsCountryHashed import statisticsCountryHashed
+from Model.statisticsCountryHashedFromComanage import statisticsCountryHashedFromComanage
+from datetime import datetime, timedelta
+from Service.ipDatabase import geoip2Database
+from Logger import log
+import ipaddress
+
+class statisticsCountryHashedController:
+  logger = log.get_logger("statisticsCountryHashedController")
+
+  @classmethod
+  def saveAllData(self, tenenv_id):
+
+    dateFrom = statisticsCountryHashed.getLastDate(tenenv_id)
+    today = date.today()
+    dateTo = today.strftime('%Y-%m-%d 23:59:59')
+     # we dont have any country statistics saved
+    if dateFrom[0][0] == None:
+      result = ipStatistics.getStatsByDate(None, dateTo)
+    else:
+      dayAfter = dateFrom[0][0] + timedelta(days=1)
+      dayFrom = dayAfter.strftime('%Y-%m-%d 00:00:00')
+
+      yesterday = date.today() - timedelta(days=1)
+      dateTo = yesterday.strftime('%Y-%m-%d 23:59:59')
+
+      result = ipStatistics.getStatsByDate(dayFrom, dateTo)
+    statisticsCountryHashedController.prepareData(result, tenenv_id)
+
+    # now we would like to save those data we dont have ip information
+    dateTo = statisticsCountryHashed.getFirstDate(tenenv_id)
+    result = statisticsDetail.getStatsByDate(dateTo[0][0].strftime('%Y-%m-%d'))
+    statisticsCountryHashedController.prepareData(result, tenenv_id)
+    dateTo = statisticsCountryHashed.getFirstDate(tenenv_id)
+    result = statistics.getStatsByDate(dateTo[0][0].strftime('%Y-%m-%d'))
+    statisticsCountryHashedController.prepareData(result, tenenv_id)
+    return result
+
+  @classmethod
+  def prepareData(self, data, tenenv_id):
+    ipDatabaseHandler = geoip2Database()
+    mappedItems = 0
+    for item in data:
+      ## find identityprovider id
+      result = identityProvidersMap.getIdpIdByIdentifier(item.sourceidp, tenenv_id)
+      if len(result) > 0:
+        #self.logger.info(result[0][0])
+        idpId = result[0][0]
+      else:
+        self.logger.error("Identity provider identifier not found {0}".format(item.sourceidp))
+        continue
+
+      ## find serviceprovider id
+      result = serviceProvidersMap.getSpIdByIdentifier(item.service, tenenv_id)
+      if len(result) > 0:
+        #self.logger.info(result[0][0])
+        spId = result[0][0]
+      else:
+        self.logger.warning("Service entityid {0} not found".format(item.service))
+        spItem = serviceProvidersMap(item.service, item.service.replace("'", "''"), tenenv_id)
+        serviceProvidersMap.save(spItem)
+        self.logger.info("Service entityid {0} created".format(item.service))
+      try:
+          if hasattr(item, "ip"):
+              ipaddr = ipaddress.ip_network(item.ip).network_address
+          else:
+              item.countrycode = countryData[0]
+              item.country = countryData[1] 
+          # get country code/ name
+          countryData = ipDatabaseHandler.getCountryFromIp(str(ipaddr), item.ipVersion)
+          if (countryData[0] == None):
+              item.countrycode = "UN"
+              item.country = "Unknown"
+          else:
+              item.countrycode = countryData[0]
+              item.country = countryData[1]       
+      except:
+          item.countrycode = "UN"
+          item.country = "Unknown"
+       ## save country if not exists and get id
+      country = countries(item.countrycode, item.country)
+      countries.save(country)
+      result = countries.getIdByCountryCode(item.countrycode)   
+      if len(result) > 0:
+        #self.logger.info(result[0][0])
+        countryId = result[0][0]
+      else:
+        self.logger.error("Country not found")
+        continue
+      self.logger.info("{0} ".format(item.date))
+
+      if hasattr(item, "count"):
+        count = item.count
+      else:
+        count = 1
+      statsCountry = statisticsCountryHashed(item.date, item.hasheduserid, idpId, spId, countryId, count, tenenv_id)
+      statisticsCountryHashed.save(statsCountry)
+      mappedItems += 1
+    self.logger.info("{0} Country Stats created".format(mappedItems))
+
+
+
+

metrics-migrate-stats/Controller/usersController.py

@@ -0,0 +1,27 @@
+from ast import Or
+from datetime import date, timedelta
+from Model.users import users
+from Model.usersFromComanage import usersFromComanage
+from Model.vos import vos
+from datetime import datetime, timedelta
+from Logger import log
+
+class usersController:
+  logger = log.get_logger("usersController")
+
+  @classmethod
+  def saveUsers(self, tenenv_id):
+
+    usersList = []
+    # get memberships with active status
+    usersComanage = usersFromComanage.getAllUsers()
+    # save memberships with active status
+    for item in usersComanage:
+        usersItem = users(item.hasheduserid, item.created, item.status, tenenv_id)
+        usersList.append(usersItem)
+    # save data to tables if any
+    if usersList:
+      users.saveAll(usersList)
+
+
+

metrics-migrate-stats/Controller/voMembershipsController.py

@@ -0,0 +1,81 @@
+from ast import Or
+from datetime import date, timedelta
+from Model.voMemberships import voMemberships
+from Model.voMembershipsFromComanage import voMembershipsFromComanage
+from Model.vos import vos
+from datetime import datetime, timedelta
+from Logger import log
+
+class voMembershipsController:
+  logger = log.get_logger("voMembershipsController")
+
+  @classmethod
+  def getVoId(self, voName, source, tenenv_id):
+    result = vos.getVoIdFromVoName(voName, source, tenenv_id)
+    return result
+
+  @classmethod
+  def getDataNotMapped(self, tenenv_id):
+    voMemberships.truncate(tenenv_id)
+
+    voMembershipsList = []
+    # dateFrom = vos.getLastDate()
+
+    # we dont have any vos saved
+    # if dateFrom[0][0] == None:
+
+    # get memberships with active status
+    activeMemberships = voMembershipsFromComanage.getAllVoMembershipsByStatus('A')
+    # save memberships with active status
+    for item in activeMemberships:
+      result = voMembershipsController.getVoId(item.voName, "registry", tenenv_id)
+
+      if len(result) > 0:
+        voId = result[0][0]
+        self.logger.info("Vo name {0} with id {1}".format(item.voName, result[0][0]))
+
+        voMembershipsItem = voMemberships(voId, item.hasheduserid, item.status, item.created, tenenv_id)
+        voMembershipsList.append(voMembershipsItem)
+    # save data to tables if any
+    if voMembershipsList:
+
+      voMemberships.saveAll(voMembershipsList)
+
+    # get memberships with grace period
+    # save only if there isnt any existing membership or status membership is not Active/ Grace Period
+    gracePeriodVoMembershipsList = []
+    graceperiodMemberships = voMembershipsFromComanage.getAllVoMembershipsByStatus('GP')
+    for item in graceperiodMemberships:
+      result = voMembershipsController.getVoId(item.voName, "registry", tenenv_id)
+      if len(result) > 0:
+        voId = result[0][0]
+        self.logger.info("Vo name {0} with id {1}".format(item.voName, result[0][0]))
+        #check if already there is a membership with active status saved
+        statusMembership = voMemberships.getMembershipStatus(voId, item.hasheduserid, tenenv_id)
+        if len(statusMembership) > 0  and (statusMembership[0][0]=='A' or statusMembership[0][0]=='GP'):
+            continue
+        voMembershipsItem = voMemberships(voId, item.hasheduserid, item.status, item.created, tenenv_id)
+        gracePeriodVoMembershipsList.append(voMembershipsItem)
+    if len(gracePeriodVoMembershipsList)>0:
+        voMemberships.saveAll(gracePeriodVoMembershipsList)
+
+    # get memberships with other statuses
+    # save only if there isnt any existing membership
+    otherStatusVoMembershipsList = []
+    otherStatusMemberships = voMembershipsFromComanage.getAllVoMembershipsByStatus('Other')
+    for item in otherStatusMemberships:
+      result = voMembershipsController.getVoId(item.voName, "registry", tenenv_id)
+      if len(result) > 0:
+        voId = result[0][0]
+        self.logger.info("Vo name {0} with id {1}".format(item.voName, result[0][0]))
+        #check if already there is a membership with active status saved
+        statusMembership = voMemberships.getMembershipStatus(voId, item.hasheduserid, tenenv_id)
+        if len(statusMembership) > 0:
+          continue
+        voMembershipsItem = voMemberships(voId, item.hasheduserid, item.status, item.created, tenenv_id)
+        otherStatusVoMembershipsList.append(voMembershipsItem)
+    if len(otherStatusVoMembershipsList)>0:
+        voMemberships.saveAll(otherStatusVoMembershipsList)
+    return result
+
+