-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Check for voPerson & migration scriptis (#42)
* check if voPerson exists * add migration scripts * minor fix * add config example * minor refactor --------- Co-authored-by: Nick Mastoris <[email protected]>
- Loading branch information
1 parent
7b96c7e
commit 6624b48
Showing
49 changed files
with
1,975 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,6 @@ | ||
log/* | ||
__pycache__ | ||
config.py | ||
.venv/* | ||
env | ||
databases/GeoLite2-Country.mmdb |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,7 @@ | ||
# Changelog | ||
|
||
All notable changes to this project will be documented in this file. | ||
|
||
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) | ||
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). | ||
|
23 changes: 23 additions & 0 deletions
23
metrics-migrate-stats/Controller/countryStatisticsController.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,23 @@ | ||
from datetime import date, timedelta | ||
from Model.ipStatistics import ipStatistics | ||
from Model.countryStatistics import countryStatistics | ||
from datetime import datetime, timedelta | ||
class countryStatisticsController: | ||
@classmethod | ||
def getDataNotMapped(self): | ||
dateFrom = countryStatistics.getLastDate() | ||
|
||
# we dont have any country statistics saved | ||
if dateFrom[0][0] == None: | ||
result = ipStatistics.getAllIpStatistics() | ||
else: | ||
dayAfter = dateFrom[0][0] + timedelta(days=1) | ||
dayFrom = dayAfter.strftime('%Y-%m-%d 00:00:00') | ||
|
||
yesterday = date.today() - timedelta(days=1) | ||
dateTo = yesterday.strftime('%Y-%m-%d 23:59:59') | ||
|
||
result = ipStatistics.getIpStatisticsByDate(dayFrom, dateTo) | ||
return result | ||
|
||
|
23 changes: 23 additions & 0 deletions
23
metrics-migrate-stats/Controller/countryStatisticsHashedUserIdsController.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,23 @@ | ||
from datetime import date, timedelta | ||
from Model.ipStatistics import ipStatistics | ||
from Model.countryStatisticsHashedUserId import countryStatisticsHashedUserId | ||
from datetime import datetime, timedelta | ||
class countryStatisticsHashedUserIdsController: | ||
@classmethod | ||
def getDataNotMapped(self): | ||
dateFrom = countryStatisticsHashedUserId.getLastDate() | ||
|
||
# we dont have any country statistics saved | ||
if dateFrom[0][0] == None: | ||
result = ipStatistics.getAllIpStatistics() | ||
else: | ||
dayAfter = dateFrom[0][0] + timedelta(days=1) | ||
dayFrom = dayAfter.strftime('%Y-%m-%d 00:00:00') | ||
|
||
yesterday = date.today() - timedelta(days=1) | ||
dateTo = yesterday.strftime('%Y-%m-%d 23:59:59') | ||
|
||
result = ipStatistics.getIpStatisticsByDate(dayFrom, dateTo) | ||
return result | ||
|
||
|
13 changes: 13 additions & 0 deletions
13
metrics-migrate-stats/Controller/identityProvidersMapController.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,13 @@ | ||
from datetime import date, timedelta | ||
from Model.identityProvidersMapFromProxy import identityProvidersMapFromProxy | ||
from Model.identityProvidersMap import identityProvidersMap | ||
from datetime import datetime, timedelta | ||
class identityProvidersMapController: | ||
|
||
@classmethod | ||
def getAllData(self): | ||
|
||
result = identityProvidersMapFromProxy.getAllIdps() | ||
return result | ||
|
||
|
13 changes: 13 additions & 0 deletions
13
metrics-migrate-stats/Controller/serviceProvidersMapController.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,13 @@ | ||
from datetime import date, timedelta | ||
from Model.serviceProvidersMapFromProxy import serviceProvidersMapFromProxy | ||
from Model.serviceProvidersMap import serviceProvidersMap | ||
from datetime import datetime, timedelta | ||
class serviceProvidersMapController: | ||
|
||
@classmethod | ||
def getAllData(self): | ||
|
||
result = serviceProvidersMapFromProxy.getAllSps() | ||
return result | ||
|
||
|
111 changes: 111 additions & 0 deletions
111
metrics-migrate-stats/Controller/statisticsCountryHashedController.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,111 @@ | ||
from datetime import date, timedelta | ||
from multiprocessing.spawn import prepare | ||
from Model.countries import countries | ||
from Model.identityProvidersMap import identityProvidersMap | ||
from Model.ipStatistics import ipStatistics | ||
from Model.statisticsDetail import statisticsDetail | ||
from Model.statistics import statistics | ||
from Model.serviceProvidersMap import serviceProvidersMap | ||
from Model.statisticsCountryHashed import statisticsCountryHashed | ||
from Model.statisticsCountryHashedFromComanage import statisticsCountryHashedFromComanage | ||
from datetime import datetime, timedelta | ||
from Service.ipDatabase import geoip2Database | ||
from Logger import log | ||
import ipaddress | ||
|
||
class statisticsCountryHashedController: | ||
logger = log.get_logger("statisticsCountryHashedController") | ||
|
||
@classmethod | ||
def saveAllData(self, tenenv_id): | ||
|
||
dateFrom = statisticsCountryHashed.getLastDate(tenenv_id) | ||
today = date.today() | ||
dateTo = today.strftime('%Y-%m-%d 23:59:59') | ||
# we dont have any country statistics saved | ||
if dateFrom[0][0] == None: | ||
result = ipStatistics.getStatsByDate(None, dateTo) | ||
else: | ||
dayAfter = dateFrom[0][0] + timedelta(days=1) | ||
dayFrom = dayAfter.strftime('%Y-%m-%d 00:00:00') | ||
|
||
yesterday = date.today() - timedelta(days=1) | ||
dateTo = yesterday.strftime('%Y-%m-%d 23:59:59') | ||
|
||
result = ipStatistics.getStatsByDate(dayFrom, dateTo) | ||
statisticsCountryHashedController.prepareData(result, tenenv_id) | ||
|
||
# now we would like to save those data we dont have ip information | ||
dateTo = statisticsCountryHashed.getFirstDate(tenenv_id) | ||
result = statisticsDetail.getStatsByDate(dateTo[0][0].strftime('%Y-%m-%d')) | ||
statisticsCountryHashedController.prepareData(result, tenenv_id) | ||
dateTo = statisticsCountryHashed.getFirstDate(tenenv_id) | ||
result = statistics.getStatsByDate(dateTo[0][0].strftime('%Y-%m-%d')) | ||
statisticsCountryHashedController.prepareData(result, tenenv_id) | ||
return result | ||
|
||
@classmethod | ||
def prepareData(self, data, tenenv_id): | ||
ipDatabaseHandler = geoip2Database() | ||
mappedItems = 0 | ||
for item in data: | ||
## find identityprovider id | ||
result = identityProvidersMap.getIdpIdByIdentifier(item.sourceidp, tenenv_id) | ||
if len(result) > 0: | ||
#self.logger.info(result[0][0]) | ||
idpId = result[0][0] | ||
else: | ||
self.logger.error("Identity provider identifier not found {0}".format(item.sourceidp)) | ||
continue | ||
|
||
## find serviceprovider id | ||
result = serviceProvidersMap.getSpIdByIdentifier(item.service, tenenv_id) | ||
if len(result) > 0: | ||
#self.logger.info(result[0][0]) | ||
spId = result[0][0] | ||
else: | ||
self.logger.warning("Service entityid {0} not found".format(item.service)) | ||
spItem = serviceProvidersMap(item.service, item.service.replace("'", "''"), tenenv_id) | ||
serviceProvidersMap.save(spItem) | ||
self.logger.info("Service entityid {0} created".format(item.service)) | ||
try: | ||
if hasattr(item, "ip"): | ||
ipaddr = ipaddress.ip_network(item.ip).network_address | ||
else: | ||
item.countrycode = countryData[0] | ||
item.country = countryData[1] | ||
# get country code/ name | ||
countryData = ipDatabaseHandler.getCountryFromIp(str(ipaddr), item.ipVersion) | ||
if (countryData[0] == None): | ||
item.countrycode = "UN" | ||
item.country = "Unknown" | ||
else: | ||
item.countrycode = countryData[0] | ||
item.country = countryData[1] | ||
except: | ||
item.countrycode = "UN" | ||
item.country = "Unknown" | ||
## save country if not exists and get id | ||
country = countries(item.countrycode, item.country) | ||
countries.save(country) | ||
result = countries.getIdByCountryCode(item.countrycode) | ||
if len(result) > 0: | ||
#self.logger.info(result[0][0]) | ||
countryId = result[0][0] | ||
else: | ||
self.logger.error("Country not found") | ||
continue | ||
self.logger.info("{0} ".format(item.date)) | ||
|
||
if hasattr(item, "count"): | ||
count = item.count | ||
else: | ||
count = 1 | ||
statsCountry = statisticsCountryHashed(item.date, item.hasheduserid, idpId, spId, countryId, count, tenenv_id) | ||
statisticsCountryHashed.save(statsCountry) | ||
mappedItems += 1 | ||
self.logger.info("{0} Country Stats created".format(mappedItems)) | ||
|
||
|
||
|
||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,27 @@ | ||
from ast import Or | ||
from datetime import date, timedelta | ||
from Model.users import users | ||
from Model.usersFromComanage import usersFromComanage | ||
from Model.vos import vos | ||
from datetime import datetime, timedelta | ||
from Logger import log | ||
|
||
class usersController: | ||
logger = log.get_logger("usersController") | ||
|
||
@classmethod | ||
def saveUsers(self, tenenv_id): | ||
|
||
usersList = [] | ||
# get memberships with active status | ||
usersComanage = usersFromComanage.getAllUsers() | ||
# save memberships with active status | ||
for item in usersComanage: | ||
usersItem = users(item.hasheduserid, item.created, item.status, tenenv_id) | ||
usersList.append(usersItem) | ||
# save data to tables if any | ||
if usersList: | ||
users.saveAll(usersList) | ||
|
||
|
||
|
81 changes: 81 additions & 0 deletions
81
metrics-migrate-stats/Controller/voMembershipsController.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,81 @@ | ||
from ast import Or | ||
from datetime import date, timedelta | ||
from Model.voMemberships import voMemberships | ||
from Model.voMembershipsFromComanage import voMembershipsFromComanage | ||
from Model.vos import vos | ||
from datetime import datetime, timedelta | ||
from Logger import log | ||
|
||
class voMembershipsController: | ||
logger = log.get_logger("voMembershipsController") | ||
|
||
@classmethod | ||
def getVoId(self, voName, source, tenenv_id): | ||
result = vos.getVoIdFromVoName(voName, source, tenenv_id) | ||
return result | ||
|
||
@classmethod | ||
def getDataNotMapped(self, tenenv_id): | ||
voMemberships.truncate(tenenv_id) | ||
|
||
voMembershipsList = [] | ||
# dateFrom = vos.getLastDate() | ||
|
||
# we dont have any vos saved | ||
# if dateFrom[0][0] == None: | ||
|
||
# get memberships with active status | ||
activeMemberships = voMembershipsFromComanage.getAllVoMembershipsByStatus('A') | ||
# save memberships with active status | ||
for item in activeMemberships: | ||
result = voMembershipsController.getVoId(item.voName, "registry", tenenv_id) | ||
|
||
if len(result) > 0: | ||
voId = result[0][0] | ||
self.logger.info("Vo name {0} with id {1}".format(item.voName, result[0][0])) | ||
|
||
voMembershipsItem = voMemberships(voId, item.hasheduserid, item.status, item.created, tenenv_id) | ||
voMembershipsList.append(voMembershipsItem) | ||
# save data to tables if any | ||
if voMembershipsList: | ||
|
||
voMemberships.saveAll(voMembershipsList) | ||
|
||
# get memberships with grace period | ||
# save only if there isnt any existing membership or status membership is not Active/ Grace Period | ||
gracePeriodVoMembershipsList = [] | ||
graceperiodMemberships = voMembershipsFromComanage.getAllVoMembershipsByStatus('GP') | ||
for item in graceperiodMemberships: | ||
result = voMembershipsController.getVoId(item.voName, "registry", tenenv_id) | ||
if len(result) > 0: | ||
voId = result[0][0] | ||
self.logger.info("Vo name {0} with id {1}".format(item.voName, result[0][0])) | ||
#check if already there is a membership with active status saved | ||
statusMembership = voMemberships.getMembershipStatus(voId, item.hasheduserid, tenenv_id) | ||
if len(statusMembership) > 0 and (statusMembership[0][0]=='A' or statusMembership[0][0]=='GP'): | ||
continue | ||
voMembershipsItem = voMemberships(voId, item.hasheduserid, item.status, item.created, tenenv_id) | ||
gracePeriodVoMembershipsList.append(voMembershipsItem) | ||
if len(gracePeriodVoMembershipsList)>0: | ||
voMemberships.saveAll(gracePeriodVoMembershipsList) | ||
|
||
# get memberships with other statuses | ||
# save only if there isnt any existing membership | ||
otherStatusVoMembershipsList = [] | ||
otherStatusMemberships = voMembershipsFromComanage.getAllVoMembershipsByStatus('Other') | ||
for item in otherStatusMemberships: | ||
result = voMembershipsController.getVoId(item.voName, "registry", tenenv_id) | ||
if len(result) > 0: | ||
voId = result[0][0] | ||
self.logger.info("Vo name {0} with id {1}".format(item.voName, result[0][0])) | ||
#check if already there is a membership with active status saved | ||
statusMembership = voMemberships.getMembershipStatus(voId, item.hasheduserid, tenenv_id) | ||
if len(statusMembership) > 0: | ||
continue | ||
voMembershipsItem = voMemberships(voId, item.hasheduserid, item.status, item.created, tenenv_id) | ||
otherStatusVoMembershipsList.append(voMembershipsItem) | ||
if len(otherStatusVoMembershipsList)>0: | ||
voMemberships.saveAll(otherStatusVoMembershipsList) | ||
return result | ||
|
||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,24 @@ | ||
from datetime import date, timedelta | ||
from Model.vosFromComanage import vosFromComanage | ||
from Model.vos import vos | ||
from datetime import datetime, timedelta | ||
|
||
class vosController: | ||
@classmethod | ||
def getDataNotMapped(self, tenenv_id): | ||
dateFrom = vos.getLastDate(tenenv_id) | ||
|
||
# we dont have any vos saved | ||
if dateFrom[0][0] == None: | ||
result = vosFromComanage.getAllVos() | ||
else: | ||
dayAfter = dateFrom[0][0] + timedelta(days=1) | ||
dayFrom = dayAfter.strftime('%Y-%m-%d 00:00:00') | ||
|
||
yesterday = date.today() - timedelta(days=1) | ||
dateTo = yesterday.strftime('%Y-%m-%d 23:59:59') | ||
|
||
result = vosFromComanage.getVosByDate(dayFrom, dateTo) | ||
return result | ||
|
||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,29 @@ | ||
import logging | ||
import sys | ||
from Utils import configParser | ||
from logging.handlers import TimedRotatingFileHandler | ||
|
||
FORMATTER = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") | ||
LOG_FILE = "{0}/{1}".format(configParser.getConfig('logging')['folder'] ,configParser.getConfig('logging')['file']) | ||
LEVEL = configParser.getConfig('logging')['level'] | ||
|
||
def get_console_handler(): | ||
console_handler = logging.StreamHandler(sys.stdout) | ||
console_handler.setFormatter(FORMATTER) | ||
return console_handler | ||
|
||
def get_file_handler(): | ||
file_handler = TimedRotatingFileHandler(LOG_FILE, when='midnight') | ||
file_handler.setFormatter(FORMATTER) | ||
return file_handler | ||
|
||
def get_logger(logger_name): | ||
|
||
logger = logging.getLogger(logger_name) | ||
logger.setLevel(LEVEL) | ||
logger.addHandler(get_console_handler()) | ||
logger.addHandler(get_file_handler()) | ||
# with this pattern, it's rarely necessary to propagate the error up to parent | ||
logger.propagate = False | ||
|
||
return logger |
Oops, something went wrong.