From a7437c3c03df27a1ba38fd75d571f11f7304a732 Mon Sep 17 00:00:00 2001 From: Nick Mastoris Date: Tue, 12 Dec 2023 12:50:52 +0200 Subject: [PATCH 1/7] add CHANGELOG for EGI Demo --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 145b6dd..6c390a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ ### Changelog +## 2023-12-12 +- Add EGI demo instance at Metrics-Dev ## 2023-11-23 From 94cc69e917358b91f8eebc211a0bfb5691bb2257 Mon Sep 17 00:00:00 2001 From: Nick Mastoris Date: Thu, 14 Dec 2023 11:18:50 +0200 Subject: [PATCH 2/7] fix CHANGELOG --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c390a7..1ad1212 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ ### Changelog -## 2023-12-12 +## 2023-12-14 - Add EGI demo instance at Metrics-Dev ## 2023-11-23 From 5eacd51b01785033cfabf1fd8bb03feb7bb76333 Mon Sep 17 00:00:00 2001 From: Nick Mastoris Date: Fri, 12 Jan 2024 10:29:56 +0200 Subject: [PATCH 3/7] Add IGTF demo instance --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ad1212..9123c4b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ ### Changelog +## 2023-01-12 +- Add IGTF demo instance at Metrics-Dev + ## 2023-12-14 - Add EGI demo instance at Metrics-Dev From 308aa6d94201ffb1ee867f51bf55232d74d31acd Mon Sep 17 00:00:00 2001 From: Nick Mastoris Date: Tue, 16 Jan 2024 12:27:08 +0200 Subject: [PATCH 4/7] add user if not exists, when ingesting a login --- app/ingester/loginsIngester.py | 12 +++++++++++- app/ingester/utilsIngester.py | 12 ++++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/app/ingester/loginsIngester.py b/app/ingester/loginsIngester.py index 559aa7b..47251f5 100644 --- a/app/ingester/loginsIngester.py +++ b/app/ingester/loginsIngester.py @@ -197,7 +197,7 @@ def ingestLoginDataPerTenenv(cls, tenenv, session): hash {0} as he is at the blacklist""". format(login[0]['voPersonId'])) continue - if (not login[0]['failedLogin'] + if ((not login[0]['failedLogin'] or login[0]['failedLogin'] == 'false') and utilsIngester.validateTenenv(login[0]['tenenvId'], session) and 'voPersonId' in login[0] and utilsIngester.validateHashedUser(login[0]['voPersonId'], @@ -237,6 +237,16 @@ def ingestLoginDataPerTenenv(cls, tenenv, session): loginMappedItems += 1 else: cls.logger.warning("The record {0} was not imported due to validation errors".format(repr(login[0]))) + cls.logger.warning("validateTenenv:") + cls.logger.warning(utilsIngester.validateTenenv(login[0]['tenenvId'], session)) + cls.logger.warning("voPersonId in login[0]:") + cls.logger.warning('voPersonId' in login[0]) + cls.logger.warning("validateHashedUser:") + cls.logger.warning(utilsIngester.validateHashedUser(login[0]['voPersonId'], + login[0]['tenenvId'], + session)) + cls.logger.warning("validate if login is successful:") + cls.logger.warning(not login[0]['failedLogin'] or login[0]['failedLogin'] == 'false') cls.logger.info(""" {0} new logins ingested""".format(loginMappedItems)) diff --git a/app/ingester/utilsIngester.py b/app/ingester/utilsIngester.py index 9b27666..8d71273 100644 --- a/app/ingester/utilsIngester.py +++ b/app/ingester/utilsIngester.py @@ -58,6 +58,14 @@ def validateHashedUser(cls, hashedUser, tenenvId, session): ) ).one() except NoResultFound: - cls.logger.info("User {0} not found".format(hashedUser)) - return False + cls.logger.info("""User {0} not found, we are going to create it + with default values.""".format(hashedUser)) + now = date.today().strftime('%Y-%m-%d %H:%M:%S') + session.exec("""INSERT INTO users(hasheduserid, created, updated, + status, tenenv_id) + VALUES ('{0}','{1}','{1}', '{2}', {3}) + """. format( + hashedUser, now, 'A', + tenenvId)) + session.commit() return True From 81eb2c61fdc4c1e745b8f5c0c0035fd1ea948dba Mon Sep 17 00:00:00 2001 From: Nick Mastoris Date: Tue, 16 Jan 2024 12:32:01 +0200 Subject: [PATCH 5/7] change info to warning message --- app/ingester/utilsIngester.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/ingester/utilsIngester.py b/app/ingester/utilsIngester.py index 8d71273..8f00dbb 100644 --- a/app/ingester/utilsIngester.py +++ b/app/ingester/utilsIngester.py @@ -58,7 +58,7 @@ def validateHashedUser(cls, hashedUser, tenenvId, session): ) ).one() except NoResultFound: - cls.logger.info("""User {0} not found, we are going to create it + cls.logger.warning("""User {0} not found, we are going to create it with default values.""".format(hashedUser)) now = date.today().strftime('%Y-%m-%d %H:%M:%S') session.exec("""INSERT INTO users(hasheduserid, created, updated, From d46fce121daf3c356a90e0f3a92aeecc6295431c Mon Sep 17 00:00:00 2001 From: Nick Mastoris Date: Wed, 17 Jan 2024 12:00:41 +0200 Subject: [PATCH 6/7] fix warning messages --- app/ingester/loginsIngester.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/app/ingester/loginsIngester.py b/app/ingester/loginsIngester.py index 47251f5..a179ba2 100644 --- a/app/ingester/loginsIngester.py +++ b/app/ingester/loginsIngester.py @@ -241,10 +241,11 @@ def ingestLoginDataPerTenenv(cls, tenenv, session): cls.logger.warning(utilsIngester.validateTenenv(login[0]['tenenvId'], session)) cls.logger.warning("voPersonId in login[0]:") cls.logger.warning('voPersonId' in login[0]) - cls.logger.warning("validateHashedUser:") - cls.logger.warning(utilsIngester.validateHashedUser(login[0]['voPersonId'], - login[0]['tenenvId'], - session)) + if ('voPersonId' in login[0] and utilsIngester.validateTenenv(login[0]['tenenvId'], session)): + cls.logger.warning("validateHashedUser:") + cls.logger.warning(utilsIngester.validateHashedUser(login[0]['voPersonId'], + login[0]['tenenvId'], + session)) cls.logger.warning("validate if login is successful:") cls.logger.warning(not login[0]['failedLogin'] or login[0]['failedLogin'] == 'false') From ad8a586ddb39ff086edcd1a899ebff2ec69c3056 Mon Sep 17 00:00:00 2001 From: lionick <44701697+lionick@users.noreply.github.com> Date: Mon, 20 May 2024 11:48:41 +0300 Subject: [PATCH 7/7] Check if voPersonId exists at the record (#52) --- app/ingester/usersIngester.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/ingester/usersIngester.py b/app/ingester/usersIngester.py index 20fcd44..542ba36 100644 --- a/app/ingester/usersIngester.py +++ b/app/ingester/usersIngester.py @@ -48,6 +48,9 @@ def ingestUserDataPerTenenv(cls, tenenv, session): cls.logger.error(""" user status '{0}' is not valid """.format(user[0]['status'])) continue + if ('voPersonId' not in user[0]): + cls.logger.warning("""voPersonId not found at record. Ignoring...""") + continue if (user[0]['voPersonId'] in hashed_user_ids): cls.logger.info("""Ignore this user with hash {0} as he is at the blacklist""". format(user[0]['voPersonId']))