Various checks added at ingestion process

CHANGELOG.md

@@ -1,5 +1,10 @@
 ### Changelog
 
+## 2023-01-12
+- Add IGTF demo instance at Metrics-Dev
+
+## 2023-12-14
+- Add EGI demo instance at Metrics-Dev
 
 ## 2023-11-23
 

app/ingester/loginsIngester.py

@@ -197,7 +197,7 @@ def ingestLoginDataPerTenenv(cls, tenenv, session):
                     hash {0} as he is at the blacklist""". format(login[0]['voPersonId']))
                 continue
 
-            if (not login[0]['failedLogin']
+            if ((not login[0]['failedLogin'] or login[0]['failedLogin'] == 'false')
                 and utilsIngester.validateTenenv(login[0]['tenenvId'], session)
                 and 'voPersonId' in login[0]
                 and utilsIngester.validateHashedUser(login[0]['voPersonId'],
@@ -237,6 +237,17 @@ def ingestLoginDataPerTenenv(cls, tenenv, session):
                 loginMappedItems += 1
             else:
                 cls.logger.warning("The record {0} was not imported due to validation errors".format(repr(login[0])))
+                cls.logger.warning("validateTenenv:")
+                cls.logger.warning(utilsIngester.validateTenenv(login[0]['tenenvId'], session))
+                cls.logger.warning("voPersonId in login[0]:")
+                cls.logger.warning('voPersonId' in login[0])
+                if ('voPersonId' in login[0] and utilsIngester.validateTenenv(login[0]['tenenvId'], session)):
+                    cls.logger.warning("validateHashedUser:")
+                    cls.logger.warning(utilsIngester.validateHashedUser(login[0]['voPersonId'],
+                                                      login[0]['tenenvId'],
+                                                      session))
+                cls.logger.warning("validate if login is successful:")
+                cls.logger.warning(not login[0]['failedLogin'] or login[0]['failedLogin'] == 'false')
 
         cls.logger.info("""
             {0} new logins ingested""".format(loginMappedItems))

app/ingester/usersIngester.py

@@ -48,6 +48,9 @@ def ingestUserDataPerTenenv(cls, tenenv, session):
                 cls.logger.error("""
                     user status '{0}' is not valid """.format(user[0]['status']))
                 continue
+            if ('voPersonId' not in user[0]):
+                cls.logger.warning("""voPersonId not found at record. Ignoring...""")
+                continue
             if (user[0]['voPersonId'] in hashed_user_ids):
                 cls.logger.info("""Ignore this user with
                     hash {0} as he is at the blacklist""". format(user[0]['voPersonId']))

app/ingester/utilsIngester.py

@@ -58,6 +58,14 @@ def validateHashedUser(cls, hashedUser, tenenvId, session):
                     )
             ).one()
         except NoResultFound:
-            cls.logger.info("User {0} not found".format(hashedUser))
-            return False
+            cls.logger.warning("""User {0} not found, we are going to create it
+                            with default values.""".format(hashedUser))
+            now = date.today().strftime('%Y-%m-%d %H:%M:%S')
+            session.exec("""INSERT INTO users(hasheduserid, created, updated,
+                         status, tenenv_id)
+                VALUES ('{0}','{1}','{1}', '{2}', {3})
+                """. format(
+                hashedUser, now, 'A',
+                tenenvId))
+            session.commit()
         return True