YUPANA #60 Set configs for CLOWDER_ENABLED (#372)

* YUPANA #60 Set configs for CLOWDER_ENABLED

* Add disable check for unpacked tuple

* Revisit Kafka Topics for Clowder and Legacy

Pipfile

@@ -8,6 +8,7 @@ name = "pypi"
 [packages]
 
 aiokafka = "==0.6.0"
+app-common-python = "~=0.1.10"
 djangorestframework = "==3.11.0"
 Django = "==2.2.13"
 django-environ = "==0.4.5"

yupana/config/settings/base.py

@@ -35,8 +35,56 @@
 from botocore.exceptions import ClientError
 from .env import ENVIRONMENT
 
-ROOT_DIR = environ.Path(__file__) - 4
-APPS_DIR = ROOT_DIR.path('yupana')
+CLOWDER_ENABLED = True if os.getenv("CLOWDER_ENABLED", default="False").lower() in ["true", "t", "yes", "y"] else False
+
+ENGINES = {
+    'sqlite': 'django.db.backends.sqlite3',
+    'postgresql': 'django.db.backends.postgresql',
+    'mysql': 'django.db.backends.mysql',
+}
+
+SERVICE_NAME = ENVIRONMENT.get_value('DATABASE_SERVICE_NAME', default='').upper().replace('-', '_')
+
+if CLOWDER_ENABLED:
+    LOG.info("Using Clowder Operator...")
+    from app_common_python import LoadedConfig, KafkaTopics
+    CW_AWS_ACCESS_KEY_ID = LoadedConfig.logging.cloudwatch.accessKeyId
+    CW_AWS_SECRET_ACCESS_KEY = LoadedConfig.logging.cloudwatch.secretAccessKey
+    CW_AWS_REGION = LoadedConfig.logging.cloudwatch.region
+    CW_LOG_GROUP = LoadedConfig.logging.cloudwatch.logGroup
+    DB_NAME = LoadedConfig.database.name
+    DB_USER = LoadedConfig.database.username
+    DB_PASSWORD = LoadedConfig.database.password
+    DB_HOST = LoadedConfig.database.hostname
+    DB_PORT = LoadedConfig.database.port
+    INSIGHTS_KAFKA_ADDRESS = LoadedConfig.kafka.brokers[0].hostname + ":" + str(LoadedConfig.kafka.brokers[0].port)
+    QPC_TOPIC = KafkaTopics["platform.upload.qpc"].name
+    UPLOAD_TOPIC = KafkaTopics["platform.inventory.host-ingress"].name
+    VALIDATION_TOPIC = KafkaTopics["platform.upload.validation"].name
+else:
+    CW_AWS_ACCESS_KEY_ID = ENVIRONMENT.get_value('CW_AWS_ACCESS_KEY_ID', default=None)
+    CW_AWS_SECRET_ACCESS_KEY = ENVIRONMENT.get_value('CW_AWS_SECRET_ACCESS_KEY', default=None)
+    CW_AWS_REGION = ENVIRONMENT.get_value('CW_AWS_REGION', default='us-east-1')
+    CW_LOG_GROUP = ENVIRONMENT.get_value('CW_LOG_GROUP', default='platform-dev')
+    ROOT_DIR = environ.Path(__file__) - 4
+    APPS_DIR = ROOT_DIR.path('yupana')
+    if SERVICE_NAME:
+        ENGINE = ENGINES.get(ENVIRONMENT.get_value('DATABASE_ENGINE'), ENGINES['postgresql'])
+    else:
+        ENGINE = ENGINES['sqlite']
+    DB_NAME = ENVIRONMENT.get_value('DATABASE_NAME', default=None)
+    if not DB_NAME and ENGINE == ENGINES['sqlite']:
+        DB_NAME = os.path.join(APPS_DIR, 'db.sqlite3')
+    DB_USER = ENVIRONMENT.get_value('DATABASE_USER', default=None)
+    DB_PASSWORD = ENVIRONMENT.get_value('DATABASE_PASSWORD', default=None)
+    DB_HOST = ENVIRONMENT.get_value('{}_SERVICE_HOST'.format(SERVICE_NAME), default=None)
+    DB_PORT = ENVIRONMENT.get_value('{}_SERVICE_PORT'.format(SERVICE_NAME), default=None)
+    INSIGHTS_KAFKA_HOST = os.getenv('INSIGHTS_KAFKA_HOST', 'localhost')
+    INSIGHTS_KAFKA_PORT = os.getenv('INSIGHTS_KAFKA_PORT', '29092')
+    INSIGHTS_KAFKA_ADDRESS = f'{INSIGHTS_KAFKA_HOST}:{INSIGHTS_KAFKA_PORT}'
+    QPC_TOPIC = os.getenv('QPC_TOPIC', 'platform.upload.qpc')
+    UPLOAD_TOPIC = os.getenv('UPLOAD_TOPIC', 'platform.inventory.host-ingress')
+    VALIDATION_TOPIC = os.getenv('VALIDATION_TOPIC', 'platform.upload.validation')
 
 # Quick-start development settings - unsuitable for production
 # See https://docs.djangoproject.com/en/2.1/howto/deployment/checklist/
@@ -94,12 +142,6 @@
 # https://docs.djangoproject.com/en/dev/topics/logging/
 # https://docs.python.org/3.6/library/logging.html
 
-# cloudwatch logging variables
-CW_AWS_ACCESS_KEY_ID = ENVIRONMENT.get_value('CW_AWS_ACCESS_KEY_ID', default=None)
-CW_AWS_SECRET_ACCESS_KEY = ENVIRONMENT.get_value('CW_AWS_SECRET_ACCESS_KEY', default=None)
-CW_AWS_REGION = ENVIRONMENT.get_value('CW_AWS_REGION', default='us-east-1')
-CW_LOG_GROUP = ENVIRONMENT.get_value('CW_LOG_GROUP', default='platform-dev')
-
 LOGGING_LEVEL = os.getenv('DJANGO_LOG_LEVEL', 'INFO')
 LOGGING_HANDLERS = os.getenv('DJANGO_LOG_HANDLERS', 'console').split(',')
 LOGGING_FORMATTER = os.getenv('DJANGO_LOG_FORMATTER', 'simple')
@@ -239,35 +281,16 @@
 
 # Database
 # https://docs.djangoproject.com/en/2.1/ref/settings/#databases
-ENGINES = {
-    'sqlite': 'django.db.backends.sqlite3',
-    'postgresql': 'django.db.backends.postgresql',
-    'mysql': 'django.db.backends.mysql',
-}
-
-SERVICE_NAME = ENVIRONMENT.get_value('DATABASE_SERVICE_NAME',
-                                     default='').upper().replace('-', '_')
-if SERVICE_NAME:
-    ENGINE = ENGINES.get(ENVIRONMENT.get_value('DATABASE_ENGINE'),
-                         ENGINES['postgresql'])
-else:
-    ENGINE = ENGINES['sqlite']
-
-NAME = ENVIRONMENT.get_value('DATABASE_NAME', default=None)
-
-if not NAME and ENGINE == ENGINES['sqlite']:
-    NAME = os.path.join(APPS_DIR, 'db.sqlite3')
 
 DATABASES = {
-    'ENGINE': ENGINE,
-    'NAME': NAME,
-    'USER': ENVIRONMENT.get_value('DATABASE_USER', default=None),
-    'PASSWORD': ENVIRONMENT.get_value('DATABASE_PASSWORD', default=None),
-    'HOST': ENVIRONMENT.get_value('{}_SERVICE_HOST'.format(SERVICE_NAME),
-                                  default=None),
-    'PORT': ENVIRONMENT.get_value('{}_SERVICE_PORT'.format(SERVICE_NAME),
-                                  default=None),
+    'NAME': DB_NAME,
+    'USER': DB_USER,
+    'PASSWORD': DB_PASSWORD,
+    'HOST': DB_HOST,
+    'PORT': DB_PORT,
 }
+if not CLOWDER_ENABLED:
+    DATABASES['ENGINE'] = ENGINE
 
 # add ssl cert if specified
 DATABASE_CERT = ENVIRONMENT.get_value('DATABASE_SERVICE_CERT', default=None)
@@ -337,15 +360,6 @@
 # pylint: disable=simplifiable-if-expression
 INGEST_OVERRIDE = False if os.getenv('INITIAL_INGEST_OVERRIDE', 'False') == 'False' else True
 
-# Insights Kafka messaging address
-INSIGHTS_KAFKA_HOST = os.getenv('INSIGHTS_KAFKA_HOST', 'localhost')
-
-# Insights Kafka messaging address
-INSIGHTS_KAFKA_PORT = os.getenv('INSIGHTS_KAFKA_PORT', '29092')
-
-# Insights Kafka server address
-INSIGHTS_KAFKA_ADDRESS = f'{INSIGHTS_KAFKA_HOST}:{INSIGHTS_KAFKA_PORT}'
-
 # override max_request_size for kafka producer to 2(MB)
 KAFKA_PRODUCER_OVERRIDE_MAX_REQUEST_SIZE = os.getenv(
     'KAFKA_PRODUCER_OVERRIDE_MAX_REQUEST_SIZE', 2097152

yupana/processor/report_consumer.py

@@ -34,12 +34,11 @@
 
 from api.models import Report
 from api.serializers import ReportSerializer
-from config.settings.base import INSIGHTS_KAFKA_ADDRESS
+from config.settings.base import INSIGHTS_KAFKA_ADDRESS, QPC_TOPIC
 
 LOG = logging.getLogger(__name__)
 
 REPORT_PENDING_QUEUE = asyncio.Queue()
-QPC_TOPIC = 'platform.upload.qpc'
 
 MSG_UPLOADS = Counter('yupana_message_uploads',
                       'Number of messages uploaded to qpc topic',

yupana/processor/report_processor.py

@@ -47,10 +47,10 @@
 from config.settings.base import (INSIGHTS_KAFKA_ADDRESS,
                                   MAX_HOSTS_PER_REP,
                                   RETRIES_ALLOWED,
-                                  RETRY_TIME)
+                                  RETRY_TIME,
+                                  VALIDATION_TOPIC)
 
 LOG = logging.getLogger(__name__)
-VALIDATION_TOPIC = 'platform.upload.validation'
 SUCCESS_CONFIRM_STATUS = 'success'
 FAILURE_CONFIRM_STATUS = 'failure'
 RETRIES_ALLOWED = int(RETRIES_ALLOWED)

yupana/processor/report_slice_processor.py

@@ -43,7 +43,8 @@
                                   INSIGHTS_KAFKA_ADDRESS,
                                   KAFKA_PRODUCER_OVERRIDE_MAX_REQUEST_SIZE,
                                   RETRIES_ALLOWED,
-                                  RETRY_TIME)
+                                  RETRY_TIME,
+                                  UPLOAD_TOPIC)
 
 LOG = logging.getLogger(__name__)
 
@@ -52,7 +53,6 @@
 FAILED_UPLOAD = 'UPLOAD'
 RETRIES_ALLOWED = int(RETRIES_ALLOWED)
 RETRY_TIME = int(RETRY_TIME)
-UPLOAD_TOPIC = 'platform.inventory.host-ingress'  # placeholder topic
 OS_RELEASE_PATTERN = re.compile(
     r'(?P<name>[a-zA-Z\s]*)?\s*((?P<major>\d*)(\.?(?P<minor>\d*)(\.?(?P<patch>\d*))?)?)\s*'
     r'(\((?P<code>\S*)\))?'

yupana/processor/tests_report_processor.py

@@ -1241,7 +1241,7 @@ def test_download_and_validate_contents_invalid_report(self):
             'hosts': {self.uuid: {'key': 'value'}}}
         self.processor.report_or_slice = self.report_record
         with self.assertRaises(msg_handler.QPCReportException):
-            _, _ = self.processor._validate_report_details()
+            _, _ = self.processor._validate_report_details()  # pylint: disable=unbalanced-tuple-unpacking
 
     def test_download_contents_raises_error(self):
         """Test to verify downloading contents fails when error is raised."""