diff --git a/Pipfile b/Pipfile index 3ec7e385..846cc0a8 100644 --- a/Pipfile +++ b/Pipfile @@ -8,6 +8,7 @@ name = "pypi" [packages] aiokafka = "==0.6.0" +app-common-python = "~=0.1.10" djangorestframework = "==3.11.0" Django = "==2.2.13" django-environ = "==0.4.5" diff --git a/yupana/config/settings/base.py b/yupana/config/settings/base.py index fe5519d9..5b0a6e56 100644 --- a/yupana/config/settings/base.py +++ b/yupana/config/settings/base.py @@ -35,8 +35,56 @@ from botocore.exceptions import ClientError from .env import ENVIRONMENT -ROOT_DIR = environ.Path(__file__) - 4 -APPS_DIR = ROOT_DIR.path('yupana') +CLOWDER_ENABLED = True if os.getenv("CLOWDER_ENABLED", default="False").lower() in ["true", "t", "yes", "y"] else False + +ENGINES = { + 'sqlite': 'django.db.backends.sqlite3', + 'postgresql': 'django.db.backends.postgresql', + 'mysql': 'django.db.backends.mysql', +} + +SERVICE_NAME = ENVIRONMENT.get_value('DATABASE_SERVICE_NAME', default='').upper().replace('-', '_') + +if CLOWDER_ENABLED: + LOG.info("Using Clowder Operator...") + from app_common_python import LoadedConfig, KafkaTopics + CW_AWS_ACCESS_KEY_ID = LoadedConfig.logging.cloudwatch.accessKeyId + CW_AWS_SECRET_ACCESS_KEY = LoadedConfig.logging.cloudwatch.secretAccessKey + CW_AWS_REGION = LoadedConfig.logging.cloudwatch.region + CW_LOG_GROUP = LoadedConfig.logging.cloudwatch.logGroup + DB_NAME = LoadedConfig.database.name + DB_USER = LoadedConfig.database.username + DB_PASSWORD = LoadedConfig.database.password + DB_HOST = LoadedConfig.database.hostname + DB_PORT = LoadedConfig.database.port + INSIGHTS_KAFKA_ADDRESS = LoadedConfig.kafka.brokers[0].hostname + ":" + str(LoadedConfig.kafka.brokers[0].port) + QPC_TOPIC = KafkaTopics["platform.upload.qpc"].name + UPLOAD_TOPIC = KafkaTopics["platform.inventory.host-ingress"].name + VALIDATION_TOPIC = KafkaTopics["platform.upload.validation"].name +else: + CW_AWS_ACCESS_KEY_ID = ENVIRONMENT.get_value('CW_AWS_ACCESS_KEY_ID', default=None) + CW_AWS_SECRET_ACCESS_KEY = ENVIRONMENT.get_value('CW_AWS_SECRET_ACCESS_KEY', default=None) + CW_AWS_REGION = ENVIRONMENT.get_value('CW_AWS_REGION', default='us-east-1') + CW_LOG_GROUP = ENVIRONMENT.get_value('CW_LOG_GROUP', default='platform-dev') + ROOT_DIR = environ.Path(__file__) - 4 + APPS_DIR = ROOT_DIR.path('yupana') + if SERVICE_NAME: + ENGINE = ENGINES.get(ENVIRONMENT.get_value('DATABASE_ENGINE'), ENGINES['postgresql']) + else: + ENGINE = ENGINES['sqlite'] + DB_NAME = ENVIRONMENT.get_value('DATABASE_NAME', default=None) + if not DB_NAME and ENGINE == ENGINES['sqlite']: + DB_NAME = os.path.join(APPS_DIR, 'db.sqlite3') + DB_USER = ENVIRONMENT.get_value('DATABASE_USER', default=None) + DB_PASSWORD = ENVIRONMENT.get_value('DATABASE_PASSWORD', default=None) + DB_HOST = ENVIRONMENT.get_value('{}_SERVICE_HOST'.format(SERVICE_NAME), default=None) + DB_PORT = ENVIRONMENT.get_value('{}_SERVICE_PORT'.format(SERVICE_NAME), default=None) + INSIGHTS_KAFKA_HOST = os.getenv('INSIGHTS_KAFKA_HOST', 'localhost') + INSIGHTS_KAFKA_PORT = os.getenv('INSIGHTS_KAFKA_PORT', '29092') + INSIGHTS_KAFKA_ADDRESS = f'{INSIGHTS_KAFKA_HOST}:{INSIGHTS_KAFKA_PORT}' + QPC_TOPIC = os.getenv('QPC_TOPIC', 'platform.upload.qpc') + UPLOAD_TOPIC = os.getenv('UPLOAD_TOPIC', 'platform.inventory.host-ingress') + VALIDATION_TOPIC = os.getenv('VALIDATION_TOPIC', 'platform.upload.validation') # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/2.1/howto/deployment/checklist/ @@ -94,12 +142,6 @@ # https://docs.djangoproject.com/en/dev/topics/logging/ # https://docs.python.org/3.6/library/logging.html -# cloudwatch logging variables -CW_AWS_ACCESS_KEY_ID = ENVIRONMENT.get_value('CW_AWS_ACCESS_KEY_ID', default=None) -CW_AWS_SECRET_ACCESS_KEY = ENVIRONMENT.get_value('CW_AWS_SECRET_ACCESS_KEY', default=None) -CW_AWS_REGION = ENVIRONMENT.get_value('CW_AWS_REGION', default='us-east-1') -CW_LOG_GROUP = ENVIRONMENT.get_value('CW_LOG_GROUP', default='platform-dev') - LOGGING_LEVEL = os.getenv('DJANGO_LOG_LEVEL', 'INFO') LOGGING_HANDLERS = os.getenv('DJANGO_LOG_HANDLERS', 'console').split(',') LOGGING_FORMATTER = os.getenv('DJANGO_LOG_FORMATTER', 'simple') @@ -239,35 +281,16 @@ # Database # https://docs.djangoproject.com/en/2.1/ref/settings/#databases -ENGINES = { - 'sqlite': 'django.db.backends.sqlite3', - 'postgresql': 'django.db.backends.postgresql', - 'mysql': 'django.db.backends.mysql', -} - -SERVICE_NAME = ENVIRONMENT.get_value('DATABASE_SERVICE_NAME', - default='').upper().replace('-', '_') -if SERVICE_NAME: - ENGINE = ENGINES.get(ENVIRONMENT.get_value('DATABASE_ENGINE'), - ENGINES['postgresql']) -else: - ENGINE = ENGINES['sqlite'] - -NAME = ENVIRONMENT.get_value('DATABASE_NAME', default=None) - -if not NAME and ENGINE == ENGINES['sqlite']: - NAME = os.path.join(APPS_DIR, 'db.sqlite3') DATABASES = { - 'ENGINE': ENGINE, - 'NAME': NAME, - 'USER': ENVIRONMENT.get_value('DATABASE_USER', default=None), - 'PASSWORD': ENVIRONMENT.get_value('DATABASE_PASSWORD', default=None), - 'HOST': ENVIRONMENT.get_value('{}_SERVICE_HOST'.format(SERVICE_NAME), - default=None), - 'PORT': ENVIRONMENT.get_value('{}_SERVICE_PORT'.format(SERVICE_NAME), - default=None), + 'NAME': DB_NAME, + 'USER': DB_USER, + 'PASSWORD': DB_PASSWORD, + 'HOST': DB_HOST, + 'PORT': DB_PORT, } +if not CLOWDER_ENABLED: + DATABASES['ENGINE'] = ENGINE # add ssl cert if specified DATABASE_CERT = ENVIRONMENT.get_value('DATABASE_SERVICE_CERT', default=None) @@ -337,15 +360,6 @@ # pylint: disable=simplifiable-if-expression INGEST_OVERRIDE = False if os.getenv('INITIAL_INGEST_OVERRIDE', 'False') == 'False' else True -# Insights Kafka messaging address -INSIGHTS_KAFKA_HOST = os.getenv('INSIGHTS_KAFKA_HOST', 'localhost') - -# Insights Kafka messaging address -INSIGHTS_KAFKA_PORT = os.getenv('INSIGHTS_KAFKA_PORT', '29092') - -# Insights Kafka server address -INSIGHTS_KAFKA_ADDRESS = f'{INSIGHTS_KAFKA_HOST}:{INSIGHTS_KAFKA_PORT}' - # override max_request_size for kafka producer to 2(MB) KAFKA_PRODUCER_OVERRIDE_MAX_REQUEST_SIZE = os.getenv( 'KAFKA_PRODUCER_OVERRIDE_MAX_REQUEST_SIZE', 2097152 diff --git a/yupana/processor/report_consumer.py b/yupana/processor/report_consumer.py index 02348447..75b21546 100644 --- a/yupana/processor/report_consumer.py +++ b/yupana/processor/report_consumer.py @@ -34,12 +34,11 @@ from api.models import Report from api.serializers import ReportSerializer -from config.settings.base import INSIGHTS_KAFKA_ADDRESS +from config.settings.base import INSIGHTS_KAFKA_ADDRESS, QPC_TOPIC LOG = logging.getLogger(__name__) REPORT_PENDING_QUEUE = asyncio.Queue() -QPC_TOPIC = 'platform.upload.qpc' MSG_UPLOADS = Counter('yupana_message_uploads', 'Number of messages uploaded to qpc topic', diff --git a/yupana/processor/report_processor.py b/yupana/processor/report_processor.py index b0544dbb..e15fddc6 100644 --- a/yupana/processor/report_processor.py +++ b/yupana/processor/report_processor.py @@ -47,10 +47,10 @@ from config.settings.base import (INSIGHTS_KAFKA_ADDRESS, MAX_HOSTS_PER_REP, RETRIES_ALLOWED, - RETRY_TIME) + RETRY_TIME, + VALIDATION_TOPIC) LOG = logging.getLogger(__name__) -VALIDATION_TOPIC = 'platform.upload.validation' SUCCESS_CONFIRM_STATUS = 'success' FAILURE_CONFIRM_STATUS = 'failure' RETRIES_ALLOWED = int(RETRIES_ALLOWED) diff --git a/yupana/processor/report_slice_processor.py b/yupana/processor/report_slice_processor.py index 98d48963..cacfcb26 100644 --- a/yupana/processor/report_slice_processor.py +++ b/yupana/processor/report_slice_processor.py @@ -43,7 +43,8 @@ INSIGHTS_KAFKA_ADDRESS, KAFKA_PRODUCER_OVERRIDE_MAX_REQUEST_SIZE, RETRIES_ALLOWED, - RETRY_TIME) + RETRY_TIME, + UPLOAD_TOPIC) LOG = logging.getLogger(__name__) @@ -52,7 +53,6 @@ FAILED_UPLOAD = 'UPLOAD' RETRIES_ALLOWED = int(RETRIES_ALLOWED) RETRY_TIME = int(RETRY_TIME) -UPLOAD_TOPIC = 'platform.inventory.host-ingress' # placeholder topic OS_RELEASE_PATTERN = re.compile( r'(?P[a-zA-Z\s]*)?\s*((?P\d*)(\.?(?P\d*)(\.?(?P\d*))?)?)\s*' r'(\((?P\S*)\))?' diff --git a/yupana/processor/tests_report_processor.py b/yupana/processor/tests_report_processor.py index 3af9ae17..1a83ab38 100644 --- a/yupana/processor/tests_report_processor.py +++ b/yupana/processor/tests_report_processor.py @@ -1241,7 +1241,7 @@ def test_download_and_validate_contents_invalid_report(self): 'hosts': {self.uuid: {'key': 'value'}}} self.processor.report_or_slice = self.report_record with self.assertRaises(msg_handler.QPCReportException): - _, _ = self.processor._validate_report_details() + _, _ = self.processor._validate_report_details() # pylint: disable=unbalanced-tuple-unpacking def test_download_contents_raises_error(self): """Test to verify downloading contents fails when error is raised."""