SFR-2316: Create a local backend for frontend CI (#450)

api/app.py

@@ -46,7 +46,7 @@ def run(self):
             self.app.config['DEBUG'] = True
             self.app.run(host='0.0.0.0', port=5050)
 
-        elif 'local' in os.environ['ENVIRONMENT']:
+        elif 'local' in os.environ['ENVIRONMENT'] or os.environ['ENVIRONMENT'] == 'frontend-ci':
             logger.debug('Starting dev server on port 5050')
 
             self.app.config['ENV'] = 'development'

api/blueprints/drbSearch.py

@@ -57,16 +57,19 @@ def query():
         filtered_formats = APIUtils.formatFilters(terms)
 
         works = db_client.fetchSearchedWorks(results)
+
+        # Depending on the version of elastic search, hits will either be an integer or a dictionary
+        total_hits = search_result.hits.total  if isinstance(search_result.hits.total, int) else search_result.hits.total.value
 
         facets = APIUtils.formatAggregationResult(search_result.aggregations.to_dict())
         paging = APIUtils.formatPagingOptions(
             search_page + 1, 
             search_size, 
-            search_result.hits.total.value
+            total_hits
         )
 
         data_block = {
-            'totalWorks': search_result.hits.total.value,
+            'totalWorks': total_hits,
             'works': APIUtils.formatWorkOutput(
                 works, 
                 results, 

config/frontend-ci.yaml

@@ -0,0 +1,17 @@
+LOG_LEVEL: INFO
+
+POSTGRES_HOST: sfr-new-metadata-production-cluster.cluster-cvy7z512hcjg.us-east-1.rds.amazonaws.com
+POSTGRES_NAME: dcdw_qa
+POSTGRES_PORT: '5432'
+
+REDIS_HOST: localhost
+REDIS_PORT: '6379'
+
+ELASTICSEARCH_INDEX: drb_dcdw_qa
+ELASTICSEARCH_SCHEME: https
+ELASTICSEARCH_HOST: vpc-drb-search-production-5ptfzwisshcadbbaph35dqilva.us-east-1.es.amazonaws.com
+ELASTICSEARCH_PORT: '9200'
+ELASTICSEARCH_TIMEOUT: '30'
+
+READER_VERSION: 'v2'
+API_PROXY_CORS_ALLOWED: '*'

managers/elasticsearch.py

@@ -1,10 +1,12 @@
 import os
 
+import boto3
 from elasticsearch.client import IngestClient
-from elasticsearch import Elasticsearch
+from elasticsearch import Elasticsearch, RequestsHttpConnection
 from elasticsearch.helpers import bulk
 from elasticsearch_dsl import connections, Index
 from elastic_transport import ConnectionTimeout
+from requests_aws4auth import AWS4Auth
 
 from model import ESWork
 from logger import create_log
@@ -26,12 +28,17 @@ def createElasticConnection(self, scheme=None, host=None, port=None, user=None,
         user = user or os.environ.get('ELASTICSEARCH_USER', None)
         pswd = pswd or os.environ.get('ELASTICSEARCH_PSWD', None)
         timeout = int(os.environ.get('ELASTICSEARCH_TIMEOUT', 5))
+        environment = os.environ.get('ENVIRONMENT', None)
 
         creds = '{}:{}@'.format(user, pswd) if user and pswd else ''
 
         #Allowing multple hosts for a ES connection
         multHosts = []
 
+        if environment == 'frontend-ci':
+            self._create_ci_connection(scheme=scheme, host=host, timeout=timeout)
+            return
+
         if ',' not in host:
             host = '{}://{}{}:{}'.format(scheme, creds, host, port)
 
@@ -318,6 +325,25 @@ def _deleteGenerator(self, uuids):
                 '_id': uuid
             }
 
+    def _create_ci_connection(self, scheme: str, host: str, timeout: int):
+        host = '{}://{}'.format(scheme, host)
+
+        session = boto3.Session()
+        credentials = session.get_credentials()
+        awsauth = AWS4Auth(credentials.access_key, credentials.secret_key, 'us-east-1', 'es', session_token=credentials.token)
+
+        connection_config = {
+            'hosts': [host],
+            'timeout': timeout,
+            'http_auth': awsauth,
+            'connection_class': RequestsHttpConnection,
+            'use_ssl': True,
+            'verify_certs': True,
+        }
+
+        self.client = connections.create_connection(**connection_config)
+        self.es = Elasticsearch(**connection_config)
+
     @staticmethod
     def _splitWorkBatch(works):
         workCount = len(works)

requirements.txt

@@ -22,6 +22,7 @@ pyyaml==6.0
 redis==4.5.4
 requests==2.28.2
 requests_oauthlib==1.3.1
+requests-aws4auth==1.3.1
 scikit-learn==1.2.2
 sqlalchemy==2.0.20
 waitress==2.1.2