WIP redis sentinel

stackl/agent/agent/config.py

@@ -8,6 +8,7 @@
     import importlib_metadata as metadata
 
 from pydantic import BaseSettings
+from typing import List, Tuple
 
 
 class Settings(BaseSettings):
@@ -19,8 +20,11 @@ class Settings(BaseSettings):
     agent_name: str = "common"
     agent_type: str = "mock"
     redis_host: str = "localhost"
+    redis_hosts: List[Tuple[str, int]] = []  # '[["localhost", 26379]]'
     redis_port: int = 6379
     redis_password: str = None
+    redis_sentinel: bool = False
+    redis_sentinel_master: str = ""
     secret_handler: str = "base64"
     loglevel: str = "INFO"
     max_jobs: int = 10

stackl/agent/agent/main.py

@@ -85,6 +85,13 @@ class AgentSettings:
     queue_name = config.settings.agent_name
     job_timeout = config.settings.job_timeout
     max_jobs = config.settings.max_jobs
-    redis_settings = RedisSettings(host=config.settings.redis_host,
-                                   port=config.settings.redis_port,
-                                   password=config.settings.redis_password)
+    if config.settings.redis_hosts:
+        redis_settings = RedisSettings(
+            host=config.settings.redis_hosts,
+            password=config.settings.redis_password,
+            sentinel_master=config.settings.redis_sentinel_master,
+            sentinel=config.settings.redis_sentinel)
+    else:
+        redis_settings = RedisSettings(host=config.settings.redis_host,
+                                       port=config.settings.redis_port,
+                                       password=config.settings.redis_password)

stackl/core/core/config.py

@@ -5,6 +5,7 @@
 
 from loguru import logger
 from pydantic import BaseSettings
+from typing import List, Tuple
 
 
 class InterceptHandler(logging.Handler):
@@ -40,10 +41,12 @@ class Settings(BaseSettings):  # pylint: disable=too-few-public-methods
     stackl_datastore_path: str = "/lfs-store"
 
     # Redis options
-    stackl_redis_type: str = "real"
+    stackl_redis_type: str = "sentinel"
     stackl_redis_host: str = "localhost"
     stackl_redis_password: str = None
     stackl_redis_port: int = 6379
+    stackl_redis_sentinel_master: str = "stackl"
+    stackl_redis_hosts: List[Tuple[str, int]] = []  # '[["localhost", 26379]]'
 
     # OPA options
     stackl_opa_host: str = "http://localhost:8181"

stackl/core/core/datastore/redis_store.py

@@ -4,6 +4,7 @@
 import json
 
 import redis
+from redis.sentinel import Sentinel
 from loguru import logger
 from redislite import Redis
 
@@ -14,18 +15,32 @@
 
 class RedisStore(DataStore):
     """Implementation of Redis datastore"""
-
     def __init__(self):
         super().__init__()
         if config.settings.stackl_redis_type == "fake":
             logger.info("Using fake client")
 
             self.redis = Redis()
+        elif config.settings.stackl_redis_type == "sentinel":
+            logger.info("Using Sentinel to discover Redis")
+            sentinel = Sentinel(config.settings.stackl_redis_hosts)
+            redis_connection = sentinel.discover_master(
+                config.settings.stackl_redis_sentinel_master)
+            logger.info(f"Connecting to Redis at {redis_connection}")
+            self.redis = redis.Redis(
+                host=redis_connection[0],
+                port=redis_connection[1],
+                password=config.settings.stackl_redis_password,
+                db=0)
+            logger.info("Connection successful")
+
         else:
-            self.redis = redis.Redis(host=config.settings.stackl_redis_host,
-                                     port=config.settings.stackl_redis_port,
-                                     password=config.settings.stackl_redis_password,
-                                     db=0)
+            logger.info("Using single Redis instance")
+            self.redis = redis.Redis(
+                host=config.settings.stackl_redis_host,
+                port=config.settings.stackl_redis_port,
+                password=config.settings.stackl_redis_password,
+                db=0)
 
     def get(self, **keys):
         """Gets a document from a redis instance"""
@@ -48,7 +63,8 @@ def get_all(self, category, document_type, wildcard_prefix=""):
         """Gets all documents of a type from a Redis"""
         document_key = f"{category}/{document_type}/{wildcard_prefix}*"
         logger.debug(
-            f"[RedisStore] get_all in '{document_key}' for type '{document_type}'")
+            f"[RedisStore] get_all in '{document_key}' for type '{document_type}'"
+        )
         content = []
         for key in self.redis.scan_iter(document_key):
             content.append(json.loads(self.redis.get(key)))
@@ -61,7 +77,8 @@ def get_history(self, category, document_type, name):
         """Gets the snapshots of document from Redis"""
         document_key = category + '/' + document_type + '/' + name
         logger.debug(
-            f"[RedisStore] get_history in '{document_key}' for type '{document_type}'")
+            f"[RedisStore] get_history in '{document_key}' for type '{document_type}'"
+        )
         content = []
         for key in self.redis.scan_iter(document_key):
             content.append(json.loads(self.redis.get(key)))