feat: unify ElasticSearch and OpenSearch configuration

Changes:
- set the same cluster and service name
- unify tutor plugin configuration variables

TODO:
- add internal_users.yml > admin > hash for a custom default password

README.md

@@ -208,7 +208,7 @@ To enable set `elasticsearch.enabled=true` in your `values.yaml` and deploy the
 For each instance you would like to enable this on, set the configuration values in the respective `config.yml`:
 
 ```yaml
-K8S_HARMONY_ENABLE_SHARED_ELASTICSEARCH: true
+K8S_HARMONY_ENABLE_SHARED_HARMONY_SEARCH: true
 RUN_ELASTICSEARCH: false
 ```
 

harmony-chart/templates/elasticsearch/secrets.yaml

@@ -1,6 +1,6 @@
 ---
 {{- $ca := genCA "elasticca" 1825 }}
-{{- $cert := genSignedCert "elasticsearch-master.{{ Release.Namespace }}.local" nil (list "elasticsearch-master.{{ Release.Namespace }}.local") 1825 $ca }}
+{{- $cert := genSignedCert "harmony-search-cluster-master.{{ Release.Namespace }}.local" nil (list "harmony-search-cluster-master.{{ Release.Namespace }}.local") 1825 $ca }}
 apiVersion: v1
 kind: Secret
 metadata:

harmony-chart/values.yaml

@@ -18,16 +18,18 @@ cert-manager:
 elasticsearch:
   enabled: false
 
+  clusterName: "harmony-search-cluster"
+
   # Operators will need to add/update the following setting in each
   # of their instances by running the commands:
   # ```
-  # tutor config save --set K8S_HARMONY_ENABLE_SHARED_ELASTICSEARCH=true --set RUN_ELASTICSEARCH=false
+  # tutor config save --set K8S_HARMONY_ENABLE_SHARED_HARMONY_SEARCH=true --set RUN_ELASTICSEARCH=false
   # tutor harmony create-elasticsearch-user
   # ```
   # RUN_ELASTICSEARCH: false
-  # ELASTICSEARCH_PREFIX_INDEX: "username-"
-  # K8S_HARMONY_USE_SHARED_ELASTICSEARCH: true
-  # ELASTICSEARCH_AUTH: "username:actual_password"
+  # HARMONY_SEARCH_INDEX_PREFIX: "username-"
+  # K8S_HARMONY_ENABLE_SHARED_HARMONY_SEARCH: true
+  # HARMONY_SEARCH_HTTP_AUTH: "username:actual_password"
 
   # We will create the relevant certs, because they need to shared
   # with pods in other namespaces.
@@ -52,6 +54,7 @@ elasticsearch:
 
   esConfig:
     "elasticsearch.yml": |
+      cluster.name: harmony-search-cluster
       xpack.security.enabled: true
       xpack.security.http.ssl.enabled: true
       xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certs/tls.key
@@ -82,24 +85,26 @@ vpa:
 # Multi-tenant OpenSearch
 opensearch:
   enabled: false
+  clusterName: "harmony-search-cluster"
+  masterService: "harmony-search-cluster"
 
   # Operators will need to add/update the following setting in each
   # of their instances by running the commands:
   # ```
-  # tutor config save --set K8S_HARMONY_ENABLE_SHARED_OPENSEARCH=true --set RUN_ELASTICSEARCH=false
+  # tutor config save --set K8S_HARMONY_ENABLE_SHARED_HARMONY_SEARCH=true --set RUN_ELASTICSEARCH=false
   # tutor harmony create-opensearch-user
   # ```
   # RUN_ELASTICSEARCH: false
-  # ELASTICSEARCH_PREFIX_INDEX: "username-"
+  # HARMONY_SEARCH_INDEX_PREFIX: "username-"
   # K8S_HARMONY_USE_SHARED_OPENSEARCH: true
-  # ELASTICSEARCH_AUTH: "username:actual_password"
+  # HARMONY_SEARCH_HTTP_AUTH: "username:actual_password"
 
 # Allows you to add any config files in {{ .Values.opensearchHome }}/config
 opensearchHome: /usr/share/opensearch
 # such as opensearch.yml and log4j2.properties
 config:
   opensearch.yml: |
-    cluster.name: opensearch-cluster
+    cluster.name: harmony-search-cluster
     network.host: 0.0.0.0
     plugins:
       security:

tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/commands.py

@@ -4,8 +4,8 @@
 from tutor import config as tutor_config
 from tutor import env as tutor_env
 from tutor.commands.k8s import K8sContext, kubectl_exec
-from .elasticsearch import ElasticSearchAPI
-from .opensearch import OpenSearchAPI
+from .harmony_search.elasticsearch import ElasticSearchAPI
+from .harmony_search.opensearch import OpenSearchAPI
 
 @click.group(help="Commands and subcommands of the openedx-k8s-harmony.")
 @click.pass_context
@@ -22,10 +22,10 @@ def create_elasticsearch_user(context: click.Context):
     config = tutor_config.load(context.root)
     namespace = config["K8S_HARMONY_NAMESPACE"]
     api = ElasticSearchAPI(namespace)
-    username, password = config["ELASTICSEARCH_HTTP_AUTH"].split(":", 1)
+    username, password = config["HARMONY_SEARCH_HTTP_AUTH"].split(":", 1)
     role_name = f"{username}_role"
 
-    prefix = config["ELASTICSEARCH_INDEX_PREFIX"]
+    prefix = config["HARMONY_SEARCH_INDEX_PREFIX"]
     api.post(
         f"_security/role/{role_name}",
         {"indices": [{"names": [f"{prefix}*"], "privileges": ["all"]}]},
@@ -50,10 +50,10 @@ def create_opensearch_user(context: click.Context):
     config = tutor_config.load(context.root)
     namespace = config["K8S_HARMONY_NAMESPACE"]
     api = OpenSearchAPI(namespace)
-    username, password = config["OPENSEARCH_HTTP_AUTH"].split(":", 1)
+    username, password = config["HARMONY_SEARCH_HTTP_AUTH"].split(":", 1)
     role_name = f"{username}_role"
 
-    prefix = config["OPENSEARCH_INDEX_PREFIX"]
+    prefix = config["HARMONY_SEARCH_INDEX_PREFIX"]
     api.put(
         f"_plugins/_security/api/roles/{role_name}",
         {"index_permissions": [{

tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/elasticsearch.py → tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/harmony_search/base.py

@@ -4,9 +4,9 @@
 from tutor import utils
 
 
-class ElasticSearchAPI:
+class BaseSearchAPI:
     """
-    Helper class to interact with the ElasticSearch
+    Helper class to interact with the HarmonySearch
     API on the deployed cluster.
     """
 
@@ -18,18 +18,19 @@ def __init__(self, namespace):
             "--tty",
             "--namespace",
             namespace,
-            "elasticsearch-master-0",
+            "harmony-search-cluster-master-0",
             "--",
             "bash",
             "-c",
         ]
-        self._curl_base = ["curl", "--insecure", "-u", "elastic:${ELASTIC_PASSWORD}"]
+        # Must be specified by subclasses
+        self._curl_base = None
 
     def run_command(self, curl_options) -> typing.Union[dict, bytes]:
         """
-        Invokes a curl command on the first Elasticsearch pod.
+        Invokes a curl command on the first HarmonySearch pod.
 
-        If possible returns the parsed json from the Elasticsearch response.
+        If possible returns the parsed json from the HarmonySearch response.
         Otherwise, the raw bytes from the curl command are returned.
         """
         response = utils.check_output(
@@ -42,20 +43,20 @@ def run_command(self, curl_options) -> typing.Union[dict, bytes]:
 
     def get(self, endpoint):
         """
-        Runs a GET request on the Elasticsearch cluster with the specified
+        Runs a GET request on the HarmonySearch cluster with the specified
         endpoint.
 
-        If possible returns the parsed json from the Elasticsearch response.
+        If possible returns the parsed json from the HarmonySearch response.
         Otherwise, the raw bytes from the curl command are returned.
         """
         return self.run_command(["-XGET", f"https://localhost:9200/{endpoint}"])
 
     def post(self, endpoint: str, data: dict) -> typing.Union[dict, bytes]:
         """
-        Runs a POST request on the Elasticsearch cluster with the specified
+        Runs a POST request on the HarmonySearch cluster with the specified
         endpoint.
 
-        If possible returns the parsed json from the Elasticsearch response.
+        If possible returns the parsed json from the HarmonySearch response.
         Otherwise, the raw bytes from the curl command are returned.
         """
         return self.run_command(
@@ -68,3 +69,22 @@ def post(self, endpoint: str, data: dict) -> typing.Union[dict, bytes]:
                 '"Content-Type: application/json"',
             ]
         )
+
+    def put(self, endpoint: str, data: dict) -> typing.Union[dict, bytes]:
+        """
+        Runs a PUT request on the HarmonySearch cluster with the specified
+        endpoint.
+
+        If possible returns the parsed json from the HarmonySearch response.
+        Otherwise, the raw bytes from the curl command are returned.
+        """
+        return self.run_command(
+            [
+                "-XPUT",
+                f"https://localhost:9200/{endpoint}",
+                "-d",
+                f"'{json.dumps(data)}'",
+                "-H",
+                '"Content-Type: application/json"',
+            ]
+        )

tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/harmony_search/elasticsearch.py

@@ -0,0 +1,12 @@
+from .base import BaseSearchAPI
+
+
+class ElasticSearchAPI(BaseSearchAPI):
+    """
+    Helper class to interact with the ElasticSearch
+    API on the deployed cluster.
+    """
+
+    def __init__(self, namespace):
+        super().__init__(namespace)
+        self._curl_base = ["curl", "--insecure", "-u", "elastic:${ELASTIC_PASSWORD}"]

tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/harmony_search/opensearch.py

@@ -0,0 +1,12 @@
+from .base import BaseSearchAPI
+
+
+class OpenSearchAPI(BaseSearchAPI):
+    """
+    Helper class to interact with the OpenSearch
+    API on the deployed cluster.
+    """
+
+    def __init__(self, namespace):
+        super().__init__(namespace)
+        self._curl_base = ["curl", "--insecure", "-u", "admin:admin"]

tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/patches/openedx-common-settings

@@ -1,23 +1,10 @@
-# ElasticSearch is a prefered engine.
-# If both are turned on, ELASTICSEARCH will be used.
-{% if K8S_HARMONY_ENABLE_SHARED_OPENSEARCH %}
-ELASTICSEARCH_INDEX_PREFIX = "{{OPENSEARCH_INDEX_PREFIX}}"
+{% if K8S_HARMONY_ENABLE_SHARED_HARMONY_SEARCH %}
+ELASTICSEARCH_INDEX_PREFIX = "{{HARMONY_SEARCH_INDEX_PREFIX}}"
 ELASTIC_SEARCH_CONFIG = [{
   "use_ssl": True,
-  "host": "opensearch-cluster-master.{{K8S_HARMONY_NAMESPACE}}.svc.cluster.local",
+  "host": "harmony-search-cluster-master.{{K8S_HARMONY_NAMESPACE}}.svc.cluster.local",
   "verify_certs": False,
   "port": 9200,
-  "http_auth": "{{ OPENSEARCH_HTTP_AUTH }}"
-}]
-{% endif %}
-
-{% if K8S_HARMONY_ENABLE_SHARED_ELASTICSEARCH %}
-ELASTICSEARCH_INDEX_PREFIX = "{{ELASTICSEARCH_INDEX_PREFIX}}"
-ELASTIC_SEARCH_CONFIG = [{
-  "use_ssl": True,
-  "host": "elasticsearch-master.{{K8S_HARMONY_NAMESPACE}}.svc.cluster.local",
-  "verify_certs": False,
-  "port": 9200,
-  "http_auth": "{{ ELASTICSEARCH_HTTP_AUTH }}"
+  "http_auth": "{{ HARMONY_SEARCH_HTTP_AUTH }}"
 }]
 {% endif %}

tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/patches/openedx-dockerfile-post-python-requirements

@@ -1,4 +1,4 @@
-{% if K8S_HARMONY_ENABLE_SHARED_ELASTICSEARCH or K8S_HARMONY_ENABLE_SHARED_OPENSEARCH %}
+{% if K8S_HARMONY_ENABLE_SHARED_HARMONY_SEARCH %}
 # This is needed otherwise the previously installed edx-search
 # package doesn't get replaced. Once the below branch is merged
 # upstream it will no longer be needed.

tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/plugin.py

@@ -21,8 +21,7 @@
         # The workaround is to manually add a list of hosts to be routed to the caddy
         # instance.
         "INGRESS_HOST_LIST": [],
-        "ENABLE_SHARED_ELASTICSEARCH": False,
-        "ENABLE_SHARED_OPENSEARCH": False,
+        "K8S_HARMONY_ENABLE_SHARED_HARMONY_SEARCH": False,
     },
     "overrides": {
         # Don't use Caddy as a per-instance external web proxy, but do still use it
@@ -32,10 +31,8 @@
         "ENABLE_HTTPS": True,
     },
     "unique": {
-        "ELASTICSEARCH_HTTP_AUTH": "{{K8S_NAMESPACE}}:{{ 24|random_string }}",
-        "ELASTICSEARCH_INDEX_PREFIX": "{{K8S_NAMESPACE}}-{{ 4|random_string|lower }}-",
-        "OPENSEARCH_HTTP_AUTH": "{{K8S_NAMESPACE}}:{{ 24|random_string }}",
-        "OPENSEARCH_INDEX_PREFIX": "{{K8S_NAMESPACE}}-{{ 4|random_string|lower }}-",
+        "HARMONY_SEARCH_HTTP_AUTH": "{{K8S_NAMESPACE}}:{{ 24|random_string }}",
+        "HARMONY_SEARCH_INDEX_PREFIX": "{{K8S_NAMESPACE}}-{{ 4|random_string|lower }}-",
     },
 }