From 799dd4bd488e9dbdd60401b721eea292d6d43706 Mon Sep 17 00:00:00 2001 From: Gabor Boros Date: Tue, 30 Apr 2024 19:49:54 +0400 Subject: [PATCH] fix: ensure ES/OS cluster setup for shared usage Signed-off-by: Gabor Boros --- README.md | 13 ++- charts/harmony-chart/Chart.lock | 2 +- charts/harmony-chart/Chart.yaml | 2 +- .../templates/elasticsearch/secrets.yaml | 5 +- .../templates/opensearch/secrets.yaml | 9 +- charts/harmony-chart/values.yaml | 15 +--- .../tutor_k8s_harmony_plugin/__about__.py | 2 +- .../tutor_k8s_harmony_plugin/commands.py | 18 ++-- .../harmony_search/base.py | 66 ++++++++++---- .../harmony_search/elasticsearch.py | 3 +- .../harmony_search/opensearch.py | 4 +- .../patches/discovery-common-settings | 14 +++ .../patches/kustomization | 33 +++++++ .../patches/openedx-common-settings | 17 ++-- .../tutor_k8s_harmony_plugin/plugin.py | 88 ++++++++++++++++--- .../k8s/deployment-revision-history.yaml | 8 ++ .../shared-search-cert-patch-volumeless.yaml | 17 ++++ .../k8s/shared-search-cert-patch.yaml | 17 ++++ 18 files changed, 265 insertions(+), 68 deletions(-) create mode 100644 tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/patches/discovery-common-settings create mode 100644 tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/patches/kustomization create mode 100644 tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/templates/k8s_harmony/k8s/deployment-revision-history.yaml create mode 100644 tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/templates/k8s_harmony/k8s/shared-search-cert-patch-volumeless.yaml create mode 100644 tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/templates/k8s_harmony/k8s/shared-search-cert-patch.yaml diff --git a/README.md b/README.md index 531dd4b..175bf5b 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ In particular, this project aims to provide the following benefits to Open edX o * Ingress controller: [ingress-nginx](https://kubernetes.github.io/ingress-nginx/) * Automatic HTTPS cert provisioning: [cert-manager](https://cert-manager.io/) * Autoscaling: `metrics-server` and `vertical-pod-autoscaler` - * Search index: ElasticSearch (support for OpenSearch is planned) + * Search index: ElasticSearch or OpenSearch * Monitoring: TODO * Database clusters: TODO (for now we recommend provisioning managed MySQL/MongoDB database clusters from your cloud provider using OpenTofu or a tool like [Grove](https://grove.opencraft.com/).) * Where possible, we try to configure these systems to **auto-detect** newly deployed Open edX instances and adapt to them automatically; where that isn't possible, Tutor plugins are used so that the instances self-register or self-provision the shared resources as needed. @@ -292,11 +292,18 @@ To enable set `elasticsearch.enabled=true` in your `values.yaml` and deploy the For each instance you would like to enable this on, set the configuration values in the respective `config.yml`: ```yaml -K8S_HARMONY_ENABLE_SHARED_HARMONY_SEARCH: true RUN_ELASTICSEARCH: false +K8S_HARMONY_ENABLE_SHARED_SEARCH_CLUSTER: true ``` -* And create the user on the cluster with `tutor k8s harmony create-elasticsearch-user`. +* Create the user on the cluster with `tutor k8s harmony create-elasticsearch-user`. +* Copy the Elasticsearch CA certificate to the instance's namespace where `$INSTANCE_NAMESPACE` is where the instance is installed in. + ```shell + kubectl get secret "search-cluster-certificates-elasticsearch" -n "harmony" -o "yaml" | \ + grep -v '^\s*namespace:\s' | \ + sed s/-elasticsearch//g |\ + kubectl apply -n "$INSTANCE_NAMESPACE" --force -f - + ``` * Rebuild your Open edX image `tutor images build openedx`. * Finally, redeploy your changes: `tutor k8s start && tutor k8s init`. diff --git a/charts/harmony-chart/Chart.lock b/charts/harmony-chart/Chart.lock index b489bca..5ab58fe 100644 --- a/charts/harmony-chart/Chart.lock +++ b/charts/harmony-chart/Chart.lock @@ -33,4 +33,4 @@ dependencies: repository: https://openfaas.github.io/faas-netes version: 14.2.34 digest: sha256:b636bd16d732d51544ca7223f460e22f45a7132e31e874a789c5fc0cac460a45 -generated: "2024-04-26T06:09:47.906542+04:00" +generated: "2024-05-02T12:32:49.796635+04:00" diff --git a/charts/harmony-chart/Chart.yaml b/charts/harmony-chart/Chart.yaml index 9f86329..adadf12 100644 --- a/charts/harmony-chart/Chart.yaml +++ b/charts/harmony-chart/Chart.yaml @@ -5,7 +5,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes to the chart and its # templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.7.3 +version: 0.7.4 # This is the version number of the application being deployed. This version number should be incremented each time you # make changes to the application. Versions are not expected to follow Semantic Versioning. They should reflect the # version the application is using. It is recommended to use it with quotes. diff --git a/charts/harmony-chart/templates/elasticsearch/secrets.yaml b/charts/harmony-chart/templates/elasticsearch/secrets.yaml index 3ffcdb6..08c0135 100644 --- a/charts/harmony-chart/templates/elasticsearch/secrets.yaml +++ b/charts/harmony-chart/templates/elasticsearch/secrets.yaml @@ -5,12 +5,13 @@ apiVersion: v1 kind: Secret metadata: - name: elasticsearch-certificates + name: search-cluster-certificates-elasticsearch type: Opaque data: "ca.crt": {{ $ca.Cert | b64enc | toYaml | indent 4}} "tls.key": {{ $cert.Key | b64enc | toYaml | indent 4}} - "tls.crt": {{ print $cert.Cert $ca.Cert | b64enc | toYaml | indent 4}} + "tls.crt": {{ print $cert.Cert | b64enc | toYaml | indent 4}} + "chain.crt": {{ print $cert.Cert $ca.Cert | b64enc | toYaml | indent 4}} --- apiVersion: v1 kind: Secret diff --git a/charts/harmony-chart/templates/opensearch/secrets.yaml b/charts/harmony-chart/templates/opensearch/secrets.yaml index 40bbf00..41aafe8 100644 --- a/charts/harmony-chart/templates/opensearch/secrets.yaml +++ b/charts/harmony-chart/templates/opensearch/secrets.yaml @@ -1,16 +1,17 @@ --- {{- $ca := genCA "opensearchca" 1825 }} -{{- $cn := printf "opensearch-master.%s.local" .Release.Namespace }} +{{- $cn := printf "harmony-search-cluster.%s.svc.cluster.local" .Release.Namespace }} {{- $cert := genSignedCert $cn nil (list $cn) 1825 $ca }} apiVersion: v1 kind: Secret metadata: - name: opensearch-certificates + name: search-cluster-certificates-opensearch type: Opaque data: "ca.crt": {{ $ca.Cert | b64enc | toYaml | indent 4}} "tls.key": {{ $cert.Key | b64enc | toYaml | indent 4}} - "tls.crt": {{ print $cert.Cert $ca.Cert | b64enc | toYaml | indent 4}} + "tls.crt": {{ print $cert.Cert | b64enc | toYaml | indent 4}} + "chain.crt": {{ print $cert.Cert $ca.Cert | b64enc | toYaml | indent 4}} --- {{- $password := randAlphaNum 32 }} {{- $password_bcrypt := $password | bcrypt }} @@ -20,5 +21,5 @@ metadata: name: opensearch-credentials type: Opaque data: - harmony_password: {{ $password | b64enc | quote }} + password: {{ $password | b64enc | quote }} internal_users.yml: {{ printf "---\n_meta:\n type: \"internalusers\"\n config_version: 2\n\nharmony:\n hash: \"%s\"\n reserved: true\n backend_roles:\n - \"admin\"\n description: \"Harmony admin user\"\n" $password_bcrypt | b64enc | quote }} diff --git a/charts/harmony-chart/values.yaml b/charts/harmony-chart/values.yaml index 498b35b..0734cd2 100644 --- a/charts/harmony-chart/values.yaml +++ b/charts/harmony-chart/values.yaml @@ -47,9 +47,7 @@ elasticsearch: # tutor harmony create-elasticsearch-user # ``` # RUN_ELASTICSEARCH: false - # HARMONY_SEARCH_INDEX_PREFIX: "username-" # K8S_HARMONY_ENABLE_SHARED_HARMONY_SEARCH: true - # HARMONY_SEARCH_HTTP_AUTH: "username:actual_password" # We will create the relevant certs, because they need to shared # with pods in other namespaces. @@ -60,7 +58,7 @@ elasticsearch: # This secret will contain the http certificates. secretMounts: - name: elasticsearch-certificates - secretName: elasticsearch-certificates + secretName: search-cluster-certificates-elasticsearch path: /usr/share/elasticsearch/config/certs defaultMode: 0777 @@ -78,7 +76,7 @@ elasticsearch: xpack.security.enabled: true xpack.security.http.ssl.enabled: true xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certs/tls.key - xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certs/tls.crt + xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certs/chain.crt xpack.security.transport.ssl.enabled: true xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certs/tls.key xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certs/tls.crt @@ -98,14 +96,12 @@ opensearch: # tutor harmony create-opensearch-user # ``` # RUN_ELASTICSEARCH: false - # HARMONY_SEARCH_INDEX_PREFIX: "username-" # K8S_HARMONY_USE_SHARED_OPENSEARCH: true - # HARMONY_SEARCH_HTTP_AUTH: "username:actual_password" # # This secret will contain the ssl certificates. secretMounts: - name: opensearch-certificates - secretName: opensearch-certificates + secretName: search-cluster-certificates-elasticsearch path: /usr/share/opensearch/config/certs defaultMode: 0777 @@ -125,11 +121,6 @@ opensearch: extraEnvs: - name: DISABLE_INSTALL_DEMO_CONFIG value: "true" - - name: HARMONY_PASSWORD - valueFrom: - secretKeyRef: - name: opensearch-credentials - key: harmony_password # Allows you to add any config files in {{ .Values.opensearchHome }}/config opensearchHome: /usr/share/opensearch diff --git a/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/__about__.py b/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/__about__.py index a08b09c..dba3a77 100644 --- a/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/__about__.py +++ b/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/__about__.py @@ -1 +1 @@ -__version__ = "17.0.0" +__version__ = "17.0.1" diff --git a/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/commands.py b/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/commands.py index 1ad23e4..af8266a 100644 --- a/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/commands.py +++ b/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/commands.py @@ -21,13 +21,21 @@ def create_elasticsearch_user(context: click.Context): config = tutor_config.load(context.root) namespace = config["K8S_HARMONY_NAMESPACE"] api = ElasticSearchAPI(namespace) - username, password = config["HARMONY_SEARCH_HTTP_AUTH"].split(":", 1) + username, password = config["K8S_HARMONY_SEARCH_CLUSTER_HTTP_AUTH"].split(":", 1) role_name = f"{username}_role" - prefix = config["HARMONY_SEARCH_INDEX_PREFIX"] + prefix = config["K8S_HARMONY_SEARCH_CLUSTER_INDEX_PREFIX"] api.post( f"_security/role/{role_name}", - {"indices": [{"names": [f"{prefix}*"], "privileges": ["all"]}]}, + { + "cluster": ["monitor"], + "indices": [ + { + "names": [f"{prefix}*"], + "privileges": ["all"], + }, + ], + }, ) api.post( @@ -50,10 +58,10 @@ def create_opensearch_user(context: click.Context): config = tutor_config.load(context.root) namespace = config["K8S_HARMONY_NAMESPACE"] api = OpenSearchAPI(namespace) - username, password = config["HARMONY_SEARCH_HTTP_AUTH"].split(":", 1) + username, password = config["K8S_HARMONY_SEARCH_CLUSTER_HTTP_AUTH"].split(":", 1) role_name = f"{username}_role" - prefix = config["HARMONY_SEARCH_INDEX_PREFIX"] + prefix = config["K8S_HARMONY_SEARCH_CLUSTER_INDEX_PREFIX"] api.put( f"_plugins/_security/api/roles/{role_name}", { diff --git a/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/harmony_search/base.py b/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/harmony_search/base.py index be8d95e..42eabe7 100644 --- a/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/harmony_search/base.py +++ b/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/harmony_search/base.py @@ -1,5 +1,6 @@ import json import typing +import base64 from tutor import utils @@ -11,36 +12,65 @@ class BaseSearchAPI: """ def __init__(self, namespace): - self._command_base = [ - "kubectl", - "exec", - "--stdin", - "--tty", - "--namespace", - namespace, - "harmony-search-cluster-master-0", - "--", - "bash", - "-c", - ] + self._command_base = ["kubectl", "--namespace", namespace] + self._exec_command = [*self._command_base, "exec", "--stdin", "--tty"] + # Must be specified by subclasses self._curl_base = None - def run_command(self, curl_options) -> typing.Union[dict, bytes]: + def run_kubectl_command(self, cmd: list = None, opts: list = None) -> bytes: + """ + Invokes a kubectl command in a pre-defined namespace. + """ + if cmd is None: + cmd = self._command_base + + if opts is None: + opts = list() + + call_args = list([x for x in [*cmd, " ".join(opts)] if x]) + return utils.check_output(*call_args) + + def run_curl_command(self, curl_options) -> typing.Union[dict, bytes]: """ Invokes a curl command on the first HarmonySearch pod. If possible returns the parsed json from the HarmonySearch response. Otherwise, the raw bytes from the curl command are returned. """ - response = utils.check_output( - *self._command_base, " ".join(self._curl_base + curl_options) + container = "harmony-search-cluster-master-0" + response = self.run_kubectl_command( + cmd=[*self._exec_command, container, "--", "bash", "-c"], + opts=self._curl_base + curl_options, ) + try: return json.loads(response) except (TypeError, ValueError): return response + def get_cluster_password(self, secret_name: str, field: str = "password") -> str: + """ + Returns the search admin password for the cluster. + + Read the kubernetes opaque secret and return the value at the given + `field` from the `secret_name`. + """ + password = self.run_kubectl_command( + [ + *self._command_base, + "get", + "secret", + secret_name, + "-o", + f"jsonpath={{.data.{field}}}", + ] + ) + + print("PASS", password) + + return base64.b64decode(password).decode() + def get(self, endpoint): """ Runs a GET request on the HarmonySearch cluster with the specified @@ -49,7 +79,7 @@ def get(self, endpoint): If possible returns the parsed json from the HarmonySearch response. Otherwise, the raw bytes from the curl command are returned. """ - return self.run_command(["-XGET", f"https://localhost:9200/{endpoint}"]) + return self.run_curl_command(["-XGET", f"https://localhost:9200/{endpoint}"]) def post(self, endpoint: str, data: dict) -> typing.Union[dict, bytes]: """ @@ -59,7 +89,7 @@ def post(self, endpoint: str, data: dict) -> typing.Union[dict, bytes]: If possible returns the parsed json from the HarmonySearch response. Otherwise, the raw bytes from the curl command are returned. """ - return self.run_command( + return self.run_curl_command( [ "-XPOST", f"https://localhost:9200/{endpoint}", @@ -78,7 +108,7 @@ def put(self, endpoint: str, data: dict) -> typing.Union[dict, bytes]: If possible returns the parsed json from the HarmonySearch response. Otherwise, the raw bytes from the curl command are returned. """ - return self.run_command( + return self.run_curl_command( [ "-XPUT", f"https://localhost:9200/{endpoint}", diff --git a/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/harmony_search/elasticsearch.py b/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/harmony_search/elasticsearch.py index 14a6331..0eda583 100644 --- a/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/harmony_search/elasticsearch.py +++ b/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/harmony_search/elasticsearch.py @@ -9,4 +9,5 @@ class ElasticSearchAPI(BaseSearchAPI): def __init__(self, namespace): super().__init__(namespace) - self._curl_base = ["curl", "--insecure", "-u", "elastic:${ELASTIC_PASSWORD}"] + cluster_password = self.get_cluster_password("elasticsearch-credentials") + self._curl_base = ["curl", "--insecure", "-u", f"elastic:{cluster_password}"] diff --git a/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/harmony_search/opensearch.py b/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/harmony_search/opensearch.py index ca25572..c7d3094 100644 --- a/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/harmony_search/opensearch.py +++ b/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/harmony_search/opensearch.py @@ -9,5 +9,5 @@ class OpenSearchAPI(BaseSearchAPI): def __init__(self, namespace): super().__init__(namespace) - # TODO: Make this configurable - self._curl_base = ["curl", "--insecure", "-u", "harmony:${HARMONY_PASSWORD}"] + cluster_password = self.get_cluster_password("opensearch-credentials") + self._curl_base = ["curl", "--insecure", "-u", f"harmony:{cluster_password}"] diff --git a/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/patches/discovery-common-settings b/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/patches/discovery-common-settings new file mode 100644 index 0000000..83d840a --- /dev/null +++ b/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/patches/discovery-common-settings @@ -0,0 +1,14 @@ +{%- if is_plugin_loaded("discovery") and K8S_HARMONY_ENABLE_SHARED_SEARCH_CLUSTER %} +import os +import ssl + +with open(os.getenv("ELASTICSEARCH_CA_PATH")) as ca_cert: + ELASTICSEARCH_CA_CERT = ca_cert.read() + +ELASTICSEARCH_DSL['default'].update({ + "use_ssl": True, + "hosts": "{{ ELASTICSEARCH_HOST }}", + "http_auth": "{{ K8S_HARMONY_SEARCH_CLUSTER_HTTP_AUTH }}", + "ssl_context": ssl.create_default_context(cadata=ELASTICSEARCH_CA_CERT), +}) +{% endif %} diff --git a/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/patches/kustomization b/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/patches/kustomization new file mode 100644 index 0000000..a2d91c1 --- /dev/null +++ b/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/patches/kustomization @@ -0,0 +1,33 @@ +# Most of the open edX resources have volumes and volumeMounts, which means that +# we can add an extra volume to their volume list. However, in some cases, like +# the forum and forum-job, the resource does not define any volumes. This means, +# we cannot have a common solution as adding extra volumes to non-existing list +# breaks, while adding a volume definition to existing volumes replaces the +# original set of volumes. We have to manually distinguish between resources. +{%- set HAS_VOLUME = ["lms", "cms", "lms-job", "cms-job"] %} +{%- set VOLUMELESS = [] %} + +{%- if is_plugin_loaded("discovery") %} + {%- set HAS_VOLUME = HAS_VOLUME + ["discovery", "discovery-job"] %} +{% endif %} + +{%- if is_plugin_loaded("forum") %} + {%- set VOLUMELESS = VOLUMELESS + ["forum", "forum-job"] %} +{% endif %} + +patches: +- path: plugins/k8s_harmony/k8s/deployment-revision-history.yaml + target: + kind: Deployment +{%- for res in HAS_VOLUME %} +- path: plugins/k8s_harmony/k8s/shared-search-cert-patch.yaml + target: + kind: {% if "-job" in res %}Job{% else %}Deployment{% endif %} + name: {{ res }}.* +{%- endfor %} +{%- for res in VOLUMELESS %} +- path: plugins/k8s_harmony/k8s/shared-search-cert-patch-volumeless.yaml + target: + kind: {% if "-job" in res %}Job{% else %}Deployment{% endif %} + name: {{ res }}.* +{%- endfor %} diff --git a/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/patches/openedx-common-settings b/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/patches/openedx-common-settings index aeee87a..50bd6b9 100644 --- a/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/patches/openedx-common-settings +++ b/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/patches/openedx-common-settings @@ -1,10 +1,17 @@ -{% if K8S_HARMONY_ENABLE_SHARED_HARMONY_SEARCH %} -ELASTIC_SEARCH_INDEX_PREFIX = "{{ HARMONY_SEARCH_INDEX_PREFIX }}" +{%- if K8S_HARMONY_ENABLE_SHARED_SEARCH_CLUSTER %} +import os +import ssl + +with open(os.getenv("ELASTICSEARCH_CA_PATH")) as ca_cert: + ELASTICSEARCH_CA_CERT = ca_cert.read() + +ELASTIC_SEARCH_INDEX_PREFIX = "{{ K8S_HARMONY_SEARCH_CLUSTER_INDEX_PREFIX }}" + ELASTIC_SEARCH_CONFIG = [{ "use_ssl": True, - "host": "{{ K8S_HARMONY_ELASTIC_HOST }}", - "verify_certs": False, + "host": "{{ ELASTICSEARCH_HOST }}", "port": 9200, - "http_auth": "{{ HARMONY_SEARCH_HTTP_AUTH }}" + "http_auth": "{{ K8S_HARMONY_SEARCH_CLUSTER_HTTP_AUTH }}", + "ssl_context": ssl.create_default_context(cadata=ELASTICSEARCH_CA_CERT), }] {% endif %} diff --git a/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/plugin.py b/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/plugin.py index ff04423..d9ae51f 100644 --- a/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/plugin.py +++ b/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/plugin.py @@ -4,13 +4,28 @@ import pkg_resources from tutor import hooks as tutor_hooks +try: + from tutorforum.hooks import FORUM_ENV +except ImportError: + FORUM_ENV = None + from . import commands from .__about__ import __version__ +PLUGIN_DIR_NAME = "tutor_k8s_harmony_plugin" + + +def is_plugin_loaded(plugin_name: str) -> bool: + """ + Check if the provided plugin is loaded. + """ + + return plugin_name in tutor_hooks.Filters.PLUGINS_LOADED.iterate() + + config = { "defaults": { "VERSION": __version__, - "ELASTIC_HOST": "harmony-search-cluster.{{ K8S_HARMONY_NAMESPACE }}.svc.cluster.local", # This plugin assumes you are using ingress-nginx as an ingress controller to provide # you with a central load balancer. The standard Ingress object uses annotations to # trigger the generation of certificates using cert-manager. @@ -22,7 +37,8 @@ # The workaround is to manually add a list of hosts to be routed to the caddy # instance. "INGRESS_HOST_LIST": [], - "ENABLE_SHARED_HARMONY_SEARCH": False, + "ENABLE_SHARED_SEARCH_CLUSTER": False, + "DEPLOYMENT_REVISION_HISTORY_LIMIT": 10, }, "overrides": { # Don't use Caddy as a per-instance external web proxy, but do still use it @@ -30,10 +46,24 @@ "ENABLE_WEB_PROXY": False, # We are using HTTPS "ENABLE_HTTPS": True, + # We are using HTTPS + "ELASTICSEARCH_SCHEME": "https", + # Override the elasticsearch host to point to the internal cluster + "ELASTICSEARCH_HOST": "harmony-search-cluster.{{ K8S_HARMONY_NAMESPACE }}.svc.cluster.local", + # The list of indexes is defined in: + # https://github.com/openedx/course-discovery/blob/master/course_discovery/settings/base.py + # We need to keep a copy of this list so that we can prefix the index names when using shared ES. + "DISCOVERY_INDEX_OVERRIDES": { + "course_discovery.apps.course_metadata.search_indexes.documents.course": "{{ K8S_HARMONY_SEARCH_CLUSTER_INDEX_PREFIX }}course", + "course_discovery.apps.course_metadata.search_indexes.documents.course_run": "{{ K8S_HARMONY_SEARCH_CLUSTER_INDEX_PREFIX }}course_run", + "course_discovery.apps.course_metadata.search_indexes.documents.learner_pathway": "{{ K8S_HARMONY_SEARCH_CLUSTER_INDEX_PREFIX }}learner_pathway", + "course_discovery.apps.course_metadata.search_indexes.documents.person": "{{ K8S_HARMONY_SEARCH_CLUSTER_INDEX_PREFIX }}person", + "course_discovery.apps.course_metadata.search_indexes.documents.program": "{{ K8S_HARMONY_SEARCH_CLUSTER_INDEX_PREFIX }}program", + }, }, "unique": { - "HARMONY_SEARCH_HTTP_AUTH": "{{ K8S_NAMESPACE }}:{{ 24|random_string }}", - "HARMONY_SEARCH_INDEX_PREFIX": "{{ K8S_NAMESPACE }}-{{ 4|random_string|lower }}-", + "K8S_HARMONY_SEARCH_CLUSTER_HTTP_AUTH": "{{ K8S_NAMESPACE }}:{{ 24|random_string }}", + "K8S_HARMONY_SEARCH_CLUSTER_INDEX_PREFIX": "{{ K8S_NAMESPACE }}-{{ 4|random_string|lower }}-", }, } @@ -45,14 +75,46 @@ tutor_hooks.Filters.CONFIG_OVERRIDES.add_items(list(config["overrides"].items())) tutor_hooks.Filters.CONFIG_UNIQUE.add_items(list(config["unique"].items())) -# Load all patches from the "patches" folder -for path in glob( - os.path.join( - pkg_resources.resource_filename("tutor_k8s_harmony_plugin", "patches"), - "*", - ) -): +tutor_hooks.Filters.CLI_COMMANDS.add_item(commands.harmony) + +tutor_hooks.Filters.ENV_TEMPLATE_ROOTS.add_item( + pkg_resources.resource_filename(PLUGIN_DIR_NAME, "templates") +) + +tutor_hooks.Filters.ENV_TEMPLATE_TARGETS.add_items( + [ + ("k8s_harmony/k8s", "plugins"), + ] +) + +tutor_hooks.Filters.ENV_TEMPLATE_VARIABLES.add_item( + ("is_plugin_loaded", is_plugin_loaded) +) + +# The patches from this plugin should be loaded after all other plugins so that +# the overrides configured for individual instances are themselves not overriden +# by the default patches of other plugins. The priority of these patches are +# therefore set to 100. +patches = pkg_resources.resource_filename(PLUGIN_DIR_NAME, "patches") +for path in glob(os.path.join(patches, "*")): with open(path, encoding="utf-8") as patch_file: - tutor_hooks.Filters.ENV_PATCHES.add_item((os.path.basename(path), patch_file.read())) + tutor_hooks.Filters.ENV_PATCHES.add_item( + (os.path.basename(path), patch_file.read()), priority=100 + ) -tutor_hooks.Filters.CLI_COMMANDS.add_item(commands.harmony) +if FORUM_ENV: + + @FORUM_ENV.add() + def _add_forum_env_vars(env_vars: dict): + """ + Override froum env vars to configure the search cluster. + + The default Elasticsearch configuraiton does not allow HTTP auth or CA + cert path configuration. This needs to be done through overriding the + default env values. + """ + return { + **env_vars, + "SEARCH_SERVER": "{{ ELASTICSEARCH_SCHEME }}://{{ K8S_HARMONY_SEARCH_CLUSTER_HTTP_AUTH and (K8S_HARMONY_SEARCH_CLUSTER_HTTP_AUTH + '@') }}{{ ELASTICSEARCH_HOST }}:{{ ELASTICSEARCH_PORT }}", + "ELASTICSEARCH_INDEX_PREFIX": '{{ K8S_HARMONY_SEARCH_CLUSTER_INDEX_PREFIX|default("") }}', + } diff --git a/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/templates/k8s_harmony/k8s/deployment-revision-history.yaml b/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/templates/k8s_harmony/k8s/deployment-revision-history.yaml new file mode 100644 index 0000000..e0c49f6 --- /dev/null +++ b/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/templates/k8s_harmony/k8s/deployment-revision-history.yaml @@ -0,0 +1,8 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + # This name does not have to match anything as we are using this yaml to + # override resources using kustomize "patches", not patch strategic merge. + name: x-revision-history +spec: + revisionHistoryLimit: {{ K8S_HARMONY_DEPLOYMENT_REVISION_HISTORY_LIMIT }} diff --git a/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/templates/k8s_harmony/k8s/shared-search-cert-patch-volumeless.yaml b/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/templates/k8s_harmony/k8s/shared-search-cert-patch-volumeless.yaml new file mode 100644 index 0000000..c96d4e2 --- /dev/null +++ b/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/templates/k8s_harmony/k8s/shared-search-cert-patch-volumeless.yaml @@ -0,0 +1,17 @@ +- op: add + path: /spec/template/spec/containers/0/env/- + value: + name: ELASTICSEARCH_CA_PATH + value: /etc/ssl/certs/search-cluster.pem +- op: add + path: /spec/template/spec/containers/0/volumeMounts + value: + - mountPath: /etc/ssl/certs/search-cluster.pem + name: search-cluster-certs + subPath: ca.crt +- op: add + path: /spec/template/spec/volumes + value: + - name: search-cluster-certs + secret: + secretName: search-cluster-certificates diff --git a/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/templates/k8s_harmony/k8s/shared-search-cert-patch.yaml b/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/templates/k8s_harmony/k8s/shared-search-cert-patch.yaml new file mode 100644 index 0000000..a7289de --- /dev/null +++ b/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/templates/k8s_harmony/k8s/shared-search-cert-patch.yaml @@ -0,0 +1,17 @@ +- op: add + path: /spec/template/spec/containers/0/env/- + value: + name: ELASTICSEARCH_CA_PATH + value: /etc/ssl/certs/search-cluster.pem +- op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + mountPath: /etc/ssl/certs/search-cluster.pem + name: search-cluster-certs + subPath: ca.crt +- op: add + path: /spec/template/spec/volumes/- + value: + name: search-cluster-certs + secret: + secretName: search-cluster-certificates