diff --git a/README.md b/README.md index 25a4b73..1bddf14 100644 --- a/README.md +++ b/README.md @@ -132,14 +132,15 @@ HTTPS and is more complicated due to the need to use tunnelling.* 1. First, [install `minikube`](https://minikube.sigs.k8s.io/docs/start/) if you don't have it already. 2. Run `minikube start` (you can also use `minikube dashboard` to access the Kubernetes dashboard). -3. Run\ +3. Run `helm dependency update ./harmony-chart` to build helm dependencies locally. +4. Run\ `helm install --namespace harmony --create-namespace -f values-minikube.yaml harmony ./harmony-chart` -4. Run `minikube tunnel` (you may need to enter a password), and then you should be able to access the cluster (see +5. Run `minikube tunnel` (you may need to enter a password), and then you should be able to access the cluster (see "External IP" below). If this approach is not working, an alternative is to run\ `minikube service harmony-ingress-nginx-controller -n harmony`\ and then go to the URL it says, e.g. `http://127.0.0.1:52806` plus `/cluster-echo-test` (e.g. `http://127.0.0.1:52806/cluster-echo-test`) -5. In this case, skip step 2 ("Get the external IP") and use `127.0.0.1` as the external IP. You will need to remember +6. In this case, skip step 2 ("Get the external IP") and use `127.0.0.1` as the external IP. You will need to remember to include the port numbers shown above when accessing the instances. diff --git a/harmony-chart/Chart.lock b/harmony-chart/Chart.lock index 6614621..ef761e9 100644 --- a/harmony-chart/Chart.lock +++ b/harmony-chart/Chart.lock @@ -14,5 +14,8 @@ dependencies: - name: vertical-pod-autoscaler repository: https://cowboysysop.github.io/charts/ version: 6.0.3 -digest: sha256:4915c21724a8c4693f749aab2c311336990b48e16e90884d7b9ceca42eef69f8 -generated: "2023-04-13T23:07:43.616994532-05:00" +- name: opensearch + repository: https://opensearch-project.github.io/helm-charts + version: 2.11.4 +digest: sha256:e55d718bc0033d348cbabefe6d570b65ed3f26c463ace08201b3dd11b36f5a68 +generated: "2023-04-24T18:23:43.967524+03:00" diff --git a/harmony-chart/Chart.yaml b/harmony-chart/Chart.yaml index 165f759..120e44b 100644 --- a/harmony-chart/Chart.yaml +++ b/harmony-chart/Chart.yaml @@ -42,3 +42,8 @@ dependencies: repository: https://cowboysysop.github.io/charts/ alias: vpa condition: vpa.enabled + +- name: opensearch + version: "2.11.4" + condition: opensearch.enabled + repository: https://opensearch-project.github.io/helm-charts diff --git a/harmony-chart/charts/opensearch-2.11.4.tgz b/harmony-chart/charts/opensearch-2.11.4.tgz new file mode 100644 index 0000000..6151fd5 Binary files /dev/null and b/harmony-chart/charts/opensearch-2.11.4.tgz differ diff --git a/harmony-chart/templates/opensearch/secrets.yaml b/harmony-chart/templates/opensearch/secrets.yaml new file mode 100644 index 0000000..ece73da --- /dev/null +++ b/harmony-chart/templates/opensearch/secrets.yaml @@ -0,0 +1,12 @@ +--- +{{- $ca := genCA "opensearchca" 1825 }} +{{- $cert := genSignedCert "opensearch-master.{{ Release.Namespace }}.local" nil (list "opensearch-master.{{ Release.Namespace }}.local") 1825 $ca }} +apiVersion: v1 +kind: Secret +metadata: + name: opensearch-certificates +type: Opaque +data: + "ca.crt": {{ $ca.Cert | b64enc | toYaml | indent 4}} + "tls.key": {{ $cert.Key | b64enc | toYaml | indent 4}} + "tls.crt": {{ print $cert.Cert $ca.Cert | b64enc | toYaml | indent 4}} diff --git a/harmony-chart/values.yaml b/harmony-chart/values.yaml index 06d4969..b7ddfbd 100644 --- a/harmony-chart/values.yaml +++ b/harmony-chart/values.yaml @@ -77,3 +77,80 @@ vpa: # for all available options admissionController: replicaCount: 1 + + +# Multi-tenant OpenSearch +opensearch: + enabled: false + + # Operators will need to add/update the following setting in each + # of their instances by running the commands: + # ``` + # tutor config save --set K8S_HARMONY_ENABLE_SHARED_OPENSEARCH=true --set RUN_ELASTICSEARCH=false + # tutor harmony create-opensearch-user + # ``` + # RUN_ELASTICSEARCH: false + # ELASTICSEARCH_PREFIX_INDEX: "username-" + # K8S_HARMONY_USE_SHARED_OPENSEARCH: true + # ELASTICSEARCH_AUTH: "username:actual_password" + +# Allows you to add any config files in {{ .Values.opensearchHome }}/config +opensearchHome: /usr/share/opensearch +# such as opensearch.yml and log4j2.properties +config: + opensearch.yml: | + cluster.name: opensearch-cluster + network.host: 0.0.0.0 + plugins: + security: + ssl: + transport: + pemcert_filepath: certs/tls.crt + pemkey_filepath: certs/tls.key + pemtrustedcas_filepath: certs/ca.crt + enforce_hostname_verification: false + http: + enabled: true + pemcert_filepath: certs/tls.crt + pemkey_filepath: certs/tls.key + pemtrustedcas_filepath: certs/ca.crt + allow_unsafe_democertificates: false + allow_default_init_securityindex: true + audit.type: internal_opensearch + enable_snapshot_restore_privilege: true + check_snapshot_restore_write_privileges: true + restapi: + roles_enabled: ["all_access", "security_rest_api_access"] + system_indices: + enabled: true + indices: + [ + ".opendistro-alerting-config", + ".opendistro-alerting-alert*", + ".opendistro-anomaly-results*", + ".opendistro-anomaly-detector*", + ".opendistro-anomaly-checkpoints", + ".opendistro-anomaly-detection-state", + ".opendistro-reports-*", + ".opendistro-notifications-*", + ".opendistro-notebooks", + ".opendistro-asynchronous-search-response*", + ] + + # # This secret will contain the ssl certificates. + secretMounts: + - name: openseach-certificates + secretName: openseach-certificates + path: /usr/share/opensearch/config/certs + defaultMode: 0777 + + resources: + requests: + cpu: "100m" + memory: "512Mi" + limits: + cpu: "500m" + memory: "700Mi" + + persistence: + size: 30Gi diff --git a/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/commands.py b/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/commands.py index 643d019..aa2e1cc 100644 --- a/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/commands.py +++ b/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/commands.py @@ -5,6 +5,7 @@ from tutor import env as tutor_env from tutor.commands.k8s import K8sContext, kubectl_exec from .elasticsearch import ElasticSearchAPI +from .opensearch import OpenSearchAPI @click.group(help="Commands and subcommands of the openedx-k8s-harmony.") @click.pass_context @@ -40,5 +41,44 @@ def create_elasticsearch_user(context: click.Context): }, ) +@click.command(help="Create or update Opensearch users") +@click.pass_obj +def create_opensearch_user(context: click.Context): + """ + Creates or updates the Opensearch user + """ + config = tutor_config.load(context.root) + namespace = config["K8S_HARMONY_NAMESPACE"] + api = OpenSearchAPI(namespace) + username, password = config["OPENSEARCH_HTTP_AUTH"].split(":", 1) + role_name = f"{username}_role" + + prefix = config["OPENSEARCH_INDEX_PREFIX"] + api.put( + f"_plugins/_security/api/roles/{role_name}", + {"index_permissions": [{ + "index_patterns": [ + f"{prefix}*" + ], + "allowed_actions": [ + "read", + "write", + "create_index", + "manage", + "manage_ilm", + "all" + ] + }]}, + ) + + api.put( + f"_plugins/_security/api/internalusers/{username}", + { + "password": password, + "opendistro_security_roles": [role_name], + }, + ) + harmony.add_command(create_elasticsearch_user) +harmony.add_command(create_opensearch_user) diff --git a/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/opensearch.py b/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/opensearch.py new file mode 100644 index 0000000..7fc296f --- /dev/null +++ b/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/opensearch.py @@ -0,0 +1,70 @@ +import json +import typing + +from tutor import utils + + +class OpenSearchAPI: + """ + Helper class to interact with the OpenSearch + API on the deployed cluster. + """ + + def __init__(self, namespace): + self._command_base = [ + "kubectl", + "exec", + "--stdin", + "--tty", + "--namespace", + namespace, + "opensearch-cluster-master-0", + "--", + "bash", + "-c", + ] + self._curl_base = ["curl", "--insecure", "-u", "admin:admin"] + + def run_command(self, curl_options) -> typing.Union[dict, bytes]: + """ + Invokes a curl command on the first Opensearch pod. + + If possible returns the parsed json from the Opensearch response. + Otherwise, the raw bytes from the curl command are returned. + """ + response = utils.check_output( + *self._command_base, " ".join(self._curl_base + curl_options) + ) + try: + return json.loads(response) + except (TypeError, ValueError): + return response + + def get(self, endpoint): + """ + Runs a GET request on the Opensearch cluster with the specified + endpoint. + + If possible returns the parsed json from the Opensearch response. + Otherwise, the raw bytes from the curl command are returned. + """ + return self.run_command(["-XGET", f"https://opensearch-cluster-master:9200/{endpoint}"]) + + def put(self, endpoint: str, data: dict) -> typing.Union[dict, bytes]: + """ + Runs a POST request on the Opensearch cluster with the specified + endpoint. + + If possible returns the parsed json from the Opensearch response. + Otherwise, the raw bytes from the curl command are returned. + """ + return self.run_command( + [ + "-XPUT", + f"https://opensearch-cluster-master:9200/{endpoint}", + "-d", + f"'{json.dumps(data)}'", + "-H", + '"Content-Type: application/json"', + ] + ) diff --git a/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/patches/openedx-common-settings b/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/patches/openedx-common-settings index c2e5611..285a3cf 100644 --- a/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/patches/openedx-common-settings +++ b/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/patches/openedx-common-settings @@ -1,3 +1,16 @@ +# ElasticSearch is a prefered engine. +# If both are turned on, ELASTICSEARCH will be used. +{% if K8S_HARMONY_ENABLE_SHARED_OPENSEARCH %} +ELASTICSEARCH_INDEX_PREFIX = "{{OPENSEARCH_INDEX_PREFIX}}" +ELASTIC_SEARCH_CONFIG = [{ + "use_ssl": True, + "host": "opensearch-cluster-master.{{K8S_HARMONY_NAMESPACE}}.svc.cluster.local", + "verify_certs": False, + "port": 9200, + "http_auth": "{{ OPENSEARCH_HTTP_AUTH }}" +}] +{% endif %} + {% if K8S_HARMONY_ENABLE_SHARED_ELASTICSEARCH %} ELASTICSEARCH_INDEX_PREFIX = "{{ELASTICSEARCH_INDEX_PREFIX}}" ELASTIC_SEARCH_CONFIG = [{ diff --git a/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/patches/openedx-dockerfile-post-python-requirements b/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/patches/openedx-dockerfile-post-python-requirements index 660080a..e7efbda 100644 --- a/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/patches/openedx-dockerfile-post-python-requirements +++ b/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/patches/openedx-dockerfile-post-python-requirements @@ -1,4 +1,4 @@ -{% if K8S_HARMONY_ENABLE_SHARED_ELASTICSEARCH %} +{% if K8S_HARMONY_ENABLE_SHARED_ELASTICSEARCH or K8S_HARMONY_ENABLE_SHARED_OPENSEARCH %} # This is needed otherwise the previously installed edx-search # package doesn't get replaced. Once the below branch is merged # upstream it will no longer be needed. diff --git a/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/plugin.py b/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/plugin.py index 568f14a..813a4de 100644 --- a/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/plugin.py +++ b/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/plugin.py @@ -22,6 +22,7 @@ # instance. "INGRESS_HOST_LIST": [], "ENABLE_SHARED_ELASTICSEARCH": False, + "ENABLE_SHARED_OPENSEARCH": False, }, "overrides": { # Don't use Caddy as a per-instance external web proxy, but do still use it @@ -33,6 +34,8 @@ "unique": { "ELASTICSEARCH_HTTP_AUTH": "{{K8S_NAMESPACE}}:{{ 24|random_string }}", "ELASTICSEARCH_INDEX_PREFIX": "{{K8S_NAMESPACE}}-{{ 4|random_string|lower }}-", + "OPENSEARCH_HTTP_AUTH": "{{K8S_NAMESPACE}}:{{ 24|random_string }}", + "OPENSEARCH_INDEX_PREFIX": "{{K8S_NAMESPACE}}-{{ 4|random_string|lower }}-", }, } diff --git a/values-example.yaml b/values-example.yaml index 22eb500..0df602e 100644 --- a/values-example.yaml +++ b/values-example.yaml @@ -13,3 +13,6 @@ metricsserver: enabled: false vpa: enabled: false + +opensearch: + enabled: false diff --git a/values-minikube.yaml b/values-minikube.yaml index 3f12c49..c739977 100644 --- a/values-minikube.yaml +++ b/values-minikube.yaml @@ -1,3 +1,24 @@ # Disable HTTPS cert provisioning for testing with minikube cert-manager: enabled: false + +elasticsearch: + enabled: false + + # TODO: move this to a separate PR + # Permit co-located instances for solitary minikube virtual machines. + antiAffinity: "soft" + + volumeClaimTemplate: + resources: + requests: + storage: 8Gi + +opensearch: + enabled: false + + # Permit co-located instances for solitary minikube virtual machines. + antiAffinity: "soft" + + persistence: + size: 8Gi