diff --git a/README.md b/README.md index f57c741..08a528e 100644 --- a/README.md +++ b/README.md @@ -217,7 +217,7 @@ To enable set `elasticsearch.enabled=true` in your `values.yaml` and deploy the For each instance you would like to enable this on, set the configuration values in the respective `config.yml`: ```yaml -K8S_HARMONY_ENABLE_SHARED_ELASTICSEARCH: true +K8S_HARMONY_ENABLE_SHARED_HARMONY_SEARCH: true RUN_ELASTICSEARCH: false ``` diff --git a/charts/harmony-chart/Chart.lock b/charts/harmony-chart/Chart.lock index 6614621..4bf4ac8 100644 --- a/charts/harmony-chart/Chart.lock +++ b/charts/harmony-chart/Chart.lock @@ -14,5 +14,8 @@ dependencies: - name: vertical-pod-autoscaler repository: https://cowboysysop.github.io/charts/ version: 6.0.3 -digest: sha256:4915c21724a8c4693f749aab2c311336990b48e16e90884d7b9ceca42eef69f8 -generated: "2023-04-13T23:07:43.616994532-05:00" +- name: opensearch + repository: https://opensearch-project.github.io/helm-charts + version: 2.13.3 +digest: sha256:11b69b1ea771337b1e7cf8497ee342a25b095b86899b8cee716be8cc9f955559 +generated: "2023-07-01T19:23:29.18815+03:00" diff --git a/charts/harmony-chart/Chart.yaml b/charts/harmony-chart/Chart.yaml index 165f759..ea7ea57 100644 --- a/charts/harmony-chart/Chart.yaml +++ b/charts/harmony-chart/Chart.yaml @@ -11,7 +11,7 @@ version: 0.1.0 # version the application is using. It is recommended to use it with quotes. # # In our case, this represents the version of Tutor that this chart is compatible with. -appVersion: "14.1.1" +appVersion: "16.0.2" dependencies: # This is just info for the "helm dependency update" command, which will update the ./charts/ directory when run, using @@ -42,3 +42,8 @@ dependencies: repository: https://cowboysysop.github.io/charts/ alias: vpa condition: vpa.enabled + +- name: opensearch + version: "2.13.3" + condition: opensearch.enabled + repository: https://opensearch-project.github.io/helm-charts diff --git a/charts/harmony-chart/templates/elasticsearch/secrets.yaml b/charts/harmony-chart/templates/elasticsearch/secrets.yaml index e25eecf..cf70d9e 100644 --- a/charts/harmony-chart/templates/elasticsearch/secrets.yaml +++ b/charts/harmony-chart/templates/elasticsearch/secrets.yaml @@ -1,6 +1,7 @@ --- {{- $ca := genCA "elasticca" 1825 }} -{{- $cert := genSignedCert "elasticsearch-master.{{ Release.Namespace }}.local" nil (list "elasticsearch-master.{{ Release.Namespace }}.local") 1825 $ca }} +{{- $cn := printf "elasticsearch-master.%s.local" .Release.Namespace }} +{{- $cert := genSignedCert $cn nil (list $cn) 1825 $ca }} apiVersion: v1 kind: Secret metadata: diff --git a/charts/harmony-chart/templates/opensearch/secrets.yaml b/charts/harmony-chart/templates/opensearch/secrets.yaml new file mode 100644 index 0000000..40bbf00 --- /dev/null +++ b/charts/harmony-chart/templates/opensearch/secrets.yaml @@ -0,0 +1,24 @@ +--- +{{- $ca := genCA "opensearchca" 1825 }} +{{- $cn := printf "opensearch-master.%s.local" .Release.Namespace }} +{{- $cert := genSignedCert $cn nil (list $cn) 1825 $ca }} +apiVersion: v1 +kind: Secret +metadata: + name: opensearch-certificates +type: Opaque +data: + "ca.crt": {{ $ca.Cert | b64enc | toYaml | indent 4}} + "tls.key": {{ $cert.Key | b64enc | toYaml | indent 4}} + "tls.crt": {{ print $cert.Cert $ca.Cert | b64enc | toYaml | indent 4}} +--- +{{- $password := randAlphaNum 32 }} +{{- $password_bcrypt := $password | bcrypt }} +apiVersion: v1 +kind: Secret +metadata: + name: opensearch-credentials +type: Opaque +data: + harmony_password: {{ $password | b64enc | quote }} + internal_users.yml: {{ printf "---\n_meta:\n type: \"internalusers\"\n config_version: 2\n\nharmony:\n hash: \"%s\"\n reserved: true\n backend_roles:\n - \"admin\"\n description: \"Harmony admin user\"\n" $password_bcrypt | b64enc | quote }} diff --git a/charts/harmony-chart/values.yaml b/charts/harmony-chart/values.yaml index 06d4969..bcf37f4 100644 --- a/charts/harmony-chart/values.yaml +++ b/charts/harmony-chart/values.yaml @@ -18,16 +18,19 @@ cert-manager: elasticsearch: enabled: false + clusterName: "harmony-search-cluster" + masterService: "harmony-search-cluster" + # Operators will need to add/update the following setting in each # of their instances by running the commands: # ``` - # tutor config save --set K8S_HARMONY_ENABLE_SHARED_ELASTICSEARCH=true --set RUN_ELASTICSEARCH=false + # tutor config save --set K8S_HARMONY_ENABLE_SHARED_HARMONY_SEARCH=true --set RUN_ELASTICSEARCH=false # tutor harmony create-elasticsearch-user # ``` # RUN_ELASTICSEARCH: false - # ELASTICSEARCH_PREFIX_INDEX: "username-" - # K8S_HARMONY_USE_SHARED_ELASTICSEARCH: true - # ELASTICSEARCH_AUTH: "username:actual_password" + # HARMONY_SEARCH_INDEX_PREFIX: "username-" + # K8S_HARMONY_ENABLE_SHARED_HARMONY_SEARCH: true + # HARMONY_SEARCH_HTTP_AUTH: "username:actual_password" # We will create the relevant certs, because they need to shared # with pods in other namespaces. @@ -52,6 +55,7 @@ elasticsearch: esConfig: "elasticsearch.yml": | + cluster.name: harmony-search-cluster xpack.security.enabled: true xpack.security.http.ssl.enabled: true xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certs/tls.key @@ -77,3 +81,105 @@ vpa: # for all available options admissionController: replicaCount: 1 + + +# Multi-tenant OpenSearch +opensearch: + enabled: false + clusterName: "harmony-search-cluster" + masterService: "harmony-search-cluster" + + # Operators will need to add/update the following setting in each + # of their instances by running the commands: + # ``` + # tutor config save --set K8S_HARMONY_ENABLE_SHARED_HARMONY_SEARCH=true --set RUN_ELASTICSEARCH=false + # tutor harmony create-opensearch-user + # ``` + # RUN_ELASTICSEARCH: false + # HARMONY_SEARCH_INDEX_PREFIX: "username-" + # K8S_HARMONY_USE_SHARED_OPENSEARCH: true + # HARMONY_SEARCH_HTTP_AUTH: "username:actual_password" + + + # # This secret will contain the ssl certificates. + secretMounts: + - name: opensearch-certificates + secretName: opensearch-certificates + path: /usr/share/opensearch/config/certs + defaultMode: 0777 + + resources: + requests: + cpu: "1000m" + memory: "100Mi" + + persistence: + size: 30Gi + + # Set vm.max_map_count + # Default value is 262144 + sysctlInit: + enabled: true + + extraEnvs: + - name: DISABLE_INSTALL_DEMO_CONFIG + value: "true" + - name: HARMONY_PASSWORD + valueFrom: + secretKeyRef: + name: opensearch-credentials + key: harmony_password + + # Allows you to add any config files in {{ .Values.opensearchHome }}/config + opensearchHome: /usr/share/opensearch + # such as opensearch.yml and log4j2.properties + + securityConfig: + enabled: true + internalUsersSecret: opensearch-credentials + + config: + opensearch.yml: | + cluster.name: harmony-search-cluster + network.host: 0.0.0.0 + plugins: + security: + ssl: + transport: + enabled: true + pemcert_filepath: certs/tls.crt + pemkey_filepath: certs/tls.key + pemtrustedcas_filepath: certs/ca.crt + enforce_hostname_verification: false + http: + enabled: true + pemcert_filepath: certs/tls.crt + pemkey_filepath: certs/tls.key + pemtrustedcas_filepath: certs/ca.crt + authcz: + admin_dn: + - CN=opensearch-master.*.local + nodes_dn: + - CN=opensearch-master.*.local + allow_unsafe_democertificates: false + allow_default_init_securityindex: true + audit.type: internal_opensearch + enable_snapshot_restore_privilege: true + check_snapshot_restore_write_privileges: true + restapi: + roles_enabled: ["all_access", "security_rest_api_access"] + system_indices: + enabled: true + indices: + [ + ".opendistro-alerting-config", + ".opendistro-alerting-alert*", + ".opendistro-anomaly-results*", + ".opendistro-anomaly-detector*", + ".opendistro-anomaly-checkpoints", + ".opendistro-anomaly-detection-state", + ".opendistro-reports-*", + ".opendistro-notifications-*", + ".opendistro-notebooks", + ".opendistro-asynchronous-search-response*", + ] diff --git a/harmony-chart/charts/opensearch-2.11.4.tgz b/harmony-chart/charts/opensearch-2.11.4.tgz new file mode 100644 index 0000000..6151fd5 Binary files /dev/null and b/harmony-chart/charts/opensearch-2.11.4.tgz differ diff --git a/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/commands.py b/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/commands.py index 643d019..047c4bf 100644 --- a/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/commands.py +++ b/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/commands.py @@ -4,7 +4,8 @@ from tutor import config as tutor_config from tutor import env as tutor_env from tutor.commands.k8s import K8sContext, kubectl_exec -from .elasticsearch import ElasticSearchAPI +from .harmony_search.elasticsearch import ElasticSearchAPI +from .harmony_search.opensearch import OpenSearchAPI @click.group(help="Commands and subcommands of the openedx-k8s-harmony.") @click.pass_context @@ -21,10 +22,10 @@ def create_elasticsearch_user(context: click.Context): config = tutor_config.load(context.root) namespace = config["K8S_HARMONY_NAMESPACE"] api = ElasticSearchAPI(namespace) - username, password = config["ELASTICSEARCH_HTTP_AUTH"].split(":", 1) + username, password = config["HARMONY_SEARCH_HTTP_AUTH"].split(":", 1) role_name = f"{username}_role" - prefix = config["ELASTICSEARCH_INDEX_PREFIX"] + prefix = config["HARMONY_SEARCH_INDEX_PREFIX"] api.post( f"_security/role/{role_name}", {"indices": [{"names": [f"{prefix}*"], "privileges": ["all"]}]}, @@ -40,5 +41,44 @@ def create_elasticsearch_user(context: click.Context): }, ) +@click.command(help="Create or update Opensearch users") +@click.pass_obj +def create_opensearch_user(context: click.Context): + """ + Creates or updates the Opensearch user + """ + config = tutor_config.load(context.root) + namespace = config["K8S_HARMONY_NAMESPACE"] + api = OpenSearchAPI(namespace) + username, password = config["HARMONY_SEARCH_HTTP_AUTH"].split(":", 1) + role_name = f"{username}_role" + + prefix = config["HARMONY_SEARCH_INDEX_PREFIX"] + api.put( + f"_plugins/_security/api/roles/{role_name}", + {"index_permissions": [{ + "index_patterns": [ + f"{prefix}*" + ], + "allowed_actions": [ + "read", + "write", + "create_index", + "manage", + "manage_ilm", + "all" + ] + }]}, + ) + + api.put( + f"_plugins/_security/api/internalusers/{username}", + { + "password": password, + "opendistro_security_roles": [role_name], + }, + ) + harmony.add_command(create_elasticsearch_user) +harmony.add_command(create_opensearch_user) diff --git a/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/elasticsearch.py b/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/harmony_search/base.py similarity index 56% rename from tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/elasticsearch.py rename to tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/harmony_search/base.py index c654e31..57c3bca 100644 --- a/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/elasticsearch.py +++ b/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/harmony_search/base.py @@ -4,9 +4,9 @@ from tutor import utils -class ElasticSearchAPI: +class BaseSearchAPI: """ - Helper class to interact with the ElasticSearch + Helper class to interact with the HarmonySearch API on the deployed cluster. """ @@ -18,18 +18,19 @@ def __init__(self, namespace): "--tty", "--namespace", namespace, - "elasticsearch-master-0", + "harmony-search-cluster-master-0", "--", "bash", "-c", ] - self._curl_base = ["curl", "--insecure", "-u", "elastic:${ELASTIC_PASSWORD}"] + # Must be specified by subclasses + self._curl_base = None def run_command(self, curl_options) -> typing.Union[dict, bytes]: """ - Invokes a curl command on the first Elasticsearch pod. + Invokes a curl command on the first HarmonySearch pod. - If possible returns the parsed json from the Elasticsearch response. + If possible returns the parsed json from the HarmonySearch response. Otherwise, the raw bytes from the curl command are returned. """ response = utils.check_output( @@ -42,20 +43,20 @@ def run_command(self, curl_options) -> typing.Union[dict, bytes]: def get(self, endpoint): """ - Runs a GET request on the Elasticsearch cluster with the specified + Runs a GET request on the HarmonySearch cluster with the specified endpoint. - If possible returns the parsed json from the Elasticsearch response. + If possible returns the parsed json from the HarmonySearch response. Otherwise, the raw bytes from the curl command are returned. """ return self.run_command(["-XGET", f"https://localhost:9200/{endpoint}"]) def post(self, endpoint: str, data: dict) -> typing.Union[dict, bytes]: """ - Runs a POST request on the Elasticsearch cluster with the specified + Runs a POST request on the HarmonySearch cluster with the specified endpoint. - If possible returns the parsed json from the Elasticsearch response. + If possible returns the parsed json from the HarmonySearch response. Otherwise, the raw bytes from the curl command are returned. """ return self.run_command( @@ -68,3 +69,22 @@ def post(self, endpoint: str, data: dict) -> typing.Union[dict, bytes]: '"Content-Type: application/json"', ] ) + + def put(self, endpoint: str, data: dict) -> typing.Union[dict, bytes]: + """ + Runs a PUT request on the HarmonySearch cluster with the specified + endpoint. + + If possible returns the parsed json from the HarmonySearch response. + Otherwise, the raw bytes from the curl command are returned. + """ + return self.run_command( + [ + "-XPUT", + f"https://localhost:9200/{endpoint}", + "-d", + f"'{json.dumps(data)}'", + "-H", + '"Content-Type: application/json"', + ] + ) diff --git a/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/harmony_search/elasticsearch.py b/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/harmony_search/elasticsearch.py new file mode 100644 index 0000000..14a6331 --- /dev/null +++ b/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/harmony_search/elasticsearch.py @@ -0,0 +1,12 @@ +from .base import BaseSearchAPI + + +class ElasticSearchAPI(BaseSearchAPI): + """ + Helper class to interact with the ElasticSearch + API on the deployed cluster. + """ + + def __init__(self, namespace): + super().__init__(namespace) + self._curl_base = ["curl", "--insecure", "-u", "elastic:${ELASTIC_PASSWORD}"] diff --git a/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/harmony_search/opensearch.py b/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/harmony_search/opensearch.py new file mode 100644 index 0000000..ca25572 --- /dev/null +++ b/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/harmony_search/opensearch.py @@ -0,0 +1,13 @@ +from .base import BaseSearchAPI + + +class OpenSearchAPI(BaseSearchAPI): + """ + Helper class to interact with the OpenSearch + API on the deployed cluster. + """ + + def __init__(self, namespace): + super().__init__(namespace) + # TODO: Make this configurable + self._curl_base = ["curl", "--insecure", "-u", "harmony:${HARMONY_PASSWORD}"] diff --git a/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/patches/openedx-common-settings b/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/patches/openedx-common-settings index c2e5611..5c6e4a4 100644 --- a/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/patches/openedx-common-settings +++ b/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/patches/openedx-common-settings @@ -1,10 +1,10 @@ -{% if K8S_HARMONY_ENABLE_SHARED_ELASTICSEARCH %} -ELASTICSEARCH_INDEX_PREFIX = "{{ELASTICSEARCH_INDEX_PREFIX}}" +{% if K8S_HARMONY_ENABLE_SHARED_HARMONY_SEARCH %} +ELASTIC_SEARCH_INDEX_PREFIX = "{{HARMONY_SEARCH_INDEX_PREFIX}}" ELASTIC_SEARCH_CONFIG = [{ "use_ssl": True, - "host": "elasticsearch-master.{{K8S_HARMONY_NAMESPACE}}.svc.cluster.local", + "host": "harmony-search-cluster.{{K8S_HARMONY_NAMESPACE}}.svc.cluster.local", "verify_certs": False, "port": 9200, - "http_auth": "{{ ELASTICSEARCH_HTTP_AUTH }}" + "http_auth": "{{ HARMONY_SEARCH_HTTP_AUTH }}" }] {% endif %} diff --git a/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/patches/openedx-dockerfile-post-python-requirements b/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/patches/openedx-dockerfile-post-python-requirements deleted file mode 100644 index 660080a..0000000 --- a/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/patches/openedx-dockerfile-post-python-requirements +++ /dev/null @@ -1,7 +0,0 @@ -{% if K8S_HARMONY_ENABLE_SHARED_ELASTICSEARCH %} -# This is needed otherwise the previously installed edx-search -# package doesn't get replaced. Once the below branch is merged -# upstream it will no longer be needed. -RUN pip uninstall -y edx-search -RUN pip install --upgrade git+https://github.com/open-craft/edx-search.git@keith/prefixed-index-names -{% endif %} diff --git a/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/plugin.py b/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/plugin.py index 568f14a..9de94ca 100644 --- a/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/plugin.py +++ b/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/plugin.py @@ -21,7 +21,7 @@ # The workaround is to manually add a list of hosts to be routed to the caddy # instance. "INGRESS_HOST_LIST": [], - "ENABLE_SHARED_ELASTICSEARCH": False, + "ENABLE_SHARED_HARMONY_SEARCH": False, }, "overrides": { # Don't use Caddy as a per-instance external web proxy, but do still use it @@ -31,8 +31,8 @@ "ENABLE_HTTPS": True, }, "unique": { - "ELASTICSEARCH_HTTP_AUTH": "{{K8S_NAMESPACE}}:{{ 24|random_string }}", - "ELASTICSEARCH_INDEX_PREFIX": "{{K8S_NAMESPACE}}-{{ 4|random_string|lower }}-", + "HARMONY_SEARCH_HTTP_AUTH": "{{K8S_NAMESPACE}}:{{ 24|random_string }}", + "HARMONY_SEARCH_INDEX_PREFIX": "{{K8S_NAMESPACE}}-{{ 4|random_string|lower }}-", }, } diff --git a/values-example.yaml b/values-example.yaml index 22eb500..0df602e 100644 --- a/values-example.yaml +++ b/values-example.yaml @@ -13,3 +13,6 @@ metricsserver: enabled: false vpa: enabled: false + +opensearch: + enabled: false diff --git a/values-minikube.yaml b/values-minikube.yaml index 3f12c49..c739977 100644 --- a/values-minikube.yaml +++ b/values-minikube.yaml @@ -1,3 +1,24 @@ # Disable HTTPS cert provisioning for testing with minikube cert-manager: enabled: false + +elasticsearch: + enabled: false + + # TODO: move this to a separate PR + # Permit co-located instances for solitary minikube virtual machines. + antiAffinity: "soft" + + volumeClaimTemplate: + resources: + requests: + storage: 8Gi + +opensearch: + enabled: false + + # Permit co-located instances for solitary minikube virtual machines. + antiAffinity: "soft" + + persistence: + size: 8Gi