HBASE-27833 Introduce zookeeper-ha_ensemble component

.gitignore

@@ -21,3 +21,11 @@ linklint-*.zip
 linklint/
 .checkstyle
 **/.checkstyle
+
+# kubernetes download artifacts
+*kubectl*
+*checksums.txt
+awscli-exe*
+*aws_cli_public_key
+*.tar.gz
+*.sig

hbase-kubernetes-deployment/README.md

@@ -0,0 +1,27 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# HBase Kubernetes Deployment
+
+This module contains Kubernetes configurations suitable for deploying ZooKeeper, HDFS, and HBase.
+It supports several physical topologies, including:
+ * a minimum deployment footprint consisting of a single instance of each architectural component.
+ * a high-avaiability deployment footprint consisting of redundancies for each architectural
+   component.
+ * a deployment where HBase region servers and HDFS data nodes share a pod, enabling short-circuit
+   read between them.

hbase-kubernetes-deployment/components/zookeeper/README.md

@@ -0,0 +1,23 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+Uses latest image from the apache zookeeper project.
+There is then a start script in 'single-instance/start.sh'
+which sets parameters for the zookeeper image and process
+on startup. Currently only logs to STDOUT/STDERR; there
+are no files in /var/log/zookeeper.

hbase-kubernetes-deployment/components/zookeeper/ha-ensemble/kustomization.yaml

@@ -0,0 +1,45 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+apiVersion: kustomize.config.k8s.io/v1alpha1
+kind: Component
+
+replicas:
+- name: zookeeper
+  count: 5
+
+configMapGenerator:
+- name: zookeeper-quorum
+  behavior: merge
+  literals:
+  # Hard-coded. Default we expect a simple standalone zk at this location.
+  # One define is for hbase, the other for hadoop.
+  - HBASE_ZOOKEEPER_QUORUM="zookeeper-0,zookeeper-1,zookeeper-2,zookeeper-3,zookeeper-4"
+  - HA_ZOOKEEPER_QUORUM="zookeeper-0:2181,zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181,zookeeper-4:2181"
+  options:
+    disableNameSuffixHash: true
+
+patches:
+- target:
+    kind: StatefulSet
+    name: zookeeper
+  path: patch-zookeeper.yaml
+
+components:
+- ../single-instance
+
+resources:
+- zk-service.yaml

hbase-kubernetes-deployment/components/zookeeper/ha-ensemble/patch-zookeeper.yaml

@@ -0,0 +1,51 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+- # increase container resource allocations
+  op: replace
+  path: /spec/template/spec/containers/0/resources
+  value:
+    requests:
+      cpu: '1.0'
+      memory: 1Gi
+    limits:
+      cpu: '2.0'
+      memory: 4Gi
+- # update the launch command
+  op: add
+  path: /spec/template/spec/containers/0/command
+  value:
+  - sh
+  - -c
+  - |-
+    # Make the heap be a fraction of the requested memory size.
+    # Make 'servers' count agree with replicas count.
+    /usr/bin/zookeeper/start.sh --servers=5 --percentage=50 --max_client_cnxns=300 --quorum_listen_on_all_ips=true
+- # increase storage volume size
+  op: replace
+  path: /spec/volumeClaimTemplates/0/spec/resources/requests/storage
+  value: 5Gi
+- # specify pod scheduling anti-affinity
+  op: add
+  path: /spec/template/spec/affinity
+  value:
+    podAntiAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+      - labelSelector:
+          matchLabels:
+            app: hadoop
+            role: zookeeper
+        topologyKey: kubernetes.io/hostname

hbase-kubernetes-deployment/components/zookeeper/ha-ensemble/zk-service.yaml

@@ -0,0 +1,130 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Put a Service in front of each zookeeper so ip doesn't change on clients/admin.
+# The service is at zookeeper-?.svc.domain. The pod
+# it is proxying will be at zookeeper-?.${env.ZOOKEEPER_HEADLESS_SERVICE}.NAMESPACE.svc.domain.
+# TODO: Port for prometheus to scape.
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: zookeeper-0
+  labels:
+    # So helm prometheus standalone install will scrape this zk; TODO
+    # jmxexporter: enabled
+spec:
+  ports:
+  - port: 2181
+    name: client
+  - port: 2888
+    name: server
+  - port: 3888
+    name: leader-election
+  - port: 8080
+    name: http
+  selector:
+    role: zookeeper
+    statefulset.kubernetes.io/pod-name: zookeeper-0
+    cluster: zookeeper
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: zookeeper-1
+  labels:
+    # So helm prometheus standalone install will scrape this zk; TODO
+    # jmxexporter: enabled
+spec:
+  ports:
+  - port: 2181
+    name: client
+  - port: 2888
+    name: server
+  - port: 3888
+    name: leader-election
+  - port: 8080
+    name: http
+  selector:
+    role: zookeeper
+    statefulset.kubernetes.io/pod-name: zookeeper-1
+    cluster: zookeeper
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: zookeeper-2
+  labels:
+    # So helm prometheus standalone install will scrape this zk; TODO
+    # jmxexporter: enabled
+spec:
+  ports:
+  - port: 2181
+    name: client
+  - port: 2888
+    name: server
+  - port: 3888
+    name: leader-election
+  - port: 8080
+    name: http
+  selector:
+    role: zookeeper
+    statefulset.kubernetes.io/pod-name: zookeeper-2
+    cluster: zookeeper
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: zookeeper-3
+  labels:
+    # So helm prometheus standalone install will scrape this zk; TODO
+    # jmxexporter: enabled
+spec:
+  ports:
+  - port: 2181
+    name: client
+  - port: 2888
+    name: server
+  - port: 3888
+    name: leader-election
+  - port: 8080
+    name: http
+  selector:
+    role: zookeeper
+    statefulset.kubernetes.io/pod-name: zookeeper-3
+    cluster: zookeeper
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: zookeeper-4
+  labels:
+    # So helm prometheus standalone install will scrape this zk; TODO
+    # jmxexporter: enabled
+spec:
+  ports:
+  - port: 2181
+    name: client
+  - port: 2888
+    name: server
+  - port: 3888
+    name: leader-election
+  - port: 8080
+    name: http
+  selector:
+    role: zookeeper
+    statefulset.kubernetes.io/pod-name: zookeeper-4
+    cluster: zookeeper

hbase-kubernetes-deployment/components/zookeeper/single-instance/kustomization.yaml

@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+apiVersion: kustomize.config.k8s.io/v1alpha1
+kind: Component
+
+configMapGenerator:
+- name: zookeeper-scripts
+  files:
+  - start.sh
+# Publish where the zk ensemble can be found.
+- name: zookeeper-quorum
+  literals:
+    # Hard-coded. Default we expect a simple standalone zk at this location.
+    # One define is for hbase, the other for hadoop.
+    - HBASE_ZOOKEEPER_QUORUM="zookeeper-0.zookeeper-headless"
+    - HA_ZOOKEEPER_QUORUM="zookeeper-0.zookeeper-headless:2181"
+  options:
+    disableNameSuffixHash: true
+
+resources:
+- zookeeper.yaml