From 229c302aa9db19ad481dc9474f56f29fed8b1fb3 Mon Sep 17 00:00:00 2001 From: Nick Dimiduk Date: Tue, 9 May 2023 13:12:10 +0200 Subject: [PATCH] HBASE-27832 Introduce hbase overlay --- .../hbase/hadoop-metrics2-hbase.properties | 46 ++++ .../components/hbase/hbase-site.xml | 75 ++++++ .../components/hbase/kustomization.yaml | 40 +++ .../components/hbase/log4j.properties.hbase | 68 +++++ .../components/hbase/log4j2.properties.hbase | 167 ++++++++++++ .../components/hbase/m-service.yaml | 29 ++ .../components/hbase/m-statefulset.yaml | 250 ++++++++++++++++++ .../components/hbase/rs-service.yaml | 29 ++ .../components/hbase/rs-statefulset.yaml | 237 +++++++++++++++++ .../overlays/hbase/kustomization.yaml | 25 ++ .../overlays_hbase/00-assert-hbase.yaml | 34 +++ .../overlays_hbase/00-kustomize.yaml | 21 ++ .../overlays_hbase/kustomization.yaml | 22 ++ .../unit/overlays_hbase/00-assert-hbase.yaml | 72 +++++ .../unit/overlays_hbase/00-kustomize.yaml | 21 ++ .../unit/overlays_hbase/kustomization.yaml | 21 ++ 16 files changed, 1157 insertions(+) create mode 100644 hbase-kubernetes-deployment/components/hbase/hadoop-metrics2-hbase.properties create mode 100644 hbase-kubernetes-deployment/components/hbase/hbase-site.xml create mode 100644 hbase-kubernetes-deployment/components/hbase/kustomization.yaml create mode 100644 hbase-kubernetes-deployment/components/hbase/log4j.properties.hbase create mode 100644 hbase-kubernetes-deployment/components/hbase/log4j2.properties.hbase create mode 100644 hbase-kubernetes-deployment/components/hbase/m-service.yaml create mode 100644 hbase-kubernetes-deployment/components/hbase/m-statefulset.yaml create mode 100644 hbase-kubernetes-deployment/components/hbase/rs-service.yaml create mode 100644 hbase-kubernetes-deployment/components/hbase/rs-statefulset.yaml create mode 100644 hbase-kubernetes-deployment/overlays/hbase/kustomization.yaml create mode 100644 hbase-kubernetes-deployment/tests/integration/overlays_hbase/00-assert-hbase.yaml create mode 100644 hbase-kubernetes-deployment/tests/integration/overlays_hbase/00-kustomize.yaml create mode 100644 hbase-kubernetes-deployment/tests/integration/overlays_hbase/kustomization.yaml create mode 100644 hbase-kubernetes-deployment/tests/unit/overlays_hbase/00-assert-hbase.yaml create mode 100644 hbase-kubernetes-deployment/tests/unit/overlays_hbase/00-kustomize.yaml create mode 100644 hbase-kubernetes-deployment/tests/unit/overlays_hbase/kustomization.yaml diff --git a/hbase-kubernetes-deployment/components/hbase/hadoop-metrics2-hbase.properties b/hbase-kubernetes-deployment/components/hbase/hadoop-metrics2-hbase.properties new file mode 100644 index 0000000000..922822310a --- /dev/null +++ b/hbase-kubernetes-deployment/components/hbase/hadoop-metrics2-hbase.properties @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + # syntax: [prefix].[source|sink].[instance].[options] + # See javadoc of package-info.java for org.apache.hadoop.metrics2 for details + + *.sink.file*.class=org.apache.hadoop.metrics2.sink.FileSink + # default sampling period + *.period=10 + + # Below are some examples of sinks that could be used + # to monitor different hbase daemons. + + # hbase.sink.file-all.class=org.apache.hadoop.metrics2.sink.FileSink + # hbase.sink.file-all.filename=all.metrics + + # hbase.sink.file0.class=org.apache.hadoop.metrics2.sink.FileSink + # hbase.sink.file0.context=hmaster + # hbase.sink.file0.filename=master.metrics + + # hbase.sink.file1.class=org.apache.hadoop.metrics2.sink.FileSink + # hbase.sink.file1.context=thrift-one + # hbase.sink.file1.filename=thrift-one.metrics + + # hbase.sink.file2.class=org.apache.hadoop.metrics2.sink.FileSink + # hbase.sink.file2.context=thrift-two + # hbase.sink.file2.filename=thrift-one.metrics + + # hbase.sink.file3.class=org.apache.hadoop.metrics2.sink.FileSink + # hbase.sink.file3.context=rest + # hbase.sink.file3.filename=rest.metrics + + # Read from src rather than copy over like this. diff --git a/hbase-kubernetes-deployment/components/hbase/hbase-site.xml b/hbase-kubernetes-deployment/components/hbase/hbase-site.xml new file mode 100644 index 0000000000..e6917fadff --- /dev/null +++ b/hbase-kubernetes-deployment/components/hbase/hbase-site.xml @@ -0,0 +1,75 @@ + + + + + + hbase.cluster.distributed + true + + + hbase.rootdir + hdfs://${env.HADOOP_SERVICE}:8020/hbase + + + hbase.zookeeper.quorum + ${env.HBASE_ZOOKEEPER_QUORUM} + + + + hbase.unsafe.regionserver.hostname.disable.master.reversedns + true + + + dfs.client.use.datanode.hostname + true + + + hbase.ssl.enabled + + ${env.HBASE_SSL_ENABLED} + + + hbase.master.cleaner.interval + + 10000 + + + hbase.regionserver.handler.count + 45 + + + hbase.netty.eventloop.rpcserver.thread.count + 1 + See the end of https://issues.apache.org/jira/browse/HBASE-27112. Default + is 2xCPU_COUNT which seems way too much. 1 thread seems fine for the siri workload at least. + + + hbase.netty.worker.count + 1 + See the end of https://issues.apache.org/jira/browse/HBASE-27112. Default + is 2xCPU_COUNT which seems way too much. 1 thread seems fine for the siri workload at least. + + + hbase.regionserver.throughput.controller + org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController + + diff --git a/hbase-kubernetes-deployment/components/hbase/kustomization.yaml b/hbase-kubernetes-deployment/components/hbase/kustomization.yaml new file mode 100644 index 0000000000..98ef68a0ec --- /dev/null +++ b/hbase-kubernetes-deployment/components/hbase/kustomization.yaml @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +commonLabels: + app: hadoop + +configMapGenerator: +- name: hbase-configuration + files: + - hbase-site.xml + - hadoop-metrics2-hbase.properties + - log4j2.properties=log4j2.properties.hbase +- name: hbck2-configuration + # Add an hbck2-configuration. Uses log4j1 vs log4j2. + files: + - hbase-site.xml + - hadoop-metrics2-hbase.properties + - log4j.properties=log4j.properties.hbase + +resources: +- m-service.yaml +- m-statefulset.yaml +- rs-service.yaml +- rs-statefulset.yaml diff --git a/hbase-kubernetes-deployment/components/hbase/log4j.properties.hbase b/hbase-kubernetes-deployment/components/hbase/log4j.properties.hbase new file mode 100644 index 0000000000..c322699ced --- /dev/null +++ b/hbase-kubernetes-deployment/components/hbase/log4j.properties.hbase @@ -0,0 +1,68 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Define some default values that can be overridden by system properties +hbase.root.logger=INFO,console +hbase.log.dir=. +hbase.log.file=hbase.log + +# Define the root logger to the system property "hbase.root.logger". +log4j.rootLogger=${hbase.root.logger} + +# Logging Threshold +log4j.threshold=ALL + +# +# Daily Rolling File Appender +# +log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender +log4j.appender.DRFA.File=${hbase.log.dir}/${hbase.log.file} + +# Rollver at midnight +log4j.appender.DRFA.DatePattern=.yyyy-MM-dd + +# 30-day backup +#log4j.appender.DRFA.MaxBackupIndex=30 +log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout +# Debugging Pattern format +log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %C{2}(%L): %m%n + + +# +# console +# Add "console" to rootlogger above if you want to use this +# +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %C{2}(%L): %m%n + +# Custom Logging levels + +#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG + +log4j.logger.org.apache.hadoop=WARN +log4j.logger.org.apache.zookeeper=ERROR +log4j.logger.org.apache.hadoop.hbase=DEBUG + +#These settings are workarounds against spurious logs from the minicluster. +#See HBASE-4709 +log4j.logger.org.apache.hadoop.metrics2.impl.MetricsConfig=WARN +log4j.logger.org.apache.hadoop.metrics2.impl.MetricsSinkAdapter=WARN +log4j.logger.org.apache.hadoop.metrics2.impl.MetricsSystemImpl=WARN +log4j.logger.org.apache.hadoop.metrics2.util.MBeans=WARN +# Enable this to get detailed connection error/retry logging. +# log4j.logger.org.apache.hadoop.hbase.client.ConnectionImplementation=TRACE diff --git a/hbase-kubernetes-deployment/components/hbase/log4j2.properties.hbase b/hbase-kubernetes-deployment/components/hbase/log4j2.properties.hbase new file mode 100644 index 0000000000..d1acef75f8 --- /dev/null +++ b/hbase-kubernetes-deployment/components/hbase/log4j2.properties.hbase @@ -0,0 +1,167 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +status = warn +dest = err +name = PropertiesConfig +# refresh configuration from this file every 5 minutes +monitorInterval = 300 + +# Console appender +appender.console.type = Console +appender.console.target = SYSTEM_ERR +appender.console.name = console +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{ISO8601} %-5p [%t] %c{2}: %.-20000m%n + +# Daily Rolling File Appender +appender.DRFA.type = RollingFile +appender.DRFA.name = DRFA +appender.DRFA.fileName = ${sys:hbase.log.dir:-.}/${sys:hbase.log.file:-hbase.log} +appender.DRFA.filePattern = ${sys:hbase.log.dir:-.}/${sys:hbase.log.file:-hbase.log}.%d{yyyy-MM-dd} +appender.DRFA.createOnDemand = true +appender.DRFA.layout.type = PatternLayout +appender.DRFA.layout.pattern = %d{ISO8601} %-5p [%t] %c{2}: %.-20000m%n +appender.DRFA.policies.type = Policies +appender.DRFA.policies.time.type = TimeBasedTriggeringPolicy +appender.DRFA.policies.time.interval = 1 +appender.DRFA.policies.time.modulate = true +appender.DRFA.policies.size.type = SizeBasedTriggeringPolicy +appender.DRFA.policies.size.size = ${sys:hbase.log.maxfilesize:-256MB} +appender.DRFA.strategy.type = DefaultRolloverStrategy +appender.DRFA.strategy.max = ${sys:hbase.log.maxbackupindex:-20} + +# Rolling File Appender +appender.RFA.type = RollingFile +appender.RFA.name = RFA +appender.RFA.fileName = ${sys:hbase.log.dir:-.}/${sys:hbase.log.file:-hbase.log} +appender.RFA.filePattern = ${sys:hbase.log.dir:-.}/${sys:hbase.log.file:-hbase.log}.%i +appender.RFA.createOnDemand = true +appender.RFA.layout.type = PatternLayout +appender.RFA.layout.pattern = %d{ISO8601} %-5p [%t] %c{2}: %.-20000m%n +appender.RFA.policies.type = Policies +appender.RFA.policies.size.type = SizeBasedTriggeringPolicy +appender.RFA.policies.size.size = ${sys:hbase.log.maxfilesize:-256MB} +appender.RFA.strategy.type = DefaultRolloverStrategy +appender.RFA.strategy.max = ${sys:hbase.log.maxbackupindex:-20} + +# Security Audit Appender +appender.RFAS.type = RollingFile +appender.RFAS.name = RFAS +appender.RFAS.fileName = ${sys:hbase.log.dir:-.}/${sys:hbase.security.log.file:-SecurityAuth.audit} +appender.RFAS.filePattern = ${sys:hbase.log.dir:-.}/${sys:hbase.security.log.file:-SecurityAuth.audit}.%i +appender.RFAS.createOnDemand = true +appender.RFAS.layout.type = PatternLayout +appender.RFAS.layout.pattern = %d{ISO8601} %-5p [%t] %c{2}: %.-20000m%n +appender.RFAS.policies.type = Policies +appender.RFAS.policies.size.type = SizeBasedTriggeringPolicy +appender.RFAS.policies.size.size = ${sys:hbase.security.log.maxfilesize:-256MB} +appender.RFAS.strategy.type = DefaultRolloverStrategy +appender.RFAS.strategy.max = ${sys:hbase.security.log.maxbackupindex:-20} + +# Http Access Log RFA, uncomment this if you want an http access.log +# appender.AccessRFA.type = RollingFile +# appender.AccessRFA.name = AccessRFA +# appender.AccessRFA.fileName = /var/log/hbase/access.log +# appender.AccessRFA.filePattern = /var/log/hbase/access.log.%i +# appender.AccessRFA.createOnDemand = true +# appender.AccessRFA.layout.type = PatternLayout +# appender.AccessRFA.layout.pattern = %m%n +# appender.AccessRFA.policies.type = Policies +# appender.AccessRFA.policies.size.type = SizeBasedTriggeringPolicy +# appender.AccessRFA.policies.size.size = 200MB +# appender.AccessRFA.strategy.type = DefaultRolloverStrategy +# appender.AccessRFA.strategy.max = 10 + +# Null Appender +appender.NullAppender.type = Null +appender.NullAppender.name = NullAppender + +rootLogger = ${sys:hbase.root.logger:-INFO,console} + +logger.SecurityLogger.name = SecurityLogger +logger.SecurityLogger = ${sys:hbase.security.logger:-INFO,console} +logger.SecurityLogger.additivity = false + +# Custom Logging levels +# logger.zookeeper.name = org.apache.zookeeper +# logger.zookeeper.level = ERROR + +# logger.FSNamesystem.name = org.apache.hadoop.fs.FSNamesystem +# logger.FSNamesystem.level = DEBUG + +# logger.hbase.name = org.apache.hadoop.hbase +# logger.hbase.level = DEBUG + +# logger.META.name = org.apache.hadoop.hbase.META +# logger.META.level = DEBUG + +# Make these two classes below DEBUG to see more zk debug. +# logger.ZKUtil.name = org.apache.hadoop.hbase.zookeeper.ZKUtil +# logger.ZKUtil.level = DEBUG + +# logger.ZKWatcher.name = org.apache.hadoop.hbase.zookeeper.ZKWatcher +# logger.ZKWatcher.level = DEBUG + +logger.dfs.name = org.apache.hadoop.hdfs +logger.dfs.level = DEBUG + +# Prevent metrics subsystem start/stop messages (HBASE-17722) +logger.MetricsConfig.name = org.apache.hadoop.metrics2.impl.MetricsConfig +logger.MetricsConfig.level = WARN + +logger.MetricsSinkAdapte.name = org.apache.hadoop.metrics2.impl.MetricsSinkAdapter +logger.MetricsSinkAdapte.level = WARN + +logger.MetricsSystemImpl.name = org.apache.hadoop.metrics2.impl.MetricsSystemImpl +logger.MetricsSystemImpl.level = WARN + +# Disable request log by default, you can enable this by changing the appender +logger.http.name = http.requests +logger.http.additivity = false +logger.http = INFO,NullAppender +# Replace the above with this configuration if you want an http access.log +# logger.http = INFO,AccessRFA +# +# + +# Turn off performanceadvisory... it logs when a block is remote +# complaining short-circuit read is UNUSABLE when the block is +# remote (duh!). +logger.PerformanceAdvisory.name = org.apache.hadoop.util.PerformanceAdvisory +logger.PerformanceAdvisory.level = INFO + +# Turn down CodecPool log level so we avoid messages like the below +# 2022-04-05 23:34:34,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=25,queue=0,port=16020] compress.CodecPool: Got recycled decompressor +# +logger.CodecPool.name = org.apache.hadoop.io.compress.CodecPool +logger.CodecPool.level = INFO + +# Turn off noise. +logger.hadoopipc.name = org.apache.hadoop.ipc +logger.hadoopipc.level = WARN +logger.jetty.name = org.eclipse.jetty +logger.jetty.level = INFO +logger.jetty3rdparty.name = org.apache.hbase.thirdparty.org.eclipse.jetty +logger.jetty3rdparty.level = INFO +logger.hdfs.name = org.apache.hadoop.hdfs +logger.hdfs.level = INFO +logger.metrics2.name = org.apache.hadoop.metrics2 +logger.metrics2.level = INFO +logger.jsonbean.name = org.apache.hadoop.hbase.util.JSONBean +logger.jsonbean.level = INFO +logger.scheduledchore.name = org.apache.hadoop.hbase.ScheduledChore +logger.scheduledchore.level = INFO diff --git a/hbase-kubernetes-deployment/components/hbase/m-service.yaml b/hbase-kubernetes-deployment/components/hbase/m-service.yaml new file mode 100644 index 0000000000..5146fcf862 --- /dev/null +++ b/hbase-kubernetes-deployment/components/hbase/m-service.yaml @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +kind: Service +apiVersion: v1 +metadata: + name: master + labels: + jmxexporter: enabled +spec: + selector: + role: master + clusterIP: None + ports: + - name: jmxexporter + port: 7000 diff --git a/hbase-kubernetes-deployment/components/hbase/m-statefulset.yaml b/hbase-kubernetes-deployment/components/hbase/m-statefulset.yaml new file mode 100644 index 0000000000..c867f69ac3 --- /dev/null +++ b/hbase-kubernetes-deployment/components/hbase/m-statefulset.yaml @@ -0,0 +1,250 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: master + labels: + role: master +spec: + podManagementPolicy: Parallel + replicas: 1 + selector: + matchLabels: + role: master + serviceName: hadoop + template: + metadata: + labels: + role: master + spec: + serviceAccountName: hadoop + containers: + - image: hbase + name: hbase + command: + - /bin/bash + - -c + - |- + # Shell context so we can pull in the environment variables set in the container and + # via the env and envFrom. + # See https://stackoverflow.com/questions/57885828/netty-cannot-access-class-jdk-internal-misc-unsafe + HBASE_CLASSPATH=/etc/hadoop \ + HBASE_OPTS=" \ + -XX:MaxRAMPercentage=${JVM_HEAP_PERCENTAGE_OF_RESOURCE_LIMIT} \ + -XX:InitialRAMPercentage=${JVM_HEAP_PERCENTAGE_OF_RESOURCE_LIMIT} \ + -javaagent:${JMX_PROMETHEUS_JAR}=7000:/tmp/scratch/jmxexporter.yaml \ + -Djava.security.properties=/tmp/scratch/java.security \ + -Djava.library.path=${HADOOP_HOME}/lib/native --add-opens java.base/jdk.internal.misc=ALL-UNNAMED \ + -Djava.util.logging.config.file=${HBASE_CONF_DIR}/logging.properties \ + -Dio.netty.tryReflectionSetAccessible=true \ + -Xlog:gc:${HBASE_LOG_DIR}/gc.log:time,uptime:filecount=10,filesize=100M" \ + hbase-daemon.sh --config /etc/hbase foreground_start master \ + ${FAILOVER_PROXY_PROVIDER} + # For now, just fetch local /jmx + # Says kubelet only exposes failures, not success: https://stackoverflow.com/questions/34455040/kubernetes-liveness-probe-logging + # TODO: Use https://hbase.apache.org/devapidocs/org/apache/hadoop/hbase/HealthChecker.html + livenessProbe: + httpGet: + path: /jmx?qry=java.lang:type=OperatingSystem + port: 16010 + initialDelaySeconds: 10 + periodSeconds: 10 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /jmx?qry=java.lang:type=OperatingSystem + port: 16010 + initialDelaySeconds: 10 + periodSeconds: 10 + failureThreshold: 3 + startupProbe: + httpGet: + path: /jmx?qry=java.lang:type=OperatingSystem + port: 16010 + initialDelaySeconds: 10 + failureThreshold: 30 + periodSeconds: 10 + resources: + requests: + memory: "1.5Gi" + cpu: "0.1" + limits: + memory: "2Gi" + cpu: "1.0" + envFrom: + - configMapRef: + name: environment + - configMapRef: + name: zookeeper-quorum + - configMapRef: + name: jaegertracing + optional: true + - configMapRef: + name: xraytracing + optional: true + env: +# not honored by opentelemetry-java/v1.0.1 +# - name: OTEL_EXPERIMENTAL_SDK_ENABLED +# # flip this switch to disable tracing +# value: "true" +# not honored by opentelemetry-java/v1.0.1 +# - name: OTEL_SERVICE_NAME +# value: "hbase-master" +# - name: OTEL_RESOURCE_ATTRIBUTES +# value: "service.name=hbase-master" +# - name: HBASE_TRACE_OPTS +# value: "true" +# - name: HBASE_SHELL_OPTS +# # it appears that system properties override environment variables, so naming the shell +# # can via system property will override the master name provided by environment variable +# value: "-Dotel.resource.attributes=service.name=hbase-shell" + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - containerPort: 16010 + name: http + - containerPort: 16000 + name: rpc + volumeMounts: + - name: hbase-configuration + mountPath: /etc/hbase + readOnly: true + - name: hbck2-configuration + mountPath: /etc/hbck2 + readOnly: true + - mountPath: /etc/hadoop + name: hadoop-configuration + - mountPath: /var/log/hbase + name: hbase-logs + - mountPath: /tmp/scratch + name: scratch + - mountPath: /tmp/scripts + name: scripts + initContainers: + - image: hbase + name: bootstrapper + imagePullPolicy: IfNotPresent + command: + - /bin/bash + - -c + - |- + set -ex + env | sort + # Currently, waits till NNs and all DNs are up. Waiting on all DNs is extreme + # but will do for now; could just wait on 5 or 10 or so. + /tmp/scripts/jmxping.sh namenode ${HADOOP_SERVICE} + # TODO: Should we check if ha and if so, if a NN active... get a report on health? + /tmp/scripts/jmxping.sh datanode ${HADOOP_SERVICE} 3 + cp /tmp/global-files/* /tmp/scratch/ + # Write the USER hbase is running as into temporary file for use by next init container + echo ${USER} > /tmp/scratch/hbaseuser.txt + # Ditto for the location of the hbase dir. + hbase org.apache.hadoop.hbase.util.HBaseConfTool hbase.rootdir > /tmp/scratch/hbaserootdir.txt + cat /tmp/scratch/hbaserootdir.txt + securityContext: + # Run bootstrapper as root so can set ${USER} owner on data volume + allowPrivilegeEscalation: false + runAsUser: 0 + resources: + requests: + cpu: '0.2' + memory: 256Mi + limits: + cpu: '0.5' + memory: 512Mi + envFrom: + - configMapRef: + name: environment + volumeMounts: + - mountPath: /tmp/scripts + name: scripts + # Scratch dir is a location where init containers place items for later use + # by the main containers when they run. + - mountPath: /tmp/scratch + name: scratch + - mountPath: /tmp/global-files + name: global-files + - name: hbase-configuration + mountPath: /etc/hbase + readOnly: true + - image: hadoop + name: prepare-hdfs-for-hbase + command: + - /bin/bash + - -c + - |- + set -ex + # This container runs as the HDFS ${USER}/super-user. + # Make sure that this image is the same as that of the cluster hbase is to + # run on (else below will fail w/ permissions issues). + HBASE_USER=$(cat /tmp/scratch/hbaseuser.txt) + HBASE_ROOTDIR=$(cat /tmp/scratch/hbaserootdir.txt) + hdfs --config /etc/hadoop dfs -mkdir ${HBASE_ROOTDIR} || echo $? + hdfs --config /etc/hadoop dfs -chown -R ${HBASE_USER} ${HBASE_ROOTDIR} + resources: + requests: + cpu: '0.2' + memory: 256Mi + limits: + cpu: '0.5' + memory: 512Mi + envFrom: + - configMapRef: + name: environment + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + # Scratch dir is a location where init containers place items for later use + # by the main containers when they run. + - mountPath: /tmp/scratch + name: scratch + - mountPath: /etc/hadoop + name: hadoop-configuration + - mountPath: /var/log/hadoop + name: hadoop-logs + restartPolicy: Always + volumes: + - name: hbase-configuration + configMap: + name: hbase-configuration + - name: hbck2-configuration + configMap: + name: hbck2-configuration + - configMap: + name: hadoop-configuration + name: hadoop-configuration + - configMap: + name: global-files + name: global-files + - emptyDir: {} + name: hbase-logs + - emptyDir: {} + name: hadoop-logs + - configMap: + name: scripts + defaultMode: 0555 + name: scripts + # Scratch dir is a location where init containers place items for later use + # by the main containers when they run. + - emptyDir: {} + name: scratch diff --git a/hbase-kubernetes-deployment/components/hbase/rs-service.yaml b/hbase-kubernetes-deployment/components/hbase/rs-service.yaml new file mode 100644 index 0000000000..762a677e7d --- /dev/null +++ b/hbase-kubernetes-deployment/components/hbase/rs-service.yaml @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +kind: Service +apiVersion: v1 +metadata: + name: regionserver + labels: + jmxexporter: enabled +spec: + selector: + role: regionserver + clusterIP: None + ports: + - name: jmxexporter + port: 7000 diff --git a/hbase-kubernetes-deployment/components/hbase/rs-statefulset.yaml b/hbase-kubernetes-deployment/components/hbase/rs-statefulset.yaml new file mode 100644 index 0000000000..1b6ece2abf --- /dev/null +++ b/hbase-kubernetes-deployment/components/hbase/rs-statefulset.yaml @@ -0,0 +1,237 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: regionserver + labels: + role: regionserver +spec: + podManagementPolicy: Parallel + replicas: 1 + selector: + matchLabels: + role: regionserver + serviceName: hadoop + template: + metadata: + labels: + role: regionserver + spec: + serviceAccountName: hadoop + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + role: regionserver + topologyKey: kubernetes.io/hostname + containers: + - image: hbase + name: hbase + command: + - /bin/bash + - -c + - |- + # Shell context so we can pull in the environment variables set in the container and + # via the env and envFrom. + # + # See https://stackoverflow.com/questions/57885828/netty-cannot-access-class-jdk-internal-misc-unsafe + HBASE_CLASSPATH=/etc/hadoop \ + HBASE_OPTS=" \ + -XX:MaxRAMPercentage=${JVM_HEAP_PERCENTAGE_OF_RESOURCE_LIMIT} \ + -XX:InitialRAMPercentage=${JVM_HEAP_PERCENTAGE_OF_RESOURCE_LIMIT} \ + -javaagent:${JMX_PROMETHEUS_JAR}=7000:/tmp/scratch/jmxexporter.yaml \ + -Djava.security.properties=/tmp/scratch/java.security \ + -Djava.library.path=/var/lib/hadoop-native-lib/native \ + --add-opens java.base/jdk.internal.misc=ALL-UNNAMED \ + -Djava.util.logging.config.file=${HBASE_CONF_DIR}/logging.properties \ + -Dio.netty.tryReflectionSetAccessible=true \ + -Xlog:gc:${HBASE_LOG_DIR}/gc.log:time,uptime:filecount=10,filesize=100M" \ + hbase-daemon.sh --config /etc/hbase foreground_start regionserver \ + ${FAILOVER_PROXY_PROVIDER} # &> ${HBASE_LOG_DIR}/regionserver.$(date -u +"%Y-%m-%dT%H%M%SZ").out + # For now, just fetch local /jmx + # Says kubelet only exposes failures, not success: https://stackoverflow.com/questions/34455040/kubernetes-liveness-probe-logging + # TODO: Use https://hbase.apache.org/devapidocs/org/apache/hadoop/hbase/HealthChecker.html + livenessProbe: + httpGet: + path: /jmx?qry=java.lang:type=OperatingSystem + port: 16030 + initialDelaySeconds: 10 + periodSeconds: 10 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /jmx?qry=java.lang:type=OperatingSystem + port: 16030 + initialDelaySeconds: 10 + periodSeconds: 10 + failureThreshold: 3 + startupProbe: + httpGet: + path: /jmx?qry=java.lang:type=OperatingSystem + port: 16030 + initialDelaySeconds: 10 + failureThreshold: 30 + periodSeconds: 10 + resources: + requests: + memory: "2Gi" + cpu: "0.2" + limits: + memory: "3Gi" + cpu: "1.0" + envFrom: + - configMapRef: + name: environment + - configMapRef: + name: zookeeper-quorum + - configMapRef: + name: jaegertracing + optional: true + - configMapRef: + name: xraytracing + optional: true + env: +# not honored by opentelemetry-java/v1.0.1 +# - name: OTEL_EXPERIMENTAL_SDK_ENABLED +# # flip this switch to disable tracing +# value: "true" +# not honored by opentelemetry-java/v1.0.1 +# - name: OTEL_SERVICE_NAME +# value: "hbase-regionserver" +# - name: HBASE_TRACE_OPTS +# value: "true" +# - name: OTEL_RESOURCE_ATTRIBUTES +# value: "service.name=hbase-regionserver" +# - name: HBASE_SHELL_OPTS +# # it appears that system properties override environment variables, so naming the shell +# # can via system property will override the master name provided by environment variable +# value: "-Dotel.service.name=hbase-shell" + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - containerPort: 16030 + name: http + - containerPort: 16020 + name: rpc + volumeMounts: + - name: hbase-configuration + mountPath: /etc/hbase + readOnly: true + - name: hbck2-configuration + mountPath: /etc/hbck2 + readOnly: true + - mountPath: /etc/hadoop + name: hadoop-configuration + - mountPath: /var/log/hbase + name: hbase-logs + - mountPath: /tmp/scratch + name: scratch + - mountPath: /tmp/scripts + name: scripts + - mountPath: /var/lib/hadoop-native-lib + name: hadoop-native-lib + initContainers: + - image: hbase + name: bootstrapper + imagePullPolicy: IfNotPresent + command: + - /bin/bash + - -c + - |- + set -ex + # Wait on a single master to be up... + /tmp/scripts/jmxping.sh master ${HADOOP_SERVICE} 1 + cp /tmp/global-files/* /tmp/scratch/ + securityContext: + # Run bootstrapper as root so can set ${USER} owner on data volume + allowPrivilegeEscalation: false + runAsUser: 0 + resources: + requests: + cpu: '0.2' + memory: 256Mi + limits: + cpu: '0.5' + memory: 512Mi + envFrom: + - configMapRef: + name: environment + volumeMounts: + - mountPath: /tmp/scripts + name: scripts + # Scratch dir is a location where init containers place items for later use + # by the main containers when they run. + - mountPath: /tmp/scratch + name: scratch + - mountPath: /tmp/global-files + name: global-files + - image: hadoop + name: copy-hadoop-native-lib + command: + # Runs as the image/hdfs user. + - /bin/bash + - -c + - |- + set -xe + if [ -d ~/hadoop/lib/native ]; then + cp -r ~/hadoop/lib/native /var/lib/hadoop-native-lib + else + echo "Native dir not found at ~/hadoop/lib/native" + fi + resources: + requests: + cpu: '0.2' + memory: 256Mi + limits: + cpu: '0.5' + memory: 512Mi + envFrom: + - configMapRef: + name: environment + volumeMounts: + - mountPath: /var/lib/hadoop-native-lib + name: hadoop-native-lib + restartPolicy: Always + volumes: + - name: hbase-configuration + configMap: + name: hbase-configuration + - name: hbck2-configuration + configMap: + name: hbck2-configuration + - configMap: + name: hadoop-configuration + name: hadoop-configuration + - configMap: + name: global-files + name: global-files + - emptyDir: {} + name: hbase-logs + - configMap: + name: scripts + defaultMode: 0555 + name: scripts + # Scratch dir is a location where init containers place items for later use + # by the main containers when they run. + - emptyDir: {} + name: scratch + - emptyDir: {} + name: hadoop-native-lib diff --git a/hbase-kubernetes-deployment/overlays/hbase/kustomization.yaml b/hbase-kubernetes-deployment/overlays/hbase/kustomization.yaml new file mode 100644 index 0000000000..e91ad8a6bb --- /dev/null +++ b/hbase-kubernetes-deployment/overlays/hbase/kustomization.yaml @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../hdfs + +components: +- ../../components/zookeeper/single-instance +- ../../components/hbase diff --git a/hbase-kubernetes-deployment/tests/integration/overlays_hbase/00-assert-hbase.yaml b/hbase-kubernetes-deployment/tests/integration/overlays_hbase/00-assert-hbase.yaml new file mode 100644 index 0000000000..62e4aabbf2 --- /dev/null +++ b/hbase-kubernetes-deployment/tests/integration/overlays_hbase/00-assert-hbase.yaml @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Asserts on the HBase portion of the deployment. +# +--- +# assert that there is a `StatefulSet` named "master" that has one live instance +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: master +status: + availableReplicas: 1 +--- +# assert that there is a `StatefulSet` named "regionserver" that has one live instance +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: regionserver +status: + availableReplicas: 1 diff --git a/hbase-kubernetes-deployment/tests/integration/overlays_hbase/00-kustomize.yaml b/hbase-kubernetes-deployment/tests/integration/overlays_hbase/00-kustomize.yaml new file mode 100644 index 0000000000..b365471f5f --- /dev/null +++ b/hbase-kubernetes-deployment/tests/integration/overlays_hbase/00-kustomize.yaml @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep + +commands: +- script: ../../bin/kustomize_into_tmpdir.sh diff --git a/hbase-kubernetes-deployment/tests/integration/overlays_hbase/kustomization.yaml b/hbase-kubernetes-deployment/tests/integration/overlays_hbase/kustomization.yaml new file mode 100644 index 0000000000..115caa7495 --- /dev/null +++ b/hbase-kubernetes-deployment/tests/integration/overlays_hbase/kustomization.yaml @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../test_base +- ../../../overlays/hbase diff --git a/hbase-kubernetes-deployment/tests/unit/overlays_hbase/00-assert-hbase.yaml b/hbase-kubernetes-deployment/tests/unit/overlays_hbase/00-assert-hbase.yaml new file mode 100644 index 0000000000..27e809dcd7 --- /dev/null +++ b/hbase-kubernetes-deployment/tests/unit/overlays_hbase/00-assert-hbase.yaml @@ -0,0 +1,72 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Asserts on the HBase portion of the deployment. +# +--- +# assert that there is a `ConfigMap` named "hbase-configuration-XXX" +# TODO: kuttl does not support generated names +#apiVersion: v1 +#kind: ConfigMap +#metadata: +# name: hbase-configuration-c94h8k249d +--- +# assert that there is a `ConfigMap` named "hbck2-configuration-XXX" +# TODO: kuttl does not support generated names +#apiVersion: v1 +#kind: ConfigMap +#metadata: +# name: hbck2-configuration-c94h8k249d +--- +# assert that there is a `StatefulSet` named "master" that it provides pods labeled role:master +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: master +spec: + template: + metadata: + labels: + role: master +--- +# assert that there is a `Service` named "master" pointing to pods labeled role:master +apiVersion: v1 +kind: Service +metadata: + name: master +spec: + selector: + role: master +--- +# assert that there is a `StatefulSet` named "regionserver" that it provides pods labeled role:regionserver +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: regionserver +spec: + template: + metadata: + labels: + role: regionserver +--- +# assert that there is a `Service` named "regionserver" pointing to pods labeled role:regionserver +apiVersion: v1 +kind: Service +metadata: + name: regionserver +spec: + selector: + role: regionserver diff --git a/hbase-kubernetes-deployment/tests/unit/overlays_hbase/00-kustomize.yaml b/hbase-kubernetes-deployment/tests/unit/overlays_hbase/00-kustomize.yaml new file mode 100644 index 0000000000..b365471f5f --- /dev/null +++ b/hbase-kubernetes-deployment/tests/unit/overlays_hbase/00-kustomize.yaml @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep + +commands: +- script: ../../bin/kustomize_into_tmpdir.sh diff --git a/hbase-kubernetes-deployment/tests/unit/overlays_hbase/kustomization.yaml b/hbase-kubernetes-deployment/tests/unit/overlays_hbase/kustomization.yaml new file mode 100644 index 0000000000..7f33c58571 --- /dev/null +++ b/hbase-kubernetes-deployment/tests/unit/overlays_hbase/kustomization.yaml @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../../overlays/hbase