HBASE-27832 Introduce hbase overlay

hbase-kubernetes-deployment/components/hbase/hadoop-metrics2-hbase.properties

@@ -0,0 +1,46 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+    # syntax: [prefix].[source|sink].[instance].[options]
+    # See javadoc of package-info.java for org.apache.hadoop.metrics2 for details
+
+    *.sink.file*.class=org.apache.hadoop.metrics2.sink.FileSink
+    # default sampling period
+    *.period=10
+
+    # Below are some examples of sinks that could be used
+    # to monitor different hbase daemons.
+
+    # hbase.sink.file-all.class=org.apache.hadoop.metrics2.sink.FileSink
+    # hbase.sink.file-all.filename=all.metrics
+
+    # hbase.sink.file0.class=org.apache.hadoop.metrics2.sink.FileSink
+    # hbase.sink.file0.context=hmaster
+    # hbase.sink.file0.filename=master.metrics
+
+    # hbase.sink.file1.class=org.apache.hadoop.metrics2.sink.FileSink
+    # hbase.sink.file1.context=thrift-one
+    # hbase.sink.file1.filename=thrift-one.metrics
+
+    # hbase.sink.file2.class=org.apache.hadoop.metrics2.sink.FileSink
+    # hbase.sink.file2.context=thrift-two
+    # hbase.sink.file2.filename=thrift-one.metrics
+
+    # hbase.sink.file3.class=org.apache.hadoop.metrics2.sink.FileSink
+    # hbase.sink.file3.context=rest
+    # hbase.sink.file3.filename=rest.metrics
+
+  # Read from src rather than copy over like this.

hbase-kubernetes-deployment/components/hbase/hbase-site.xml

@@ -0,0 +1,75 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration>
+  <property>
+    <name>hbase.cluster.distributed</name>
+    <value>true</value>
+  </property>
+  <property>
+    <name>hbase.rootdir</name>
+    <value>hdfs://${env.HADOOP_SERVICE}:8020/hbase</value>
+  </property>
+  <property>
+    <name>hbase.zookeeper.quorum</name>
+    <value>${env.HBASE_ZOOKEEPER_QUORUM}</value>
+  </property>
+  <property>
+    <!--In k8s context, master notion of name doesn't always agree; use RS name.
+        Seems to fix case where RS was showing up w/ IP instead of name because
+        Master told it use its understanding of what its name was. See
+        HBASE-18226 and then HBASE-24667-->
+    <name>hbase.unsafe.regionserver.hostname.disable.master.reversedns</name>
+    <value>true</value>
+  </property>
+  <property>
+    <name>dfs.client.use.datanode.hostname</name>
+    <value>true</value>
+  </property>
+  <property>
+    <name>hbase.ssl.enabled</name>
+      <!-- Needs upstream fix. See https://github.com/eclipse/jetty.project/issues/4385-->
+    <value>${env.HBASE_SSL_ENABLED}</value>
+  </property>
+  <property>
+    <name>hbase.master.cleaner.interval</name>
+    <!-- Needs to clean up oldWAL and archived hfiles every 10 seconds -->
+    <value>10000</value>
+  </property>
+  <property>
+    <name>hbase.regionserver.handler.count</name>
+    <value>45</value>
+  </property>
+  <property>
+    <name>hbase.netty.eventloop.rpcserver.thread.count</name>
+    <value>1</value>
+    <description>See the end of https://issues.apache.org/jira/browse/HBASE-27112. Default
+  is 2xCPU_COUNT which seems way too much. 1 thread seems fine for the siri workload at least.</description>
+  </property>
+  <property>
+    <name>hbase.netty.worker.count</name>
+    <value>1</value>
+    <description>See the end of https://issues.apache.org/jira/browse/HBASE-27112. Default
+  is 2xCPU_COUNT which seems way too much. 1 thread seems fine for the siri workload at least.</description>
+  </property>
+  <property>
+    <name>hbase.regionserver.throughput.controller</name>
+    <value>org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController</value>
+  </property>
+</configuration>

hbase-kubernetes-deployment/components/hbase/kustomization.yaml

@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+apiVersion: kustomize.config.k8s.io/v1alpha1
+kind: Component
+
+commonLabels:
+  app: hadoop
+
+configMapGenerator:
+- name: hbase-configuration
+  files:
+  - hbase-site.xml
+  - hadoop-metrics2-hbase.properties
+  - log4j2.properties=log4j2.properties.hbase
+- name: hbck2-configuration
+  # Add an hbck2-configuration. Uses log4j1 vs log4j2.
+  files:
+  - hbase-site.xml
+  - hadoop-metrics2-hbase.properties
+  - log4j.properties=log4j.properties.hbase
+
+resources:
+- m-service.yaml
+- m-statefulset.yaml
+- rs-service.yaml
+- rs-statefulset.yaml

hbase-kubernetes-deployment/components/hbase/log4j.properties.hbase

@@ -0,0 +1,68 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Define some default values that can be overridden by system properties
+hbase.root.logger=INFO,console
+hbase.log.dir=.
+hbase.log.file=hbase.log
+
+# Define the root logger to the system property "hbase.root.logger".
+log4j.rootLogger=${hbase.root.logger}
+
+# Logging Threshold
+log4j.threshold=ALL
+
+#
+# Daily Rolling File Appender
+#
+log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.DRFA.File=${hbase.log.dir}/${hbase.log.file}
+
+# Rollver at midnight
+log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
+
+# 30-day backup
+#log4j.appender.DRFA.MaxBackupIndex=30
+log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
+# Debugging Pattern format
+log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %C{2}(%L): %m%n
+
+
+#
+# console
+# Add "console" to rootlogger above if you want to use this
+#
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %C{2}(%L): %m%n
+
+# Custom Logging levels
+
+#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG
+
+log4j.logger.org.apache.hadoop=WARN
+log4j.logger.org.apache.zookeeper=ERROR
+log4j.logger.org.apache.hadoop.hbase=DEBUG
+
+#These settings are workarounds against spurious logs from the minicluster.
+#See HBASE-4709
+log4j.logger.org.apache.hadoop.metrics2.impl.MetricsConfig=WARN
+log4j.logger.org.apache.hadoop.metrics2.impl.MetricsSinkAdapter=WARN
+log4j.logger.org.apache.hadoop.metrics2.impl.MetricsSystemImpl=WARN
+log4j.logger.org.apache.hadoop.metrics2.util.MBeans=WARN
+# Enable this to get detailed connection error/retry logging.
+# log4j.logger.org.apache.hadoop.hbase.client.ConnectionImplementation=TRACE