Add support for 2.4.17, 2.5.3 and 2.5.3-hadoop3

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## [Unreleased]
 
+### Added
+
+- Support for HBase `2.4.17`, `2.5.3` and `2.5.3-hadoop3` ([#353]).
+
 ### Changed
 
 - Operator-rs: `0.40.2` -> `0.41.0` ([#349]).
@@ -16,6 +20,7 @@
 
 [#349]: https://github.com/stackabletech/hbase-operator/pull/349
 [#351]: https://github.com/stackabletech/hbase-operator/pull/351
+[#353]: https://github.com/stackabletech/hbase-operator/pull/353
 [#356]: https://github.com/stackabletech/hbase-operator/pull/356
 [#357]: https://github.com/stackabletech/hbase-operator/pull/357
 

docs/modules/hbase/partials/supported-versions.adoc

@@ -7,3 +7,6 @@
 - 2.4.9
 - 2.4.11
 - 2.4.12
+- 2.4.17
+- 2.5.3
+- 2.5.3-hadoop3

rust/crd/src/lib.rs

@@ -460,3 +460,71 @@ impl HbaseCluster {
         fragment::validate(conf_rolegroup).context(FragmentValidationFailureSnafu)
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_config_merging() {
+        let input = r#"
+        apiVersion: hbase.stackable.tech/v1alpha1
+        kind: HbaseCluster
+        metadata:
+          name: simple-hbase
+        spec:
+          image:
+            productVersion: 2.4.12
+            stackableVersion: "23.4"
+          clusterConfig:
+            hdfsConfigMapName: simple-hdfs
+            zookeeperConfigMapName: simple-znode
+          masters:
+            config:
+              logging:
+                enableVectorAgent: true
+            roleGroups:
+              default:
+                replicas: 1
+              disable-logging:
+                replicas: 1
+                config:
+                  logging:
+                    enableVectorAgent: false
+          regionServers:
+            roleGroups:
+              default:
+                replicas: 1
+          restServers:
+            roleGroups:
+              default:
+                replicas: 1
+        "#;
+        let hbase: HbaseCluster = serde_yaml::from_str(input).expect("illegal test input");
+        let master_default_config = hbase
+            .merged_config(
+                &HbaseRole::Master,
+                "default",
+                &hbase.spec.cluster_config.hdfs_config_map_name,
+            )
+            .unwrap();
+        let master_disable_logging_config = hbase
+            .merged_config(
+                &HbaseRole::Master,
+                "disable-logging",
+                &hbase.spec.cluster_config.hdfs_config_map_name,
+            )
+            .unwrap();
+        let region_server_default_config = hbase
+            .merged_config(
+                &HbaseRole::RegionServer,
+                "default",
+                &hbase.spec.cluster_config.hdfs_config_map_name,
+            )
+            .unwrap();
+
+        assert!(master_default_config.logging.enable_vector_agent);
+        assert!(!master_disable_logging_config.logging.enable_vector_agent);
+        assert!(!region_server_default_config.logging.enable_vector_agent);
+    }
+}

rust/operator-binary/src/hbase_controller.rs

@@ -2,9 +2,7 @@
 
 use crate::{
     discovery::build_discovery_configmap,
-    product_logging::{
-        extend_role_group_config_map, resolve_vector_aggregator_address, LOG4J_CONFIG_FILE,
-    },
+    product_logging::{extend_role_group_config_map, resolve_vector_aggregator_address},
     zookeeper::{self, ZookeeperConnectionInformation},
     OPERATOR_NAME,
 };
@@ -475,6 +473,7 @@ fn build_rolegroup_config_map(
         .add_data(HBASE_ENV_SH, write_hbase_env_sh(hbase_env_config.iter()));
 
     extend_role_group_config_map(
+        &resolved_product_image.product_version,
         rolegroup,
         vector_aggregator_address,
         &config.logging,
@@ -634,7 +633,7 @@ fn build_rolegroup_statefulset(
                 HDFS_DISCOVERY_TMP_DIR, CONFIG_DIR_NAME
             ),
             format!("cp {}/* {}", HBASE_CONFIG_TMP_DIR, CONFIG_DIR_NAME),
-            format!("cp {HBASE_LOG_CONFIG_TMP_DIR}/{LOG4J_CONFIG_FILE} {CONFIG_DIR_NAME}",),
+            format!("cp {HBASE_LOG_CONFIG_TMP_DIR}/log4j*.properties {CONFIG_DIR_NAME}"),
             format!(
                 "bin/hbase {} start",
                 match role {

rust/operator-binary/src/product_logging.rs

@@ -7,7 +7,9 @@ use stackable_operator::{
     kube::ResourceExt,
     product_logging::{
         self,
-        spec::{ContainerLogConfig, ContainerLogConfigChoice, Logging},
+        spec::{
+            AutomaticContainerLogConfig, ContainerLogConfig, ContainerLogConfigChoice, Logging,
+        },
     },
     role_utils::RoleGroupRef,
 };
@@ -38,8 +40,63 @@ type Result<T, E = Error> = std::result::Result<T, E>;
 
 const VECTOR_AGGREGATOR_CM_ENTRY: &str = "ADDRESS";
 const CONSOLE_CONVERSION_PATTERN: &str = "%d{ISO8601} %-5p [%t] %c{2}: %.1000m%n";
-const HBASE_LOG_FILE: &str = "hbase.log4j.xml";
-pub const LOG4J_CONFIG_FILE: &str = "log4j.properties";
+
+// TODO: Move to operator-rs when other products switch from log4j1 to log4j2.
+#[derive(Debug, PartialEq, Eq)]
+pub enum Log4JVersion {
+    Log4J1,
+    Log4J2,
+}
+
+impl Log4JVersion {
+    fn config_file_name(&self) -> &str {
+        match &self {
+            Log4JVersion::Log4J1 => "log4j.properties",
+            Log4JVersion::Log4J2 => "log4j2.properties",
+        }
+    }
+
+    fn log_file_name(&self) -> &str {
+        match &self {
+            Log4JVersion::Log4J1 => "hbase.log4j.xml",
+            Log4JVersion::Log4J2 => "hbase.log4j2.xml",
+        }
+    }
+
+    fn create_log_config(
+        &self,
+        log_dir: &str,
+        max_size_in_mib: u32,
+        console_conversion_pattern: &str,
+        config: &AutomaticContainerLogConfig,
+    ) -> String {
+        match &self {
+            Log4JVersion::Log4J1 => product_logging::framework::create_log4j_config(
+                log_dir,
+                self.log_file_name(),
+                max_size_in_mib,
+                console_conversion_pattern,
+                config,
+            ),
+            Log4JVersion::Log4J2 => product_logging::framework::create_log4j2_config(
+                log_dir,
+                self.log_file_name(),
+                max_size_in_mib,
+                console_conversion_pattern,
+                config,
+            ),
+        }
+    }
+}
+
+fn calculate_log4j_version(product_version: &str) -> Log4JVersion {
+    // The first version we support is 2.4.x, so we don't need to care about versions below that
+    if product_version.starts_with("2.4.") {
+        Log4JVersion::Log4J1
+    } else {
+        Log4JVersion::Log4J2
+    }
+}
 
 /// Return the address of the Vector aggregator if the corresponding ConfigMap name is given in the
 /// cluster spec
@@ -78,6 +135,7 @@ pub async fn resolve_vector_aggregator_address(
 
 /// Extend the role group ConfigMap with logging and Vector configurations
 pub fn extend_role_group_config_map(
+    product_version: &str,
     rolegroup: &RoleGroupRef<HbaseCluster>,
     vector_aggregator_address: Option<&str>,
     logging: &Logging<Container>,
@@ -87,11 +145,11 @@ pub fn extend_role_group_config_map(
         choice: Some(ContainerLogConfigChoice::Automatic(log_config)),
     }) = logging.containers.get(&Container::Hbase)
     {
+        let log4j_version = calculate_log4j_version(product_version);
         cm_builder.add_data(
-            LOG4J_CONFIG_FILE,
-            product_logging::framework::create_log4j_config(
+            log4j_version.config_file_name(),
+            log4j_version.create_log_config(
                 &format!("{STACKABLE_LOG_DIR}/hbase"),
-                HBASE_LOG_FILE,
                 MAX_HBASE_LOG_FILES_SIZE_IN_MIB,
                 CONSOLE_CONVERSION_PATTERN,
                 log_config,
@@ -121,3 +179,19 @@ pub fn extend_role_group_config_map(
 
     Ok(())
 }
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_calculate_log4j_version() {
+        assert_eq!(calculate_log4j_version("2.4.0"), Log4JVersion::Log4J1);
+        assert_eq!(calculate_log4j_version("2.4.17"), Log4JVersion::Log4J1);
+        assert_eq!(calculate_log4j_version("2.5.0"), Log4JVersion::Log4J2);
+        assert_eq!(calculate_log4j_version("2.5.1"), Log4JVersion::Log4J2);
+        assert_eq!(calculate_log4j_version("2.6.0"), Log4JVersion::Log4J2);
+        assert_eq!(calculate_log4j_version("3.0.0"), Log4JVersion::Log4J2);
+        assert_eq!(calculate_log4j_version("42.43.44"), Log4JVersion::Log4J2);
+    }
+}

tests/test-definition.yaml

@@ -2,17 +2,16 @@
 dimensions:
   - name: hbase
     values:
-      - 2.4.8-stackable0.0.0-dev
-      - 2.4.9-stackable0.0.0-dev
-      - 2.4.11-stackable0.0.0-dev
-      - 2.4.12-stackable0.0.0-dev
+      - 2.4.17-stackable0.0.0-dev
+      - 2.5.3-stackable0.0.0-dev
+      - 2.5.3-hadoop3-stackable0.0.0-dev
   - name: hbase-latest
     values:
-      - 2.4.12-stackable0.0.0-dev
+      - 2.5.3-hadoop3-stackable0.0.0-dev
   - name: hdfs
     values:
       - 3.2.2-stackable0.0.0-dev
-      - 3.3.3-stackable0.0.0-dev
+      - 3.3.4-stackable0.0.0-dev
   - name: hdfs-latest
     values:
       - 3.3.4-stackable0.0.0-dev