[Merged by Bors] - fix: Increase default pvc size to 1Gi as required by IONOS

docs/modules/hdfs/pages/usage-guide/resources.adoc

@@ -69,7 +69,14 @@ The hdfs-operator will re-create the StatefulSet automatically.
 
 include::home:concepts:stackable_resource_requests.adoc[]
 
-If no resource requests are configured explicitly, the HDFS operator uses the following defaults:
+A minimal HA setup consisting of 3 journalnodes, 2 namenodes and 2 datanodes has the following https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/[resource requirements]:
+
+* `2950m` CPU request
+* `8300m` CPU limit
+* `6528m` memory request and limit
+* `27648Mi` persistent storage
+
+Corresponding to the values above, the operator uses the following resource defaults:
 
 [source,yaml]
 ----
@@ -84,7 +91,7 @@ spec:
           limit: 512Mi
         storage:
           data:
-            capacity: 512Mi
+            capacity: 1Gi
   nameNodes:
     config:
       resources:
@@ -95,7 +102,7 @@ spec:
           limit: 1024Mi
         storage:
           data:
-            capacity: 2048Mi
+            capacity: 2Gi
   dataNodes:
     config:
       resources:

rust/crd/src/lib.rs

@@ -135,13 +135,12 @@ impl CurrentlySupportedListenerClasses {
 /// This is a shared trait for all role/role-group config structs to avoid duplication
 /// when extracting role specific configuration structs like resources or logging.
 pub trait MergedConfig {
-    /// Resources shared by all roles (except datanodes).
-    /// DataNodes must use `data_node_resources`
-    fn resources(&self) -> Option<Resources<HdfsStorageConfig, NoRuntimeLimits>> {
+    fn name_node_resources(&self) -> Option<Resources<HdfsStorageConfig, NoRuntimeLimits>> {
+        None
+    }
+    fn journal_node_resources(&self) -> Option<Resources<HdfsStorageConfig, NoRuntimeLimits>> {
         None
     }
-    /// Resources for datanodes.
-    /// Other roles must use `resources`.
     fn data_node_resources(
         &self,
     ) -> Option<Resources<DataNodeStorageConfigInnerType, NoRuntimeLimits>> {
@@ -786,7 +785,7 @@ pub struct NameNodeConfig {
 }
 
 impl MergedConfig for NameNodeConfig {
-    fn resources(&self) -> Option<Resources<HdfsStorageConfig, NoRuntimeLimits>> {
+    fn name_node_resources(&self) -> Option<Resources<HdfsStorageConfig, NoRuntimeLimits>> {
         Some(self.resources.clone())
     }
 
@@ -1095,7 +1094,7 @@ pub struct JournalNodeConfig {
 }
 
 impl MergedConfig for JournalNodeConfig {
-    fn resources(&self) -> Option<Resources<HdfsStorageConfig, NoRuntimeLimits>> {
+    fn journal_node_resources(&self) -> Option<Resources<HdfsStorageConfig, NoRuntimeLimits>> {
         Some(self.resources.clone())
     }
 
@@ -1138,7 +1137,7 @@ impl JournalNodeConfigFragment {
                 },
                 storage: HdfsStorageConfigFragment {
                     data: PvcConfigFragment {
-                        capacity: Some(Quantity("512Mi".to_owned())),
+                        capacity: Some(Quantity("1Gi".to_owned())),
                         storage_class: None,
                         selectors: None,
                     },

rust/operator/src/container.rs

@@ -243,9 +243,23 @@ impl ContainerConfig {
             // We use the main app container resources here in contrast to several operators (which use
             // hardcoded resources) due to the different code structure.
             // Going forward this should be replaced by calculating init container resources in the pod builder.
-            if let Some(resources) = merged_config.resources() {
-                create_tls_cert_bundle_init_cb.resources(resources.into());
-            }
+            match role {
+                HdfsRole::NameNode => {
+                    if let Some(resources) = merged_config.name_node_resources() {
+                        create_tls_cert_bundle_init_cb.resources(resources.into());
+                    }
+                }
+                HdfsRole::DataNode => {
+                    if let Some(resources) = merged_config.data_node_resources() {
+                        create_tls_cert_bundle_init_cb.resources(resources.into());
+                    }
+                }
+                HdfsRole::JournalNode => {
+                    if let Some(resources) = merged_config.journal_node_resources() {
+                        create_tls_cert_bundle_init_cb.resources(resources.into());
+                    }
+                }
+            };
 
             pb.add_init_container(create_tls_cert_bundle_init_cb.build());
         }
@@ -325,7 +339,13 @@ impl ContainerConfig {
         merged_config: &(dyn MergedConfig + Send + 'static),
     ) -> Option<Vec<PersistentVolumeClaim>> {
         match role {
-            HdfsRole::NameNode | HdfsRole::JournalNode => merged_config.resources().map(|r| {
+            HdfsRole::NameNode => merged_config.name_node_resources().map(|r| {
+                vec![r.storage.data.build_pvc(
+                    ContainerConfig::DATA_VOLUME_MOUNT_NAME,
+                    Some(vec!["ReadWriteOnce"]),
+                )]
+            }),
+            HdfsRole::JournalNode => merged_config.journal_node_resources().map(|r| {
                 vec![r.storage.data.build_pvc(
                     ContainerConfig::DATA_VOLUME_MOUNT_NAME,
                     Some(vec!["ReadWriteOnce"]),
@@ -744,17 +764,16 @@ impl ContainerConfig {
         merged_config: &(dyn MergedConfig + Send + 'static),
     ) -> Option<ResourceRequirements> {
         match self {
-            // name node and journal node resources
-            ContainerConfig::Hdfs { role, .. } if role != &HdfsRole::DataNode => {
-                merged_config.resources().map(|c| c.into())
-            }
-            // data node resources
-            ContainerConfig::Hdfs { .. } => merged_config.data_node_resources().map(|c| c.into()),
-            // init containers inherit their respective main (app) container resources
+            ContainerConfig::Hdfs { role, .. } => match role {
+                HdfsRole::NameNode => merged_config.name_node_resources().map(|r| r.into()),
+                HdfsRole::DataNode => merged_config.data_node_resources().map(|r| r.into()),
+                HdfsRole::JournalNode => merged_config.journal_node_resources().map(|r| r.into()),
+            },
+            // Namenode init containers
             ContainerConfig::FormatNameNodes { .. } | ContainerConfig::FormatZooKeeper { .. } => {
-                merged_config.resources().map(|c| c.into())
+                merged_config.name_node_resources().map(|c| c.into())
             }
-            // name node side car zk failover
+            // Namenode sidecar containers
             ContainerConfig::Zkfc { .. } => Some(
                 ResourceRequirementsBuilder::new()
                     .with_cpu_request("100m")
@@ -763,7 +782,7 @@ impl ContainerConfig {
                     .with_memory_limit("512Mi")
                     .build(),
             ),
-            // data node init container inherit their respective main (app) container resources
+            // Datanode init containers
             ContainerConfig::WaitForNameNodes { .. } => {
                 merged_config.data_node_resources().map(|c| c.into())
             }

tests/templates/kuttl/kerberos/10-install-zk.yaml.j2

@@ -16,12 +16,6 @@ spec:
     config:
       logging:
         enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
-      resources:
-        cpu:
-          min: '300m'
-          max: '600m'
-        memory:
-          limit: '512Mi'
     roleGroups:
       default:
         replicas: 1

tests/templates/kuttl/kerberos/11-install-hdfs.yaml.j2

@@ -27,12 +27,6 @@ commands:
           config:
             logging:
               enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
-            resources:
-              cpu:
-                max: '1'
-                min: '300m'
-              memory:
-                limit: '512Mi'
           configOverrides: &configOverrides
             core-site.xml:
               hadoop.user.group.static.mapping.overrides: "dr.who=;nn=;nm=;jn=;testuser=supergroup;"
@@ -43,12 +37,6 @@ commands:
           config:
             logging:
               enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
-            resources:
-              cpu:
-                max: '1'
-                min: '300m'
-              memory:
-                limit: '512Mi'
           configOverrides: *configOverrides
           roleGroups:
             default:
@@ -57,12 +45,6 @@ commands:
           config:
             logging:
               enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
-            resources:
-              cpu:
-                max: '1'
-                min: '300m'
-              memory:
-                limit: '512Mi'
           configOverrides: *configOverrides
           roleGroups:
             default:

tests/templates/kuttl/orphaned-resources/01-install-zookeeper.yaml.j2

@@ -16,12 +16,6 @@ spec:
     config:
       logging:
         enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
-      resources:
-        cpu:
-          min: '300m'
-          max: '600m'
-        memory:
-          limit: '512Mi'
     roleGroups:
       default:
         replicas: 1

tests/templates/kuttl/orphaned-resources/02-install-hdfs.yaml.j2

@@ -17,25 +17,13 @@ spec:
     config:
       logging:
         enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
-      resources:
-        cpu:
-          max: '1'
-          min: '300m'
-        memory:
-          limit: '512Mi'
     roleGroups:
       default:
         replicas: 2
   dataNodes:
     config:
       logging:
         enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
-      resources:
-        cpu:
-          max: '1'
-          min: '300m'
-        memory:
-          limit: '512Mi'
     roleGroups:
       default:
         replicas: 1
@@ -45,12 +33,6 @@ spec:
     config:
       logging:
         enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
-      resources:
-        cpu:
-          max: '1'
-          min: '300m'
-        memory:
-          limit: '512Mi'
     roleGroups:
       default:
         replicas: 1

tests/templates/kuttl/orphaned-resources/04-change-datanode-rolegroup.yaml.j2

@@ -8,12 +8,6 @@ spec:
     config:
       logging:
         enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
-      resources:
-        cpu:
-          max: '1'
-          min: '300m'
-        memory:
-          limit: '512Mi'
     roleGroups:
       default: null
       newrolegroup:

tests/templates/kuttl/smoke/30-assert.yaml.j2

@@ -7,46 +7,6 @@ apiVersion: apps/v1
 kind: StatefulSet
 metadata:
   name: hdfs-namenode-default
-spec:
-  template:
-    spec:
-      containers:
-      - name: namenode
-        resources:
-          limits:
-            cpu: "1"
-            memory: 512Mi
-          requests:
-            cpu: 300m
-            memory: 512Mi
-{% if lookup('env', 'VECTOR_AGGREGATOR') %}
-      - name: vector
-{% endif %}
-      - name: zkfc
-        resources:
-          limits:
-            cpu: 400m
-            memory: 512Mi
-          requests:
-            cpu: 100m
-            memory: 512Mi
-      initContainers:
-      - name: format-namenodes
-        resources:
-          limits:
-            cpu: "1"
-            memory: 512Mi
-          requests:
-            cpu: 300m
-            memory: 512Mi
-      - name: format-zookeeper
-        resources:
-          limits:
-            cpu: "1"
-            memory: 512Mi
-          requests:
-            cpu: 300m
-            memory: 512Mi
 status:
   readyReplicas: 2
   replicas: 2
@@ -55,21 +15,6 @@ apiVersion: apps/v1
 kind: StatefulSet
 metadata:
   name: hdfs-journalnode-default
-spec:
-  template:
-    spec:
-      containers:
-      - name: journalnode
-        resources:
-          limits:
-            cpu: "1"
-            memory: 512Mi
-          requests:
-            cpu: 300m
-            memory: 512Mi
-{% if lookup('env', 'VECTOR_AGGREGATOR') %}
-      - name: vector
-{% endif %}
 status:
   readyReplicas: 1
   replicas: 1
@@ -78,48 +23,9 @@ apiVersion: apps/v1
 kind: StatefulSet
 metadata:
   name: hdfs-datanode-default
-spec:
-  template:
-    spec:
-      containers:
-      - name: datanode
-        resources:
-          limits:
-            cpu: "1"
-            memory: 512Mi
-          requests:
-            cpu: 400m
-            memory: 512Mi
-{% if lookup('env', 'VECTOR_AGGREGATOR') %}
-      - name: vector
-{% endif %}
-      initContainers:
-      - name: wait-for-namenodes
-        resources:
-          limits:
-            cpu: "1"
-            memory: 512Mi
-          requests:
-            cpu: 400m
-            memory: 512Mi
 status:
   readyReplicas: {{ test_scenario['values']['number-of-datanodes'] }}
   replicas: {{ test_scenario['values']['number-of-datanodes'] }}
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: hdfs-namenode-default
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: hdfs-datanode-default
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: hdfs-journalnode-default
 {% if test_scenario['values']['datanode-pvcs'] == '2hdd-1ssd' %}
 ---
 apiVersion: v1