Setting rackAwareness.enabled: true on an existing deployment doesn't enable rack awareness

[vuldin]

What happened?

Deployed with:

helm upgrade --install redpanda redpanda --repo https://charts.redpanda.com -n redpanda --wait --timeout 2h --create-namespace --set tls.enabled=false --set console.enabled=false

Then attempted to enable rack awareness:

helm upgrade --install redpanda redpanda --repo https://charts.redpanda.com -n redpanda --wait --timeout 2h --create-namespace --reuse-values --set rackAwareness.enabled=true

Rack awareness wasn't enabled.

What did you expect to happen?

Expected true to be returned:

❯ k -n redpanda exec redpanda-0 -c redpanda -it -- rpk cluster config get enable_rack_awareness
false

How can we reproduce it (as minimally and precisely as possible)?. Please include values file.

COMPUTED VALUES:
affinity: {}
auditLogging:
  clientMaxBufferSize: 16777216
  enabled: false
  enabledEventTypes: null
  excludedPrincipals: null
  excludedTopics: null
  listener: internal
  partitions: 12
  queueDrainIntervalMs: 500
  queueMaxBufferSizePerShard: 1048576
  replicationFactor: null
auth:
  sasl:
    enabled: false
    mechanism: SCRAM-SHA-512
    secretRef: redpanda-users
    users: []
clusterDomain: cluster.local
commonLabels: {}
config:
  cluster:
    default_topic_replications: 3
  node:
    crash_loop_limit: 5
  pandaproxy_client: {}
  rpk: {}
  schema_registry_client: {}
  tunable:
    compacted_log_segment_size: 67108864
    group_topic_partitions: 16
    kafka_batch_max_bytes: 1048576
    kafka_connection_rate_limit: 1000
    log_segment_size: 134217728
    log_segment_size_max: 268435456
    log_segment_size_min: 16777216
    max_compacted_log_segment_size: 536870912
    topic_partitions_per_shard: 1000
connectors:
  auth:
    sasl:
      enabled: false
      mechanism: scram-sha-512
      secretRef: ""
      userName: ""
  commonLabels: {}
  connectors:
    additionalConfiguration: ""
    bootstrapServers: ""
    brokerTLS:
      ca:
        secretNameOverwrite: ""
        secretRef: ""
      cert:
        secretNameOverwrite: ""
        secretRef: ""
      enabled: false
      key:
        secretNameOverwrite: ""
        secretRef: ""
    groupID: connectors-cluster
    producerBatchSize: 131072
    producerLingerMS: 1
    restPort: 8083
    schemaRegistryURL: ""
    secretManager:
      connectorsPrefix: ""
      consolePrefix: ""
      enabled: false
      region: ""
    storage:
      remote:
        read:
          config: false
          offset: false
          status: false
        write:
          config: false
          offset: false
          status: false
      replicationFactor:
        config: -1
        offset: -1
        status: -1
      topic:
        config: _internal_connectors_configs
        offset: _internal_connectors_offsets
        status: _internal_connectors_status
  container:
    javaGCLogEnabled: "false"
    resources:
      javaMaxHeapSize: 2G
      limits:
        cpu: 1
        memory: 2350Mi
      request:
        cpu: 1
        memory: 2350Mi
    securityContext:
      allowPrivilegeEscalation: false
  deployment:
    annotations: {}
    budget:
      maxUnavailable: 1
    create: false
    extraEnv: []
    extraEnvFrom: []
    livenessProbe:
      failureThreshold: 3
      initialDelaySeconds: 10
      periodSeconds: 10
      successThreshold: 1
      timeoutSeconds: 1
    nodeAffinity: {}
    nodeSelector: {}
    podAffinity: {}
    podAntiAffinity:
      custom: {}
      topologyKey: kubernetes.io/hostname
      type: hard
      weight: 100
    priorityClassName: ""
    progressDeadlineSeconds: 600
    readinessProbe:
      failureThreshold: 2
      initialDelaySeconds: 60
      periodSeconds: 10
      successThreshold: 3
      timeoutSeconds: 5
    restartPolicy: Always
    revisionHistoryLimit: 10
    schedulerName: ""
    securityContext:
      fsGroup: 101
      fsGroupChangePolicy: OnRootMismatch
      runAsUser: 101
    strategy:
      type: RollingUpdate
    terminationGracePeriodSeconds: 30
    tolerations: []
    topologySpreadConstraints:
    - maxSkew: 1
      topologyKey: topology.kubernetes.io/zone
      whenUnsatisfiable: ScheduleAnyway
    updateStrategy:
      type: RollingUpdate
  enabled: false
  fullnameOverride: ""
  global: {}
  image:
    pullPolicy: IfNotPresent
    repository: docker.redpanda.com/redpandadata/connectors
    tag: ""
  imagePullSecrets: []
  logging:
    level: warn
  monitoring:
    annotations: {}
    enabled: false
    labels: {}
    namespaceSelector:
      any: true
    scrapeInterval: 30s
  nameOverride: ""
  service:
    annotations: {}
    name: ""
    ports:
    - name: prometheus
      port: 9404
  serviceAccount:
    annotations: {}
    create: false
    name: ""
  storage:
    volume:
    - emptyDir:
        medium: Memory
        sizeLimit: 5Mi
      name: rp-connect-tmp
    volumeMounts:
    - mountPath: /tmp
      name: rp-connect-tmp
  test:
    create: false
  tolerations: []
console:
  config: {}
  configmap:
    create: false
  deployment:
    create: false
  enabled: false
  secret:
    create: false
    enterprise: {}
    kafka: {}
    login:
      github: {}
      google: {}
      jwtSecret: ""
      oidc: {}
      okta: {}
    redpanda:
      adminApi: {}
enterprise:
  license: ""
  licenseSecretRef: {}
external:
  enabled: true
  service:
    enabled: true
  type: NodePort
fullnameOverride: ""
image:
  pullPolicy: IfNotPresent
  repository: docker.redpanda.com/redpandadata/redpanda
  tag: ""
imagePullSecrets: []
license_key: ""
license_secret_ref: {}
listeners:
  admin:
    external:
      default:
        advertisedPorts:
        - 31644
        port: 9645
        tls:
          cert: external
    port: 9644
    tls:
      cert: default
      requireClientAuth: false
  http:
    authenticationMethod: null
    enabled: true
    external:
      default:
        advertisedPorts:
        - 30082
        authenticationMethod: null
        port: 8083
        tls:
          cert: external
          requireClientAuth: false
    kafkaEndpoint: default
    port: 8082
    tls:
      cert: default
      requireClientAuth: false
  kafka:
    authenticationMethod: null
    external:
      default:
        advertisedPorts:
        - 31092
        authenticationMethod: null
        port: 9094
        tls:
          cert: external
    port: 9093
    tls:
      cert: default
      requireClientAuth: false
  rpc:
    port: 33145
    tls:
      cert: default
      requireClientAuth: false
  schemaRegistry:
    authenticationMethod: null
    enabled: true
    external:
      default:
        advertisedPorts:
        - 30081
        authenticationMethod: null
        port: 8084
        tls:
          cert: external
          requireClientAuth: false
    kafkaEndpoint: default
    port: 8081
    tls:
      cert: default
      requireClientAuth: false
logging:
  logLevel: info
  usageStats:
    enabled: true
monitoring:
  enabled: false
  labels: {}
  scrapeInterval: 30s
nameOverride: ""
nodeSelector: {}
post_install_job:
  affinity: {}
  enabled: true
  podTemplate:
    annotations: {}
    labels: {}
    spec:
      containers:
      - env: []
        name: post-install
        securityContext: {}
      securityContext: {}
post_upgrade_job:
  affinity: {}
  enabled: true
  podTemplate:
    annotations: {}
    labels: {}
    spec:
      containers:
      - env: []
        name: post-upgrade
        securityContext: {}
      securityContext: {}
rackAwareness:
  enabled: true
  nodeAnnotation: topology.kubernetes.io/zone
rbac:
  annotations: {}
  enabled: false
resources:
  cpu:
    cores: 1
  memory:
    container:
      max: 2.5Gi
serviceAccount:
  annotations: {}
  create: false
  name: ""
statefulset:
  additionalRedpandaCmdFlags: []
  additionalSelectorLabels: {}
  annotations: {}
  budget:
    maxUnavailable: 1
  extraVolumeMounts: ""
  extraVolumes: ""
  initContainerImage:
    repository: busybox
    tag: latest
  initContainers:
    configurator:
      extraVolumeMounts: ""
      resources: {}
    extraInitContainers: ""
    fsValidator:
      enabled: false
      expectedFS: xfs
      extraVolumeMounts: ""
      resources: {}
    setDataDirOwnership:
      enabled: false
      extraVolumeMounts: ""
      resources: {}
    setTieredStorageCacheDirOwnership:
      extraVolumeMounts: ""
      resources: {}
    tuning:
      extraVolumeMounts: ""
      resources: {}
  livenessProbe:
    failureThreshold: 3
    initialDelaySeconds: 10
    periodSeconds: 10
  nodeSelector: {}
  podAffinity: {}
  podAntiAffinity:
    custom: {}
    topologyKey: kubernetes.io/hostname
    type: hard
    weight: 100
  podTemplate:
    annotations: {}
    labels: {}
    spec:
      containers:
      - env: []
        name: redpanda
        securityContext: {}
      securityContext: {}
  priorityClassName: ""
  readinessProbe:
    failureThreshold: 3
    initialDelaySeconds: 1
    periodSeconds: 10
    successThreshold: 1
  replicas: 3
  securityContext:
    fsGroup: 101
    fsGroupChangePolicy: OnRootMismatch
    runAsUser: 101
  sideCars:
    configWatcher:
      enabled: true
      extraVolumeMounts: ""
      resources: {}
      securityContext: {}
    controllers:
      createRBAC: true
      enabled: false
      healthProbeAddress: :8085
      image:
        repository: docker.redpanda.com/redpandadata/redpanda-operator
        tag: v2.1.10-23.2.18
      metricsAddress: :9082
      resources: {}
      run:
      - all
      securityContext: {}
  startupProbe:
    failureThreshold: 120
    initialDelaySeconds: 1
    periodSeconds: 10
  terminationGracePeriodSeconds: 90
  tolerations: []
  topologySpreadConstraints:
  - maxSkew: 1
    topologyKey: topology.kubernetes.io/zone
    whenUnsatisfiable: ScheduleAnyway
  updateStrategy:
    type: RollingUpdate
storage:
  hostPath: ""
  persistentVolume:
    annotations: {}
    enabled: true
    labels: {}
    nameOverwrite: ""
    size: 20Gi
    storageClass: ""
  tiered:
    config:
      cloud_storage_access_key: ""
      cloud_storage_api_endpoint: ""
      cloud_storage_azure_container: null
      cloud_storage_azure_managed_identity_id: null
      cloud_storage_azure_shared_key: null
      cloud_storage_azure_storage_account: null
      cloud_storage_bucket: ""
      cloud_storage_cache_size: 5368709120
      cloud_storage_credentials_source: config_file
      cloud_storage_enable_remote_read: true
      cloud_storage_enable_remote_write: true
      cloud_storage_enabled: false
      cloud_storage_region: ""
      cloud_storage_secret_key: ""
    credentialsSecretRef:
      accessKey:
        configurationKey: cloud_storage_access_key
      secretKey:
        configurationKey: cloud_storage_secret_key
    hostPath: ""
    mountType: emptyDir
    persistentVolume:
      annotations: {}
      labels: {}
      storageClass: ""
tests:
  enabled: true
tls:
  certs:
    default:
      caEnabled: true
    external:
      caEnabled: true
  enabled: false
tolerations: []
tuning:
  tune_aio_events: true

Chart Version(s)

❯ helm list -A                                                  
NAME            NAMESPACE       REVISION        UPDATED                                 STATUS          CHART           APP VERSION
redpanda        redpanda        2               2024-08-14 16:25:52.398057618 -0500 CDT deployed        redpanda-5.9.0  v24.2.2

Cloud provider

none (kind)

JIRA Link: K8S-319

[vuldin]

Community slack thread: https://redpandacommunity.slack.com/archives/C01AJDUT88N/p1723669069797089?thread_ts=1723669069.797089&cid=C01AJDUT88N