Unpublish tasks for non-existent nodes

[MaxRink]

Our clustera are managed by ClusterAPI, so our nodes are treated like throwaway objects and replaced instead of upgraded.
This causes issues with PSO, as PSO tried to create unpublish tasks for non-existent (and already shutdown) nodes

k --context m2m-1.refsa2.bn -n pso-system get pods -o wide
NAME                                                   READY   STATUS             RESTARTS   AGE   IP                NODE                          NOMINATED NODE   READINESS GATES
pso-csi-controller-0                                   5/6     Running            240        26h   192.168.219.111   m2m-1-md-0-8648b7b7b4-grx27   <none>           <none>
pso-csi-node-cqvz4                                     2/3     Running            240        26h   10.23.143.42      m2m-1-md-0-8648b7b7b4-grx27   <none>           <none>
pso-csi-node-fmbjh                                     2/3     Running            240        26h   10.23.143.40      m2m-1-md-0-8648b7b7b4-czgwk   <none>           <none>
pso-csi-node-g6dgb                                     2/3     Running            238        26h   10.23.143.41      m2m-1-md-0-8648b7b7b4-mb6fl   <none>           <none>
pso-csi-node-jctdn                                     2/3     CrashLoopBackOff   242        26h   10.23.143.37      m2m-1-md-0-8648b7b7b4-4pdcv   <none>           <none>
pso-db-cockroach-operator-5d84f4fd79-c9x8g             0/1     CrashLoopBackOff   247        20h   192.168.219.86    m2m-1-md-0-8648b7b7b4-grx27   <none>           <none>
pso-db-deployer-54686dd87f-mfbnf                       1/1     Running            0          26h   192.168.219.106   m2m-1-md-0-8648b7b7b4-grx27   <none>           <none>
unpublish-24742e93-2f6c-3c9a-9739-ca37cd0aa958-pcwl2   0/1     Pending            0          31s   <none>            m2m-1-md-0-7d9658dddb-668gh   <none>           <none>
unpublish-8afcc399-f47f-3cd0-be22-7d433f865f86-jfgmh   0/1     Pending            0          30s   <none>            m2m-1-md-0-7d9658dddb-7zdjk   <none>           <none>
unpublish-8e387b3e-e07a-386d-90fc-d553bd4bcb33-ggdx2   0/1     Pending            0          31s   <none>            m2m-1-md-0-7d9658dddb-7zdjk   <none>           <none>
❯ k --context m2m-1.refsa2.bn -n pso-system get nodes -o wide
NAME                          STATUS   ROLES                  AGE   VERSION    INTERNAL-IP    EXTERNAL-IP   OS-IMAGE             KERNEL-VERSION      CONTAINER-RUNTIME
m2m-1-75p9b                   Ready    control-plane,master   34d   v1.20.13   10.23.143.39   <none>        Ubuntu 20.04.3 LTS   5.11.0-40-generic   containerd://1.5.8
m2m-1-9pbnm                   Ready    control-plane,master   34d   v1.20.13   10.23.143.38   <none>        Ubuntu 20.04.3 LTS   5.11.0-40-generic   containerd://1.5.8
m2m-1-md-0-8648b7b7b4-4pdcv   Ready    <none>                 34d   v1.20.13   10.23.143.37   <none>        Ubuntu 20.04.3 LTS   5.11.0-40-generic   containerd://1.5.8
m2m-1-md-0-8648b7b7b4-czgwk   Ready    <none>                 34d   v1.20.13   10.23.143.40   <none>        Ubuntu 20.04.3 LTS   5.11.0-40-generic   containerd://1.5.8
m2m-1-md-0-8648b7b7b4-grx27   Ready    <none>                 34d   v1.20.13   10.23.143.42   <none>        Ubuntu 20.04.3 LTS   5.11.0-40-generic   containerd://1.5.8
m2m-1-md-0-8648b7b7b4-mb6fl   Ready    <none>                 29d   v1.20.13   10.23.143.41   <none>        Ubuntu 20.04.3 LTS   5.11.0-40-generic   containerd://1.5.8
m2m-1-qg75n                   Ready    control-plane,master   34d   v1.20.13   10.23.143.44   <none>        Ubuntu 20.04.3 LTS   5.11.0-40-generic   containerd://1.5.8

This leads to the cockroach operator failing, thus completely breaking PSO

k --context m2m-1.refsa2.bn -n pso-system logs pso-db-cockroach-operator-5d84f4fd79-c9x8g
{"level":"info","ts":1642074091.331381,"logger":"cmd","msg":"","is_debug_mode":true}
{"level":"info","ts":1642074091.3314283,"logger":"cmd","msg":"Go Version: go1.15.12"}
{"level":"info","ts":1642074091.3314354,"logger":"cmd","msg":"Go OS/Arch: linux/amd64"}
{"level":"info","ts":1642074091.331442,"logger":"cmd","msg":"Version of operator-sdk: v0.6.0"}
{"level":"info","ts":1642074091.3314614,"logger":"cmd","msg":"Setting max startup duration to 600 seconds"}
{"level":"info","ts":1642074091.3314645,"logger":"cmd","msg":"Setting max suspect duration to 3600 seconds"}
{"level":"info","ts":1642074091.3314803,"logger":"cmd","msg":"Did not find config file for affinity"}
{"level":"info","ts":1642074091.331488,"logger":"cmd","msg":"Successfully parsed config file","file":"affinity","value":"&Affinity{NodeAffinity:nil,PodAffinity:nil,PodAntiAffinity:nil,}"}
{"level":"info","ts":1642074091.3315244,"logger":"cmd","msg":"Did not find config file for tolerations"}
{"level":"info","ts":1642074091.3315308,"logger":"cmd","msg":"Successfully parsed config file","file":"tolerations","value":null}
{"level":"info","ts":1642074091.3317494,"logger":"cmd","msg":"Successfully parsed config file","file":"resources","value":"&ResourceRequirements{Limits:ResourceList{cpu: {{4 0} {<nil>} 4 DecimalSI},memory: {{3221225472 0} {<nil>} 3Gi BinarySI},},Requests:ResourceList{cpu: {{3 0} {<nil>} 3 DecimalSI},memory: {{2147483648 0} {<nil>} 2Gi BinarySI},},}"}
{"level":"info","ts":1642074091.3317897,"logger":"cmd","msg":"Did not find config file for nodeSelector"}
{"level":"info","ts":1642074091.331793,"logger":"cmd","msg":"Successfully parsed config file","file":"nodeSelector","value":{}}
{"level":"info","ts":1642074091.331931,"logger":"leader","msg":"Trying to become the leader."}
{"level":"debug","ts":1642074091.3319592,"logger":"k8sutil","msg":"Found namespace","Namespace":"pso-system"}
{"level":"debug","ts":1642074091.4173028,"logger":"k8sutil","msg":"Found podname","Pod.Name":"pso-db-cockroach-operator-5d84f4fd79-c9x8g"}
{"level":"debug","ts":1642074091.4314082,"logger":"k8sutil","msg":"Found Pod","Pod.Namespace":"pso-system","Pod.Name":"pso-db-cockroach-operator-5d84f4fd79-c9x8g"}
{"level":"info","ts":1642074091.4366193,"logger":"leader","msg":"Found existing lock with my name. I was likely restarted."}
{"level":"info","ts":1642074091.4366424,"logger":"leader","msg":"Continuing as the leader."}
{"level":"info","ts":1642074091.5198205,"logger":"cmd","msg":"Registering Components."}
{"level":"info","ts":1642074091.5210316,"logger":"kubebuilder.controller","msg":"Starting EventSource","controller":"intrusion-controller","source":"kind source: /, Kind="}
{"level":"info","ts":1642074091.5212226,"logger":"kubebuilder.controller","msg":"Starting EventSource","controller":"intrusion-controller","source":"kind source: /, Kind="}
{"level":"info","ts":1642074091.521362,"logger":"kubebuilder.controller","msg":"Starting EventSource","controller":"intrusion-controller","source":"kind source: /, Kind="}
{"level":"info","ts":1642074091.5214958,"logger":"kubebuilder.controller","msg":"Starting EventSource","controller":"intrusion-controller","source":"kind source: /, Kind="}
{"level":"info","ts":1642074091.5215948,"logger":"kubebuilder.controller","msg":"Starting EventSource","controller":"intrusion-controller","source":"kind source: /, Kind="}
{"level":"info","ts":1642074091.521678,"logger":"kubebuilder.controller","msg":"Starting EventSource","controller":"intrusion-controller","source":"kind source: /, Kind="}
time="2022-01-13T11:41:31Z" level=info msg="Successfully started watches." reconciler_name=controller_intrusion
{"level":"debug","ts":1642074091.5734723,"logger":"k8sutil","msg":"Found namespace","Namespace":"pso-system"}
{"level":"debug","ts":1642074091.573554,"logger":"k8sutil","msg":"Found podname","Pod.Name":"pso-db-cockroach-operator-5d84f4fd79-c9x8g"}
{"level":"debug","ts":1642074091.5765975,"logger":"k8sutil","msg":"Found Pod","Pod.Namespace":"pso-system","Pod.Name":"pso-db-cockroach-operator-5d84f4fd79-c9x8g"}
{"level":"debug","ts":1642074091.5815158,"logger":"metrics","msg":"Pods owner found","Kind":"Deployment","Name":"pso-db-cockroach-operator","Namespace":"pso-system"}
{"level":"debug","ts":1642074091.6132002,"logger":"metrics","msg":"Metrics Service object updated","Service.Name":"cockroach-operator","Service.Namespace":"pso-system"}
{"level":"info","ts":1642074091.6132355,"logger":"cmd","msg":"Starting the Cmd."}
{"level":"info","ts":1642074091.7134633,"logger":"kubebuilder.controller","msg":"Starting Controller","controller":"intrusion-controller"}
{"level":"info","ts":1642074091.813633,"logger":"kubebuilder.controller","msg":"Starting workers","controller":"intrusion-controller","worker count":1}
time="2022-01-13T11:41:31Z" level=info msg="--------------------- Begin reconcile ---------------------" reconciler_name=controller_intrusion request=pso-system/pso-db
time="2022-01-13T11:41:31Z" level=debug msg="finalizer is already in correct state" finalizer_present=true reconciler_name=controller_intrusion
time="2022-01-13T11:41:31Z" level=debug msg="Beginning to ensure child owners set" reconciler_name=controller_intrusion
time="2022-01-13T11:41:32Z" level=debug msg="Child object owners already set correctly" object_name=pure object_namespace=pso-system reconciler_name=controller_intrusion
time="2022-01-13T11:41:32Z" level=debug msg="Child object owners already set correctly" object_name=pso-db-runner object_namespace=pso-system reconciler_name=controller_intrusion
time="2022-01-13T11:41:32Z" level=debug msg="Child object owners already set correctly" object_name=pso-db-role object_namespace=pso-system reconciler_name=controller_intrusion
time="2022-01-13T11:41:32Z" level=debug msg="Child object owners already set correctly" object_name=pso-cockroach-node-certs object_namespace=pso-system reconciler_name=controller_intrusion
time="2022-01-13T11:41:32Z" level=debug msg="Child object owners already set correctly" object_name=pso-cockroach-client-certs object_namespace=pso-system reconciler_name=controller_intrusion
time="2022-01-13T11:41:32Z" level=debug msg="Child object owners already set correctly" object_name=pso-csi-server-certs object_namespace=pso-system reconciler_name=controller_intrusion
time="2022-01-13T11:41:32Z" level=debug msg="Child object owners already set correctly" object_name=pso-csi-client-certs object_namespace=pso-system reconciler_name=controller_intrusion
time="2022-01-13T11:41:32Z" level=debug msg="unpublish volume job still active" job=unpublish-24742e93-2f6c-3c9a-9739-ca37cd0aa958 reconciler_name=controller_intrusion request=pso-system/pso-db
time="2022-01-13T11:41:32Z" level=debug msg="unpublish volume job still active" job=unpublish-8afcc399-f47f-3cd0-be22-7d433f865f86 reconciler_name=controller_intrusion request=pso-system/pso-db
time="2022-01-13T11:41:32Z" level=debug msg="unpublish volume job still active" job=unpublish-8e387b3e-e07a-386d-90fc-d553bd4bcb33 reconciler_name=controller_intrusion request=pso-system/pso-db
time="2022-01-13T11:41:32Z" level=info msg="waiting to create unpublish volume job because pod still exists" pod=unpublish-8afcc399-f47f-3cd0-be22-7d433f865f86-kntqt reconciler_name=controller_intrusion request=pso-system/pso-db secret=af9064ac-6419-4790-b5f6-d13b5671be23-pso-db-0000000003
time="2022-01-13T11:41:32Z" level=info msg="waiting to create unpublish volume job because pod still exists" pod=unpublish-24742e93-2f6c-3c9a-9739-ca37cd0aa958-mxvmz reconciler_name=controller_intrusion request=pso-system/pso-db secret=af9064ac-6419-4790-b5f6-d13b5671be23-pso-db-0000000004
time="2022-01-13T11:41:32Z" level=info msg="waiting to create unpublish volume job because pod still exists" pod=unpublish-8e387b3e-e07a-386d-90fc-d553bd4bcb33-7xjd4 reconciler_name=controller_intrusion request=pso-system/pso-db secret=af9064ac-6419-4790-b5f6-d13b5671be23-pso-db-0000000005
time="2022-01-13T11:41:32Z" level=info msg="intrusion deleted, waiting for pods, secrets, and volume unpublish jobs to be cleaned up before removing finalizer" pods="[unpublish-24742e93-2f6c-3c9a-9739-ca37cd0aa958-mxvmz unpublish-8afcc399-f47f-3cd0-be22-7d433f865f86-kntqt unpublish-8e387b3e-e07a-386d-90fc-d553bd4bcb33-7xjd4]" reconciler_name=controller_intrusion secrets="[af9064ac-6419-4790-b5f6-d13b5671be23-pso-db-0000000003 af9064ac-6419-4790-b5f6-d13b5671be23-pso-db-0000000004 af9064ac-6419-4790-b5f6-d13b5671be23-pso-db-0000000005]" unpublish_jobs="[{{ } {unpublish-24742e93-2f6c-3c9a-9739-ca37cd0aa958  pso-system  fad82df7-98f4-47bc-a8b6-b5c8ec7ef73f 127437910 0 2022-01-12 09:26:56 +0000 UTC <nil> <nil> map[app:cockroachdb backendId:af9064ac-6419-4790-b5f6-d13b5671be23 intrusion:pso-db task:volumeUnpublish volume:pso-db_0000000004] map[] [{pso.purestorage.com/v1alpha1 Intrusion pso-db de2a5236-40f0-47e0-87b6-a77588077914 0xc000584530 0xc000584531}] nil [] } {0xc000584534 0xc000584538 <nil> 0xc00058453c &LabelSelector{MatchLabels:map[string]string{controller-uid: fad82df7-98f4-47bc-a8b6-b5c8ec7ef73f,},MatchExpressions:[],} <nil> {{unpublish-24742e93-2f6c-3c9a-9739-ca37cd0aa958  pso-system    0 0001-01-01 00:00:00 +0000 UTC <nil> <nil> map[app:cockroachdb backendId:af9064ac-6419-4790-b5f6-d13b5671be23 controller-uid:fad82df7-98f4-47bc-a8b6-b5c8ec7ef73f intrusion:pso-db job-name:unpublish-24742e93-2f6c-3c9a-9739-ca37cd0aa958 task:volumeUnpublish volume:pso-db_0000000004] map[] [] nil [] } {[{dev {&HostPathVolumeSource{Path:/dev,Type:*Directory,} nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil}} {sys {&HostPathVolumeSource{Path:/sys,Type:*Directory,} nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil}} {pure-lock {&HostPathVolumeSource{Path:/run/lock/pure,Type:*DirectoryOrCreate,} nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil}} {pure-json {nil nil nil nil nil &SecretVolumeSource{SecretName:af9064ac-6419-4790-b5f6-d13b5671be23-pso-db-0000000004,Items:[],DefaultMode:*420,Optional:nil,} nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil}}] [] [{unpublish-24742e93-2f6c-3c9a-9739-ca37cd0aa958 docker.io/purestorage/psctl:v1.1.0 [/bin/sh -ecx psctl --debug --namespace m2m_1_refsa2_bn volume unpublish --name pso-db_0000000004 --mountpoint /mnt/pso/db/cockroach/pso-system/pso-db-6/1e70ee6a-0a31-11ec-8196-36f29dfcb92f && nsenter --mount=/proc/1/ns/mnt -- chattr -i /mnt/pso/db/cockroach/pso-system/pso-db-6/1e70ee6a-0a31-11ec-8196-36f29dfcb92f || true && nsenter --mount=/proc/1/ns/mnt -- rmdir /mnt/pso/db/cockroach/pso-system/pso-db-6/1e70ee6a-0a31-11ec-8196-36f29dfcb92f || true] []  [] [] [{PURE_DISCOVERY_CONF /etc/pure/pure.json nil} {PURITY_HOST  &EnvVarSource{FieldRef:&ObjectFieldSelector{APIVersion:v1,FieldPath:spec.nodeName,},ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:nil,}} {PURE_FLASHARRAY_SAN_TYPE ISCSI nil} {PURE_ISCSI_LOGIN_TIMEOUT 20 nil} {PURE_ISCSI_ALLOWED_CIDRS  nil} {PURE_DATASTORE_ENABLED false nil}] {map[] map[]} [{dev false /dev  <nil>} {sys false /sys  <nil>} {pure-lock false /run/lock/pure  <nil>} {pure-json true /etc/pure  <nil>}] [] nil nil nil /dev/termination-log File IfNotPresent &SecurityContext{Capabilities:nil,Privileged:*true,SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,ReadOnlyRootFilesystem:nil,AllowPrivilegeEscalation:nil,RunAsGroup:nil,ProcMount:nil,} false false false}] OnFailure 0xc000584548 <nil> ClusterFirst map[] pure pure <nil> m2m-1-md-0-7d9658dddb-668gh false true false <nil> &PodSecurityContext{SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,SupplementalGroups:[],FSGroup:nil,RunAsGroup:nil,Sysctls:[],} []   nil default-scheduler [{ Exists   <nil>}] []  <nil> nil [] <nil> <nil>}} <nil>} {[] 2022-01-12 09:26:56 +0000 UTC <nil> 1 0 0}} {{ } {unpublish-8afcc399-f47f-3cd0-be22-7d433f865f86  pso-system  d67e1d2d-6117-4194-a12c-6799030b9538 127437937 0 2022-01-12 09:26:56 +0000 UTC <nil> <nil> map[app:cockroachdb backendId:af9064ac-6419-4790-b5f6-d13b5671be23 intrusion:pso-db task:volumeUnpublish volume:pso-db_0000000003] map[] [{pso.purestorage.com/v1alpha1 Intrusion pso-db de2a5236-40f0-47e0-87b6-a77588077914 0xc000584550 0xc000584551}] nil [] } {0xc000584554 0xc000584558 <nil> 0xc00058455c &LabelSelector{MatchLabels:map[string]string{controller-uid: d67e1d2d-6117-4194-a12c-6799030b9538,},MatchExpressions:[],} <nil> {{unpublish-8afcc399-f47f-3cd0-be22-7d433f865f86  pso-system    0 0001-01-01 00:00:00 +0000 UTC <nil> <nil> map[app:cockroachdb backendId:af9064ac-6419-4790-b5f6-d13b5671be23 controller-uid:d67e1d2d-6117-4194-a12c-6799030b9538 intrusion:pso-db job-name:unpublish-8afcc399-f47f-3cd0-be22-7d433f865f86 task:volumeUnpublish volume:pso-db_0000000003] map[] [] nil [] } {[{dev {&HostPathVolumeSource{Path:/dev,Type:*Directory,} nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil}} {sys {&HostPathVolumeSource{Path:/sys,Type:*Directory,} nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil}} {pure-lock {&HostPathVolumeSource{Path:/run/lock/pure,Type:*DirectoryOrCreate,} nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil}} {pure-json {nil nil nil nil nil &SecretVolumeSource{SecretName:af9064ac-6419-4790-b5f6-d13b5671be23-pso-db-0000000003,Items:[],DefaultMode:*420,Optional:nil,} nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil}}] [] [{unpublish-8afcc399-f47f-3cd0-be22-7d433f865f86 docker.io/purestorage/psctl:v1.1.0 [/bin/sh -ecx psctl --debug --namespace m2m_1_refsa2_bn volume unpublish --name pso-db_0000000003 --mountpoint /mnt/pso/db/cockroach/pso-system/pso-db-5/1e667563-0a31-11ec-8196-36f29dfcb92f && nsenter --mount=/proc/1/ns/mnt -- chattr -i /mnt/pso/db/cockroach/pso-system/pso-db-5/1e667563-0a31-11ec-8196-36f29dfcb92f || true && nsenter --mount=/proc/1/ns/mnt -- rmdir /mnt/pso/db/cockroach/pso-system/pso-db-5/1e667563-0a31-11ec-8196-36f29dfcb92f || true] []  [] [] [{PURE_DISCOVERY_CONF /etc/pure/pure.json nil} {PURITY_HOST  &EnvVarSource{FieldRef:&ObjectFieldSelector{APIVersion:v1,FieldPath:spec.nodeName,},ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:nil,}} {PURE_FLASHARRAY_SAN_TYPE ISCSI nil} {PURE_ISCSI_LOGIN_TIMEOUT 20 nil} {PURE_ISCSI_ALLOWED_CIDRS  nil} {PURE_DATASTORE_ENABLED false nil}] {map[] map[]} [{dev false /dev  <nil>} {sys false /sys  <nil>} {pure-lock false /run/lock/pure  <nil>} {pure-json true /etc/pure  <nil>}] [] nil nil nil /dev/termination-log File IfNotPresent &SecurityContext{Capabilities:nil,Privileged:*true,SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,ReadOnlyRootFilesystem:nil,AllowPrivilegeEscalation:nil,RunAsGroup:nil,ProcMount:nil,} false false false}] OnFailure 0xc000584568 <nil> ClusterFirst map[] pure pure <nil> m2m-1-md-0-7d9658dddb-7zdjk false true false <nil> &PodSecurityContext{SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,SupplementalGroups:[],FSGroup:nil,RunAsGroup:nil,Sysctls:[],} []   nil default-scheduler [{ Exists   <nil>}] []  <nil> nil [] <nil> <nil>}} <nil>} {[] 2022-01-12 09:26:56 +0000 UTC <nil> 1 0 0}} {{ } {unpublish-8e387b3e-e07a-386d-90fc-d553bd4bcb33  pso-system  d57d92f5-fc07-442f-a841-8c7b1ee43ec1 127436090 0 2022-01-12 09:26:56 +0000 UTC <nil> <nil> map[app:cockroachdb backendId:af9064ac-6419-4790-b5f6-d13b5671be23 intrusion:pso-db task:volumeUnpublish volume:pso-db_0000000005] map[] [{pso.purestorage.com/v1alpha1 Intrusion pso-db de2a5236-40f0-47e0-87b6-a77588077914 0xc000584570 0xc000584571}] nil [] } {0xc000584574 0xc000584578 <nil> 0xc00058457c &LabelSelector{MatchLabels:map[string]string{controller-uid: d57d92f5-fc07-442f-a841-8c7b1ee43ec1,},MatchExpressions:[],} <nil> {{unpublish-8e387b3e-e07a-386d-90fc-d553bd4bcb33  pso-system    0 0001-01-01 00:00:00 +0000 UTC <nil> <nil> map[app:cockroachdb backendId:af9064ac-6419-4790-b5f6-d13b5671be23 controller-uid:d57d92f5-fc07-442f-a841-8c7b1ee43ec1 intrusion:pso-db job-name:unpublish-8e387b3e-e07a-386d-90fc-d553bd4bcb33 task:volumeUnpublish volume:pso-db_0000000005] map[] [] nil [] } {[{dev {&HostPathVolumeSource{Path:/dev,Type:*Directory,} nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil}} {sys {&HostPathVolumeSource{Path:/sys,Type:*Directory,} nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil}} {pure-lock {&HostPathVolumeSource{Path:/run/lock/pure,Type:*DirectoryOrCreate,} nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil}} {pure-json {nil nil nil nil nil &SecretVolumeSource{SecretName:af9064ac-6419-4790-b5f6-d13b5671be23-pso-db-0000000005,Items:[],DefaultMode:*420,Optional:nil,} nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil}}] [] [{unpublish-8e387b3e-e07a-386d-90fc-d553bd4bcb33 docker.io/purestorage/psctl:v1.1.0 [/bin/sh -ecx psctl --debug --namespace m2m_1_refsa2_bn volume unpublish --name pso-db_0000000005 --mountpoint /mnt/pso/db/cockroach/pso-system/pso-db-7/1db4c9f9-0a31-11ec-8196-36f29dfcb92f && nsenter --mount=/proc/1/ns/mnt -- chattr -i /mnt/pso/db/cockroach/pso-system/pso-db-7/1db4c9f9-0a31-11ec-8196-36f29dfcb92f || true && nsenter --mount=/proc/1/ns/mnt -- rmdir /mnt/pso/db/cockroach/pso-system/pso-db-7/1db4c9f9-0a31-11ec-8196-36f29dfcb92f || true] []  [] [] [{PURE_DISCOVERY_CONF /etc/pure/pure.json nil} {PURITY_HOST  &EnvVarSource{FieldRef:&ObjectFieldSelector{APIVersion:v1,FieldPath:spec.nodeName,},ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:nil,}} {PURE_FLASHARRAY_SAN_TYPE ISCSI nil} {PURE_ISCSI_LOGIN_TIMEOUT 20 nil} {PURE_ISCSI_ALLOWED_CIDRS  nil} {PURE_DATASTORE_ENABLED false nil}] {map[] map[]} [{dev false /dev  <nil>} {sys false /sys  <nil>} {pure-lock false /run/lock/pure  <nil>} {pure-json true /etc/pure  <nil>}] [] nil nil nil /dev/termination-log File IfNotPresent &SecurityContext{Capabilities:nil,Privileged:*true,SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,ReadOnlyRootFilesystem:nil,AllowPrivilegeEscalation:nil,RunAsGroup:nil,ProcMount:nil,} false false false}] OnFailure 0xc000584588 <nil> ClusterFirst map[] pure pure <nil> m2m-1-md-0-7d9658dddb-7zdjk false true false <nil> &PodSecurityContext{SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,SupplementalGroups:[],FSGroup:nil,RunAsGroup:nil,Sysctls:[],} []   nil default-scheduler [{ Exists   <nil>}] []  <nil> nil [] <nil> <nil>}} <nil>} {[] 2022-01-12 09:26:56 +0000 UTC <nil> 1 0 0}}]"
E0113 11:41:32.216656       1 runtime.go:69] Observed a panic: "invalid memory address or nil pointer dereference" (runtime error: invalid memory address or nil pointer dereference)
/go/src/pso.purestorage.com/cockroach-operator/vendor/k8s.io/apimachinery/pkg/util/runtime/runtime.go:76
/go/src/pso.purestorage.com/cockroach-operator/vendor/k8s.io/apimachinery/pkg/util/runtime/runtime.go:65
/go/src/pso.purestorage.com/cockroach-operator/vendor/k8s.io/apimachinery/pkg/util/runtime/runtime.go:51
/usr/local/go/src/runtime/panic.go:969
/usr/local/go/src/runtime/panic.go:212
/usr/local/go/src/runtime/signal_unix.go:742
/go/src/pso.purestorage.com/cockroach-operator/pkg/apis/pso/v1alpha1/intrusion_types.go:232
/go/src/pso.purestorage.com/cockroach-operator/pkg/controller/cockroach/intrusion/intrusion_controller.go:837
/go/src/pso.purestorage.com/cockroach-operator/pkg/controller/cockroach/intrusion/intrusion_controller.go:776
/go/src/pso.purestorage.com/cockroach-operator/pkg/controller/cockroach/intrusion/intrusion_controller.go:602
/go/src/pso.purestorage.com/cockroach-operator/pkg/controller/cockroach/intrusion/cluster.go:719
/go/src/pso.purestorage.com/cockroach-operator/pkg/controller/cockroach/intrusion/intrusion_controller.go:429
/go/src/pso.purestorage.com/cockroach-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:215
/go/src/pso.purestorage.com/cockroach-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:158
/go/src/pso.purestorage.com/cockroach-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133
/go/src/pso.purestorage.com/cockroach-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:134
/go/src/pso.purestorage.com/cockroach-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
/usr/local/go/src/runtime/asm_amd64.s:1374
panic: runtime error: invalid memory address or nil pointer dereference [recovered]
	panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x0 pc=0xf6b5a7]

goroutine 638 [running]:
k8s.io/apimachinery/pkg/util/runtime.HandleCrash(0x0, 0x0, 0x0)
	/go/src/pso.purestorage.com/cockroach-operator/vendor/k8s.io/apimachinery/pkg/util/runtime/runtime.go:58 +0x10c
panic(0x161fd00, 0x22731b0)
	/usr/local/go/src/runtime/panic.go:969 +0x1b9
pso.purestorage.com/cockroach-operator/pkg/apis/pso/v1alpha1.GetNodeIdentifier(0xc000584c10, 0xa, 0x0, 0xc0005c15a0, 0x11, 0xc000626ca8, 0x4d1792)
	/go/src/pso.purestorage.com/cockroach-operator/pkg/apis/pso/v1alpha1/intrusion_types.go:232 +0x67
pso.purestorage.com/cockroach-operator/pkg/controller/cockroach/intrusion.getReplicaStatusMap(0xc0013604b0, 0x5, 0x5, 0x233d098)
	/go/src/pso.purestorage.com/cockroach-operator/pkg/controller/cockroach/intrusion/intrusion_controller.go:837 +0xf4
pso.purestorage.com/cockroach-operator/pkg/controller/cockroach/intrusion.(*ReconcileIntrusion).getReplicaStatuses(0xc0009096c0, 0xc000296500, 0x230ada0, 0x17ee38b, 0x19)
	/go/src/pso.purestorage.com/cockroach-operator/pkg/controller/cockroach/intrusion/intrusion_controller.go:776 +0xa6
pso.purestorage.com/cockroach-operator/pkg/controller/cockroach/intrusion.(*ReconcileIntrusion).updateStatus(0xc0009096c0, 0xc000296500, 0x1, 0x1)
	/go/src/pso.purestorage.com/cockroach-operator/pkg/controller/cockroach/intrusion/intrusion_controller.go:602 +0x2bd
pso.purestorage.com/cockroach-operator/pkg/controller/cockroach/intrusion.(*ReconcileIntrusion).cleanupDeletedIntrusion(0xc0009096c0, 0xc000296500, 0x0, 0x0, 0x0)
	/go/src/pso.purestorage.com/cockroach-operator/pkg/controller/cockroach/intrusion/cluster.go:719 +0x1325
pso.purestorage.com/cockroach-operator/pkg/controller/cockroach/intrusion.(*ReconcileIntrusion).Reconcile(0xc0009096c0, 0xc0005a2850, 0xa, 0xc0005a28ca, 0x6, 0xc000ca9578, 0x8, 0xc000f1a600, 0xc000fa4238)
	/go/src/pso.purestorage.com/cockroach-operator/pkg/controller/cockroach/intrusion/intrusion_controller.go:429 +0x2935
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem(0xc0002fe000, 0x0)
	/go/src/pso.purestorage.com/cockroach-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:215 +0x1de
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func1()
	/go/src/pso.purestorage.com/cockroach-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:158 +0x36
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc000f179e0)
	/go/src/pso.purestorage.com/cockroach-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133 +0x5f
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc000f179e0, 0x3b9aca00, 0x0, 0x100000000000001, 0xc000f1a0c0)
	/go/src/pso.purestorage.com/cockroach-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:134 +0x105
k8s.io/apimachinery/pkg/util/wait.Until(0xc000f179e0, 0x3b9aca00, 0xc000f1a0c0)
	/go/src/pso.purestorage.com/cockroach-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88 +0x4d
created by sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start
	/go/src/pso.purestorage.com/cockroach-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:157 +0x331