add validation of the map

Signed-off-by: Leonid Chernin <[email protected]>
ceph · Apr 18, 2024 · 78ac9ff · 78ac9ff
1 parent 1c8eaae
commit 78ac9ff
Show file tree

Hide file tree

Showing 3 changed files with 37 additions and 0 deletions.
diff --git a/src/mon/NVMeofGwMap.cc b/src/mon/NVMeofGwMap.cc
@@ -109,6 +109,7 @@ int NVMeofGwMap::process_gw_map_gw_down(const NvmeGwId &gw_id, const NvmeGroupKe
             st.standby_state(i);
         }
         propose_pending = true; // map should reflect that gw becames unavailable
+        if (propose_pending) validate_gw_map(group_key);
     }
     else {
         dout(1)  << __FUNCTION__ << "ERROR GW-id was not found in the map " << gw_id << dendl;
@@ -152,6 +153,7 @@ void NVMeofGwMap::process_gw_map_ka(const NvmeGwId &gw_id, const NvmeGroupKey& g
           fsm_handle_gw_alive (gw_id, group_key, gw_state->second, st.sm_state[i], i, last_osd_epoch, propose_pending);
         }
     }
+    if (propose_pending) validate_gw_map(group_key);
 }
 
 
@@ -192,6 +194,9 @@ void NVMeofGwMap::handle_abandoned_ana_groups(bool& propose)
                     find_failback_gw(gw_id, group_key, propose);
                 }
             }
+            if (propose) {
+                validate_gw_map(group_key);
+            }
     }
 }
 
@@ -453,6 +458,7 @@ void NVMeofGwMap::fsm_handle_gw_delete (const NvmeGwId &gw_id, const NvmeGroupKe
             ceph_assert(false);
         }
     }
+    if (map_modified) validate_gw_map(group_key);
 }
 
 void NVMeofGwMap::fsm_handle_to_expired(const NvmeGwId &gw_id, const NvmeGroupKey& group_key, NvmeAnaGrpId grpid,  bool &map_modified)
@@ -494,6 +500,7 @@ void NVMeofGwMap::fsm_handle_to_expired(const NvmeGwId &gw_id, const NvmeGroupKe
         dout(1) << " Expired GW_WAIT_FAILOVER_PREPARED timer from GW " << gw_id << " ANA groupId: "<< grpid << dendl;
         ceph_assert(false);
     }
+    if (map_modified) validate_gw_map(group_key);
 }
 
 NvmeGwCreated& NVMeofGwMap::find_already_created_gw(const NvmeGwId &gw_id, const NvmeGroupKey& group_key)
@@ -555,6 +562,34 @@ int NVMeofGwMap::blocklist_gw(const NvmeGwId &gw_id, const NvmeGroupKey& group_k
     return 0;
 }
 
+void  NVMeofGwMap::validate_gw_map(const NvmeGroupKey& group_key)
+{
+   NvmeAnaGrpId anas[MAX_SUPPORTED_ANA_GROUPS];
+   int i = 0;
+   int max_groups = 0;
+   for (auto& gw_created_pair: Created_gws[group_key]) {
+        auto& st = gw_created_pair.second;
+        anas[i++] = st.ana_grp_id;
+   }
+   max_groups = i;
+   for(int i = 0; i < max_groups; i++)
+   {
+       int ana_group = anas[i];
+       int count = 0;
+       for (auto& gw_created_pair: Created_gws[group_key]) {
+           auto& st = gw_created_pair.second;
+           if (st.sm_state[ana_group] == GW_STATES_PER_AGROUP_E::GW_ACTIVE_STATE){
+               count ++;
+               if(count == 2) {
+                   dout(1) << "number active states per ana-group " << ana_group << "more than 1 in pool-group " << group_key << dendl;
+                   dout(1) << Created_gws[group_key] << dendl;
+                   ceph_assert(false);
+               }
+           }
+       }
+   }
+}
+
 void NVMeofGwMap::update_active_timers( bool &propose_pending ){
 
     //dout(4) << __func__  <<  " called,  p_monitor: " << mon << dendl;

diff --git a/src/mon/NVMeofGwMap.h b/src/mon/NVMeofGwMap.h
@@ -64,6 +64,7 @@ class NVMeofGwMap
 
     int  get_timer   (const NvmeGwId &gw_id, const NvmeGroupKey& group_key, NvmeAnaGrpId anagrpid);
     void cancel_timer(const NvmeGwId &gw_id, const NvmeGroupKey& group_key, NvmeAnaGrpId anagrpid);
+    void validate_gw_map(const NvmeGroupKey& group_key);
 
 public:
     void encode(ceph::buffer::list &bl) const {

diff --git a/src/mon/NVMeofGwMon.cc b/src/mon/NVMeofGwMon.cc
@@ -332,6 +332,7 @@ bool NVMeofGwMon::prepare_command(MonOpRequestRef op)
 
         if( map.Created_gws[group_key].size()){
             f->open_object_section("common");
+            f->dump_unsigned("epoch", map.epoch);
             f->dump_string("pool", pool);
             f->dump_string("group", group);
             f->dump_unsigned("num gws", map.Created_gws[group_key].size());