rgw/rgw_file: Fix the incorrect lru object eviction

In func lookup_fh, when RGWFileHandle not be found in fh_cache, it
need to recycle an object and create an new RGWFileHandle. When there
are multi threads use lookup_fh to find and create RGWFileHandle concurrently,
it must to make sure evict lru object from the partiton of fh_cache which new
RGWFileHandle will be inserted to.

Fixes: https://tracker.ceph.com/issues/47235

Signed-off-by: luo rixin <[email protected]>

src/common/cohort_lru.h

@@ -42,6 +42,8 @@ namespace cohort {
 
     typedef bi::link_mode<bi::safe_link> link_mode;
 
+    class ObjectFactory; // Forward declaration
+
     class Object
     {
     private:
@@ -70,7 +72,7 @@ namespace cohort {
 
       uint32_t get_refcnt() const { return lru_refcnt; }
 
-      virtual bool reclaim() = 0;
+      virtual bool reclaim(const ObjectFactory* newobj_fac) = 0;
 
       virtual ~Object() {}
 
@@ -132,7 +134,7 @@ namespace cohort {
 		(!(o->lru_flags & FLAG_EVICTING)));
       }
 
-      Object* evict_block() {
+      Object* evict_block(const ObjectFactory* newobj_fac) {
 	uint32_t lane_ix = next_evict_lane();
 	for (int ix = 0; ix < n_lanes; ++ix,
 	       lane_ix = next_evict_lane()) {
@@ -144,7 +146,7 @@ namespace cohort {
 	    ++(o->lru_refcnt);
 	    o->lru_flags |= FLAG_EVICTING;
 	    lane.lock.unlock();
-	    if (o->reclaim()) {
+	    if (o->reclaim(newobj_fac)) {
 	      lane.lock.lock();
 	      --(o->lru_refcnt);
 	      /* assertions that o state has not changed across
@@ -236,7 +238,7 @@ namespace cohort {
       Object* insert(ObjectFactory* fac, Edge edge, uint32_t& flags) {
 	/* use supplied functor to re-use an evicted object, or
 	 * allocate a new one of the descendant type */
-	Object* o = evict_block();
+	Object* o = evict_block(fac);
 	if (o) {
 	  fac->recycle(o); /* recycle existing object */
 	  flags |= FLAG_RECYCLE;
@@ -425,7 +427,9 @@ namespace cohort {
 	  lat.lock->unlock();
 	return v;
       } /* find_latch */
-
+      bool is_same_partition(uint64_t lhs, uint64_t rhs) {
+        return ((lhs % n_part) == (rhs % n_part));
+      }
       void insert_latched(T* v, Latch& lat, uint32_t flags) {
 	(void) lat.p->tr.insert_unique_commit(*v, lat.commit_data);
 	if (flags & FLAG_UNLOCK)

src/rgw/rgw_file.cc

@@ -1207,10 +1207,19 @@ namespace rgw {
     return dar;
   } /* RGWFileHandle::decode_attrs */
 
-  bool RGWFileHandle::reclaim() {
+  bool RGWFileHandle::reclaim(const cohort::lru::ObjectFactory* newobj_fac) {
     lsubdout(fs->get_context(), rgw, 17)
       << __func__ << " " << *this
       << dendl;
+    auto factory = dynamic_cast<const RGWFileHandle::Factory*>(newobj_fac);
+    if (factory == nullptr) {
+      return false;
+    }
+    /* make sure the reclaiming object is the same partiton with newobject factory,
+     * then we can recycle the object, and replace with newobject */
+    if (!fs->fh_cache.is_same_partition(factory->fhk.fh_hk.object, fh.fh_hk.object)) {
+      return false;
+    }
     /* in the non-delete case, handle may still be in handle table */
     if (fh_hook.is_linked()) {
       /* in this case, we are being called from a context which holds

src/rgw/rgw_file.h

@@ -721,7 +721,7 @@ namespace rgw {
 
     void invalidate();
 
-    bool reclaim() override;
+    bool reclaim(const cohort::lru::ObjectFactory* newobj_fac) override;
 
     typedef cohort::lru::LRU<std::mutex> FhLRU;