mds/quiesce: use LocalLock for iquiesce and a caps mask for a lock-fr…

…ee quiesce

qa/tasks/cephfs/test_quiesce.py

@@ -188,14 +188,6 @@ def _verify_quiesce(self, rank=0, root=None, splitauth=False):
         visited = set()
         locks_expected = set([
           "iquiesce",
-          "isnap",
-          "ipolicy",
-          "ifile",
-          "inest",
-          "idft",
-          "iauth",
-          "ilink",
-          "ixattr",
         ])
         for inode in cache:
             ino = inode['ino']
@@ -212,21 +204,9 @@ def _verify_quiesce(self, rank=0, root=None, splitauth=False):
                 for lock in op['type_data']['locks']:
                     lock_type = lock['lock']['type']
                     if lock_type == "iquiesce":
-                        if ino == root_ino:
-                            self.assertEqual(lock['flags'], 1)
-                            self.assertEqual(lock['lock']['state'], 'sync')
-                        else:
-                            self.assertEqual(lock['flags'], 4)
-                            self.assertEqual(lock['lock']['state'], 'xlock')
-                    elif lock_type == "isnap":
-                        self.assertEqual(lock['flags'], 1)
-                        self.assertEqual(lock['lock']['state'][:4], 'sync')
-                    elif lock_type == "ifile":
-                        self.assertEqual(lock['flags'], 1)
-                        self.assertEqual(lock['lock']['state'][:4], 'sync')
-                    elif lock_type in ("ipolicy", "inest", "idft", "iauth", "ilink", "ixattr"):
-                        self.assertEqual(lock['flags'], 1)
-                        self.assertEqual(lock['lock']['state'][:4], 'sync')
+                        self.assertEqual(lock['flags'], 4)
+                        self.assertEqual(lock['lock']['state'], 'lock')
+                        self.assertEqual(lock['lock']['num_xlocks'], 1)
                     else:
                         # no iflock
                         self.assertFalse(lock_type.startswith("i"))

src/mds/CInode.cc

@@ -2882,6 +2882,7 @@ bool CInode::freeze_inode(int auth_pin_allowance)
   const static int lock_types[] = {
     CEPH_LOCK_IVERSION, CEPH_LOCK_IFILE, CEPH_LOCK_IAUTH, CEPH_LOCK_ILINK, CEPH_LOCK_IDFT,
     CEPH_LOCK_IXATTR, CEPH_LOCK_ISNAP, CEPH_LOCK_INEST, CEPH_LOCK_IFLOCK, CEPH_LOCK_IPOLICY, 0
+    //TODO: add iquiesce here?
   };
   for (int i = 0; lock_types[i]; ++i) {
     auto lock = get_lock(lock_types[i]);
@@ -3532,13 +3533,23 @@ void CInode::export_client_caps(map<client_t,Capability::Export>& cl)
   }
 }
 
+int CInode::get_caps_quiesce_mask() const
+{
+  if (is_quiesced()) {
+    // what we allow to our clients for a quiesced node
+    return CEPH_CAP_ANY_RD | CEPH_CAP_FILE_BUFFER | CEPH_CAP_PIN;
+  } else {
+    return CEPH_CAP_ANY;
+  }
+}
+
   // caps allowed
 int CInode::get_caps_liked() const
 {
   if (is_dir())
-    return CEPH_CAP_PIN | CEPH_CAP_ANY_EXCL | CEPH_CAP_ANY_SHARED;  // but not, say, FILE_RD|WR|WRBUFFER
+    return get_caps_quiesce_mask() & (CEPH_CAP_PIN | CEPH_CAP_ANY_EXCL | CEPH_CAP_ANY_SHARED); // but not, say, FILE_RD|WR|WRBUFFER
   else
-    return CEPH_CAP_ANY & ~CEPH_CAP_FILE_LAZYIO;
+    return get_caps_quiesce_mask() & (CEPH_CAP_ANY & ~CEPH_CAP_FILE_LAZYIO);
 }
 
 int CInode::get_caps_allowed_ever() const
@@ -3558,30 +3569,33 @@ int CInode::get_caps_allowed_ever() const
 
 int CInode::get_caps_allowed_by_type(int type) const
 {
-  return 
+  return get_caps_quiesce_mask() & (
     CEPH_CAP_PIN |
     (filelock.gcaps_allowed(type) << filelock.get_cap_shift()) |
     (authlock.gcaps_allowed(type) << authlock.get_cap_shift()) |
     (xattrlock.gcaps_allowed(type) << xattrlock.get_cap_shift()) |
-    (linklock.gcaps_allowed(type) << linklock.get_cap_shift());
+    (linklock.gcaps_allowed(type) << linklock.get_cap_shift())
+  );
 }
 
 int CInode::get_caps_careful() const
 {
-  return 
+  return get_caps_quiesce_mask() & (
     (filelock.gcaps_careful() << filelock.get_cap_shift()) |
     (authlock.gcaps_careful() << authlock.get_cap_shift()) |
     (xattrlock.gcaps_careful() << xattrlock.get_cap_shift()) |
-    (linklock.gcaps_careful() << linklock.get_cap_shift());
+    (linklock.gcaps_careful() << linklock.get_cap_shift())
+  );
 }
 
 int CInode::get_xlocker_mask(client_t client) const
 {
-  return 
+  return get_caps_quiesce_mask() & (
     (filelock.gcaps_xlocker_mask(client) << filelock.get_cap_shift()) |
     (authlock.gcaps_xlocker_mask(client) << authlock.get_cap_shift()) |
     (xattrlock.gcaps_xlocker_mask(client) << xattrlock.get_cap_shift()) |
-    (linklock.gcaps_xlocker_mask(client) << linklock.get_cap_shift());
+    (linklock.gcaps_xlocker_mask(client) << linklock.get_cap_shift())
+  );
 }
 
 int CInode::get_caps_allowed_for_client(Session *session, Capability *cap,
@@ -3679,6 +3693,14 @@ int CInode::get_caps_wanted(int *ploner, int *pother, int shift, int mask) const
       other |= p.second;
       //cout << " get_caps_wanted mds " << it->first << " " << cap_string(it->second) << endl;
     }
+
+  // we adjust wanted caps to prevent unnecessary lock transitions
+  // don't worry, when the quiesce lock is dropped
+  // the whole thing will get evaluated again, with a fixed mask
+//   loner &= get_caps_quiesce_mask();
+//   other &= get_caps_quiesce_mask();
+//   w &= get_caps_quiesce_mask();
+
   if (ploner) *ploner = (loner >> shift) & mask;
   if (pother) *pother = (other >> shift) & mask;
   return (w >> shift) & mask;

src/mds/CInode.h

@@ -661,6 +661,7 @@ class CInode : public MDSCacheObject, public InodeStoreBase, public Counter<CIno
   bool is_file() const    { return get_inode()->is_file(); }
   bool is_symlink() const { return get_inode()->is_symlink(); }
   bool is_dir() const     { return get_inode()->is_dir(); }
+  bool is_quiesced() const { return quiescelock.is_xlocked(); }
 
   bool is_head() const { return last == CEPH_NOSNAP; }
 
@@ -866,6 +867,8 @@ class CInode : public MDSCacheObject, public InodeStoreBase, public Counter<CIno
   int count_nonstale_caps();
   bool multiple_nonstale_caps();
 
+  int get_caps_quiesce_mask() const;
+
   bool is_any_caps() { return !client_caps.empty(); }
   bool is_any_nonstale_caps() { return count_nonstale_caps(); }
 
@@ -1105,7 +1108,7 @@ class CInode : public MDSCacheObject, public InodeStoreBase, public Counter<CIno
    * quiescelock.
    */
 
-  SimpleLock quiescelock; // FIXME not part of mempool
+  LocalLockC quiescelock; // FIXME not part of mempool
   LocalLockC versionlock; // FIXME not part of mempool
   SimpleLock authlock; // FIXME not part of mempool
   SimpleLock linklock; // FIXME not part of mempool