Prefetch next inode number (#5130)

Signed-off-by: Changxin Miao <[email protected]>

pkg/meta/base.go

@@ -46,8 +46,11 @@ const (
 	maxSymCacheNum = int32(10000)
 )
 
-var maxCompactSlices = 1000
-var maxSlices = 2500
+var (
+	maxCompactSlices  = 1000
+	maxSlices         = 2500
+	inodeNeedPrefetch = uint64(utils.JitterIt(inodeBatch * 0.1)) // Add jitter to reduce probability of txn conflicts
+)
 
 type engine interface {
 	// Get the value of counter name.
@@ -217,9 +220,11 @@ type baseMeta struct {
 	dirParents map[Ino]Ino    // directory inode -> parent inode
 	dirQuotas  map[Ino]*Quota // directory inode -> quota
 
-	freeMu     sync.Mutex
-	freeInodes freeID
-	freeSlices freeID
+	freeMu           sync.Mutex
+	freeInodes       freeID
+	freeSlices       freeID
+	prefetchMu       sync.Mutex
+	prefetchedInodes freeID
 
 	usedSpaceG  prometheus.Gauge
 	usedInodesG prometheus.Gauge
@@ -991,22 +996,56 @@ func (m *baseMeta) nextInode() (Ino, error) {
 	m.freeMu.Lock()
 	defer m.freeMu.Unlock()
 	if m.freeInodes.next >= m.freeInodes.maxid {
-		v, err := m.en.incrCounter("nextInode", inodeBatch)
-		if err != nil {
-			return 0, err
+
+		m.prefetchMu.Lock() // Wait until prefetchInodes() is done
+		if m.prefetchedInodes.maxid > m.freeInodes.maxid {
+			m.freeInodes = m.prefetchedInodes
+			m.prefetchedInodes = freeID{}
+		}
+		m.prefetchMu.Unlock()
+
+		if m.freeInodes.next >= m.freeInodes.maxid { // Prefetch missed, try again
+			nextInodes, err := m.allocateInodes()
+			if err != nil {
+				return 0, err
+			}
+			m.freeInodes = nextInodes
 		}
-		m.freeInodes.next = uint64(v) - inodeBatch
-		m.freeInodes.maxid = uint64(v)
 	}
 	n := m.freeInodes.next
 	m.freeInodes.next++
 	for n <= 1 {
 		n = m.freeInodes.next
 		m.freeInodes.next++
 	}
+	if m.freeInodes.maxid-m.freeInodes.next == inodeNeedPrefetch {
+		go m.prefetchInodes()
+	}
 	return Ino(n), nil
 }
 
+func (m *baseMeta) prefetchInodes() {
+	m.prefetchMu.Lock()
+	defer m.prefetchMu.Unlock()
+	if m.prefetchedInodes.maxid > m.freeInodes.maxid {
+		return // Someone else has done the job
+	}
+	nextInodes, err := m.allocateInodes()
+	if err == nil {
+		m.prefetchedInodes = nextInodes
+	} else {
+		logger.Warnf("Failed to prefetch inodes: %s, current limit: %d", err, m.freeInodes.maxid)
+	}
+}
+
+func (m *baseMeta) allocateInodes() (freeID, error) {
+	v, err := m.en.incrCounter("nextInode", inodeBatch)
+	if err != nil {
+		return freeID{}, err
+	}
+	return freeID{next: uint64(v) - inodeBatch, maxid: uint64(v)}, nil
+}
+
 func (m *baseMeta) Mknod(ctx Context, parent Ino, name string, _type uint8, mode, cumask uint16, rdev uint32, path string, inode *Ino, attr *Attr) syscall.Errno {
 	if _type < TypeFile || _type > TypeSocket {
 		return syscall.EINVAL

pkg/utils/general.go

@@ -22,6 +22,10 @@ import (
 )
 
 func SleepWithJitter(d time.Duration) {
+	time.Sleep(d + JitterIt(d))
+}
+
+func JitterIt[T float64 | time.Duration](d T) T {
 	j := int64(d / 20) // +- 5%
-	time.Sleep(d + time.Duration(rand.Int63n(2*j+1)-j))
+	return d + T(rand.Int63n(2*j+1)-j)
 }