From de11c1815a567d9542b4ab74b2dc20d2804dbd05 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=89rico=20Nogueira?= <erico.rolim@lnls.br>
Date: Mon, 23 Sep 2024 15:46:28 -0300
Subject: [PATCH] util: add file locking for PCIe access.

This way, only one application using the library can access a given
device at a time.

We use a file descriptor pointing to BAR0 (via the resource0 file)
because it's the first one to be opened (i.e. we fail early), and
because it makes the most sense semantically: the issue with sharing a
device between processes is that they will need to access different page
indexes, which are controlled by BAR0.

We chose to use Open File Description (OFD) locking because it has more
predictable behavior [1] and is POSIX, unlike the flock(2) API.

No locking is added for the serial mode because it's not used in
production, and it's believed that the kernel will already report the
device as busy when another process has already opened it.

[1] https://lwn.net/Articles/586904/
---
 util/pcie-defs.h  |  3 +++
 util/pcie-open.cc | 29 +++++++++++++++++++++++++++++
 2 files changed, 32 insertions(+)

diff --git a/util/pcie-defs.h b/util/pcie-defs.h
index 04c9ebd6..ce114ba8 100644
--- a/util/pcie-defs.h
+++ b/util/pcie-defs.h
@@ -31,6 +31,9 @@ struct pcie_bars {
     pthread_mutex_t locks[NUM_LOCKS];
 
     FILE *fserport;
+
+    /* fd holding OFD-owned lock for BAR resource files */
+    int fd_lock;
 };
 
 #endif
diff --git a/util/pcie-open.cc b/util/pcie-open.cc
index 744db2a6..e56813e4 100644
--- a/util/pcie-open.cc
+++ b/util/pcie-open.cc
@@ -28,6 +28,27 @@ void configure_mutexes(struct pcie_bars &bars)
             throw std::runtime_error("couldn't initialize mutex");
 }
 
+void lock_file(int fd)
+{
+    struct flock arg = {
+        /* exclusive lock */
+        .l_type = F_WRLCK,
+        /* lock the whole file */
+        .l_whence = SEEK_SET,
+        .l_start = 0,
+        .l_len = 0,
+        /* application is resposible for setting to 0 */
+        .l_pid = 0,
+    };
+    if (fcntl(fd, F_OFD_SETLK, &arg) < 0) {
+        /* allow program to work when F_OFD_SETLK isn't implemented */
+        if (errno == EINVAL)
+            fputs("WARNING: F_OFD_SETLK not supported!\n", stderr);
+        else
+            throw std::runtime_error(std::string("couldn't lock file"));
+    }
+}
+
 }
 
 void dev_open_slot(struct pcie_bars &bars, int slot)
@@ -69,6 +90,12 @@ void dev_open(struct pcie_bars &bars, const char *pci_address)
             if (fd < 0)
                 throw std::runtime_error(std::string("couldn't open resource file: ") + pci_address);
         }
+
+        if (bar == 0) {
+            lock_file(fd);
+            bars.fd_lock = fd;
+        }
+
         struct stat st;
         fstat(fd, &st);
         bars.sizes[i] = st.st_size;
@@ -130,6 +157,8 @@ void dev_close(struct pcie_bars &bars)
         return;
     }
 
+    close(bars.fd_lock);
+
     auto unmap = [&bars](auto &p, unsigned i) {
         munmap(const_cast<void *>(p), bars.sizes[i]);
         p = 0;