util: add file locking for PCIe access.

This way, only one application using the library can access a given
device at a time.

We use a file descriptor pointing to BAR0 (via the resource0 file)
because it's the first one to be opened (i.e. we fail early), and
because it makes the most sense semantically: the issue with sharing a
device between processes is that they will need to access different page
indexes, which are controlled by BAR0.

We chose to use Open File Description (OFD) locking because it has more
predictable behavior [1] and is POSIX, unlike the flock(2) API.

No locking is added for the serial mode because it's not used in
production, and it's believed that the kernel will already report the
device as busy when another process has already opened it.

[1] https://lwn.net/Articles/586904/

util/pcie-defs.h

@@ -31,6 +31,9 @@ struct pcie_bars {
     pthread_mutex_t locks[NUM_LOCKS];
 
     FILE *fserport;
+
+    /* fd holding OFD-owned lock for BAR resource files */
+    int fd_lock;
 };
 
 #endif

util/pcie-open.cc

@@ -28,6 +28,27 @@ void configure_mutexes(struct pcie_bars &bars)
             throw std::runtime_error("couldn't initialize mutex");
 }
 
+void lock_file(int fd)
+{
+    struct flock arg = {
+        /* exclusive lock */
+        .l_type = F_WRLCK,
+        /* lock the whole file */
+        .l_whence = SEEK_SET,
+        .l_start = 0,
+        .l_len = 0,
+        /* application is resposible for setting to 0 */
+        .l_pid = 0,
+    };
+    if (fcntl(fd, F_OFD_SETLK, &arg) < 0) {
+        /* allow program to work when F_OFD_SETLK isn't implemented */
+        if (errno == EINVAL)
+            fputs("WARNING: F_OFD_SETLK not supported!\n", stderr);
+        else
+            throw std::runtime_error(std::string("couldn't lock file"));
+    }
+}
+
 }
 
 void dev_open_slot(struct pcie_bars &bars, int slot)
@@ -69,6 +90,12 @@ void dev_open(struct pcie_bars &bars, const char *pci_address)
             if (fd < 0)
                 throw std::runtime_error(std::string("couldn't open resource file: ") + pci_address);
         }
+
+        if (bar == 0) {
+            lock_file(fd);
+            bars.fd_lock = fd;
+        }
+
         struct stat st;
         fstat(fd, &st);
         bars.sizes[i] = st.st_size;
@@ -130,6 +157,8 @@ void dev_close(struct pcie_bars &bars)
         return;
     }
 
+    close(bars.fd_lock);
+
     auto unmap = [&bars](auto &p, unsigned i) {
         munmap(const_cast<void *>(p), bars.sizes[i]);
         p = 0;