rm2fb: Hook msgsnd to support msgqueue protocol

libs/rm2fb/Client.cpp

@@ -106,6 +106,31 @@ ioctl(int fd, unsigned long request, char* ptr) {
 
   return func_ioctl(fd, request, ptr);
 }
+
+constexpr key_t rm2fb_key = 0x2257c;
+static int rm2fb_mqid = -1;
+
+int
+msgget(key_t key, int msgflg) {
+  static auto func_msgsnd = (int (*)(key_t, int))dlsym(RTLD_NEXT, "msgget");
+  int res = func_msgsnd(key, msgflg);
+  if (!inXochitl && key == rm2fb_key) {
+    rm2fb_mqid = res;
+  }
+  return res;
+}
+
+int
+msgsnd(int msqid, const void* msgp, size_t msgsz, int msgflg) {
+  if (!inXochitl && msqid == rm2fb_mqid) {
+    return handleMsgSend(msgp, msgsz);
+  }
+
+  static auto func_msgsnd =
+    (int (*)(int, const void*, size_t, int))dlsym(RTLD_NEXT, "msgsnd");
+
+  return func_msgsnd(msqid, msgp, msgsz, msgflg);
+}
 }
 
 extern "C" {
@@ -139,6 +164,9 @@ __libc_start_main(int (*_main)(int, char**, char**),
     setenv("RM2FB_ACTIVE", "1", true);
   }
 
+  // We don't support waiting with semaphores yet
+  setenv("RM2FB_NO_WAIT_IOCTL", "1", true);
+
   char pathBuffer[PATH_MAX];
   auto size = readlink("/proc/self/exe", pathBuffer, PATH_MAX);
 

libs/rm2fb/IOCTL.cpp

@@ -14,6 +14,35 @@
 #include <rm2.h>
 
 namespace {
+enum MSG_TYPE { INIT_t = 1, UPDATE_t, XO_t, WAIT_t };
+
+struct xochitl_data {
+  int x1;
+  int y1;
+  int x2;
+  int y2;
+
+  int waveform;
+  int flags;
+};
+
+struct wait_sem_data {
+  char sem_name[512];
+};
+
+struct swtfb_update {
+  long mtype;
+  struct {
+    union {
+      xochitl_data xochitl_update;
+
+      struct mxcfb_update_data update;
+      wait_sem_data wait_update;
+    };
+
+  } mdata;
+};
+
 int
 handleUpdate(const mxcfb_update_data& data) {
   const auto& rect = data.update_region;
@@ -130,3 +159,18 @@ handleIOCTL(unsigned long request, char* ptr) {
     return 0;
   }
 }
+
+int
+handleMsgSend(const void* buffer, size_t size) {
+  // NOLINTNEXTLINE
+  const auto* update = reinterpret_cast<const swtfb_update*>(buffer);
+
+  if (update->mtype != UPDATE_t) {
+    std::cerr << "Unsupported msgsnd: " << update->mtype << "\n";
+    return 0;
+  }
+
+  handleUpdate(update->mdata.update);
+
+  return 0;
+}

libs/rm2fb/IOCTL.h

@@ -1,3 +1,9 @@
 #pragma once
 
-int handleIOCTL(unsigned long request, char* ptr);
+#include <cstdlib>
+
+int
+handleIOCTL(unsigned long request, char* ptr);
+
+int
+handleMsgSend(const void* buffer, size_t size);

libs/rm2fb/SharedBuffer.h

@@ -10,7 +10,7 @@ constexpr int fb_height = 1872;
 constexpr int fb_pixel_size = sizeof(uint16_t);
 constexpr int fb_size = fb_width * fb_height * fb_pixel_size;
 
-constexpr auto default_fb_name = "/rm2fb.01";
+constexpr auto default_fb_name = "/swtfb.01";
 
 struct SharedFB {
   unistdpp::FD fd;