Added the abillity for vnc to use 9 mouse buttons instead of 7. This…

… adds the Xinput device buttons back and forward for the mouse.

Both Client and server message reader and writters as well as connections signal their support for the extra mouse buttons. The client signals the server through support of a psuedo encoding "pseudoEncodingExtendedMouseButtons". The server responds with a server message "msgTypeExtendedMouseSupport". Not sure if this is the best way to do this.
After confirming both both client and server support the extended mouse button msg, the client starts sending new message with a u16 write instead of a u8. The button layout doesn't change to maintain support for clients that don't support more mouse buttons.

common/rfb/CConnection.cxx

@@ -860,5 +860,7 @@ void CConnection::updateEncodings()
   if (qualityLevel >= 0 && qualityLevel <= 9)
       encodings.push_back(pseudoEncodingQualityLevel0 + qualityLevel);
 
+  encodings.push_back(pseudoEncodingExtendedMouseButtons);
+
   writer()->writeSetEncodings(encodings);
 }

common/rfb/CMsgHandler.cxx

@@ -80,6 +80,11 @@ void CMsgHandler::endOfContinuousUpdates()
   server.supportsContinuousUpdates = true;
 }
 
+void CMsgHandler::supportExtendedMouseButtons()
+{
+  server.supportsExtendedMouseButtons = true;
+}
+
 void CMsgHandler::supportsQEMUKeyEvent()
 {
   server.supportsQEMUKeyEvent = true;

common/rfb/CMsgHandler.h

@@ -58,6 +58,7 @@ namespace rfb {
     virtual void fence(uint32_t flags, unsigned len, const uint8_t data[]);
     virtual void endOfContinuousUpdates();
     virtual void supportsQEMUKeyEvent();
+    virtual void supportExtendedMouseButtons();
     virtual void serverInit(int width, int height,
                             const PixelFormat& pf,
                             const char* name) = 0;

common/rfb/CMsgReader.cxx

@@ -118,6 +118,9 @@ bool CMsgReader::readMsg()
     case msgTypeEndOfContinuousUpdates:
       ret = readEndOfContinuousUpdates();
       break;
+    case msgTypeExtendedMouseSupport:
+      ret = readSupportExtendedMouseButton();
+      break;
     default:
       throw Exception("Unknown message type %d", currentMsgType);
     }
@@ -454,6 +457,12 @@ bool CMsgReader::readEndOfContinuousUpdates()
   return true;
 }
 
+bool CMsgReader::readSupportExtendedMouseButton()
+{
+  handler->supportExtendedMouseButtons();
+  return true;
+}
+
 bool CMsgReader::readFramebufferUpdate()
 {
   if (!is->hasData(1 + 2))

common/rfb/CMsgReader.h

@@ -56,6 +56,7 @@ namespace rfb {
     bool readExtendedClipboard(int32_t len);
     bool readFence();
     bool readEndOfContinuousUpdates();
+    bool readSupportExtendedMouseButton();
 
     bool readFramebufferUpdate();
 

common/rfb/CMsgWriter.cxx

@@ -182,13 +182,28 @@ void CMsgWriter::writePointerEvent(const Point& pos, int buttonMask)
   if (p.x >= server->width()) p.x = server->width() - 1;
   if (p.y >= server->height()) p.y = server->height() - 1;
 
+  if (server->supportsExtendedMouseButtons)
+  {
+    writePointerEventExt(pos,buttonMask);
+    return;
+  }
+
   startMsg(msgTypePointerEvent);
   os->writeU8(buttonMask);
   os->writeU16(p.x);
   os->writeU16(p.y);
   endMsg();
 }
 
+void CMsgWriter::writePointerEventExt(const Point& p, int buttonMask)
+{
+  startMsg(msgTypePointerEventExt);
+  os->writeU16(buttonMask);
+  os->writeU16(p.x);
+  os->writeU16(p.y);
+  endMsg();
+}
+
 
 void CMsgWriter::writeClientCutText(const char* str)
 {

common/rfb/CMsgWriter.h

@@ -55,6 +55,7 @@ namespace rfb {
 
     void writeKeyEvent(uint32_t keysym, uint32_t keycode, bool down);
     void writePointerEvent(const Point& pos, int buttonMask);
+    void writePointerEventExt(const Point& pos, int buttonMask);
 
     void writeClientCutText(const char* str);
 

common/rfb/ClientParams.cxx

@@ -228,3 +228,10 @@ bool ClientParams::supportsContinuousUpdates() const
     return true;
   return false;
 }
+
+bool ClientParams::supportExtendedMouseButtons() const
+{
+  if (supportsEncoding(pseudoEncodingExtendedMouseButtons))
+    return true;
+  return false;
+}

common/rfb/ClientParams.h

@@ -101,6 +101,7 @@ namespace rfb {
     bool supportsLEDState() const;
     bool supportsFence() const;
     bool supportsContinuousUpdates() const;
+    bool supportExtendedMouseButtons() const;
 
     int compressLevel;
     int qualityLevel;

common/rfb/SConnection.cxx

@@ -446,6 +446,11 @@ void SConnection::supportsQEMUKeyEvent()
   writer()->writeQEMUKeyEvent();
 }
 
+void SConnection::supportsExtendedMouseButtons()
+{
+  writer()->writeExtendedMouseButtonSupport();
+}
+
 void SConnection::versionReceived()
 {
 }

common/rfb/SConnection.h

@@ -95,6 +95,8 @@ namespace rfb {
 
     virtual void supportsQEMUKeyEvent();
 
+    virtual void supportsExtendedMouseButtons() override;
+
 
     // Methods to be overridden in a derived class
 

common/rfb/SMsgHandler.cxx

@@ -72,6 +72,8 @@ void SMsgHandler::setEncodings(int nEncodings, const int32_t* encodings)
     supportsLEDState();
   if (client.supportsEncoding(pseudoEncodingQEMUKeyEvent) && firstQEMUKeyEvent)
     supportsQEMUKeyEvent();
+  if (client.supportsEncoding(pseudoEncodingExtendedMouseButtons))
+    supportsExtendedMouseButtons();
 }
 
 void SMsgHandler::handleClipboardCaps(uint32_t flags, const uint32_t* lengths)
@@ -153,3 +155,7 @@ void SMsgHandler::supportsLEDState()
 void SMsgHandler::supportsQEMUKeyEvent()
 {
 }
+
+void SMsgHandler::supportsExtendedMouseButtons()
+{
+}

common/rfb/SMsgHandler.h

@@ -95,6 +95,11 @@ namespace rfb {
     // handler will send a pseudo-rect back, signalling server support.
     virtual void supportsQEMUKeyEvent();
 
+    // supportsExtendedMouseButtons() is called the first time we detect that the
+    // client supports sending 16 bit mouse button state. This lets us pass more button
+    // states between server and client.
+    virtual void supportsExtendedMouseButtons();
+
     ClientParams client;
   };
 }

common/rfb/SMsgReader.cxx

@@ -99,6 +99,9 @@ bool SMsgReader::readMsg()
   case msgTypePointerEvent:
     ret = readPointerEvent();
     break;
+  case msgTypePointerEventExt:
+    ret = readPointerEvenExt();
+    break;
   case msgTypeClientCutText:
     ret = readClientCutText();
     break;
@@ -281,6 +284,16 @@ bool SMsgReader::readPointerEvent()
   return true;
 }
 
+bool SMsgReader::readPointerEvenExt()
+{
+  if (!is->hasData(2 + 2 + 2))
+    return false;
+  int mask = is->readU16();
+  int x = is->readU16();
+  int y = is->readU16();
+  handler->pointerEvent(Point(x, y), mask);
+  return true;
+}
 
 bool SMsgReader::readClientCutText()
 {

common/rfb/SMsgReader.h

@@ -53,6 +53,7 @@ namespace rfb {
 
     bool readKeyEvent();
     bool readPointerEvent();
+    bool readPointerEvenExt();
     bool readClientCutText();
     bool readExtendedClipboard(int32_t len);
 

common/rfb/SMsgWriter.cxx

@@ -303,6 +303,15 @@ void SMsgWriter::writeQEMUKeyEvent()
   needQEMUKeyEvent = true;
 }
 
+void SMsgWriter::writeExtendedMouseButtonSupport()
+{
+  if (!client->supportsEncoding(pseudoEncodingExtendedMouseButtons))
+    throw Exception("Client does not support Extended Mouse Buttons");
+
+  startMsg(msgTypeExtendedMouseSupport);
+  endMsg();
+}
+
 bool SMsgWriter::needFakeUpdate()
 {
   if (needSetDesktopName)

common/rfb/SMsgWriter.h

@@ -93,6 +93,9 @@ namespace rfb {
     // And QEMU keyboard event handshake
     void writeQEMUKeyEvent();
 
+    // let the client know we support extended mouse button support
+    void writeExtendedMouseButtonSupport();
+
     // needFakeUpdate() returns true when an immediate update is needed in
     // order to flush out pseudo-rectangles to the client.
     bool needFakeUpdate();

common/rfb/ServerParams.cxx

@@ -32,7 +32,7 @@ ServerParams::ServerParams()
   : majorVersion(0), minorVersion(0),
     supportsQEMUKeyEvent(false),
     supportsSetDesktopSize(false), supportsFence(false),
-    supportsContinuousUpdates(false),
+    supportsContinuousUpdates(false), supportsExtendedMouseButtons(false),
     width_(0), height_(0),
     ledState_(ledUnknown)
 {

common/rfb/ServerParams.h

@@ -79,6 +79,7 @@ namespace rfb {
     bool supportsSetDesktopSize;
     bool supportsFence;
     bool supportsContinuousUpdates;
+    bool supportsExtendedMouseButtons;
 
   private:
 

common/rfb/encodings.h

@@ -36,6 +36,7 @@ namespace rfb {
 
   const int pseudoEncodingXCursor = -240;
   const int pseudoEncodingCursor = -239;
+  const int pseudoEncodingExtendedMouseButtons = -241;
   const int pseudoEncodingDesktopSize = -223;
   const int pseudoEncodingLEDState = -261;
   const int pseudoEncodingExtendedDesktopSize = -308;

common/rfb/msgTypes.h

@@ -28,6 +28,8 @@ namespace rfb {
 
   const int msgTypeEndOfContinuousUpdates = 150;
 
+  const int msgTypeExtendedMouseSupport = 151;
+
   const int msgTypeServerFence = 248;
 
   // client to server
@@ -39,6 +41,8 @@ namespace rfb {
   const int msgTypeKeyEvent = 4;
   const int msgTypePointerEvent = 5;
   const int msgTypeClientCutText = 6;
+  const int msgTypePointerEventExt = 7;
+
 
   const int msgTypeEnableContinuousUpdates = 150;
 

java/com/tigervnc/rfb/CConnection.java

@@ -403,6 +403,12 @@ public void endOfContinuousUpdates()
         requestNewUpdate();
     }
   }
+
+  public void SupportExtendedMouseButton()
+  {
+    super.SupportExtendedMouseButton();
+  }
+
   // serverInit() is called when the ServerInit message is received.  The
   // derived class must call on to CConnection::serverInit().
   public void serverInit(int width, int height,
@@ -683,6 +689,8 @@ private void updateEncodings()
     encodings.add(Encodings.pseudoEncodingLastRect);
     encodings.add(Encodings.pseudoEncodingContinuousUpdates);
     encodings.add(Encodings.pseudoEncodingFence);
+    encodings.add(Encodings.pseudoEncodingExtendedMouseButtons);
+
 
     if (Decoder.supported(preferredEncoding)) {
       encodings.add(preferredEncoding);

java/com/tigervnc/rfb/CMsgHandler.java

@@ -73,6 +73,11 @@ public void endOfContinuousUpdates()
     server.supportsContinuousUpdates = true;
   }
 
+  public void SupportExtendedMouseButton()
+  {
+    server.supportsExtendedMouseButtons = true;
+  }
+
   abstract public void clientRedirect(int port, String host,
                                       String x509subject);
 

java/com/tigervnc/rfb/CMsgReader.java

@@ -80,6 +80,9 @@ public void readMsg()
       case MsgTypes.msgTypeEndOfContinuousUpdates:
         readEndOfContinuousUpdates();
         break;
+      case MsgTypes.msgTypeExtendedMouseSupport:
+        readSupportExtendedMouseButton();
+        break;
       default:
         vlog.error("unknown message type "+type);
         throw new Exception("unknown message type");
@@ -192,6 +195,11 @@ protected void readEndOfContinuousUpdates()
     handler.endOfContinuousUpdates();
   }
 
+  protected void readSupportExtendedMouseButton()
+  {
+    handler.SupportExtendedMouseButton();
+  }
+
   protected void readFramebufferUpdate()
   {
     is.skip(1);

java/com/tigervnc/rfb/CMsgWriter.java

@@ -150,13 +150,28 @@ synchronized public void writePointerEvent(Point pos, int buttonMask)
     if (p.x >= server.width()) p.x = server.width() - 1;
     if (p.y >= server.height()) p.y = server.height() - 1;
 
+    if(server.supportsExtendedMouseButtons)
+    {
+      writePointerEventExt(pos,buttonMask);
+      return;
+    }
+
     startMsg(MsgTypes.msgTypePointerEvent);
     os.writeU8(buttonMask);
     os.writeU16(p.x);
     os.writeU16(p.y);
     endMsg();
   }
 
+  synchronized public void writePointerEventExt(Point p, int buttonMask)
+  {
+    startMsg(MsgTypes.msgTypePointerEventExt);
+    os.writeU16(buttonMask);
+    os.writeU16(p.x);
+    os.writeU16(p.y);
+    endMsg();
+  }
+
   synchronized public void writeClientCutText(String str, int len)
   {
     startMsg(MsgTypes.msgTypeClientCutText);

java/com/tigervnc/rfb/Encodings.java

@@ -34,6 +34,7 @@ public class Encodings {
 
   public static final int pseudoEncodingXCursor = -240;
   public static final int pseudoEncodingCursor = -239;
+  public static final int pseudoEncodingExtendedMouseButtons = -241;
   public static final int pseudoEncodingDesktopSize = -223;
   public static final int pseudoEncodingExtendedDesktopSize = -308;
   public static final int pseudoEncodingDesktopName = -307;

java/com/tigervnc/rfb/MsgTypes.java

@@ -29,6 +29,8 @@ public class MsgTypes {
 
   public static final int msgTypeEndOfContinuousUpdates = 150;
 
+  public static final int msgTypeExtendedMouseSupport = 151;
+
   public static final int msgTypeServerFence = 248;
 
   // client to server
@@ -40,6 +42,7 @@ public class MsgTypes {
   public static final int msgTypeKeyEvent = 4;
   public static final int msgTypePointerEvent = 5;
   public static final int msgTypeClientCutText = 6;
+  public static final int msgTypePointerEventExt = 7;
 
   public static final int msgTypeEnableContinuousUpdates = 150;