Add support for HID mouse

app/meson.build

@@ -77,6 +77,7 @@ if aoa_hid_support
     src += [
         'src/aoa_hid.c',
         'src/hid_keyboard.c',
+        'src/hid_mouse.c',
     ]
 endif
 

app/scrcpy.1

@@ -96,6 +96,8 @@ The keyboard layout must be configured (once and for all) on the device, via Set
 
 However, the option is only available when the HID keyboard is enabled (or a physical keyboard is connected).
 
+Also see \fB\-\-hid\-mouse\fR.
+
 .TP
 .B \-\-legacy\-paste
 Inject computer clipboard text as a sequence of key events on Ctrl+v (like MOD+Shift+v).
@@ -120,6 +122,18 @@ Limit both the width and height of the video to \fIvalue\fR. The other dimension
 
 Default is 0 (unlimited).
 
+.TP
+.B \-M, \-\-hid\-mouse
+Simulate a physical mouse by using HID over AOAv2.
+
+In this mode, the computer mouse is captured to control the device directly (relative mouse mode).
+
+LAlt, LSuper or RSuper toggle the capture mode, to give control of the mouse back to the computer.
+
+It may only work over USB, and is currently only supported on Linux.
+
+Also see \fB\-\-hid\-keyboard\fR.
+
 .TP
 .B \-\-no\-clipboard\-autosync
 By default, scrcpy automatically synchronizes the computer clipboard to the device clipboard before injecting Ctrl+v, and the device clipboard to the computer clipboard whenever it changes.

app/src/cli.c

@@ -178,7 +178,8 @@ static const struct sc_option options[] = {
                 "directly: `adb shell am start -a "
                 "android.settings.HARD_KEYBOARD_SETTINGS`.\n"
                 "However, the option is only available when the HID keyboard "
-                "is enabled (or a physical keyboard is connected).",
+                "is enabled (or a physical keyboard is connected).\n"
+                "Also see --hid-mouse.",
     },
     {
         .shortopt = 'h',
@@ -214,6 +215,18 @@ static const struct sc_option options[] = {
         .text = "Limit the frame rate of screen capture (officially supported "
                 "since Android 10, but may work on earlier versions).",
     },
+    {
+        .shortopt = 'M',
+        .longopt = "hid-mouse",
+        .text = "Simulate a physical mouse by using HID over AOAv2.\n"
+                "In this mode, the computer mouse is captured to control the "
+                "device directly (relative mouse mode).\n"
+                "LAlt, LSuper or RSuper toggle the capture mode, to give "
+                "control of the mouse back to the computer.\n"
+                "It may only work over USB, and is currently only supported "
+                "on Linux.\n"
+                "Also see --hid-keyboard.",
+    },
     {
         .shortopt = 'm',
         .longopt = "max-size",
@@ -1315,6 +1328,15 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[],
                     return false;
                 }
                 break;
+            case 'M':
+#ifdef HAVE_AOA_HID
+                opts->mouse_input_mode = SC_MOUSE_INPUT_MODE_HID;
+#else
+                LOGE("HID over AOA (-M/--hid-mouse) is not supported on this"
+                     "platform. It is only available on Linux.");
+                return false;
+#endif
+                break;
             case OPT_LOCK_VIDEO_ORIENTATION:
                 if (!parse_lock_video_orientation(optarg,
                         &opts->lock_video_orientation)) {

app/src/hid_mouse.c

@@ -0,0 +1,267 @@
+#include "hid_mouse.h"
+
+#include <assert.h>
+
+#include "input_events.h"
+#include "util/log.h"
+
+/** Downcast mouse processor to hid_mouse */
+#define DOWNCAST(MP) container_of(MP, struct sc_hid_mouse, mouse_processor)
+
+#define HID_MOUSE_ACCESSORY_ID 2
+
+// 1 byte for buttons + padding, 1 byte for X position, 1 byte for Y position
+#define HID_MOUSE_EVENT_SIZE 4
+
+/**
+ * Mouse descriptor from the specification:
+ * <https://www.usb.org/sites/default/files/hid1_11.pdf>
+ *
+ * Appendix E (p71): §E.10 Report Descriptor (Mouse)
+ *
+ * The usage tags (like Wheel) are listed in "HID Usage Tables":
+ * <https://www.usb.org/sites/default/files/documents/hut1_12v2.pdf>
+ * §4 Generic Desktop Page (0x01) (p26)
+ */
+static const unsigned char mouse_report_desc[]  = {
+    // Usage Page (Generic Desktop)
+    0x05, 0x01,
+    // Usage (Mouse)
+    0x09, 0x02,
+
+    // Collection (Application)
+    0xA1, 0x01,
+
+    // Usage (Pointer)
+    0x09, 0x01,
+
+    // Collection (Physical)
+    0xA1, 0x00,
+
+    // Usage Page (Buttons)
+    0x05, 0x09,
+
+    // Usage Minimum (1)
+    0x19, 0x01,
+     // Usage Maximum (5)
+    0x29, 0x05,
+    // Logical Minimum (0)
+    0x15, 0x00,
+    // Logical Maximum (1)
+    0x25, 0x01,
+    // Report Count (5)
+    0x95, 0x05,
+    // Report Size (1)
+    0x75, 0x01,
+    // Input (Data, Variable, Absolute): 5 buttons bits
+    0x81, 0x02,
+
+    // Report Count (1)
+    0x95, 0x01,
+    // Report Size (3)
+    0x75, 0x03,
+    // Input (Constant): 3 bits padding
+    0x81, 0x01,
+
+    // Usage Page (Generic Desktop)
+    0x05, 0x01,
+    // Usage (X)
+    0x09, 0x30,
+    // Usage (Y)
+    0x09, 0x31,
+    // Usage (Wheel)
+    0x09, 0x38,
+    // Local Minimum (-127)
+    0x15, 0x81,
+    // Local Maximum (127)
+    0x25, 0x7F,
+    // Report Size (8)
+    0x75, 0x08,
+    // Report Count (3)
+    0x95, 0x03,
+    // Input (Data, Variable, Relative): 3 position bytes (X, Y, Wheel)
+    0x81, 0x06,
+
+    // End Collection
+    0xC0,
+
+    // End Collection
+    0xC0,
+};
+
+/**
+ * A mouse HID event is 3 bytes long:
+ *
+ *  - byte 0: buttons state
+ *  - byte 1: relative x motion (signed byte from -127 to 127)
+ *  - byte 2: relative y motion (signed byte from -127 to 127)
+ *
+ *                   7 6 5 4 3 2 1 0
+ *                  +---------------+
+ *         byte 0:  |0 0 0 . . . . .| buttons state
+ *                  +---------------+
+ *                         ^ ^ ^ ^ ^
+ *                         | | | | `- left button
+ *                         | | | `--- right button
+ *                         | | `----- middle button
+ *                         | `------- button 4
+ *                         `--------- button 5
+ *
+ *                  +---------------+
+ *         byte 1:  |. . . . . . . .| relative x motion
+ *                  +---------------+
+ *         byte 2:  |. . . . . . . .| relative y motion
+ *                  +---------------+
+ *         byte 3:  |. . . . . . . .| wheel motion (-1, 0 or 1)
+ *                  +---------------+
+ *
+ * As an example, here is the report for a motion of (x=5, y=-4) with left
+ * button pressed:
+ *
+ *                  +---------------+
+ *                  |0 0 0 0 0 0 0 1| left button pressed
+ *                  +---------------+
+ *                  |0 0 0 0 0 1 0 1| horizontal motion (x = 5)
+ *                  +---------------+
+ *                  |1 1 1 1 1 1 0 0| relative y motion (y = -4)
+ *                  +---------------+
+ *                  |0 0 0 0 0 0 0 0| wheel motion
+ *                  +---------------+
+ */
+
+static bool
+sc_hid_mouse_event_init(struct sc_hid_event *hid_event) {
+    unsigned char *buffer = calloc(1, HID_MOUSE_EVENT_SIZE);
+    if (!buffer) {
+        LOG_OOM();
+        return false;
+    }
+
+    sc_hid_event_init(hid_event, HID_MOUSE_ACCESSORY_ID, buffer,
+                      HID_MOUSE_EVENT_SIZE);
+    return true;
+}
+
+static unsigned char
+buttons_state_to_hid_buttons(uint8_t buttons_state) {
+    unsigned char c = 0;
+    if (buttons_state & SC_MOUSE_BUTTON_LEFT) {
+        c |= 1 << 0;
+    }
+    if (buttons_state & SC_MOUSE_BUTTON_RIGHT) {
+        c |= 1 << 1;
+    }
+    if (buttons_state & SC_MOUSE_BUTTON_MIDDLE) {
+        c |= 1 << 2;
+    }
+    if (buttons_state & SC_MOUSE_BUTTON_X1) {
+        c |= 1 << 3;
+    }
+    if (buttons_state & SC_MOUSE_BUTTON_X2) {
+        c |= 1 << 4;
+    }
+    return c;
+}
+
+static void
+sc_mouse_processor_process_mouse_motion(struct sc_mouse_processor *mp,
+                                    const struct sc_mouse_motion_event *event) {
+    struct sc_hid_mouse *mouse = DOWNCAST(mp);
+
+    struct sc_hid_event hid_event;
+    if (!sc_hid_mouse_event_init(&hid_event)) {
+        return;
+    }
+
+    unsigned char *buffer = hid_event.buffer;
+    buffer[0] = buttons_state_to_hid_buttons(event->buttons_state);
+    buffer[1] = CLAMP(event->xrel, -127, 127);
+    buffer[2] = CLAMP(event->yrel, -127, 127);
+    buffer[3] = 0; // wheel coordinates only used for scrolling
+
+    if (!sc_aoa_push_hid_event(mouse->aoa, &hid_event)) {
+        sc_hid_event_destroy(&hid_event);
+        LOGW("Could request HID event");
+    }
+}
+
+static void
+sc_mouse_processor_process_mouse_click(struct sc_mouse_processor *mp,
+                                   const struct sc_mouse_click_event *event) {
+    struct sc_hid_mouse *mouse = DOWNCAST(mp);
+
+    struct sc_hid_event hid_event;
+    if (!sc_hid_mouse_event_init(&hid_event)) {
+        return;
+    }
+
+    unsigned char *buffer = hid_event.buffer;
+    buffer[0] = buttons_state_to_hid_buttons(event->buttons_state);
+    buffer[1] = 0; // no x motion
+    buffer[2] = 0; // no y motion
+    buffer[3] = 0; // wheel coordinates only used for scrolling
+
+    if (!sc_aoa_push_hid_event(mouse->aoa, &hid_event)) {
+        sc_hid_event_destroy(&hid_event);
+        LOGW("Could request HID event");
+    }
+}
+
+static void
+sc_mouse_processor_process_mouse_scroll(struct sc_mouse_processor *mp,
+                                    const struct sc_mouse_scroll_event *event) {
+    struct sc_hid_mouse *mouse = DOWNCAST(mp);
+
+    struct sc_hid_event hid_event;
+    if (!sc_hid_mouse_event_init(&hid_event)) {
+        return;
+    }
+
+    unsigned char *buffer = hid_event.buffer;
+    buffer[0] = 0; // buttons state irrelevant (and unknown)
+    buffer[1] = 0; // no x motion
+    buffer[2] = 0; // no y motion
+    // In practice, vscroll is always -1, 0 or 1, but in theory other values
+    // are possible
+    buffer[3] = CLAMP(event->vscroll, -127, 127);
+    // Horizontal scrolling ignored
+
+    if (!sc_aoa_push_hid_event(mouse->aoa, &hid_event)) {
+        sc_hid_event_destroy(&hid_event);
+        LOGW("Could request HID event");
+    }
+}
+
+bool
+sc_hid_mouse_init(struct sc_hid_mouse *mouse, struct sc_aoa *aoa) {
+    mouse->aoa = aoa;
+
+    bool ok = sc_aoa_setup_hid(aoa, HID_MOUSE_ACCESSORY_ID, mouse_report_desc,
+                               ARRAY_LEN(mouse_report_desc));
+    if (!ok) {
+        LOGW("Register HID mouse failed");
+        return false;
+    }
+
+    static const struct sc_mouse_processor_ops ops = {
+        .process_mouse_motion = sc_mouse_processor_process_mouse_motion,
+        .process_mouse_click = sc_mouse_processor_process_mouse_click,
+        .process_mouse_scroll = sc_mouse_processor_process_mouse_scroll,
+        // Touch events not supported (coordinates are not relative)
+        .process_touch = NULL,
+    };
+
+    mouse->mouse_processor.ops = &ops;
+
+    mouse->mouse_processor.relative_mode = true;
+
+    return true;
+}
+
+void
+sc_hid_mouse_destroy(struct sc_hid_mouse *mouse) {
+    bool ok = sc_aoa_unregister_hid(mouse->aoa, HID_MOUSE_ACCESSORY_ID);
+    if (!ok) {
+        LOGW("Could not unregister HID");
+    }
+}

app/src/hid_mouse.h

@@ -0,0 +1,23 @@
+#ifndef HID_MOUSE_H
+#define HID_MOUSE_H
+
+#include "common.h"
+
+#include <stdbool.h>
+
+#include "aoa_hid.h"
+#include "trait/mouse_processor.h"
+
+struct sc_hid_mouse {
+    struct sc_mouse_processor mouse_processor; // mouse processor trait
+
+    struct sc_aoa *aoa;
+};
+
+bool
+sc_hid_mouse_init(struct sc_hid_mouse *mouse, struct sc_aoa *aoa);
+
+void
+sc_hid_mouse_destroy(struct sc_hid_mouse *mouse);
+
+#endif

app/src/options.h

@@ -38,6 +38,11 @@ enum sc_keyboard_input_mode {
     SC_KEYBOARD_INPUT_MODE_HID,
 };
 
+enum sc_mouse_input_mode {
+    SC_MOUSE_INPUT_MODE_INJECT,
+    SC_MOUSE_INPUT_MODE_HID,
+};
+
 enum sc_key_inject_mode {
     // Inject special keys, letters and space as key events.
     // Inject numbers and punctuation as text events.
@@ -90,6 +95,7 @@ struct scrcpy_options {
     enum sc_log_level log_level;
     enum sc_record_format record_format;
     enum sc_keyboard_input_mode keyboard_input_mode;
+    enum sc_mouse_input_mode mouse_input_mode;
     struct sc_port_range port_range;
     uint32_t tunnel_host;
     uint16_t tunnel_port;

app/src/scrcpy.c

@@ -19,6 +19,7 @@
 #include "file_handler.h"
 #ifdef HAVE_AOA_HID
 # include "hid_keyboard.h"
+# include "hid_mouse.h"
 #endif
 #include "keyboard_inject.h"
 #include "mouse_inject.h"
@@ -55,7 +56,12 @@ struct scrcpy {
         struct sc_hid_keyboard keyboard_hid;
 #endif
     };
-    struct sc_mouse_inject mouse_inject;
+    union {
+        struct sc_mouse_inject mouse_inject;
+#ifdef HAVE_AOA_HID
+        struct sc_hid_mouse mouse_hid;
+#endif
+    };
 };
 
 static inline void
@@ -330,6 +336,7 @@ scrcpy(struct scrcpy_options *options) {
 #ifdef HAVE_AOA_HID
     bool aoa_hid_initialized = false;
     bool hid_keyboard_initialized = false;
+    bool hid_mouse_initialized = false;
 #endif
     bool controller_initialized = false;
     bool controller_started = false;
@@ -451,7 +458,9 @@ scrcpy(struct scrcpy_options *options) {
 #ifdef HAVE_AOA_HID
         bool use_hid_keyboard =
             options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_HID;
-        if (use_hid_keyboard) {
+        bool use_hid_mouse =
+            options->mouse_input_mode == SC_MOUSE_INPUT_MODE_HID;
+        if (use_hid_keyboard || use_hid_mouse) {
             bool ok = sc_acksync_init(&s->acksync);
             if (!ok) {
                 goto end;
@@ -473,7 +482,16 @@ scrcpy(struct scrcpy_options *options) {
                 }
             }
 
-            bool need_aoa = hid_keyboard_initialized;
+            if (use_hid_mouse) {
+                if (sc_hid_mouse_init(&s->mouse_hid, &s->aoa)) {
+                    hid_mouse_initialized = true;
+                    mp = &s->mouse_hid.mouse_processor;
+                } else {
+                    LOGE("Could not initialized HID mouse");
+                }
+            }
+
+            bool need_aoa = hid_keyboard_initialized || hid_mouse_initialized;
 
             if (!need_aoa || !sc_aoa_start(&s->aoa)) {
                 sc_acksync_destroy(&s->acksync);
@@ -491,6 +509,10 @@ aoa_hid_end:
                     sc_hid_keyboard_destroy(&s->keyboard_hid);
                     hid_keyboard_initialized = false;
                 }
+                if (hid_mouse_initialized) {
+                    sc_hid_mouse_destroy(&s->mouse_hid);
+                    hid_mouse_initialized = false;
+                }
             }
 
             if (use_hid_keyboard && !hid_keyboard_initialized) {
@@ -498,9 +520,16 @@ aoa_hid_end:
                      "(-K/--hid-keyboard ignored)");
                 options->keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_INJECT;
             }
+
+            if (use_hid_mouse && !hid_mouse_initialized) {
+                LOGE("Fallback to default mouse injection method "
+                     "(-M/--hid-mouse ignored)");
+                options->mouse_input_mode = SC_MOUSE_INPUT_MODE_INJECT;
+            }
         }
 #else
         assert(options->keyboard_input_mode != SC_KEYBOARD_INPUT_MODE_HID);
+        assert(options->mouse_input_mode != SC_MOUSE_INPUT_MODE_HID);
 #endif
 
         // keyboard_input_mode may have been reset if HID mode failed
@@ -510,8 +539,11 @@ aoa_hid_end:
             kp = &s->keyboard_inject.key_processor;
         }
 
-        sc_mouse_inject_init(&s->mouse_inject, &s->controller);
+        // mouse_input_mode may have been reset if HID mode failed
-        mp = &s->mouse_inject.mouse_processor;
+        if (options->mouse_input_mode == SC_MOUSE_INPUT_MODE_INJECT) {
+            sc_mouse_inject_init(&s->mouse_inject, &s->controller);
+            mp = &s->mouse_inject.mouse_processor;
+        }
 
         if (!controller_init(&s->controller, s->server.control_socket,
                              acksync)) {