Force mouse source when --forward-all-clicks

Right click and middle click require the source device to be a mouse,
not a touchscreen. Therefore, the source device was changed only when a
button other than the primary button was pressed (see
adc547fa6e).

However, this led to inconsistencies between the ACTION_DOWN when a
secondary button is pressed (with a mouse as source device) and the
matching ACTION_UP when the secondary button is released (with a
touchscreen as source device, because then there is no button pressed).

To avoid the problem in all cases, force a mouse as source device when
--forward-all-clicks is set.

Concretely, for mouse events in --forward-all-clicks mode:
 - device source is set to InputDevice.SOURCE_MOUSE;
 - motion event toolType is set to MotionEvent.TOOL_TYPE_MOUSE;

Otherwise (when --forward-all-clicks is unset, or for real touch
events), finger events are injected:
 - device source is set to InputDevice.SOURCE_TOUCHSCREEN;
 - motion event toolType is set to MotionEvent.TOOL_TYPE_FINGER.

Fixes #3568 <https://github.com/Genymobile/scrcpy/issues/3568>
PR #3579 <https://github.com/Genymobile/scrcpy/pull/3579>

Co-authored-by: Romain Vimont <rom@rom1v.com>
Signed-off-by: Romain Vimont <rom@rom1v.com>

app/src/control_msg.c

@@ -61,6 +61,22 @@ static const char *const copy_key_labels[] = {
     "cut",
 };
 
+static inline const char *
+get_well_known_pointer_id_name(uint64_t pointer_id) {
+    switch (pointer_id) {
+        case POINTER_ID_MOUSE:
+            return "mouse";
+        case POINTER_ID_GENERIC_FINGER:
+            return "finger";
+        case POINTER_ID_VIRTUAL_MOUSE:
+            return "vmouse";
+        case POINTER_ID_VIRTUAL_FINGER:
+            return "vfinger";
+        default:
+            return NULL;
+    }
+}
+
 static void
 write_position(uint8_t *buf, const struct sc_position *position) {
     sc_write32be(&buf[0], position->point.x);
@@ -159,11 +175,12 @@ sc_control_msg_log(const struct sc_control_msg *msg) {
             int action = msg->inject_touch_event.action
                        & AMOTION_EVENT_ACTION_MASK;
             uint64_t id = msg->inject_touch_event.pointer_id;
-            if (id == POINTER_ID_MOUSE || id == POINTER_ID_VIRTUAL_FINGER) {
+            const char *pointer_name = get_well_known_pointer_id_name(id);
+            if (pointer_name) {
                 // string pointer id
                 LOG_CMSG("touch [id=%s] %-4s position=%" PRIi32 ",%" PRIi32
                              " pressure=%f buttons=%06lx",
-                         id == POINTER_ID_MOUSE ? "mouse" : "vfinger",
+                         pointer_name,
                          MOTIONEVENT_ACTION_LABEL(action),
                          msg->inject_touch_event.position.point.x,
                          msg->inject_touch_event.position.point.y,

app/src/control_msg.h

@@ -18,7 +18,11 @@
 #define SC_CONTROL_MSG_CLIPBOARD_TEXT_MAX_LENGTH (SC_CONTROL_MSG_MAX_SIZE - 14)
 
 #define POINTER_ID_MOUSE UINT64_C(-1)
-#define POINTER_ID_VIRTUAL_FINGER UINT64_C(-2)
+#define POINTER_ID_GENERIC_FINGER UINT64_C(-2)
+
+// Used for injecting an additional virtual pointer for pinch-to-zoom
+#define POINTER_ID_VIRTUAL_MOUSE UINT64_C(-3)
+#define POINTER_ID_VIRTUAL_FINGER UINT64_C(-4)
 
 enum sc_control_msg_type {
     SC_CONTROL_MSG_TYPE_INJECT_KEYCODE,

app/src/input_events.h

@@ -353,6 +353,7 @@ struct sc_mouse_click_event {
     struct sc_position position;
     enum sc_action action;
     enum sc_mouse_button button;
+    uint64_t pointer_id;
     uint8_t buttons_state; // bitwise-OR of sc_mouse_button values
 };
 
@@ -365,6 +366,7 @@ struct sc_mouse_scroll_event {
 
 struct sc_mouse_motion_event {
     struct sc_position position;
+    uint64_t pointer_id;
     int32_t xrel;
     int32_t yrel;
     uint8_t buttons_state; // bitwise-OR of sc_mouse_button values

app/src/input_manager.c

@@ -335,7 +335,9 @@ simulate_virtual_finger(struct sc_input_manager *im,
     msg.inject_touch_event.action = action;
     msg.inject_touch_event.position.screen_size = im->screen->frame_size;
     msg.inject_touch_event.position.point = point;
-    msg.inject_touch_event.pointer_id = POINTER_ID_VIRTUAL_FINGER;
+    msg.inject_touch_event.pointer_id =
+        im->forward_all_clicks ? POINTER_ID_VIRTUAL_MOUSE
+                               : POINTER_ID_VIRTUAL_FINGER;
     msg.inject_touch_event.pressure = up ? 0.0f : 1.0f;
     msg.inject_touch_event.buttons = 0;
 
@@ -564,6 +566,8 @@ sc_input_manager_process_mouse_motion(struct sc_input_manager *im,
                                                               event->x,
                                                               event->y),
         },
+        .pointer_id = im->forward_all_clicks ? POINTER_ID_MOUSE
+                                             : POINTER_ID_GENERIC_FINGER,
         .xrel = event->xrel,
         .yrel = event->yrel,
         .buttons_state =
@@ -687,6 +691,8 @@ sc_input_manager_process_mouse_button(struct sc_input_manager *im,
         },
         .action = sc_action_from_sdl_mousebutton_type(event->type),
         .button = sc_mouse_button_from_sdl(event->button),
+        .pointer_id = im->forward_all_clicks ? POINTER_ID_MOUSE
+                                             : POINTER_ID_GENERIC_FINGER,
         .buttons_state =
             sc_mouse_buttons_state_from_sdl(sdl_buttons_state,
                                             im->forward_all_clicks),

app/src/mouse_inject.c

@@ -69,7 +69,7 @@ sc_mouse_processor_process_mouse_motion(struct sc_mouse_processor *mp,
         .type = SC_CONTROL_MSG_TYPE_INJECT_TOUCH_EVENT,
         .inject_touch_event = {
             .action = AMOTION_EVENT_ACTION_MOVE,
-            .pointer_id = POINTER_ID_MOUSE,
+            .pointer_id = event->pointer_id,
             .position = event->position,
             .pressure = 1.f,
             .buttons = convert_mouse_buttons(event->buttons_state),
@@ -90,7 +90,7 @@ sc_mouse_processor_process_mouse_click(struct sc_mouse_processor *mp,
         .type = SC_CONTROL_MSG_TYPE_INJECT_TOUCH_EVENT,
         .inject_touch_event = {
             .action = convert_mouse_action(event->action),
-            .pointer_id = POINTER_ID_MOUSE,
+            .pointer_id = event->pointer_id,
             .position = event->position,
             .pressure = event->action == SC_ACTION_DOWN ? 1.f : 0.f,
             .buttons = convert_mouse_buttons(event->buttons_state),

server/src/main/java/com/genymobile/scrcpy/Controller.java

@@ -16,6 +16,10 @@ public class Controller {
 
     private static final int DEFAULT_DEVICE_ID = 0;
 
+    // control_msg.h values of the pointerId field in inject_touch_event message
+    private static final int POINTER_ID_MOUSE = -1;
+    private static final int POINTER_ID_VIRTUAL_MOUSE = -3;
+
     private static final ScheduledExecutorService EXECUTOR = Executors.newSingleThreadScheduledExecutor();
 
     private final Device device;
@@ -194,7 +198,19 @@ public class Controller {
         pointer.setPressure(pressure);
         pointer.setUp(action == MotionEvent.ACTION_UP);
 
+        int source;
         int pointerCount = pointersState.update(pointerProperties, pointerCoords);
+        if (pointerId == POINTER_ID_MOUSE || pointerId == POINTER_ID_VIRTUAL_MOUSE) {
+            // real mouse event (forced by the client when --forward-on-click)
+            pointerProperties[pointerIndex].toolType = MotionEvent.TOOL_TYPE_MOUSE;
+            source = InputDevice.SOURCE_MOUSE;
+        } else {
+            // POINTER_ID_GENERIC_FINGER, POINTER_ID_VIRTUAL_FINGER or real touch from device
+            pointerProperties[pointerIndex].toolType = MotionEvent.TOOL_TYPE_FINGER;
+            source = InputDevice.SOURCE_TOUCHSCREEN;
+            // Buttons must not be set for touch events
+            buttons = 0;
+        }
 
         if (pointerCount == 1) {
             if (action == MotionEvent.ACTION_DOWN) {
@@ -209,14 +225,6 @@ public class Controller {
             }
         }
 
-        // Right-click and middle-click only work if the source is a mouse
-        boolean nonPrimaryButtonPressed = (buttons & ~MotionEvent.BUTTON_PRIMARY) != 0;
-        int source = nonPrimaryButtonPressed ? InputDevice.SOURCE_MOUSE : InputDevice.SOURCE_TOUCHSCREEN;
-        if (source != InputDevice.SOURCE_MOUSE) {
-            // Buttons must not be set for touch events
-            buttons = 0;
-        }
-
         MotionEvent event = MotionEvent
                 .obtain(lastTouchDown, now, action, pointerCount, pointerProperties, pointerCoords, 0, buttons, 1f, 1f, DEFAULT_DEVICE_ID, 0, source,
                         0);