Wait SET_CLIPBOARD ack before Ctrl+v via HID

To allow seamless copy-paste, on Ctrl+v, a SET_CLIPBOARD request is
performed before injecting Ctrl+v.

But when HID keyboard is enabled, the Ctrl+v injection is not sent on
the same channel as the clipboard request, so they are not serialized,
and may occur in any order. If Ctrl+v happens to be injected before the
new clipboard content is set, then the old content is pasted instead,
which is incorrect.

To minimize the probability of occurrence of the wrong order, a delay of
2 milliseconds was added before injecting Ctrl+v. Then 5ms. But even
with 5ms, the wrong behavior sometimes happens.

To handle it properly, add an acknowledgement mechanism, so that Ctrl+v
is injected over AOA only after the SET_CLIPBOARD request has been
performed and acknowledged by the server.

Refs e4163321f0
Refs 45b0f8123a

PR #2814 <https://github.com/Genymobile/scrcpy/pull/2814>

app/src/aoa_hid.c

@@ -35,7 +35,7 @@ sc_hid_event_init(struct sc_hid_event *hid_event, uint16_t accessory_id,
     hid_event->accessory_id = accessory_id;
     hid_event->buffer = buffer;
     hid_event->size = buffer_size;
-    hid_event->delay = 0;
+    hid_event->ack_to_wait = SC_SEQUENCE_INVALID;
 }
 
 void
@@ -118,7 +118,10 @@ sc_aoa_open_usb_handle(libusb_device *device, libusb_device_handle **handle) {
 }
 
 bool
-sc_aoa_init(struct sc_aoa *aoa, const char *serial) {
+sc_aoa_init(struct sc_aoa *aoa, const char *serial,
+            struct sc_acksync *acksync) {
+    assert(acksync);
+
     cbuf_init(&aoa->queue);
 
     if (!sc_mutex_init(&aoa->mutex)) {
@@ -155,6 +158,7 @@ sc_aoa_init(struct sc_aoa *aoa, const char *serial) {
     }
 
     aoa->stopped = false;
+    aoa->acksync = acksync;
 
     return true;
 }
@@ -332,23 +336,28 @@ run_aoa_thread(void *data) {
         assert(non_empty);
         (void) non_empty;
 
-        assert(event.delay >= 0);
+        uint64_t ack_to_wait = event.ack_to_wait;
-        if (event.delay) {
-            // Wait during the specified delay before injecting the HID event
-            sc_tick deadline = sc_tick_now() + event.delay;
-            bool timed_out = false;
-            while (!aoa->stopped && !timed_out) {
-                timed_out = !sc_cond_timedwait(&aoa->event_cond, &aoa->mutex,
-                                               deadline);
-            }
-            if (aoa->stopped) {
         sc_mutex_unlock(&aoa->mutex);
+
+        if (ack_to_wait != SC_SEQUENCE_INVALID) {
+            LOGD("Waiting ack from server sequence=%" PRIu64_, ack_to_wait);
+            // Do not block the loop indefinitely if the ack never comes (it should
+            // never happen)
+            sc_tick deadline = sc_tick_now() + SC_TICK_FROM_MS(500);
+            enum sc_acksync_wait_result result =
+                sc_acksync_wait(aoa->acksync, ack_to_wait, deadline);
+
+            if (result == SC_ACKSYNC_WAIT_TIMEOUT) {
+                LOGW("Ack not received after 500ms, discarding HID event");
+                sc_hid_event_destroy(&event);
+                continue;
+            } else if (result == SC_ACKSYNC_WAIT_INTR) {
+                // stopped
+                sc_hid_event_destroy(&event);
                 break;
             }
         }
 
-        sc_mutex_unlock(&aoa->mutex);
-
         bool ok = sc_aoa_send_hid_event(aoa, &event);
         sc_hid_event_destroy(&event);
         if (!ok) {
@@ -377,6 +386,8 @@ sc_aoa_stop(struct sc_aoa *aoa) {
     aoa->stopped = true;
     sc_cond_signal(&aoa->event_cond);
     sc_mutex_unlock(&aoa->mutex);
+
+    sc_acksync_interrupt(aoa->acksync);
 }
 
 void

app/src/aoa_hid.h

@@ -6,6 +6,7 @@
 
 #include <libusb-1.0/libusb.h>
 
+#include "util/acksync.h"
 #include "util/cbuf.h"
 #include "util/thread.h"
 #include "util/tick.h"
@@ -14,7 +15,7 @@ struct sc_hid_event {
     uint16_t accessory_id;
     unsigned char *buffer;
     uint16_t size;
-    sc_tick delay;
+    uint64_t ack_to_wait;
 };
 
 // Takes ownership of buffer
@@ -36,10 +37,12 @@ struct sc_aoa {
     sc_cond event_cond;
     bool stopped;
     struct sc_hid_event_queue queue;
+
+    struct sc_acksync *acksync;
 };
 
 bool
-sc_aoa_init(struct sc_aoa *aoa, const char *serial);
+sc_aoa_init(struct sc_aoa *aoa, const char *serial, struct sc_acksync *acksync);
 
 void
 sc_aoa_destroy(struct sc_aoa *aoa);

app/src/controller.c

@@ -5,10 +5,11 @@
 #include "util/log.h"
 
 bool
-controller_init(struct controller *controller, sc_socket control_socket) {
+controller_init(struct controller *controller, sc_socket control_socket,
+                struct sc_acksync *acksync) {
     cbuf_init(&controller->queue);
 
-    bool ok = receiver_init(&controller->receiver, control_socket);
+    bool ok = receiver_init(&controller->receiver, control_socket, acksync);
     if (!ok) {
         return false;
     }

app/src/controller.h

@@ -7,6 +7,7 @@
 
 #include "control_msg.h"
 #include "receiver.h"
+#include "util/acksync.h"
 #include "util/cbuf.h"
 #include "util/net.h"
 #include "util/thread.h"
@@ -24,7 +25,8 @@ struct controller {
 };
 
 bool
-controller_init(struct controller *controller, sc_socket control_socket);
+controller_init(struct controller *controller, sc_socket control_socket,
+                struct sc_acksync *acksync);
 
 void
 controller_destroy(struct controller *controller);

app/src/hid_keyboard.c

@@ -279,7 +279,7 @@ push_mod_lock_state(struct sc_hid_keyboard *kb, uint16_t sdl_mod) {
 static void
 sc_key_processor_process_key(struct sc_key_processor *kp,
                              const SDL_KeyboardEvent *event,
-                             bool device_clipboard_set) {
+                             uint64_t ack_to_wait) {
     if (event->repeat) {
         // In USB HID protocol, key repeat is handled by the host (Android), so
         // just ignore key repeat here.
@@ -299,12 +299,12 @@ sc_key_processor_process_key(struct sc_key_processor *kp,
             }
         }
 
-        if (device_clipboard_set) {
+        if (ack_to_wait) {
             // Ctrl+v is pressed, so clipboard synchronization has been
-            // requested. Wait a bit so that the clipboard is set before
+            // requested. Wait until clipboard synchronization is acknowledged
-            // injecting Ctrl+v via HID, otherwise it would paste the old
+            // by the server, otherwise it could paste the old clipboard
-            // clipboard content.
+            // content.
-            hid_event.delay = SC_TICK_FROM_MS(5);
+            hid_event.ack_to_wait = ack_to_wait;
         }
 
         if (!sc_aoa_push_hid_event(kb->aoa, &hid_event)) {
@@ -345,6 +345,10 @@ sc_hid_keyboard_init(struct sc_hid_keyboard *kb, struct sc_aoa *aoa) {
         .process_text = sc_key_processor_process_text,
     };
 
+    // Clipboard synchronization is requested over the control socket, while HID
+    // events are sent over AOA, so it must wait for clipboard synchronization
+    // to be acknowledged by the device before injecting Ctrl+v.
+    kb->key_processor.async_paste = true;
     kb->key_processor.ops = &ops;
 
     return true;

app/src/input_manager.c

@@ -82,6 +82,8 @@ input_manager_init(struct input_manager *im, struct controller *controller,
     im->last_keycode = SDLK_UNKNOWN;
     im->last_mod = 0;
     im->key_repeat = 0;
+
+    im->next_sequence = 1; // 0 is reserved for SC_SEQUENCE_INVALID
 }
 
 static void
@@ -209,7 +211,8 @@ collapse_panels(struct controller *controller) {
 }
 
 static bool
-set_device_clipboard(struct controller *controller, bool paste) {
+set_device_clipboard(struct controller *controller, bool paste,
+                     uint64_t sequence) {
     char *text = SDL_GetClipboardText();
     if (!text) {
         LOGW("Could not get clipboard text: %s", SDL_GetError());
@@ -225,7 +228,7 @@ set_device_clipboard(struct controller *controller, bool paste) {
 
     struct control_msg msg;
     msg.type = CONTROL_MSG_TYPE_SET_CLIPBOARD;
-    msg.set_clipboard.sequence = 0; // unused for now
+    msg.set_clipboard.sequence = sequence;
     msg.set_clipboard.text = text_dup;
     msg.set_clipboard.paste = paste;
 
@@ -461,8 +464,10 @@ input_manager_process_key(struct input_manager *im,
                         // inject the text as input events
                         clipboard_paste(controller);
                     } else {
-                        // store the text in the device clipboard and paste
+                        // store the text in the device clipboard and paste,
-                        set_device_clipboard(controller, true);
+                        // without requesting an acknowledgment
+                        set_device_clipboard(controller, true,
+                                             SC_SEQUENCE_INVALID);
                     }
                 }
                 return;
@@ -511,6 +516,7 @@ input_manager_process_key(struct input_manager *im,
         return;
     }
 
+    uint64_t ack_to_wait = SC_SEQUENCE_INVALID;
     bool is_ctrl_v = ctrl && !shift && keycode == SDLK_v && down && !repeat;
     if (is_ctrl_v) {
         if (im->legacy_paste) {
@@ -518,16 +524,28 @@ input_manager_process_key(struct input_manager *im,
             clipboard_paste(controller);
             return;
         }
+
+        // Request an acknowledgement only if necessary
+        uint64_t sequence = im->kp->async_paste ? im->next_sequence
+                                                : SC_SEQUENCE_INVALID;
+
         // Synchronize the computer clipboard to the device clipboard before
         // sending Ctrl+v, to allow seamless copy-paste.
-        bool ok = set_device_clipboard(controller, false);
+        bool ok = set_device_clipboard(controller, false, sequence);
         if (!ok) {
             LOGW("Clipboard could not be synchronized, Ctrl+v not injected");
             return;
         }
+
+        if (im->kp->async_paste) {
+            // The key processor must wait for this ack before injecting Ctrl+v
+            ack_to_wait = sequence;
+            // Increment only when the request succeeded
+            ++im->next_sequence;
+        }
     }
 
-    im->kp->ops->process_key(im->kp, event, is_ctrl_v);
+    im->kp->ops->process_key(im->kp, event, ack_to_wait);
 }
 
 static void

app/src/input_manager.h

@@ -38,6 +38,8 @@ struct input_manager {
     unsigned key_repeat;
     SDL_Keycode last_keycode;
     uint16_t last_mod;
+
+    uint64_t next_sequence; // used for request acknowledgements
 };
 
 void

app/src/keyboard_inject.c

@@ -189,11 +189,11 @@ convert_input_key(const SDL_KeyboardEvent *from, struct control_msg *to,
 static void
 sc_key_processor_process_key(struct sc_key_processor *kp,
                              const SDL_KeyboardEvent *event,
-                             bool device_clipboard_set) {
+                             uint64_t ack_to_wait) {
     // The device clipboard synchronization and the key event messages are
     // serialized, there is nothing special to do to ensure that the clipboard
     // is set before injecting Ctrl+v.
-    (void) device_clipboard_set;
+    (void) ack_to_wait;
 
     struct sc_keyboard_inject *ki = DOWNCAST(kp);
 
@@ -256,5 +256,7 @@ sc_keyboard_inject_init(struct sc_keyboard_inject *ki,
         .process_text = sc_key_processor_process_text,
     };
 
+    // Key injection and clipboard synchronization are serialized
+    ki->key_processor.async_paste = false;
     ki->key_processor.ops = &ops;
 }

app/src/receiver.c

@@ -7,12 +7,16 @@
 #include "util/log.h"
 
 bool
-receiver_init(struct receiver *receiver, sc_socket control_socket) {
+receiver_init(struct receiver *receiver, sc_socket control_socket,
+              struct sc_acksync *acksync) {
     bool ok = sc_mutex_init(&receiver->mutex);
     if (!ok) {
         return false;
     }
+
     receiver->control_socket = control_socket;
+    receiver->acksync = acksync;
+
     return true;
 }
 
@@ -22,7 +26,7 @@ receiver_destroy(struct receiver *receiver) {
 }
 
 static void
-process_msg(struct device_msg *msg) {
+process_msg(struct receiver *receiver, struct device_msg *msg) {
     switch (msg->type) {
         case DEVICE_MSG_TYPE_CLIPBOARD: {
             char *current = SDL_GetClipboardText();
@@ -38,13 +42,16 @@ process_msg(struct device_msg *msg) {
             break;
         }
         case DEVICE_MSG_TYPE_ACK_CLIPBOARD:
-            // TODO
+            assert(receiver->acksync);
+            LOGD("Ack device clipboard sequence=%" PRIu64_,
+                 msg->ack_clipboard.sequence);
+            sc_acksync_ack(receiver->acksync, msg->ack_clipboard.sequence);
             break;
     }
 }
 
 static ssize_t
-process_msgs(const unsigned char *buf, size_t len) {
+process_msgs(struct receiver *receiver, const unsigned char *buf, size_t len) {
     size_t head = 0;
     for (;;) {
         struct device_msg msg;
@@ -56,7 +63,7 @@ process_msgs(const unsigned char *buf, size_t len) {
             return head;
         }
 
-        process_msg(&msg);
+        process_msg(receiver, &msg);
         device_msg_destroy(&msg);
 
         head += r;
@@ -84,7 +91,7 @@ run_receiver(void *data) {
         }
 
         head += r;
-        ssize_t consumed = process_msgs(buf, head);
+        ssize_t consumed = process_msgs(receiver, buf, head);
         if (consumed == -1) {
             // an error occurred
             break;

app/src/receiver.h

@@ -5,6 +5,7 @@
 
 #include <stdbool.h>
 
+#include "util/acksync.h"
 #include "util/net.h"
 #include "util/thread.h"
 
@@ -14,10 +15,13 @@ struct receiver {
     sc_socket control_socket;
     sc_thread thread;
     sc_mutex mutex;
+
+    struct sc_acksync *acksync;
 };
 
 bool
-receiver_init(struct receiver *receiver, sc_socket control_socket);
+receiver_init(struct receiver *receiver, sc_socket control_socket,
+              struct sc_acksync *acksync);
 
 void
 receiver_destroy(struct receiver *receiver);

app/src/scrcpy.c

@@ -27,6 +27,7 @@
 #include "screen.h"
 #include "server.h"
 #include "stream.h"
+#include "util/acksync.h"
 #include "util/log.h"
 #include "util/net.h"
 #ifdef HAVE_V4L2
@@ -46,6 +47,8 @@ struct scrcpy {
     struct file_handler file_handler;
 #ifdef HAVE_AOA_HID
     struct sc_aoa aoa;
+    // sequence/ack helper to synchronize clipboard and Ctrl+v via HID
+    struct sc_acksync acksync;
 #endif
     union {
         struct sc_keyboard_inject keyboard_inject;
@@ -340,6 +343,8 @@ scrcpy(struct scrcpy_options *options) {
     bool controller_started = false;
     bool screen_initialized = false;
 
+    struct sc_acksync *acksync = NULL;
+
     struct sc_server_params params = {
         .serial = options->serial,
         .log_level = options->log_level,
@@ -445,7 +450,18 @@ scrcpy(struct scrcpy_options *options) {
     }
 
     if (options->control) {
-        if (!controller_init(&s->controller, s->server.control_socket)) {
+#ifdef HAVE_AOA_HID
+        if (options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_HID) {
+            bool ok = sc_acksync_init(&s->acksync);
+            if (!ok) {
+                goto end;
+            }
+
+            acksync = &s->acksync;
+        }
+#endif
+        if (!controller_init(&s->controller, s->server.control_socket,
+                             acksync)) {
             goto end;
         }
         controller_initialized = true;
@@ -521,7 +537,7 @@ scrcpy(struct scrcpy_options *options) {
 #ifdef HAVE_AOA_HID
             bool aoa_hid_ok = false;
 
-            bool ok = sc_aoa_init(&s->aoa, serial);
+            bool ok = sc_aoa_init(&s->aoa, serial, acksync);
             if (!ok) {
                 goto aoa_hid_end;
             }
@@ -586,6 +602,9 @@ end:
         sc_hid_keyboard_destroy(&s->keyboard_hid);
         sc_aoa_stop(&s->aoa);
     }
+    if (acksync) {
+        sc_acksync_destroy(acksync);
+    }
 #endif
     if (controller_started) {
         controller_stop(&s->controller);

app/src/trait/key_processor.h

@@ -14,6 +14,15 @@
  * Component able to process and inject keys should implement this trait.
  */
 struct sc_key_processor {
+    /**
+     * Set by the implementation to indicate that it must explicitly wait for
+     * the clipboard to be set on the device before injecting Ctrl+v to avoid
+     * race conditions. If it is set, the input_manager will pass a valid
+     * ack_to_wait to process_key() in case of clipboard synchronization
+     * resulting of the key event.
+     */
+    bool async_paste;
+
     const struct sc_key_processor_ops *ops;
 };
 
@@ -22,13 +31,14 @@ struct sc_key_processor_ops {
     /**
      * Process the keyboard event
      *
-     * The flag `device_clipboard_set` indicates that the input manager sent a
+     * The `sequence` number (if different from `SC_SEQUENCE_INVALID`) indicates
-     * control message to synchronize the device clipboard as a result of this
+     * the acknowledgement number to wait for before injecting this event.
-     * key event.
+     * This allows to ensure that the device clipboard is set before injecting
+     * Ctrl+v on the device.
      */
     void
     (*process_key)(struct sc_key_processor *kp, const SDL_KeyboardEvent *event,
-                   bool device_clipboard_set);
+                   uint64_t ack_to_wait);
 
     void
     (*process_text)(struct sc_key_processor *kp,

app/src/util/acksync.h

@@ -9,6 +9,11 @@
 
 /**
  * Helper to wait for acknowledgments
+ *
+ * In practice, it is used to wait for device clipboard acknowledgement from the
+ * server before injecting Ctrl+v via AOA HID, in order to avoid pasting the
+ * content of the old device clipboard (if Ctrl+v was injected before the
+ * clipboard content was actually set).
  */
 struct sc_acksync {
     sc_mutex mutex;