From 5d17bcf1bc38690739c88ad2b0a069a67c8bedce Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 21 Nov 2021 17:40:11 +0100 Subject: [PATCH] Wait SET_CLIPBOARD ack before Ctrl+v via HID To allow seamless copy-paste, on Ctrl+v, a SET_CLIPBOARD request is performed before injecting Ctrl+v. But when HID keyboard is enabled, the Ctrl+v injection is not sent on the same channel as the clipboard request, so they are not serialized, and may occur in any order. If Ctrl+v happens to be injected before the new clipboard content is set, then the old content is pasted instead, which is incorrect. To minimize the probability of occurrence of the wrong order, a delay of 2 milliseconds was added before injecting Ctrl+v. Then 5ms. But even with 5ms, the wrong behavior sometimes happens. To handle it properly, add an acknowledgement mechanism, so that Ctrl+v is injected over AOA only after the SET_CLIPBOARD request has been performed and acknowledged by the server. Refs e4163321f00bb3830c6049bdb6c1515e7cc668a0 Refs 45b0f8123a52f5c73a5860d616f4ceba2766ca6a PR #2814 --- app/src/aoa_hid.c | 41 ++++++++++++++++++++++------------- app/src/aoa_hid.h | 7 ++++-- app/src/controller.c | 5 +++-- app/src/controller.h | 4 +++- app/src/hid_keyboard.c | 16 +++++++++----- app/src/input_manager.c | 30 ++++++++++++++++++++----- app/src/input_manager.h | 2 ++ app/src/keyboard_inject.c | 6 +++-- app/src/receiver.c | 19 +++++++++++----- app/src/receiver.h | 6 ++++- app/src/scrcpy.c | 23 ++++++++++++++++++-- app/src/trait/key_processor.h | 18 +++++++++++---- app/src/util/acksync.h | 5 +++++ 13 files changed, 135 insertions(+), 47 deletions(-) diff --git a/app/src/aoa_hid.c b/app/src/aoa_hid.c index 4c0b2bda..6fd32610 100644 --- a/app/src/aoa_hid.c +++ b/app/src/aoa_hid.c @@ -35,7 +35,7 @@ sc_hid_event_init(struct sc_hid_event *hid_event, uint16_t accessory_id, hid_event->accessory_id = accessory_id; hid_event->buffer = buffer; hid_event->size = buffer_size; - hid_event->delay = 0; + hid_event->ack_to_wait = SC_SEQUENCE_INVALID; } void @@ -118,7 +118,10 @@ sc_aoa_open_usb_handle(libusb_device *device, libusb_device_handle **handle) { } bool -sc_aoa_init(struct sc_aoa *aoa, const char *serial) { +sc_aoa_init(struct sc_aoa *aoa, const char *serial, + struct sc_acksync *acksync) { + assert(acksync); + cbuf_init(&aoa->queue); if (!sc_mutex_init(&aoa->mutex)) { @@ -155,6 +158,7 @@ sc_aoa_init(struct sc_aoa *aoa, const char *serial) { } aoa->stopped = false; + aoa->acksync = acksync; return true; } @@ -332,23 +336,28 @@ run_aoa_thread(void *data) { assert(non_empty); (void) non_empty; - assert(event.delay >= 0); - if (event.delay) { - // Wait during the specified delay before injecting the HID event - sc_tick deadline = sc_tick_now() + event.delay; - bool timed_out = false; - while (!aoa->stopped && !timed_out) { - timed_out = !sc_cond_timedwait(&aoa->event_cond, &aoa->mutex, - deadline); - } - if (aoa->stopped) { - sc_mutex_unlock(&aoa->mutex); + uint64_t ack_to_wait = event.ack_to_wait; + sc_mutex_unlock(&aoa->mutex); + + if (ack_to_wait != SC_SEQUENCE_INVALID) { + LOGD("Waiting ack from server sequence=%" PRIu64_, ack_to_wait); + // Do not block the loop indefinitely if the ack never comes (it should + // never happen) + sc_tick deadline = sc_tick_now() + SC_TICK_FROM_MS(500); + enum sc_acksync_wait_result result = + sc_acksync_wait(aoa->acksync, ack_to_wait, deadline); + + if (result == SC_ACKSYNC_WAIT_TIMEOUT) { + LOGW("Ack not received after 500ms, discarding HID event"); + sc_hid_event_destroy(&event); + continue; + } else if (result == SC_ACKSYNC_WAIT_INTR) { + // stopped + sc_hid_event_destroy(&event); break; } } - sc_mutex_unlock(&aoa->mutex); - bool ok = sc_aoa_send_hid_event(aoa, &event); sc_hid_event_destroy(&event); if (!ok) { @@ -377,6 +386,8 @@ sc_aoa_stop(struct sc_aoa *aoa) { aoa->stopped = true; sc_cond_signal(&aoa->event_cond); sc_mutex_unlock(&aoa->mutex); + + sc_acksync_interrupt(aoa->acksync); } void diff --git a/app/src/aoa_hid.h b/app/src/aoa_hid.h index 24cef502..e8fb9708 100644 --- a/app/src/aoa_hid.h +++ b/app/src/aoa_hid.h @@ -6,6 +6,7 @@ #include +#include "util/acksync.h" #include "util/cbuf.h" #include "util/thread.h" #include "util/tick.h" @@ -14,7 +15,7 @@ struct sc_hid_event { uint16_t accessory_id; unsigned char *buffer; uint16_t size; - sc_tick delay; + uint64_t ack_to_wait; }; // Takes ownership of buffer @@ -36,10 +37,12 @@ struct sc_aoa { sc_cond event_cond; bool stopped; struct sc_hid_event_queue queue; + + struct sc_acksync *acksync; }; bool -sc_aoa_init(struct sc_aoa *aoa, const char *serial); +sc_aoa_init(struct sc_aoa *aoa, const char *serial, struct sc_acksync *acksync); void sc_aoa_destroy(struct sc_aoa *aoa); diff --git a/app/src/controller.c b/app/src/controller.c index e486ea72..6cf3d20e 100644 --- a/app/src/controller.c +++ b/app/src/controller.c @@ -5,10 +5,11 @@ #include "util/log.h" bool -controller_init(struct controller *controller, sc_socket control_socket) { +controller_init(struct controller *controller, sc_socket control_socket, + struct sc_acksync *acksync) { cbuf_init(&controller->queue); - bool ok = receiver_init(&controller->receiver, control_socket); + bool ok = receiver_init(&controller->receiver, control_socket, acksync); if (!ok) { return false; } diff --git a/app/src/controller.h b/app/src/controller.h index e7004131..00267878 100644 --- a/app/src/controller.h +++ b/app/src/controller.h @@ -7,6 +7,7 @@ #include "control_msg.h" #include "receiver.h" +#include "util/acksync.h" #include "util/cbuf.h" #include "util/net.h" #include "util/thread.h" @@ -24,7 +25,8 @@ struct controller { }; bool -controller_init(struct controller *controller, sc_socket control_socket); +controller_init(struct controller *controller, sc_socket control_socket, + struct sc_acksync *acksync); void controller_destroy(struct controller *controller); diff --git a/app/src/hid_keyboard.c b/app/src/hid_keyboard.c index 2809ccd6..2afc09b2 100644 --- a/app/src/hid_keyboard.c +++ b/app/src/hid_keyboard.c @@ -279,7 +279,7 @@ push_mod_lock_state(struct sc_hid_keyboard *kb, uint16_t sdl_mod) { static void sc_key_processor_process_key(struct sc_key_processor *kp, const SDL_KeyboardEvent *event, - bool device_clipboard_set) { + uint64_t ack_to_wait) { if (event->repeat) { // In USB HID protocol, key repeat is handled by the host (Android), so // just ignore key repeat here. @@ -299,12 +299,12 @@ sc_key_processor_process_key(struct sc_key_processor *kp, } } - if (device_clipboard_set) { + if (ack_to_wait) { // Ctrl+v is pressed, so clipboard synchronization has been - // requested. Wait a bit so that the clipboard is set before - // injecting Ctrl+v via HID, otherwise it would paste the old - // clipboard content. - hid_event.delay = SC_TICK_FROM_MS(5); + // requested. Wait until clipboard synchronization is acknowledged + // by the server, otherwise it could paste the old clipboard + // content. + hid_event.ack_to_wait = ack_to_wait; } if (!sc_aoa_push_hid_event(kb->aoa, &hid_event)) { @@ -345,6 +345,10 @@ sc_hid_keyboard_init(struct sc_hid_keyboard *kb, struct sc_aoa *aoa) { .process_text = sc_key_processor_process_text, }; + // Clipboard synchronization is requested over the control socket, while HID + // events are sent over AOA, so it must wait for clipboard synchronization + // to be acknowledged by the device before injecting Ctrl+v. + kb->key_processor.async_paste = true; kb->key_processor.ops = &ops; return true; diff --git a/app/src/input_manager.c b/app/src/input_manager.c index e7afee77..f1715b79 100644 --- a/app/src/input_manager.c +++ b/app/src/input_manager.c @@ -82,6 +82,8 @@ input_manager_init(struct input_manager *im, struct controller *controller, im->last_keycode = SDLK_UNKNOWN; im->last_mod = 0; im->key_repeat = 0; + + im->next_sequence = 1; // 0 is reserved for SC_SEQUENCE_INVALID } static void @@ -209,7 +211,8 @@ collapse_panels(struct controller *controller) { } static bool -set_device_clipboard(struct controller *controller, bool paste) { +set_device_clipboard(struct controller *controller, bool paste, + uint64_t sequence) { char *text = SDL_GetClipboardText(); if (!text) { LOGW("Could not get clipboard text: %s", SDL_GetError()); @@ -225,7 +228,7 @@ set_device_clipboard(struct controller *controller, bool paste) { struct control_msg msg; msg.type = CONTROL_MSG_TYPE_SET_CLIPBOARD; - msg.set_clipboard.sequence = 0; // unused for now + msg.set_clipboard.sequence = sequence; msg.set_clipboard.text = text_dup; msg.set_clipboard.paste = paste; @@ -461,8 +464,10 @@ input_manager_process_key(struct input_manager *im, // inject the text as input events clipboard_paste(controller); } else { - // store the text in the device clipboard and paste - set_device_clipboard(controller, true); + // store the text in the device clipboard and paste, + // without requesting an acknowledgment + set_device_clipboard(controller, true, + SC_SEQUENCE_INVALID); } } return; @@ -511,6 +516,7 @@ input_manager_process_key(struct input_manager *im, return; } + uint64_t ack_to_wait = SC_SEQUENCE_INVALID; bool is_ctrl_v = ctrl && !shift && keycode == SDLK_v && down && !repeat; if (is_ctrl_v) { if (im->legacy_paste) { @@ -518,16 +524,28 @@ input_manager_process_key(struct input_manager *im, clipboard_paste(controller); return; } + + // Request an acknowledgement only if necessary + uint64_t sequence = im->kp->async_paste ? im->next_sequence + : SC_SEQUENCE_INVALID; + // Synchronize the computer clipboard to the device clipboard before // sending Ctrl+v, to allow seamless copy-paste. - bool ok = set_device_clipboard(controller, false); + bool ok = set_device_clipboard(controller, false, sequence); if (!ok) { LOGW("Clipboard could not be synchronized, Ctrl+v not injected"); return; } + + if (im->kp->async_paste) { + // The key processor must wait for this ack before injecting Ctrl+v + ack_to_wait = sequence; + // Increment only when the request succeeded + ++im->next_sequence; + } } - im->kp->ops->process_key(im->kp, event, is_ctrl_v); + im->kp->ops->process_key(im->kp, event, ack_to_wait); } static void diff --git a/app/src/input_manager.h b/app/src/input_manager.h index f018f98a..22d77381 100644 --- a/app/src/input_manager.h +++ b/app/src/input_manager.h @@ -38,6 +38,8 @@ struct input_manager { unsigned key_repeat; SDL_Keycode last_keycode; uint16_t last_mod; + + uint64_t next_sequence; // used for request acknowledgements }; void diff --git a/app/src/keyboard_inject.c b/app/src/keyboard_inject.c index e59b5520..4112fd4c 100644 --- a/app/src/keyboard_inject.c +++ b/app/src/keyboard_inject.c @@ -189,11 +189,11 @@ convert_input_key(const SDL_KeyboardEvent *from, struct control_msg *to, static void sc_key_processor_process_key(struct sc_key_processor *kp, const SDL_KeyboardEvent *event, - bool device_clipboard_set) { + uint64_t ack_to_wait) { // The device clipboard synchronization and the key event messages are // serialized, there is nothing special to do to ensure that the clipboard // is set before injecting Ctrl+v. - (void) device_clipboard_set; + (void) ack_to_wait; struct sc_keyboard_inject *ki = DOWNCAST(kp); @@ -256,5 +256,7 @@ sc_keyboard_inject_init(struct sc_keyboard_inject *ki, .process_text = sc_key_processor_process_text, }; + // Key injection and clipboard synchronization are serialized + ki->key_processor.async_paste = false; ki->key_processor.ops = &ops; } diff --git a/app/src/receiver.c b/app/src/receiver.c index 52e6034d..eeb206f1 100644 --- a/app/src/receiver.c +++ b/app/src/receiver.c @@ -7,12 +7,16 @@ #include "util/log.h" bool -receiver_init(struct receiver *receiver, sc_socket control_socket) { +receiver_init(struct receiver *receiver, sc_socket control_socket, + struct sc_acksync *acksync) { bool ok = sc_mutex_init(&receiver->mutex); if (!ok) { return false; } + receiver->control_socket = control_socket; + receiver->acksync = acksync; + return true; } @@ -22,7 +26,7 @@ receiver_destroy(struct receiver *receiver) { } static void -process_msg(struct device_msg *msg) { +process_msg(struct receiver *receiver, struct device_msg *msg) { switch (msg->type) { case DEVICE_MSG_TYPE_CLIPBOARD: { char *current = SDL_GetClipboardText(); @@ -38,13 +42,16 @@ process_msg(struct device_msg *msg) { break; } case DEVICE_MSG_TYPE_ACK_CLIPBOARD: - // TODO + assert(receiver->acksync); + LOGD("Ack device clipboard sequence=%" PRIu64_, + msg->ack_clipboard.sequence); + sc_acksync_ack(receiver->acksync, msg->ack_clipboard.sequence); break; } } static ssize_t -process_msgs(const unsigned char *buf, size_t len) { +process_msgs(struct receiver *receiver, const unsigned char *buf, size_t len) { size_t head = 0; for (;;) { struct device_msg msg; @@ -56,7 +63,7 @@ process_msgs(const unsigned char *buf, size_t len) { return head; } - process_msg(&msg); + process_msg(receiver, &msg); device_msg_destroy(&msg); head += r; @@ -84,7 +91,7 @@ run_receiver(void *data) { } head += r; - ssize_t consumed = process_msgs(buf, head); + ssize_t consumed = process_msgs(receiver, buf, head); if (consumed == -1) { // an error occurred break; diff --git a/app/src/receiver.h b/app/src/receiver.h index 99f128a4..3c4e8c64 100644 --- a/app/src/receiver.h +++ b/app/src/receiver.h @@ -5,6 +5,7 @@ #include +#include "util/acksync.h" #include "util/net.h" #include "util/thread.h" @@ -14,10 +15,13 @@ struct receiver { sc_socket control_socket; sc_thread thread; sc_mutex mutex; + + struct sc_acksync *acksync; }; bool -receiver_init(struct receiver *receiver, sc_socket control_socket); +receiver_init(struct receiver *receiver, sc_socket control_socket, + struct sc_acksync *acksync); void receiver_destroy(struct receiver *receiver); diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 61087681..04239bf5 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -27,6 +27,7 @@ #include "screen.h" #include "server.h" #include "stream.h" +#include "util/acksync.h" #include "util/log.h" #include "util/net.h" #ifdef HAVE_V4L2 @@ -46,6 +47,8 @@ struct scrcpy { struct file_handler file_handler; #ifdef HAVE_AOA_HID struct sc_aoa aoa; + // sequence/ack helper to synchronize clipboard and Ctrl+v via HID + struct sc_acksync acksync; #endif union { struct sc_keyboard_inject keyboard_inject; @@ -340,6 +343,8 @@ scrcpy(struct scrcpy_options *options) { bool controller_started = false; bool screen_initialized = false; + struct sc_acksync *acksync = NULL; + struct sc_server_params params = { .serial = options->serial, .log_level = options->log_level, @@ -445,7 +450,18 @@ scrcpy(struct scrcpy_options *options) { } if (options->control) { - if (!controller_init(&s->controller, s->server.control_socket)) { +#ifdef HAVE_AOA_HID + if (options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_HID) { + bool ok = sc_acksync_init(&s->acksync); + if (!ok) { + goto end; + } + + acksync = &s->acksync; + } +#endif + if (!controller_init(&s->controller, s->server.control_socket, + acksync)) { goto end; } controller_initialized = true; @@ -521,7 +537,7 @@ scrcpy(struct scrcpy_options *options) { #ifdef HAVE_AOA_HID bool aoa_hid_ok = false; - bool ok = sc_aoa_init(&s->aoa, serial); + bool ok = sc_aoa_init(&s->aoa, serial, acksync); if (!ok) { goto aoa_hid_end; } @@ -586,6 +602,9 @@ end: sc_hid_keyboard_destroy(&s->keyboard_hid); sc_aoa_stop(&s->aoa); } + if (acksync) { + sc_acksync_destroy(acksync); + } #endif if (controller_started) { controller_stop(&s->controller); diff --git a/app/src/trait/key_processor.h b/app/src/trait/key_processor.h index 1f8132f2..f4afe27b 100644 --- a/app/src/trait/key_processor.h +++ b/app/src/trait/key_processor.h @@ -14,6 +14,15 @@ * Component able to process and inject keys should implement this trait. */ struct sc_key_processor { + /** + * Set by the implementation to indicate that it must explicitly wait for + * the clipboard to be set on the device before injecting Ctrl+v to avoid + * race conditions. If it is set, the input_manager will pass a valid + * ack_to_wait to process_key() in case of clipboard synchronization + * resulting of the key event. + */ + bool async_paste; + const struct sc_key_processor_ops *ops; }; @@ -22,13 +31,14 @@ struct sc_key_processor_ops { /** * Process the keyboard event * - * The flag `device_clipboard_set` indicates that the input manager sent a - * control message to synchronize the device clipboard as a result of this - * key event. + * The `sequence` number (if different from `SC_SEQUENCE_INVALID`) indicates + * the acknowledgement number to wait for before injecting this event. + * This allows to ensure that the device clipboard is set before injecting + * Ctrl+v on the device. */ void (*process_key)(struct sc_key_processor *kp, const SDL_KeyboardEvent *event, - bool device_clipboard_set); + uint64_t ack_to_wait); void (*process_text)(struct sc_key_processor *kp, diff --git a/app/src/util/acksync.h b/app/src/util/acksync.h index 1fd34444..58ab1b35 100644 --- a/app/src/util/acksync.h +++ b/app/src/util/acksync.h @@ -9,6 +9,11 @@ /** * Helper to wait for acknowledgments + * + * In practice, it is used to wait for device clipboard acknowledgement from the + * server before injecting Ctrl+v via AOA HID, in order to avoid pasting the + * content of the old device clipboard (if Ctrl+v was injected before the + * clipboard content was actually set). */ struct sc_acksync { sc_mutex mutex;