From 207082977a01b931c4451c1954c8197878c3ba10 Mon Sep 17 00:00:00 2001 From: Alynx Zhou Date: Fri, 10 Sep 2021 18:57:35 +0800 Subject: [PATCH] Add support for USB HID keyboard over AOAv2 This provides a better input experience, by simulating a physical keyboard. It converts SDL keyboard events to proper HID events, and send them over AOAv2. This is a rewriting and bugfix of the origin code from @amosbird: The feature is enabled the command line option -K or --hid-keyboard, and is only available on Linux, over USB. Refs Refs PR #2632 Signed-off-by: Romain Vimont --- BUILD.md | 10 +- README.md | 39 +++++ app/meson.build | 16 +- app/scrcpy.1 | 8 + app/src/aoa_hid.c | 368 +++++++++++++++++++++++++++++++++++++++++ app/src/aoa_hid.h | 65 ++++++++ app/src/cli.c | 14 +- app/src/hid_keyboard.c | 302 +++++++++++++++++++++++++++++++++ app/src/hid_keyboard.h | 42 +++++ app/src/scrcpy.c | 77 ++++++++- app/src/scrcpy.h | 9 +- 11 files changed, 939 insertions(+), 11 deletions(-) create mode 100644 app/src/aoa_hid.c create mode 100644 app/src/aoa_hid.h create mode 100644 app/src/hid_keyboard.c create mode 100644 app/src/hid_keyboard.h diff --git a/BUILD.md b/BUILD.md index 69475f2c..5edb5a21 100644 --- a/BUILD.md +++ b/BUILD.md @@ -14,7 +14,8 @@ First, you need to install the required packages: # for Debian/Ubuntu sudo apt install ffmpeg libsdl2-2.0-0 adb wget \ gcc git pkg-config meson ninja-build libsdl2-dev \ - libavcodec-dev libavdevice-dev libavformat-dev libavutil-dev + libavcodec-dev libavdevice-dev libavformat-dev libavutil-dev \ + libusb-1.0-0 libusb-dev ``` Then clone the repo and execute the installation script @@ -88,11 +89,12 @@ Install the required packages from your package manager. ```bash # runtime dependencies -sudo apt install ffmpeg libsdl2-2.0-0 adb +sudo apt install ffmpeg libsdl2-2.0-0 adb libusb-1.0-0 # client build dependencies sudo apt install gcc git pkg-config meson ninja-build libsdl2-dev \ - libavcodec-dev libavdevice-dev libavformat-dev libavutil-dev + libavcodec-dev libavdevice-dev libavformat-dev libavutil-dev \ + libusb-dev # server build dependencies sudo apt install openjdk-11-jdk @@ -114,7 +116,7 @@ pip3 install meson sudo dnf install https://download1.rpmfusion.org/free/fedora/rpmfusion-free-release-$(rpm -E %fedora).noarch.rpm # client build dependencies -sudo dnf install SDL2-devel ffms2-devel meson gcc make +sudo dnf install SDL2-devel ffms2-devel libusb-devel meson gcc make # server build dependencies sudo dnf install java-devel diff --git a/README.md b/README.md index 84dd5ff5..fdd74087 100644 --- a/README.md +++ b/README.md @@ -673,6 +673,39 @@ content (if supported by the app) relative to the center of the screen. Concretely, scrcpy generates additional touch events from a "virtual finger" at a location inverted through the center of the screen. +#### Physical keyboard simulation (HID) + +By default, scrcpy uses Android key or text injection: it works everywhere, but +is limited to ASCII. + +On Linux, scrcpy can simulate a USB physical keyboard on Android to provide a +better input experience (using [USB HID over AOAv2][hid-aoav2]): the virtual +keyboard is disabled and it works for all characters and IME. + +[hid-aoav2]: https://source.android.com/devices/accessories/aoa2#hid-support + +However, it only works if the device is connected by USB, and is currently only +supported on Linux. + +To enable this mode: + +```bash +scrcpy --hid-keyboard +scrcpy -K # short version +``` + +If it fails for some reason (for example because the device is not connected via +USB), it automatically fallbacks to the default mode (with a log in the +console). This allows to use the same command line options when connected over +USB and TCP/IP. + +In this mode, raw key events (scancodes) are sent to the device, independently +of the host key mapping. Therefore, if your keyboard layout does not match, it +must be configured on the Android device, in Settings → System → Languages and +input → [Physical keyboard]. + +[Physical keyboard]: https://github.com/Genymobile/scrcpy/pull/2632#issuecomment-923756915 + #### Text injection preference @@ -692,6 +725,9 @@ scrcpy --prefer-text (but this will break keyboard behavior in games) +This option has no effect on HID keyboard (all key events are sent as +scancodes in this mode). + [textevents]: https://blog.rom1v.com/2018/03/introducing-scrcpy/#handle-text-input [prefertext]: https://github.com/Genymobile/scrcpy/issues/650#issuecomment-512945343 @@ -707,6 +743,9 @@ To avoid forwarding repeated key events: scrcpy --no-key-repeat ``` +This option has no effect on HID keyboard (key repeat is handled by Android +directly in this mode). + #### Right-click and middle-click diff --git a/app/meson.build b/app/meson.build index 46d3994f..f82d37b3 100644 --- a/app/meson.build +++ b/app/meson.build @@ -42,6 +42,14 @@ if v4l2_support src += [ 'src/v4l2_sink.c' ] endif +aoa_hid_support = host_machine.system() == 'linux' +if aoa_hid_support + src += [ + 'src/aoa_hid.c', + 'src/hid_keyboard.c', + ] +endif + check_functions = [ 'strdup' ] @@ -62,8 +70,11 @@ if not get_option('crossbuild_windows') dependencies += dependency('libavdevice') endif -else + if aoa_hid_support + dependencies += dependency('libusb-1.0') + endif +else # cross-compile mingw32 build (from Linux to Windows) prebuilt_sdl2 = meson.get_cross_property('prebuilt_sdl2') sdl2_bin_dir = meson.current_source_dir() + '/../prebuilt-deps/' + prebuilt_sdl2 + '/bin' @@ -140,6 +151,9 @@ conf.set('SERVER_DEBUGGER_METHOD_NEW', get_option('server_debugger_method') == ' # enable V4L2 support (linux only) conf.set('HAVE_V4L2', v4l2_support) +# enable HID over AOA support (linux only) +conf.set('HAVE_AOA_HID', aoa_hid_support) + configure_file(configuration: conf, output: 'config.h') src_dir = include_directories('src') diff --git a/app/scrcpy.1 b/app/scrcpy.1 index 1b69a065..46db6e1d 100644 --- a/app/scrcpy.1 +++ b/app/scrcpy.1 @@ -82,6 +82,14 @@ Start in fullscreen. .B \-h, \-\-help Print this help. +.TP +.B \-K, \-\-hid\-keyboard +Simulate a physical keyboard by using HID over AOAv2. + +This provides a better experience for IME users, and allows to generate non-ASCII characters, contrary to the default injection method. + +It may only work over USB, and is currently only supported on Linux. + .TP .B \-\-legacy\-paste Inject computer clipboard text as a sequence of key events on Ctrl+v (like MOD+Shift+v). diff --git a/app/src/aoa_hid.c b/app/src/aoa_hid.c new file mode 100644 index 00000000..7fc0f34c --- /dev/null +++ b/app/src/aoa_hid.c @@ -0,0 +1,368 @@ +#include "util/log.h" + +#include +#include + +#include "aoa_hid.h" + +// See . +#define ACCESSORY_REGISTER_HID 54 +#define ACCESSORY_SET_HID_REPORT_DESC 56 +#define ACCESSORY_SEND_HID_EVENT 57 +#define ACCESSORY_UNREGISTER_HID 55 + +#define DEFAULT_TIMEOUT 1000 + +static void +sc_hid_event_log(const struct sc_hid_event *event) { + // HID Event: [00] FF FF FF FF... + assert(event->size); + unsigned buffer_size = event->size * 3 + 1; + char *buffer = malloc(buffer_size); + if (!buffer) { + return; + } + for (unsigned i = 0; i < event->size; ++i) { + snprintf(buffer + i * 3, 4, " %02x", event->buffer[i]); + } + LOGV("HID Event: [%d]%s", event->accessory_id, buffer); + free(buffer); +} + +void +sc_hid_event_init(struct sc_hid_event *hid_event, uint16_t accessory_id, + unsigned char *buffer, uint16_t buffer_size) { + hid_event->accessory_id = accessory_id; + hid_event->buffer = buffer; + hid_event->size = buffer_size; +} + +void +sc_hid_event_destroy(struct sc_hid_event *hid_event) { + free(hid_event->buffer); +} + +static inline void +log_libusb_error(enum libusb_error errcode) { + LOGW("libusb error: %s", libusb_strerror(errcode)); +} + +static bool +accept_device(libusb_device *device, const char *serial) { + // do not log any USB error in this function, it is expected that many USB + // devices available on the computer have permission restrictions + + struct libusb_device_descriptor desc; + libusb_get_device_descriptor(device, &desc); + + if (!desc.iSerialNumber) { + return false; + } + + libusb_device_handle *handle; + int result = libusb_open(device, &handle); + if (result < 0) { + return false; + } + + char buffer[128]; + result = libusb_get_string_descriptor_ascii(handle, desc.iSerialNumber, + (unsigned char *) buffer, + sizeof(buffer)); + libusb_close(handle); + if (result < 0) { + return false; + } + + buffer[sizeof(buffer) - 1] = '\0'; // just in case + + // accept the device if its serial matches + return !strcmp(buffer, serial); +} + +static libusb_device * +sc_aoa_find_usb_device(const char *serial) { + if (!serial) { + return NULL; + } + + libusb_device **list; + libusb_device *result = NULL; + ssize_t count = libusb_get_device_list(NULL, &list); + if (count < 0) { + log_libusb_error((enum libusb_error) count); + return NULL; + } + + for (size_t i = 0; i < (size_t) count; ++i) { + libusb_device *device = list[i]; + + if (accept_device(device, serial)) { + result = libusb_ref_device(device); + break; + } + } + libusb_free_device_list(list, 1); + return result; +} + +static int +sc_aoa_open_usb_handle(libusb_device *device, libusb_device_handle **handle) { + int result = libusb_open(device, handle); + if (result < 0) { + log_libusb_error((enum libusb_error) result); + return result; + } + return 0; +} + +bool +sc_aoa_init(struct sc_aoa *aoa, const char *serial) { + cbuf_init(&aoa->queue); + + if (!sc_mutex_init(&aoa->mutex)) { + return false; + } + + if (!sc_cond_init(&aoa->event_cond)) { + sc_mutex_destroy(&aoa->mutex); + return false; + } + + if (libusb_init(&aoa->usb_context) != LIBUSB_SUCCESS) { + sc_cond_destroy(&aoa->event_cond); + sc_mutex_destroy(&aoa->mutex); + return false; + } + + aoa->usb_device = sc_aoa_find_usb_device(serial); + if (!aoa->usb_device) { + LOGW("USB device of serial %s not found", serial); + libusb_exit(aoa->usb_context); + sc_mutex_destroy(&aoa->mutex); + sc_cond_destroy(&aoa->event_cond); + return false; + } + + if (sc_aoa_open_usb_handle(aoa->usb_device, &aoa->usb_handle) < 0) { + LOGW("Open USB handle failed"); + libusb_unref_device(aoa->usb_device); + libusb_exit(aoa->usb_context); + sc_cond_destroy(&aoa->event_cond); + sc_mutex_destroy(&aoa->mutex); + return false; + } + + aoa->stopped = false; + + return true; +} + +void +sc_aoa_destroy(struct sc_aoa *aoa) { + // Destroy remaining events + struct sc_hid_event event; + while (cbuf_take(&aoa->queue, &event)) { + sc_hid_event_destroy(&event); + } + + libusb_close(aoa->usb_handle); + libusb_unref_device(aoa->usb_device); + libusb_exit(aoa->usb_context); + sc_cond_destroy(&aoa->event_cond); + sc_mutex_destroy(&aoa->mutex); +} + +static bool +sc_aoa_register_hid(struct sc_aoa *aoa, uint16_t accessory_id, + uint16_t report_desc_size) { + uint8_t request_type = LIBUSB_ENDPOINT_OUT | LIBUSB_REQUEST_TYPE_VENDOR; + uint8_t request = ACCESSORY_REGISTER_HID; + // + // value (arg0): accessory assigned ID for the HID device + // index (arg1): total length of the HID report descriptor + uint16_t value = accessory_id; + uint16_t index = report_desc_size; + unsigned char *buffer = NULL; + uint16_t length = 0; + int result = libusb_control_transfer(aoa->usb_handle, request_type, request, + value, index, buffer, length, + DEFAULT_TIMEOUT); + if (result < 0) { + log_libusb_error((enum libusb_error) result); + return false; + } + + return true; +} + +static bool +sc_aoa_set_hid_report_desc(struct sc_aoa *aoa, uint16_t accessory_id, + const unsigned char *report_desc, + uint16_t report_desc_size) { + uint8_t request_type = LIBUSB_ENDPOINT_OUT | LIBUSB_REQUEST_TYPE_VENDOR; + uint8_t request = ACCESSORY_SET_HID_REPORT_DESC; + /** + * If the HID descriptor is longer than the endpoint zero max packet size, + * the descriptor will be sent in multiple ACCESSORY_SET_HID_REPORT_DESC + * commands. The data for the descriptor must be sent sequentially + * if multiple packets are needed. + * + * + * libusb handles packet abstraction internally, so we don't need to care + * about bMaxPacketSize0 here. + * + * See + */ + // value (arg0): accessory assigned ID for the HID device + // index (arg1): offset of data (buffer) in descriptor + uint16_t value = accessory_id; + uint16_t index = 0; + // libusb_control_transfer expects a pointer to non-const + unsigned char *buffer = (unsigned char *) report_desc; + uint16_t length = report_desc_size; + int result = libusb_control_transfer(aoa->usb_handle, request_type, request, + value, index, buffer, length, + DEFAULT_TIMEOUT); + if (result < 0) { + log_libusb_error((enum libusb_error) result); + return false; + } + + return true; +} + +bool +sc_aoa_setup_hid(struct sc_aoa *aoa, uint16_t accessory_id, + const unsigned char *report_desc, uint16_t report_desc_size) { + bool ok = sc_aoa_register_hid(aoa, accessory_id, report_desc_size); + if (!ok) { + return false; + } + + ok = sc_aoa_set_hid_report_desc(aoa, accessory_id, report_desc, + report_desc_size); + if (!ok) { + if (!sc_aoa_unregister_hid(aoa, accessory_id)) { + LOGW("Could not unregister HID"); + } + return false; + } + + return true; +} + +static bool +sc_aoa_send_hid_event(struct sc_aoa *aoa, const struct sc_hid_event *event) { + uint8_t request_type = LIBUSB_ENDPOINT_OUT | LIBUSB_REQUEST_TYPE_VENDOR; + uint8_t request = ACCESSORY_SEND_HID_EVENT; + // + // value (arg0): accessory assigned ID for the HID device + // index (arg1): 0 (unused) + uint16_t value = event->accessory_id; + uint16_t index = 0; + unsigned char *buffer = event->buffer; + uint16_t length = event->size; + int result = libusb_control_transfer(aoa->usb_handle, request_type, request, + value, index, buffer, length, + DEFAULT_TIMEOUT); + if (result < 0) { + log_libusb_error((enum libusb_error) result); + return false; + } + + return true; +} + +bool +sc_aoa_unregister_hid(struct sc_aoa *aoa, const uint16_t accessory_id) { + uint8_t request_type = LIBUSB_ENDPOINT_OUT | LIBUSB_REQUEST_TYPE_VENDOR; + uint8_t request = ACCESSORY_UNREGISTER_HID; + // + // value (arg0): accessory assigned ID for the HID device + // index (arg1): 0 + uint16_t value = accessory_id; + uint16_t index = 0; + unsigned char *buffer = NULL; + uint16_t length = 0; + int result = libusb_control_transfer(aoa->usb_handle, request_type, request, + value, index, buffer, length, + DEFAULT_TIMEOUT); + if (result < 0) { + log_libusb_error((enum libusb_error) result); + return false; + } + + return true; +} + +bool +sc_aoa_push_hid_event(struct sc_aoa *aoa, const struct sc_hid_event *event) { + if (sc_get_log_level() <= SC_LOG_LEVEL_VERBOSE) { + sc_hid_event_log(event); + } + + sc_mutex_lock(&aoa->mutex); + bool was_empty = cbuf_is_empty(&aoa->queue); + bool res = cbuf_push(&aoa->queue, *event); + if (was_empty) { + sc_cond_signal(&aoa->event_cond); + } + sc_mutex_unlock(&aoa->mutex); + return res; +} + +static int +run_aoa_thread(void *data) { + struct sc_aoa *aoa = data; + + for (;;) { + sc_mutex_lock(&aoa->mutex); + while (!aoa->stopped && cbuf_is_empty(&aoa->queue)) { + sc_cond_wait(&aoa->event_cond, &aoa->mutex); + } + if (aoa->stopped) { + // Stop immediately, do not process further events + sc_mutex_unlock(&aoa->mutex); + break; + } + struct sc_hid_event event; + bool non_empty = cbuf_take(&aoa->queue, &event); + assert(non_empty); + (void) non_empty; + sc_mutex_unlock(&aoa->mutex); + + bool ok = sc_aoa_send_hid_event(aoa, &event); + sc_hid_event_destroy(&event); + if (!ok) { + LOGW("Could not send HID event to USB device"); + } + } + return 0; +} + +bool +sc_aoa_start(struct sc_aoa *aoa) { + LOGD("Starting AOA thread"); + + bool ok = sc_thread_create(&aoa->thread, run_aoa_thread, "aoa_thread", aoa); + if (!ok) { + LOGC("Could not start AOA thread"); + return false; + } + + return true; +} + +void +sc_aoa_stop(struct sc_aoa *aoa) { + sc_mutex_lock(&aoa->mutex); + aoa->stopped = true; + sc_cond_signal(&aoa->event_cond); + sc_mutex_unlock(&aoa->mutex); +} + +void +sc_aoa_join(struct sc_aoa *aoa) { + sc_thread_join(&aoa->thread, NULL); +} diff --git a/app/src/aoa_hid.h b/app/src/aoa_hid.h new file mode 100644 index 00000000..11b879ce --- /dev/null +++ b/app/src/aoa_hid.h @@ -0,0 +1,65 @@ +#ifndef SC_AOA_HID_H +#define SC_AOA_HID_H + +#include +#include + +#include + +#include "scrcpy.h" +#include "util/cbuf.h" +#include "util/thread.h" + +struct sc_hid_event { + uint16_t accessory_id; + unsigned char *buffer; + uint16_t size; +}; + +// Takes ownership of buffer +void +sc_hid_event_init(struct sc_hid_event *hid_event, uint16_t accessory_id, + unsigned char *buffer, uint16_t buffer_size); + +void +sc_hid_event_destroy(struct sc_hid_event *hid_event); + +struct sc_hid_event_queue CBUF(struct sc_hid_event, 64); + +struct sc_aoa { + libusb_context *usb_context; + libusb_device *usb_device; + libusb_device_handle *usb_handle; + sc_thread thread; + sc_mutex mutex; + sc_cond event_cond; + bool stopped; + struct sc_hid_event_queue queue; +}; + +bool +sc_aoa_init(struct sc_aoa *aoa, const char *serial); + +void +sc_aoa_destroy(struct sc_aoa *aoa); + +bool +sc_aoa_start(struct sc_aoa *aoa); + +void +sc_aoa_stop(struct sc_aoa *aoa); + +void +sc_aoa_join(struct sc_aoa *aoa); + +bool +sc_aoa_setup_hid(struct sc_aoa *aoa, uint16_t accessory_id, + const unsigned char *report_desc, uint16_t report_desc_size); + +bool +sc_aoa_unregister_hid(struct sc_aoa *aoa, uint16_t accessory_id); + +bool +sc_aoa_push_hid_event(struct sc_aoa *aoa, const struct sc_hid_event *event); + +#endif diff --git a/app/src/cli.c b/app/src/cli.c index d22096ca..a0a508af 100644 --- a/app/src/cli.c +++ b/app/src/cli.c @@ -76,6 +76,14 @@ scrcpy_print_usage(const char *arg0) { " -f, --fullscreen\n" " Start in fullscreen.\n" "\n" + " -K, --hid-keyboard\n" + " Simulate a physical keyboard by using HID over AOAv2.\n" + " It provides a better experience for IME users, and allows to\n" + " generate non-ASCII characters, contrary to the default\n" + " injection method.\n" + " It may only work over USB, and is currently only supported\n" + " on Linux.\n" + "\n" " -h, --help\n" " Print this help.\n" "\n" @@ -738,6 +746,7 @@ scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) { OPT_FORWARD_ALL_CLICKS}, {"fullscreen", no_argument, NULL, 'f'}, {"help", no_argument, NULL, 'h'}, + {"hid-keyboard", no_argument, NULL, 'K'}, {"legacy-paste", no_argument, NULL, OPT_LEGACY_PASTE}, {"lock-video-orientation", optional_argument, NULL, OPT_LOCK_VIDEO_ORIENTATION}, @@ -784,7 +793,7 @@ scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) { optind = 0; // reset to start from the first argument in tests int c; - while ((c = getopt_long(argc, argv, "b:c:fF:hm:nNp:r:s:StTvV:w", + while ((c = getopt_long(argc, argv, "b:c:fF:hKm:nNp:r:s:StTvV:w", long_options, NULL)) != -1) { switch (c) { case 'b': @@ -817,6 +826,9 @@ scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) { case 'h': args->help = true; break; + case 'K': + opts->keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_HID; + break; case OPT_MAX_FPS: if (!parse_max_fps(optarg, &opts->max_fps)) { return false; diff --git a/app/src/hid_keyboard.c b/app/src/hid_keyboard.c new file mode 100644 index 00000000..2599e1d2 --- /dev/null +++ b/app/src/hid_keyboard.c @@ -0,0 +1,302 @@ +#include "hid_keyboard.h" + +#include +#include + +#include "util/log.h" + +/** Downcast key processor to hid_keyboard */ +#define DOWNCAST(KP) container_of(KP, struct sc_hid_keyboard, key_processor) + +#define HID_KEYBOARD_ACCESSORY_ID 1 + +#define HID_MODIFIER_NONE 0x00 +#define HID_MODIFIER_LEFT_CONTROL (1 << 0) +#define HID_MODIFIER_LEFT_SHIFT (1 << 1) +#define HID_MODIFIER_LEFT_ALT (1 << 2) +#define HID_MODIFIER_LEFT_GUI (1 << 3) +#define HID_MODIFIER_RIGHT_CONTROL (1 << 4) +#define HID_MODIFIER_RIGHT_SHIFT (1 << 5) +#define HID_MODIFIER_RIGHT_ALT (1 << 6) +#define HID_MODIFIER_RIGHT_GUI (1 << 7) + +#define HID_KEYBOARD_INDEX_MODIFIER 0 +#define HID_KEYBOARD_INDEX_KEYS 2 + +// USB HID protocol says 6 keys in an event is the requirement for BIOS +// keyboard support, though OS could support more keys via modifying the report +// desc. 6 should be enough for scrcpy. +#define HID_KEYBOARD_MAX_KEYS 6 +#define HID_KEYBOARD_EVENT_SIZE (2 + HID_KEYBOARD_MAX_KEYS) + +#define HID_RESERVED 0x00 +#define HID_ERROR_ROLL_OVER 0x01 + +/** + * For HID over AOAv2, only report descriptor is needed. + * + * The specification is available here: + * + * + * In particular, read: + * - 6.2.2 Report Descriptor + * - Appendix B.1 Protocol 1 (Keyboard) + * - Appendix C: Keyboard Implementation + * + * Normally a basic HID keyboard uses 8 bytes: + * Modifier Reserved Key Key Key Key Key Key + * + * You can dump your device's report descriptor with: + * + * sudo usbhid-dump -m vid:pid -e descriptor + * + * (change vid:pid' to your device's vendor ID and product ID). + */ +static const unsigned char keyboard_report_desc[] = { + // Usage Page (Generic Desktop) + 0x05, 0x01, + // Usage (Keyboard) + 0x09, 0x06, + + // Collection (Application) + 0xA1, 0x01, + + // Usage Page (Key Codes) + 0x05, 0x07, + // Usage Minimum (224) + 0x19, 0xE0, + // Usage Maximum (231) + 0x29, 0xE7, + // Logical Minimum (0) + 0x15, 0x00, + // Logical Maximum (1) + 0x25, 0x01, + // Report Size (1) + 0x75, 0x01, + // Report Count (8) + 0x95, 0x08, + // Input (Data, Variable, Absolute): Modifier byte + 0x81, 0x02, + + // Report Size (8) + 0x75, 0x08, + // Report Count (1) + 0x95, 0x01, + // Input (Constant): Reserved byte + 0x81, 0x01, + + // Usage Page (LEDs) + 0x05, 0x08, + // Usage Minimum (1) + 0x19, 0x01, + // Usage Maximum (5) + 0x29, 0x05, + // Report Size (1) + 0x75, 0x01, + // Report Count (5) + 0x95, 0x05, + // Output (Data, Variable, Absolute): LED report + 0x91, 0x02, + + // Report Size (3) + 0x75, 0x03, + // Report Count (1) + 0x95, 0x01, + // Output (Constant): LED report padding + 0x91, 0x01, + + // Usage Page (Key Codes) + 0x05, 0x07, + // Usage Minimum (0) + 0x19, 0x00, + // Usage Maximum (101) + 0x29, SC_HID_KEYBOARD_KEYS - 1, + // Logical Minimum (0) + 0x15, 0x00, + // Logical Maximum(101) + 0x25, SC_HID_KEYBOARD_KEYS - 1, + // Report Size (8) + 0x75, 0x08, + // Report Count (6) + 0x95, HID_KEYBOARD_MAX_KEYS, + // Input (Data, Array): Keys + 0x81, 0x00, + + // End Collection + 0xC0 +}; + +static unsigned char +sdl_keymod_to_hid_modifiers(SDL_Keymod mod) { + unsigned char modifiers = HID_MODIFIER_NONE; + if (mod & KMOD_LCTRL) { + modifiers |= HID_MODIFIER_LEFT_CONTROL; + } + if (mod & KMOD_LSHIFT) { + modifiers |= HID_MODIFIER_LEFT_SHIFT; + } + if (mod & KMOD_LALT) { + modifiers |= HID_MODIFIER_LEFT_ALT; + } + if (mod & KMOD_LGUI) { + modifiers |= HID_MODIFIER_LEFT_GUI; + } + if (mod & KMOD_RCTRL) { + modifiers |= HID_MODIFIER_RIGHT_CONTROL; + } + if (mod & KMOD_RSHIFT) { + modifiers |= HID_MODIFIER_RIGHT_SHIFT; + } + if (mod & KMOD_RALT) { + modifiers |= HID_MODIFIER_RIGHT_ALT; + } + if (mod & KMOD_RGUI) { + modifiers |= HID_MODIFIER_RIGHT_GUI; + } + return modifiers; +} + +static bool +sc_hid_keyboard_event_init(struct sc_hid_event *hid_event) { + unsigned char *buffer = malloc(HID_KEYBOARD_EVENT_SIZE); + if (!buffer) { + return false; + } + + buffer[HID_KEYBOARD_INDEX_MODIFIER] = HID_MODIFIER_NONE; + buffer[1] = HID_RESERVED; + memset(&buffer[HID_KEYBOARD_INDEX_KEYS], 0, HID_KEYBOARD_MAX_KEYS); + + sc_hid_event_init(hid_event, HID_KEYBOARD_ACCESSORY_ID, buffer, + HID_KEYBOARD_EVENT_SIZE); + return true; +} + +static inline bool +scancode_is_modifier(SDL_Scancode scancode) { + return scancode >= SDL_SCANCODE_LCTRL && scancode <= SDL_SCANCODE_RGUI; +} + +static bool +convert_hid_keyboard_event(struct sc_hid_keyboard *kb, + struct sc_hid_event *hid_event, + const SDL_KeyboardEvent *event) { + SDL_Scancode scancode = event->keysym.scancode; + assert(scancode >= 0); + + // SDL also generates events when only modifiers are pressed, we cannot + // ignore them totally, for example press 'a' first then press 'Control', + // if we ignore 'Control' event, only 'a' is sent. + if (scancode >= SC_HID_KEYBOARD_KEYS && !scancode_is_modifier(scancode)) { + // Scancode to ignore + return false; + } + + if (!sc_hid_keyboard_event_init(hid_event)) { + LOGW("Could not initialize HID keyboard event"); + return false; + } + + unsigned char modifiers = sdl_keymod_to_hid_modifiers(event->keysym.mod); + + if (scancode < SC_HID_KEYBOARD_KEYS) { + // Pressed is true and released is false + kb->keys[scancode] = (event->type == SDL_KEYDOWN); + LOGV("keys[%02x] = %s", scancode, + kb->keys[scancode] ? "true" : "false"); + } + + hid_event->buffer[HID_KEYBOARD_INDEX_MODIFIER] = modifiers; + + unsigned char *keys_buffer = &hid_event->buffer[HID_KEYBOARD_INDEX_KEYS]; + // Re-calculate pressed keys every time + int keys_pressed_count = 0; + for (int i = 0; i < SC_HID_KEYBOARD_KEYS; ++i) { + if (kb->keys[i]) { + // USB HID protocol says that if keys exceeds report count, a + // phantom state should be reported + if (keys_pressed_count >= HID_KEYBOARD_MAX_KEYS) { + // Pantom state: + // - Modifiers + // - Reserved + // - ErrorRollOver * HID_MAX_KEYS + memset(keys_buffer, HID_ERROR_ROLL_OVER, HID_KEYBOARD_MAX_KEYS); + goto end; + } + + keys_buffer[keys_pressed_count] = i; + ++keys_pressed_count; + } + } + +end: + LOGV("hid keyboard: key %-4s scancode=%02x (%u) mod=%02x", + event->type == SDL_KEYDOWN ? "down" : "up", event->keysym.scancode, + event->keysym.scancode, modifiers); + + return true; +} + +static void +sc_key_processor_process_key(struct sc_key_processor *kp, + const SDL_KeyboardEvent *event) { + if (event->repeat) { + // In USB HID protocol, key repeat is handled by the host (Android), so + // just ignore key repeat here. + return; + } + + struct sc_hid_keyboard *kb = DOWNCAST(kp); + + struct sc_hid_event hid_event; + // Not all keys are supported, just ignore unsupported keys + if (convert_hid_keyboard_event(kb, &hid_event, event)) { + if (!sc_aoa_push_hid_event(kb->aoa, &hid_event)) { + sc_hid_event_destroy(&hid_event); + LOGW("Could request HID event"); + } + } +} + +static void +sc_key_processor_process_text(struct sc_key_processor *kp, + const SDL_TextInputEvent *event) { + (void) kp; + (void) event; + + // Never forward text input via HID (all the keys are injected separately) +} + +bool +sc_hid_keyboard_init(struct sc_hid_keyboard *kb, struct sc_aoa *aoa) { + kb->aoa = aoa; + + bool ok = sc_aoa_setup_hid(aoa, HID_KEYBOARD_ACCESSORY_ID, + keyboard_report_desc, + ARRAY_LEN(keyboard_report_desc)); + if (!ok) { + LOGW("Register HID keyboard failed"); + return false; + } + + // Reset all states + memset(kb->keys, false, SC_HID_KEYBOARD_KEYS); + + static const struct sc_key_processor_ops ops = { + .process_key = sc_key_processor_process_key, + .process_text = sc_key_processor_process_text, + }; + + kb->key_processor.ops = &ops; + + return true; +} + +void +sc_hid_keyboard_destroy(struct sc_hid_keyboard *kb) { + // Unregister HID keyboard so the soft keyboard shows again on Android + bool ok = sc_aoa_unregister_hid(kb->aoa, HID_KEYBOARD_ACCESSORY_ID); + if (!ok) { + LOGW("Could not unregister HID keyboard"); + } +} diff --git a/app/src/hid_keyboard.h b/app/src/hid_keyboard.h new file mode 100644 index 00000000..d8276cad --- /dev/null +++ b/app/src/hid_keyboard.h @@ -0,0 +1,42 @@ +#ifndef SC_HID_KEYBOARD_H +#define SC_HID_KEYBOARD_H + +#include "common.h" + +#include + +#include "aoa_hid.h" +#include "trait/key_processor.h" + +// See "SDL2/SDL_scancode.h". +// Maybe SDL_Keycode is used by most people, but SDL_Scancode is taken from USB +// HID protocol. +// 0x65 is Application, typically AT-101 Keyboard ends here. +#define SC_HID_KEYBOARD_KEYS 0x66 + +/** + * HID keyboard events are sequence-based, every time keyboard state changes + * it sends an array of currently pressed keys, the host is responsible for + * compare events and determine which key becomes pressed and which key becomes + * released. In order to convert SDL_KeyboardEvent to HID events, we first use + * an array of keys to save each keys' state. And when a SDL_KeyboardEvent was + * emitted, we updated our state, and then we use a loop to generate HID + * events. The sequence of array elements is unimportant and when too much keys + * pressed at the same time (more than report count), we should generate + * phantom state. Don't forget that modifiers should be updated too, even for + * phantom state. + */ +struct sc_hid_keyboard { + struct sc_key_processor key_processor; // key processor trait + + struct sc_aoa *aoa; + bool keys[SC_HID_KEYBOARD_KEYS]; +}; + +bool +sc_hid_keyboard_init(struct sc_hid_keyboard *kb, struct sc_aoa *aoa); + +void +sc_hid_keyboard_destroy(struct sc_hid_keyboard *kb); + +#endif diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 50250e2c..c4041562 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -18,6 +18,9 @@ #include "events.h" #include "file_handler.h" #include "input_manager.h" +#ifdef HAVE_AOA_HID +# include "hid_keyboard.h" +#endif #include "keyboard_inject.h" #include "mouse_inject.h" #include "recorder.h" @@ -41,7 +44,15 @@ struct scrcpy { #endif struct controller controller; struct file_handler file_handler; - struct sc_keyboard_inject keyboard_inject; +#ifdef HAVE_AOA_HID + struct sc_aoa aoa; +#endif + union { + struct sc_keyboard_inject keyboard_inject; +#ifdef HAVE_AOA_HID + struct sc_hid_keyboard keyboard_hid; +#endif + }; struct sc_mouse_inject mouse_inject; struct input_manager input_manager; }; @@ -243,7 +254,7 @@ stream_on_eos(struct stream *stream, void *userdata) { } bool -scrcpy(const struct scrcpy_options *options) { +scrcpy(struct scrcpy_options *options) { static struct scrcpy scrcpy; struct scrcpy *s = &scrcpy; @@ -260,6 +271,9 @@ scrcpy(const struct scrcpy_options *options) { bool v4l2_sink_initialized = false; #endif bool stream_started = false; +#ifdef HAVE_AOA_HID + bool aoa_hid_initialized = false; +#endif bool controller_initialized = false; bool controller_started = false; bool screen_initialized = false; @@ -419,8 +433,50 @@ scrcpy(const struct scrcpy_options *options) { struct sc_mouse_processor *mp = NULL; if (options->control) { - sc_keyboard_inject_init(&s->keyboard_inject, &s->controller, options); - kp = &s->keyboard_inject.key_processor; + if (options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_HID) { +#ifdef HAVE_AOA_HID + bool aoa_hid_ok = false; + if (!sc_aoa_init(&s->aoa, options->serial)) { + goto aoa_hid_end; + } + + if (!sc_hid_keyboard_init(&s->keyboard_hid, &s->aoa)) { + sc_aoa_destroy(&s->aoa); + goto aoa_hid_end; + } + + if (!sc_aoa_start(&s->aoa)) { + sc_hid_keyboard_destroy(&s->keyboard_hid); + sc_aoa_destroy(&s->aoa); + goto aoa_hid_end; + } + + aoa_hid_ok = true; + kp = &s->keyboard_hid.key_processor; + + aoa_hid_initialized = true; + +aoa_hid_end: + if (!aoa_hid_ok) { + LOGE("Failed to enable HID over AOA, " + "fallback to default keyboard injection method " + "(-K/--hid-keyboard ignored)"); + options->keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_INJECT; + } +#else + LOGE("HID over AOA is not supported on this platform, " + "fallback to default keyboard injection method " + "(-K/--hid-keyboard ignored)"); + options->keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_INJECT; +#endif + } + + // keyboard_input_mode may have been reset if HID mode failed + if (options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_INJECT) { + sc_keyboard_inject_init(&s->keyboard_inject, &s->controller, + options); + kp = &s->keyboard_inject.key_processor; + } sc_mouse_inject_init(&s->mouse_inject, &s->controller, &s->screen); mp = &s->mouse_inject.mouse_processor; @@ -439,6 +495,12 @@ scrcpy(const struct scrcpy_options *options) { end: // The stream is not stopped explicitly, because it will stop by itself on // end-of-stream +#ifdef HAVE_AOA_HID + if (aoa_hid_initialized) { + sc_hid_keyboard_destroy(&s->keyboard_hid); + sc_aoa_stop(&s->aoa); + } +#endif if (controller_started) { controller_stop(&s->controller); } @@ -466,6 +528,13 @@ end: } #endif +#ifdef HAVE_AOA_HID + if (aoa_hid_initialized) { + sc_aoa_join(&s->aoa); + sc_aoa_destroy(&s->aoa); + } +#endif + // Destroy the screen only after the stream is guaranteed to be finished, // because otherwise the screen could receive new frames after destruction if (screen_initialized) { diff --git a/app/src/scrcpy.h b/app/src/scrcpy.h index 8b76fb25..8cf4a917 100644 --- a/app/src/scrcpy.h +++ b/app/src/scrcpy.h @@ -33,6 +33,11 @@ enum sc_lock_video_orientation { SC_LOCK_VIDEO_ORIENTATION_3, }; +enum sc_keyboard_input_mode { + SC_KEYBOARD_INPUT_MODE_INJECT, + SC_KEYBOARD_INPUT_MODE_HID, +}; + #define SC_MAX_SHORTCUT_MODS 8 enum sc_shortcut_mod { @@ -68,6 +73,7 @@ struct scrcpy_options { const char *v4l2_device; enum sc_log_level log_level; enum sc_record_format record_format; + enum sc_keyboard_input_mode keyboard_input_mode; struct sc_port_range port_range; struct sc_shortcut_mods shortcut_mods; uint16_t max_size; @@ -112,6 +118,7 @@ struct scrcpy_options { .v4l2_device = NULL, \ .log_level = SC_LOG_LEVEL_INFO, \ .record_format = SC_RECORD_FORMAT_AUTO, \ + .keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_INJECT, \ .port_range = { \ .first = DEFAULT_LOCAL_PORT_RANGE_FIRST, \ .last = DEFAULT_LOCAL_PORT_RANGE_LAST, \ @@ -151,6 +158,6 @@ struct scrcpy_options { } bool -scrcpy(const struct scrcpy_options *options); +scrcpy(struct scrcpy_options *options); #endif