Android/scrcpy
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity

Add support for USB HID keyboard over AOAv2

Browse source 
This provides a better input experience, by simulating a physical
keyboard. It converts SDL keyboard events to proper HID events, and send
them over AOAv2.

This is a rewriting and bugfix of the origin code from @amosbird:
<https://github.com/Genymobile/scrcpy/issues/279#issuecomment-453819354>

The feature is enabled the command line option -K or --hid-keyboard,
and is only available on Linux, over USB.

Refs <https://source.android.com/devices/accessories/aoa2#hid-support>
Refs <https://www.usb.org/sites/default/files/hid1_11.pdf>

PR #2632 <https://github.com/Genymobile/scrcpy/pull/2632>

Signed-off-by: Romain Vimont <rom@rom1v.com>
This commit is contained in:
Alynx Zhou 2021-09-10 18:57:35 +08:00 committed by Romain Vimont
parent f7d1efdf1d
commit 207082977a
11 changed files with 939 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
BUILD.md
View file

@ -14,7 +14,8 @@ First, you need to install the required packages:
# for Debian/Ubuntu
sudo apt install ffmpeg libsdl2-2.0-0 adb wget \
gcc git pkg-config meson ninja-build libsdl2-dev \
libavcodec-dev libavdevice-dev libavformat-dev libavutil-dev
libavcodec-dev libavdevice-dev libavformat-dev libavutil-dev \
libusb-1.0-0 libusb-dev
```
Then clone the repo and execute the installation script
@ -88,11 +89,12 @@ Install the required packages from your package manager.
```bash
# runtime dependencies
sudo apt install ffmpeg libsdl2-2.0-0 adb
sudo apt install ffmpeg libsdl2-2.0-0 adb libusb-1.0-0
# client build dependencies
sudo apt install gcc git pkg-config meson ninja-build libsdl2-dev \
libavcodec-dev libavdevice-dev libavformat-dev libavutil-dev
libavcodec-dev libavdevice-dev libavformat-dev libavutil-dev \
libusb-dev
# server build dependencies
sudo apt install openjdk-11-jdk
@ -114,7 +116,7 @@ pip3 install meson
sudo dnf install https://download1.rpmfusion.org/free/fedora/rpmfusion-free-release-$(rpm -E %fedora).noarch.rpm
# client build dependencies
sudo dnf install SDL2-devel ffms2-devel meson gcc make
sudo dnf install SDL2-devel ffms2-devel libusb-devel meson gcc make
# server build dependencies
sudo dnf install java-devel

39
README.md
View file

@ -673,6 +673,39 @@ content (if supported by the app) relative to the center of the screen.
Concretely, scrcpy generates additional touch events from a "virtual finger" at
a location inverted through the center of the screen.
#### Physical keyboard simulation (HID)
By default, scrcpy uses Android key or text injection: it works everywhere, but
is limited to ASCII.
On Linux, scrcpy can simulate a USB physical keyboard on Android to provide a
better input experience (using [USB HID over AOAv2][hid-aoav2]): the virtual
keyboard is disabled and it works for all characters and IME.
[hid-aoav2]: https://source.android.com/devices/accessories/aoa2#hid-support
However, it only works if the device is connected by USB, and is currently only
supported on Linux.
To enable this mode:
```bash
scrcpy --hid-keyboard
scrcpy -K # short version
```
If it fails for some reason (for example because the device is not connected via
USB), it automatically fallbacks to the default mode (with a log in the
console). This allows to use the same command line options when connected over
USB and TCP/IP.
In this mode, raw key events (scancodes) are sent to the device, independently
of the host key mapping. Therefore, if your keyboard layout does not match, it
must be configured on the Android device, in Settings → System → Languages and
input → [Physical keyboard].
[Physical keyboard]: https://github.com/Genymobile/scrcpy/pull/2632#issuecomment-923756915
#### Text injection preference
@ -692,6 +725,9 @@ scrcpy --prefer-text
(but this will break keyboard behavior in games)
This option has no effect on HID keyboard (all key events are sent as
scancodes in this mode).
[textevents]: https://blog.rom1v.com/2018/03/introducing-scrcpy/#handle-text-input
[prefertext]: https://github.com/Genymobile/scrcpy/issues/650#issuecomment-512945343
@ -707,6 +743,9 @@ To avoid forwarding repeated key events:
scrcpy --no-key-repeat
```
This option has no effect on HID keyboard (key repeat is handled by Android
directly in this mode).
#### Right-click and middle-click

16
app/meson.build
View file

@ -42,6 +42,14 @@ if v4l2_support
src += [ 'src/v4l2_sink.c' ]
endif
aoa_hid_support = host_machine.system() == 'linux'
if aoa_hid_support
src += [
'src/aoa_hid.c',
'src/hid_keyboard.c',
]
endif
check_functions = [
'strdup'
]
@ -62,8 +70,11 @@ if not get_option('crossbuild_windows')
dependencies += dependency('libavdevice')
endif
else
if aoa_hid_support
dependencies += dependency('libusb-1.0')
endif
else
# cross-compile mingw32 build (from Linux to Windows)
prebuilt_sdl2 = meson.get_cross_property('prebuilt_sdl2')
sdl2_bin_dir = meson.current_source_dir() + '/../prebuilt-deps/' + prebuilt_sdl2 + '/bin'
@ -140,6 +151,9 @@ conf.set('SERVER_DEBUGGER_METHOD_NEW', get_option('server_debugger_method') == '
# enable V4L2 support (linux only)
conf.set('HAVE_V4L2', v4l2_support)
# enable HID over AOA support (linux only)
conf.set('HAVE_AOA_HID', aoa_hid_support)
configure_file(configuration: conf, output: 'config.h')
src_dir = include_directories('src')

8
app/scrcpy.1
View file

@ -82,6 +82,14 @@ Start in fullscreen.
.B \-h, \-\-help
Print this help.
.TP
.B \-K, \-\-hid\-keyboard
Simulate a physical keyboard by using HID over AOAv2.
This provides a better experience for IME users, and allows to generate non-ASCII characters, contrary to the default injection method.
It may only work over USB, and is currently only supported on Linux.
.TP
.B \-\-legacy\-paste
Inject computer clipboard text as a sequence of key events on Ctrl+v (like MOD+Shift+v).

368
app/src/aoa_hid.c Normal file
View file

@ -0,0 +1,368 @@
#include "util/log.h"
#include <assert.h>
#include <stdio.h>
#include "aoa_hid.h"
// See <https://source.android.com/devices/accessories/aoa2#hid-support>.
#define ACCESSORY_REGISTER_HID 54
#define ACCESSORY_SET_HID_REPORT_DESC 56
#define ACCESSORY_SEND_HID_EVENT 57
#define ACCESSORY_UNREGISTER_HID 55
#define DEFAULT_TIMEOUT 1000
static void
sc_hid_event_log(const struct sc_hid_event *event) {
// HID Event: [00] FF FF FF FF...
assert(event->size);
unsigned buffer_size = event->size * 3 + 1;
char *buffer = malloc(buffer_size);
if (!buffer) {
return;
}
for (unsigned i = 0; i < event->size; ++i) {
snprintf(buffer + i * 3, 4, " %02x", event->buffer[i]);
}
LOGV("HID Event: [%d]%s", event->accessory_id, buffer);
free(buffer);
}
void
sc_hid_event_init(struct sc_hid_event *hid_event, uint16_t accessory_id,
unsigned char *buffer, uint16_t buffer_size) {
hid_event->accessory_id = accessory_id;
hid_event->buffer = buffer;
hid_event->size = buffer_size;
}
void
sc_hid_event_destroy(struct sc_hid_event *hid_event) {
free(hid_event->buffer);
}
static inline void
log_libusb_error(enum libusb_error errcode) {
LOGW("libusb error: %s", libusb_strerror(errcode));
}
static bool
accept_device(libusb_device *device, const char *serial) {
// do not log any USB error in this function, it is expected that many USB
// devices available on the computer have permission restrictions
struct libusb_device_descriptor desc;
libusb_get_device_descriptor(device, &desc);
if (!desc.iSerialNumber) {
return false;
}
libusb_device_handle *handle;
int result = libusb_open(device, &handle);
if (result < 0) {
return false;
}
char buffer[128];
result = libusb_get_string_descriptor_ascii(handle, desc.iSerialNumber,
(unsigned char *) buffer,
sizeof(buffer));
libusb_close(handle);
if (result < 0) {
return false;
}
buffer[sizeof(buffer) - 1] = '\0'; // just in case
// accept the device if its serial matches
return !strcmp(buffer, serial);
}
static libusb_device *
sc_aoa_find_usb_device(const char *serial) {
if (!serial) {
return NULL;
}
libusb_device **list;
libusb_device *result = NULL;
ssize_t count = libusb_get_device_list(NULL, &list);
if (count < 0) {
log_libusb_error((enum libusb_error) count);
return NULL;
}
for (size_t i = 0; i < (size_t) count; ++i) {
libusb_device *device = list[i];
if (accept_device(device, serial)) {
result = libusb_ref_device(device);
break;
}
}
libusb_free_device_list(list, 1);
return result;
}
static int
sc_aoa_open_usb_handle(libusb_device *device, libusb_device_handle **handle) {
int result = libusb_open(device, handle);
if (result < 0) {
log_libusb_error((enum libusb_error) result);
return result;
}
return 0;
}
bool
sc_aoa_init(struct sc_aoa *aoa, const char *serial) {
cbuf_init(&aoa->queue);
if (!sc_mutex_init(&aoa->mutex)) {
return false;
}
if (!sc_cond_init(&aoa->event_cond)) {
sc_mutex_destroy(&aoa->mutex);
return false;
}
if (libusb_init(&aoa->usb_context) != LIBUSB_SUCCESS) {
sc_cond_destroy(&aoa->event_cond);
sc_mutex_destroy(&aoa->mutex);
return false;
}
aoa->usb_device = sc_aoa_find_usb_device(serial);
if (!aoa->usb_device) {
LOGW("USB device of serial %s not found", serial);
libusb_exit(aoa->usb_context);
sc_mutex_destroy(&aoa->mutex);
sc_cond_destroy(&aoa->event_cond);
return false;
}
if (sc_aoa_open_usb_handle(aoa->usb_device, &aoa->usb_handle) < 0) {
LOGW("Open USB handle failed");
libusb_unref_device(aoa->usb_device);
libusb_exit(aoa->usb_context);
sc_cond_destroy(&aoa->event_cond);
sc_mutex_destroy(&aoa->mutex);
return false;
}
aoa->stopped = false;
return true;
}
void
sc_aoa_destroy(struct sc_aoa *aoa) {
// Destroy remaining events
struct sc_hid_event event;
while (cbuf_take(&aoa->queue, &event)) {
sc_hid_event_destroy(&event);
}
libusb_close(aoa->usb_handle);
libusb_unref_device(aoa->usb_device);
libusb_exit(aoa->usb_context);
sc_cond_destroy(&aoa->event_cond);
sc_mutex_destroy(&aoa->mutex);
}
static bool
sc_aoa_register_hid(struct sc_aoa *aoa, uint16_t accessory_id,
uint16_t report_desc_size) {
uint8_t request_type = LIBUSB_ENDPOINT_OUT | LIBUSB_REQUEST_TYPE_VENDOR;
uint8_t request = ACCESSORY_REGISTER_HID;
// <https://source.android.com/devices/accessories/aoa2.html#hid-support>
// value (arg0): accessory assigned ID for the HID device
// index (arg1): total length of the HID report descriptor
uint16_t value = accessory_id;
uint16_t index = report_desc_size;
unsigned char *buffer = NULL;
uint16_t length = 0;
int result = libusb_control_transfer(aoa->usb_handle, request_type, request,
value, index, buffer, length,
DEFAULT_TIMEOUT);
if (result < 0) {
log_libusb_error((enum libusb_error) result);
return false;
}
return true;
}
static bool
sc_aoa_set_hid_report_desc(struct sc_aoa *aoa, uint16_t accessory_id,
const unsigned char *report_desc,
uint16_t report_desc_size) {
uint8_t request_type = LIBUSB_ENDPOINT_OUT | LIBUSB_REQUEST_TYPE_VENDOR;
uint8_t request = ACCESSORY_SET_HID_REPORT_DESC;
/**
* If the HID descriptor is longer than the endpoint zero max packet size,
* the descriptor will be sent in multiple ACCESSORY_SET_HID_REPORT_DESC
* commands. The data for the descriptor must be sent sequentially
* if multiple packets are needed.
* <https://source.android.com/devices/accessories/aoa2.html#hid-support>
*
* libusb handles packet abstraction internally, so we don't need to care
* about bMaxPacketSize0 here.
*
* See <https://libusb.sourceforge.io/api-1.0/libusb_packetoverflow.html>
*/
// value (arg0): accessory assigned ID for the HID device
// index (arg1): offset of data (buffer) in descriptor
uint16_t value = accessory_id;
uint16_t index = 0;
// libusb_control_transfer expects a pointer to non-const
unsigned char *buffer = (unsigned char *) report_desc;
uint16_t length = report_desc_size;
int result = libusb_control_transfer(aoa->usb_handle, request_type, request,
value, index, buffer, length,
DEFAULT_TIMEOUT);
if (result < 0) {
log_libusb_error((enum libusb_error) result);
return false;
}
return true;
}
bool
sc_aoa_setup_hid(struct sc_aoa *aoa, uint16_t accessory_id,
const unsigned char *report_desc, uint16_t report_desc_size) {
bool ok = sc_aoa_register_hid(aoa, accessory_id, report_desc_size);
if (!ok) {
return false;
}
ok = sc_aoa_set_hid_report_desc(aoa, accessory_id, report_desc,
report_desc_size);
if (!ok) {
if (!sc_aoa_unregister_hid(aoa, accessory_id)) {
LOGW("Could not unregister HID");
}
return false;
}
return true;
}
static bool
sc_aoa_send_hid_event(struct sc_aoa *aoa, const struct sc_hid_event *event) {
uint8_t request_type = LIBUSB_ENDPOINT_OUT | LIBUSB_REQUEST_TYPE_VENDOR;
uint8_t request = ACCESSORY_SEND_HID_EVENT;
// <https://source.android.com/devices/accessories/aoa2.html#hid-support>
// value (arg0): accessory assigned ID for the HID device
// index (arg1): 0 (unused)
uint16_t value = event->accessory_id;
uint16_t index = 0;
unsigned char *buffer = event->buffer;
uint16_t length = event->size;
int result = libusb_control_transfer(aoa->usb_handle, request_type, request,
value, index, buffer, length,
DEFAULT_TIMEOUT);
if (result < 0) {
log_libusb_error((enum libusb_error) result);
return false;
}
return true;
}
bool
sc_aoa_unregister_hid(struct sc_aoa *aoa, const uint16_t accessory_id) {
uint8_t request_type = LIBUSB_ENDPOINT_OUT | LIBUSB_REQUEST_TYPE_VENDOR;
uint8_t request = ACCESSORY_UNREGISTER_HID;
// <https://source.android.com/devices/accessories/aoa2.html#hid-support>
// value (arg0): accessory assigned ID for the HID device
// index (arg1): 0
uint16_t value = accessory_id;
uint16_t index = 0;
unsigned char *buffer = NULL;
uint16_t length = 0;
int result = libusb_control_transfer(aoa->usb_handle, request_type, request,
value, index, buffer, length,
DEFAULT_TIMEOUT);
if (result < 0) {
log_libusb_error((enum libusb_error) result);
return false;
}
return true;
}
bool
sc_aoa_push_hid_event(struct sc_aoa *aoa, const struct sc_hid_event *event) {
if (sc_get_log_level() <= SC_LOG_LEVEL_VERBOSE) {
sc_hid_event_log(event);
}
sc_mutex_lock(&aoa->mutex);
bool was_empty = cbuf_is_empty(&aoa->queue);
bool res = cbuf_push(&aoa->queue, *event);
if (was_empty) {
sc_cond_signal(&aoa->event_cond);
}
sc_mutex_unlock(&aoa->mutex);
return res;
}
static int
run_aoa_thread(void *data) {
struct sc_aoa *aoa = data;
for (;;) {
sc_mutex_lock(&aoa->mutex);
while (!aoa->stopped && cbuf_is_empty(&aoa->queue)) {
sc_cond_wait(&aoa->event_cond, &aoa->mutex);
}
if (aoa->stopped) {
// Stop immediately, do not process further events
sc_mutex_unlock(&aoa->mutex);
break;
}
struct sc_hid_event event;
bool non_empty = cbuf_take(&aoa->queue, &event);
assert(non_empty);
(void) non_empty;
sc_mutex_unlock(&aoa->mutex);
bool ok = sc_aoa_send_hid_event(aoa, &event);
sc_hid_event_destroy(&event);
if (!ok) {
LOGW("Could not send HID event to USB device");
}
}
return 0;
}
bool
sc_aoa_start(struct sc_aoa *aoa) {
LOGD("Starting AOA thread");
bool ok = sc_thread_create(&aoa->thread, run_aoa_thread, "aoa_thread", aoa);
if (!ok) {
LOGC("Could not start AOA thread");
return false;
}
return true;
}
void
sc_aoa_stop(struct sc_aoa *aoa) {
sc_mutex_lock(&aoa->mutex);
aoa->stopped = true;
sc_cond_signal(&aoa->event_cond);
sc_mutex_unlock(&aoa->mutex);
}
void
sc_aoa_join(struct sc_aoa *aoa) {
sc_thread_join(&aoa->thread, NULL);
}

65
app/src/aoa_hid.h Normal file
View file

@ -0,0 +1,65 @@
#ifndef SC_AOA_HID_H
#define SC_AOA_HID_H
#include <stdint.h>
#include <stdbool.h>
#include <libusb-1.0/libusb.h>
#include "scrcpy.h"
#include "util/cbuf.h"
#include "util/thread.h"
struct sc_hid_event {
uint16_t accessory_id;
unsigned char *buffer;
uint16_t size;
};
// Takes ownership of buffer
void
sc_hid_event_init(struct sc_hid_event *hid_event, uint16_t accessory_id,
unsigned char *buffer, uint16_t buffer_size);
void
sc_hid_event_destroy(struct sc_hid_event *hid_event);
struct sc_hid_event_queue CBUF(struct sc_hid_event, 64);
struct sc_aoa {
libusb_context *usb_context;
libusb_device *usb_device;
libusb_device_handle *usb_handle;
sc_thread thread;
sc_mutex mutex;
sc_cond event_cond;
bool stopped;
struct sc_hid_event_queue queue;
};
bool
sc_aoa_init(struct sc_aoa *aoa, const char *serial);
void
sc_aoa_destroy(struct sc_aoa *aoa);
bool
sc_aoa_start(struct sc_aoa *aoa);
void
sc_aoa_stop(struct sc_aoa *aoa);
void
sc_aoa_join(struct sc_aoa *aoa);
bool
sc_aoa_setup_hid(struct sc_aoa *aoa, uint16_t accessory_id,
const unsigned char *report_desc, uint16_t report_desc_size);
bool
sc_aoa_unregister_hid(struct sc_aoa *aoa, uint16_t accessory_id);
bool
sc_aoa_push_hid_event(struct sc_aoa *aoa, const struct sc_hid_event *event);
#endif

14
app/src/cli.c
View file

@ -76,6 +76,14 @@ scrcpy_print_usage(const char *arg0) {
" -f, --fullscreen\n"
" Start in fullscreen.\n"
"\n"
" -K, --hid-keyboard\n"
" Simulate a physical keyboard by using HID over AOAv2.\n"
" It provides a better experience for IME users, and allows to\n"
" generate non-ASCII characters, contrary to the default\n"
" injection method.\n"
" It may only work over USB, and is currently only supported\n"
" on Linux.\n"
"\n"
" -h, --help\n"
" Print this help.\n"
"\n"
@ -738,6 +746,7 @@ scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) {
OPT_FORWARD_ALL_CLICKS},
{"fullscreen", no_argument, NULL, 'f'},
{"help", no_argument, NULL, 'h'},
{"hid-keyboard", no_argument, NULL, 'K'},
{"legacy-paste", no_argument, NULL, OPT_LEGACY_PASTE},
{"lock-video-orientation", optional_argument, NULL,
OPT_LOCK_VIDEO_ORIENTATION},
@ -784,7 +793,7 @@ scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) {
optind = 0; // reset to start from the first argument in tests
int c;
while ((c = getopt_long(argc, argv, "b:c:fF:hm:nNp:r:s:StTvV:w",
while ((c = getopt_long(argc, argv, "b:c:fF:hKm:nNp:r:s:StTvV:w",
long_options, NULL)) != -1) {
switch (c) {
case 'b':
@ -817,6 +826,9 @@ scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) {
case 'h':
args->help = true;
break;
case 'K':
opts->keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_HID;
break;
case OPT_MAX_FPS:
if (!parse_max_fps(optarg, &opts->max_fps)) {
return false;

302
app/src/hid_keyboard.c Normal file
View file

@ -0,0 +1,302 @@
#include "hid_keyboard.h"
#include <assert.h>
#include <SDL2/SDL_events.h>
#include "util/log.h"
/** Downcast key processor to hid_keyboard */
#define DOWNCAST(KP) container_of(KP, struct sc_hid_keyboard, key_processor)
#define HID_KEYBOARD_ACCESSORY_ID 1
#define HID_MODIFIER_NONE 0x00
#define HID_MODIFIER_LEFT_CONTROL (1 << 0)
#define HID_MODIFIER_LEFT_SHIFT (1 << 1)
#define HID_MODIFIER_LEFT_ALT (1 << 2)
#define HID_MODIFIER_LEFT_GUI (1 << 3)
#define HID_MODIFIER_RIGHT_CONTROL (1 << 4)
#define HID_MODIFIER_RIGHT_SHIFT (1 << 5)
#define HID_MODIFIER_RIGHT_ALT (1 << 6)
#define HID_MODIFIER_RIGHT_GUI (1 << 7)
#define HID_KEYBOARD_INDEX_MODIFIER 0
#define HID_KEYBOARD_INDEX_KEYS 2
// USB HID protocol says 6 keys in an event is the requirement for BIOS
// keyboard support, though OS could support more keys via modifying the report
// desc. 6 should be enough for scrcpy.
#define HID_KEYBOARD_MAX_KEYS 6
#define HID_KEYBOARD_EVENT_SIZE (2 + HID_KEYBOARD_MAX_KEYS)
#define HID_RESERVED 0x00
#define HID_ERROR_ROLL_OVER 0x01
/**
* For HID over AOAv2, only report descriptor is needed.
*
* The specification is available here:
* <https://www.usb.org/sites/default/files/hid1_11.pdf>
*
* In particular, read:
* - 6.2.2 Report Descriptor
* - Appendix B.1 Protocol 1 (Keyboard)
* - Appendix C: Keyboard Implementation
*
* Normally a basic HID keyboard uses 8 bytes:
* Modifier Reserved Key Key Key Key Key Key
*
* You can dump your device's report descriptor with:
*
* sudo usbhid-dump -m vid:pid -e descriptor
*
* (change vid:pid' to your device's vendor ID and product ID).
*/
static const unsigned char keyboard_report_desc[] = {
// Usage Page (Generic Desktop)
0x05, 0x01,
// Usage (Keyboard)
0x09, 0x06,
// Collection (Application)
0xA1, 0x01,
// Usage Page (Key Codes)
0x05, 0x07,
// Usage Minimum (224)
0x19, 0xE0,
// Usage Maximum (231)
0x29, 0xE7,
// Logical Minimum (0)
0x15, 0x00,
// Logical Maximum (1)
0x25, 0x01,
// Report Size (1)
0x75, 0x01,
// Report Count (8)
0x95, 0x08,
// Input (Data, Variable, Absolute): Modifier byte
0x81, 0x02,
// Report Size (8)
0x75, 0x08,
// Report Count (1)
0x95, 0x01,
// Input (Constant): Reserved byte
0x81, 0x01,
// Usage Page (LEDs)
0x05, 0x08,
// Usage Minimum (1)
0x19, 0x01,
// Usage Maximum (5)
0x29, 0x05,
// Report Size (1)
0x75, 0x01,
// Report Count (5)
0x95, 0x05,
// Output (Data, Variable, Absolute): LED report
0x91, 0x02,
// Report Size (3)
0x75, 0x03,
// Report Count (1)
0x95, 0x01,
// Output (Constant): LED report padding
0x91, 0x01,
// Usage Page (Key Codes)
0x05, 0x07,
// Usage Minimum (0)
0x19, 0x00,
// Usage Maximum (101)
0x29, SC_HID_KEYBOARD_KEYS - 1,
// Logical Minimum (0)
0x15, 0x00,
// Logical Maximum(101)
0x25, SC_HID_KEYBOARD_KEYS - 1,
// Report Size (8)
0x75, 0x08,
// Report Count (6)
0x95, HID_KEYBOARD_MAX_KEYS,
// Input (Data, Array): Keys
0x81, 0x00,
// End Collection
0xC0
};
static unsigned char
sdl_keymod_to_hid_modifiers(SDL_Keymod mod) {
unsigned char modifiers = HID_MODIFIER_NONE;
if (mod & KMOD_LCTRL) {
modifiers |= HID_MODIFIER_LEFT_CONTROL;
}
if (mod & KMOD_LSHIFT) {
modifiers |= HID_MODIFIER_LEFT_SHIFT;
}
if (mod & KMOD_LALT) {
modifiers |= HID_MODIFIER_LEFT_ALT;
}
if (mod & KMOD_LGUI) {
modifiers |= HID_MODIFIER_LEFT_GUI;
}
if (mod & KMOD_RCTRL) {
modifiers |= HID_MODIFIER_RIGHT_CONTROL;
}
if (mod & KMOD_RSHIFT) {
modifiers |= HID_MODIFIER_RIGHT_SHIFT;
}
if (mod & KMOD_RALT) {
modifiers |= HID_MODIFIER_RIGHT_ALT;
}
if (mod & KMOD_RGUI) {
modifiers |= HID_MODIFIER_RIGHT_GUI;
}
return modifiers;
}
static bool
sc_hid_keyboard_event_init(struct sc_hid_event *hid_event) {
unsigned char *buffer = malloc(HID_KEYBOARD_EVENT_SIZE);
if (!buffer) {
return false;
}
buffer[HID_KEYBOARD_INDEX_MODIFIER] = HID_MODIFIER_NONE;
buffer[1] = HID_RESERVED;
memset(&buffer[HID_KEYBOARD_INDEX_KEYS], 0, HID_KEYBOARD_MAX_KEYS);
sc_hid_event_init(hid_event, HID_KEYBOARD_ACCESSORY_ID, buffer,
HID_KEYBOARD_EVENT_SIZE);
return true;
}
static inline bool
scancode_is_modifier(SDL_Scancode scancode) {
return scancode >= SDL_SCANCODE_LCTRL && scancode <= SDL_SCANCODE_RGUI;
}
static bool
convert_hid_keyboard_event(struct sc_hid_keyboard *kb,
struct sc_hid_event *hid_event,
const SDL_KeyboardEvent *event) {
SDL_Scancode scancode = event->keysym.scancode;
assert(scancode >= 0);
// SDL also generates events when only modifiers are pressed, we cannot
// ignore them totally, for example press 'a' first then press 'Control',
// if we ignore 'Control' event, only 'a' is sent.
if (scancode >= SC_HID_KEYBOARD_KEYS && !scancode_is_modifier(scancode)) {
// Scancode to ignore
return false;
}
if (!sc_hid_keyboard_event_init(hid_event)) {
LOGW("Could not initialize HID keyboard event");
return false;
}
unsigned char modifiers = sdl_keymod_to_hid_modifiers(event->keysym.mod);
if (scancode < SC_HID_KEYBOARD_KEYS) {
// Pressed is true and released is false
kb->keys[scancode] = (event->type == SDL_KEYDOWN);
LOGV("keys[%02x] = %s", scancode,
kb->keys[scancode] ? "true" : "false");
}
hid_event->buffer[HID_KEYBOARD_INDEX_MODIFIER] = modifiers;
unsigned char *keys_buffer = &hid_event->buffer[HID_KEYBOARD_INDEX_KEYS];
// Re-calculate pressed keys every time
int keys_pressed_count = 0;
for (int i = 0; i < SC_HID_KEYBOARD_KEYS; ++i) {
if (kb->keys[i]) {
// USB HID protocol says that if keys exceeds report count, a
// phantom state should be reported
if (keys_pressed_count >= HID_KEYBOARD_MAX_KEYS) {
// Pantom state:
// - Modifiers
// - Reserved
// - ErrorRollOver * HID_MAX_KEYS
memset(keys_buffer, HID_ERROR_ROLL_OVER, HID_KEYBOARD_MAX_KEYS);
goto end;
}
keys_buffer[keys_pressed_count] = i;
++keys_pressed_count;
}
}
end:
LOGV("hid keyboard: key %-4s scancode=%02x (%u) mod=%02x",
event->type == SDL_KEYDOWN ? "down" : "up", event->keysym.scancode,
event->keysym.scancode, modifiers);
return true;
}
static void
sc_key_processor_process_key(struct sc_key_processor *kp,
const SDL_KeyboardEvent *event) {
if (event->repeat) {
// In USB HID protocol, key repeat is handled by the host (Android), so
// just ignore key repeat here.
return;
}
struct sc_hid_keyboard *kb = DOWNCAST(kp);
struct sc_hid_event hid_event;
// Not all keys are supported, just ignore unsupported keys
if (convert_hid_keyboard_event(kb, &hid_event, event)) {
if (!sc_aoa_push_hid_event(kb->aoa, &hid_event)) {
sc_hid_event_destroy(&hid_event);
LOGW("Could request HID event");
}
}
}
static void
sc_key_processor_process_text(struct sc_key_processor *kp,
const SDL_TextInputEvent *event) {
(void) kp;
(void) event;
// Never forward text input via HID (all the keys are injected separately)
}
bool
sc_hid_keyboard_init(struct sc_hid_keyboard *kb, struct sc_aoa *aoa) {
kb->aoa = aoa;
bool ok = sc_aoa_setup_hid(aoa, HID_KEYBOARD_ACCESSORY_ID,
keyboard_report_desc,
ARRAY_LEN(keyboard_report_desc));
if (!ok) {
LOGW("Register HID keyboard failed");
return false;
}
// Reset all states
memset(kb->keys, false, SC_HID_KEYBOARD_KEYS);
static const struct sc_key_processor_ops ops = {
.process_key = sc_key_processor_process_key,
.process_text = sc_key_processor_process_text,
};
kb->key_processor.ops = &ops;
return true;
}
void
sc_hid_keyboard_destroy(struct sc_hid_keyboard *kb) {
// Unregister HID keyboard so the soft keyboard shows again on Android
bool ok = sc_aoa_unregister_hid(kb->aoa, HID_KEYBOARD_ACCESSORY_ID);
if (!ok) {
LOGW("Could not unregister HID keyboard");
}
}

42
app/src/hid_keyboard.h Normal file
View file

@ -0,0 +1,42 @@
#ifndef SC_HID_KEYBOARD_H
#define SC_HID_KEYBOARD_H
#include "common.h"
#include <stdbool.h>
#include "aoa_hid.h"
#include "trait/key_processor.h"
// See "SDL2/SDL_scancode.h".
// Maybe SDL_Keycode is used by most people, but SDL_Scancode is taken from USB
// HID protocol.
// 0x65 is Application, typically AT-101 Keyboard ends here.
#define SC_HID_KEYBOARD_KEYS 0x66
/**
* HID keyboard events are sequence-based, every time keyboard state changes
* it sends an array of currently pressed keys, the host is responsible for
* compare events and determine which key becomes pressed and which key becomes
* released. In order to convert SDL_KeyboardEvent to HID events, we first use
* an array of keys to save each keys' state. And when a SDL_KeyboardEvent was
* emitted, we updated our state, and then we use a loop to generate HID
* events. The sequence of array elements is unimportant and when too much keys
* pressed at the same time (more than report count), we should generate
* phantom state. Don't forget that modifiers should be updated too, even for
* phantom state.
*/
struct sc_hid_keyboard {
struct sc_key_processor key_processor; // key processor trait
struct sc_aoa *aoa;
bool keys[SC_HID_KEYBOARD_KEYS];
};
bool
sc_hid_keyboard_init(struct sc_hid_keyboard *kb, struct sc_aoa *aoa);
void
sc_hid_keyboard_destroy(struct sc_hid_keyboard *kb);
#endif

77
app/src/scrcpy.c
View file

@ -18,6 +18,9 @@
#include "events.h"
#include "file_handler.h"
#include "input_manager.h"
#ifdef HAVE_AOA_HID
# include "hid_keyboard.h"
#endif
#include "keyboard_inject.h"
#include "mouse_inject.h"
#include "recorder.h"
@ -41,7 +44,15 @@ struct scrcpy {
#endif
struct controller controller;
struct file_handler file_handler;
struct sc_keyboard_inject keyboard_inject;
#ifdef HAVE_AOA_HID
struct sc_aoa aoa;
#endif
union {
struct sc_keyboard_inject keyboard_inject;
#ifdef HAVE_AOA_HID
struct sc_hid_keyboard keyboard_hid;
#endif
};
struct sc_mouse_inject mouse_inject;
struct input_manager input_manager;
};
@ -243,7 +254,7 @@ stream_on_eos(struct stream *stream, void *userdata) {
}
bool
scrcpy(const struct scrcpy_options *options) {
scrcpy(struct scrcpy_options *options) {
static struct scrcpy scrcpy;
struct scrcpy *s = &scrcpy;
@ -260,6 +271,9 @@ scrcpy(const struct scrcpy_options *options) {
bool v4l2_sink_initialized = false;
#endif
bool stream_started = false;
#ifdef HAVE_AOA_HID
bool aoa_hid_initialized = false;
#endif
bool controller_initialized = false;
bool controller_started = false;
bool screen_initialized = false;
@ -419,8 +433,50 @@ scrcpy(const struct scrcpy_options *options) {
struct sc_mouse_processor *mp = NULL;
if (options->control) {
sc_keyboard_inject_init(&s->keyboard_inject, &s->controller, options);
kp = &s->keyboard_inject.key_processor;
if (options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_HID) {
#ifdef HAVE_AOA_HID
bool aoa_hid_ok = false;
if (!sc_aoa_init(&s->aoa, options->serial)) {
goto aoa_hid_end;
}
if (!sc_hid_keyboard_init(&s->keyboard_hid, &s->aoa)) {
sc_aoa_destroy(&s->aoa);
goto aoa_hid_end;
}
if (!sc_aoa_start(&s->aoa)) {
sc_hid_keyboard_destroy(&s->keyboard_hid);
sc_aoa_destroy(&s->aoa);
goto aoa_hid_end;
}
aoa_hid_ok = true;
kp = &s->keyboard_hid.key_processor;
aoa_hid_initialized = true;
aoa_hid_end:
if (!aoa_hid_ok) {
LOGE("Failed to enable HID over AOA, "
"fallback to default keyboard injection method "
"(-K/--hid-keyboard ignored)");
options->keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_INJECT;
}
#else
LOGE("HID over AOA is not supported on this platform, "
"fallback to default keyboard injection method "
"(-K/--hid-keyboard ignored)");
options->keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_INJECT;
#endif
}
// keyboard_input_mode may have been reset if HID mode failed
if (options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_INJECT) {
sc_keyboard_inject_init(&s->keyboard_inject, &s->controller,
options);
kp = &s->keyboard_inject.key_processor;
}
sc_mouse_inject_init(&s->mouse_inject, &s->controller, &s->screen);
mp = &s->mouse_inject.mouse_processor;
@ -439,6 +495,12 @@ scrcpy(const struct scrcpy_options *options) {
end:
// The stream is not stopped explicitly, because it will stop by itself on
// end-of-stream
#ifdef HAVE_AOA_HID
if (aoa_hid_initialized) {
sc_hid_keyboard_destroy(&s->keyboard_hid);
sc_aoa_stop(&s->aoa);
}
#endif
if (controller_started) {
controller_stop(&s->controller);
}
@ -466,6 +528,13 @@ end:
}
#endif
#ifdef HAVE_AOA_HID
if (aoa_hid_initialized) {
sc_aoa_join(&s->aoa);
sc_aoa_destroy(&s->aoa);
}
#endif
// Destroy the screen only after the stream is guaranteed to be finished,
// because otherwise the screen could receive new frames after destruction
if (screen_initialized) {

9
app/src/scrcpy.h
View file

@ -33,6 +33,11 @@ enum sc_lock_video_orientation {
SC_LOCK_VIDEO_ORIENTATION_3,
};
enum sc_keyboard_input_mode {
SC_KEYBOARD_INPUT_MODE_INJECT,
SC_KEYBOARD_INPUT_MODE_HID,
};
#define SC_MAX_SHORTCUT_MODS 8
enum sc_shortcut_mod {
@ -68,6 +73,7 @@ struct scrcpy_options {
const char *v4l2_device;
enum sc_log_level log_level;
enum sc_record_format record_format;
enum sc_keyboard_input_mode keyboard_input_mode;
struct sc_port_range port_range;
struct sc_shortcut_mods shortcut_mods;
uint16_t max_size;
@ -112,6 +118,7 @@ struct scrcpy_options {
.v4l2_device = NULL, \
.log_level = SC_LOG_LEVEL_INFO, \
.record_format = SC_RECORD_FORMAT_AUTO, \
.keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_INJECT, \
.port_range = { \
.first = DEFAULT_LOCAL_PORT_RANGE_FIRST, \
.last = DEFAULT_LOCAL_PORT_RANGE_LAST, \
@ -151,6 +158,6 @@ struct scrcpy_options {
}
bool
scrcpy(const struct scrcpy_options *options);
scrcpy(struct scrcpy_options *options);
#endif