Add OTG mode

Add an option --otg to run scrcpy with only physical keyboard and mouse
simulation (HID over AOA), without mirroring and without requiring adb.

To avoid adding complexity into the scrcpy initialization and screen
implementation, OTG mode is implemented totally separately, with a
separate window.

PR #2974 <https://github.com/Genymobile/scrcpy/pull/2974>

app/meson.build

@@ -78,6 +78,8 @@ if usb_support
         'src/usb/aoa_hid.c',
         'src/usb/hid_keyboard.c',
         'src/usb/hid_mouse.c',
+        'src/usb/scrcpy_otg.c',
+        'src/usb/screen_otg.c',
         'src/usb/usb.c',
     ]
 endif

app/scrcpy.1

@@ -162,6 +162,18 @@ Do not forward repeated key events when a key is held down.
 .B \-\-no\-mipmaps
 If the renderer is OpenGL 3.0+ or OpenGL ES 2.0+, then mipmaps are automatically generated to improve downscaling quality. This option disables the generation of mipmaps.
 
+.TP
+.B \-\-otg
+Run in OTG mode: simulate physical keyboard and mouse, as if the computer keyboard and mouse were plugged directly to the device via an OTG cable.
+
+In this mode, adb (USB debugging) is not necessary, and mirroring is disabled.
+
+LAlt, LSuper or RSuper toggle the mouse capture mode, to give control of the mouse back to the computer.
+
+It may only work over USB, and is currently only supported on Linux.
+
+See \fB\-\-hid\-keyboard\fR and \fB\-\-hid\-mouse\fR.
+
 .TP
 .BI "\-p, \-\-port " port[:port]
 Set the TCP port (range) used by the client to listen.

app/src/cli.c

@@ -53,6 +53,7 @@
 #define OPT_TCPIP                  1033
 #define OPT_RAW_KEY_EVENTS         1034
 #define OPT_NO_DOWNSIZE_ON_ERROR   1035
+#define OPT_OTG                    1036
 
 struct sc_option {
     char shortopt;
@@ -276,6 +277,20 @@ static const struct sc_option options[] = {
                 "mipmaps are automatically generated to improve downscaling "
                 "quality. This option disables the generation of mipmaps.",
     },
+    {
+        .longopt_id = OPT_OTG,
+        .longopt = "otg",
+        .text = "Run in OTG mode: simulate physical keyboard and mouse, "
+                "as if the computer keyboard and mouse were plugged directly "
+                "to the device via an OTG cable.\n"
+                "In this mode, adb (USB debugging) is not necessary, and "
+                "mirroring is disabled.\n"
+                "LAlt, LSuper or RSuper toggle the mouse capture mode, to give "
+                "control of the mouse back to the computer.\n"
+                "It may only work over USB, and is currently only supported "
+                "on Linux.\n"
+                "See --hid-keyboard and --hid-mouse.",
+    },
     {
         .shortopt = 'p',
         .longopt = "port",
@@ -1500,6 +1515,15 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[],
             case OPT_NO_DOWNSIZE_ON_ERROR:
                 opts->downsize_on_error = false;
                 break;
+            case OPT_OTG:
+#ifdef HAVE_USB
+                opts->otg = true;
+                break;
+#else
+                LOGE("OTG mode (--otg) is not supported on this platform. It "
+                     "is only available on Linux.");
+                return false;
+#endif
             case OPT_V4L2_SINK:
 #ifdef HAVE_V4L2
                 opts->v4l2_device = optarg;
@@ -1610,6 +1634,43 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[],
         }
     }
 
+#ifdef HAVE_USB
+    if (opts->otg) {
+        // OTG mode is compatible with only very few options.
+        // Only report obvious errors.
+        if (opts->record_filename) {
+            LOGE("OTG mode: could not record");
+            return false;
+        }
+        if (opts->turn_screen_off) {
+            LOGE("OTG mode: could not turn screen off");
+            return false;
+        }
+        if (opts->stay_awake) {
+            LOGE("OTG mode: could not stay awake");
+            return false;
+        }
+        if (opts->show_touches) {
+            LOGE("OTG mode: could not request to show touches");
+            return false;
+        }
+        if (opts->power_off_on_close) {
+            LOGE("OTG mode: could not request power off on close");
+            return false;
+        }
+        if (opts->display_id) {
+            LOGE("OTG mode: could not select display");
+            return false;
+        }
+#ifdef HAVE_V4L2
+        if (opts->v4l2_device) {
+            LOGE("OTG mode: could not sink to V4L2 device");
+            return false;
+        }
+#endif
+    }
+#endif
+
     return true;
 }
 

app/src/events.h

@@ -2,3 +2,4 @@
 #define EVENT_STREAM_STOPPED           (SDL_USEREVENT + 1)
 #define EVENT_SERVER_CONNECTION_FAILED (SDL_USEREVENT + 2)
 #define EVENT_SERVER_CONNECTED         (SDL_USEREVENT + 3)
+#define EVENT_USB_DEVICE_DISCONNECTED  (SDL_USEREVENT + 4)

app/src/main.c

@@ -13,6 +13,7 @@
 #include "cli.h"
 #include "options.h"
 #include "scrcpy.h"
+#include "usb/scrcpy_otg.h"
 #include "util/log.h"
 
 static void
@@ -88,9 +89,14 @@ main(int argc, char *argv[]) {
         return 1;
     }
 
-    int res = scrcpy(&args.opts) ? 0 : 1;
+#ifdef HAVE_USB
+    bool ok = args.opts.otg ? scrcpy_otg(&args.opts)
+                            : scrcpy(&args.opts);
+#else
+    bool ok = scrcpy(&args.opts);
+#endif
 
     avformat_network_deinit(); // ignore failure
 
-    return res;
+    return ok ? 0 : 1;
 }

app/src/options.c

@@ -37,6 +37,9 @@ const struct scrcpy_options scrcpy_options_default = {
     .display_id = 0,
     .display_buffer = 0,
     .v4l2_buffer = 0,
+#ifdef HAVE_USB
+    .otg = false,
+#endif
     .show_touches = false,
     .fullscreen = false,
     .always_on_top = false,

app/src/options.h

@@ -112,6 +112,9 @@ struct scrcpy_options {
     uint32_t display_id;
     sc_tick display_buffer;
     sc_tick v4l2_buffer;
+#ifdef HAVE_USB
+    bool otg;
+#endif
     bool show_touches;
     bool fullscreen;
     bool always_on_top;

app/src/usb/scrcpy_otg.c

@@ -0,0 +1,213 @@
+#include "scrcpy_otg.h"
+
+#include <SDL2/SDL.h>
+
+#include "events.h"
+#include "screen_otg.h"
+#include "util/log.h"
+
+struct scrcpy_otg {
+    struct sc_usb usb;
+    struct sc_aoa aoa;
+    struct sc_hid_keyboard keyboard;
+    struct sc_hid_mouse mouse;
+
+    struct sc_screen_otg screen_otg;
+};
+
+static void
+sc_usb_on_disconnected(struct sc_usb *usb, void *userdata) {
+    (void) usb;
+    (void) userdata;
+
+    SDL_Event event;
+    event.type = EVENT_USB_DEVICE_DISCONNECTED;
+    int ret = SDL_PushEvent(&event);
+    if (ret < 0) {
+        LOGE("Could not post USB disconnection event: %s", SDL_GetError());
+    }
+}
+
+static bool
+event_loop(struct scrcpy_otg *s) {
+    SDL_Event event;
+    while (SDL_WaitEvent(&event)) {
+        switch (event.type) {
+            case EVENT_USB_DEVICE_DISCONNECTED:
+                LOGW("Device disconnected");
+                return false;
+            case SDL_QUIT:
+                LOGD("User requested to quit");
+                return true;
+            default:
+                sc_screen_otg_handle_event(&s->screen_otg, &event);
+                break;
+        }
+    }
+    return false;
+}
+
+bool
+scrcpy_otg(struct scrcpy_options *options) {
+    static struct scrcpy_otg scrcpy_otg;
+    struct scrcpy_otg *s = &scrcpy_otg;
+
+    const char *serial = options->serial;
+
+    // Minimal SDL initialization
+    if (SDL_Init(SDL_INIT_EVENTS)) {
+        LOGC("Could not initialize SDL: %s", SDL_GetError());
+        return false;
+    }
+
+    atexit(SDL_Quit);
+
+    bool ret = false;
+
+    struct sc_hid_keyboard *keyboard = NULL;
+    struct sc_hid_mouse *mouse = NULL;
+    bool usb_device_initialized = false;
+    bool usb_connected = false;
+    bool aoa_started = false;
+    bool aoa_initialized = false;
+
+    static const struct sc_usb_callbacks cbs = {
+        .on_disconnected = sc_usb_on_disconnected,
+    };
+    bool ok = sc_usb_init(&s->usb);
+    if (!ok) {
+        return false;
+    }
+
+    struct sc_usb_device usb_devices[16];
+    ssize_t count = sc_usb_find_devices(&s->usb, serial, usb_devices,
+                                        ARRAY_LEN(usb_devices));
+    if (count < 0) {
+        LOGE("Could not list USB devices");
+        goto end;
+    }
+
+    if (count == 0) {
+        if (serial) {
+            LOGE("Could not find USB device %s", serial);
+        } else {
+            LOGE("Could not find any USB device");
+        }
+        goto end;
+    }
+
+    if (count > 1) {
+        if (serial) {
+            LOGE("Multiple (%d) USB devices with serial %s:", (int) count,
+                 serial);
+        } else {
+            LOGE("Multiple (%d) USB devices:", (int) count);
+        }
+        for (size_t i = 0; i < (size_t) count; ++i) {
+            struct sc_usb_device *d = &usb_devices[i];
+            LOGE("    %-18s (%04" PRIx16 ":%04" PRIx16 ")  %s %s",
+                 d->serial, d->vid, d->pid, d->manufacturer, d->product);
+        }
+        if (!serial) {
+            LOGE("Specify the device via -s or --serial");
+        }
+        sc_usb_device_destroy_all(usb_devices, count);
+        goto end;
+    }
+    usb_device_initialized = true;
+
+    struct sc_usb_device *usb_device = &usb_devices[0];
+
+    LOGI("USB device: %s (%04" PRIx16 ":%04" PRIx16 ") %s %s",
+         usb_device->serial, usb_device->vid, usb_device->pid,
+         usb_device->manufacturer, usb_device->product);
+
+    ok = sc_usb_connect(&s->usb, usb_device->device, &cbs, NULL);
+    if (!ok) {
+        goto end;
+    }
+    usb_connected = true;
+
+    ok = sc_aoa_init(&s->aoa, &s->usb, NULL);
+    if (!ok) {
+        goto end;
+    }
+    aoa_initialized = true;
+
+    ok = sc_hid_keyboard_init(&s->keyboard, &s->aoa);
+    if (!ok) {
+        goto end;
+    }
+    keyboard = &s->keyboard;
+
+    ok = sc_hid_mouse_init(&s->mouse, &s->aoa);
+    if (!ok) {
+        goto end;
+    }
+    mouse = &s->mouse;
+
+    ok = sc_aoa_start(&s->aoa);
+    if (!ok) {
+        goto end;
+    }
+    aoa_started = true;
+
+    const char *window_title = options->window_title;
+    if (!window_title) {
+        window_title = usb_device->product ? usb_device->product : "scrcpy";
+    }
+
+    struct sc_screen_otg_params params = {
+        .keyboard = keyboard,
+        .mouse = mouse,
+        .window_title = window_title,
+        .always_on_top = options->always_on_top,
+        .window_x = options->window_x,
+        .window_y = options->window_y,
+        .window_borderless = options->window_borderless,
+    };
+
+    ok = sc_screen_otg_init(&s->screen_otg, &params);
+    if (!ok) {
+        goto end;
+    }
+
+    // usb_device not needed anymore
+    sc_usb_device_destroy(usb_device);
+    usb_device_initialized = false;
+
+    ret = event_loop(s);
+    LOGD("quit...");
+
+end:
+    if (aoa_started) {
+        sc_aoa_stop(&s->aoa);
+    }
+    sc_usb_stop(&s->usb);
+
+    if (mouse) {
+        sc_hid_mouse_destroy(&s->mouse);
+    }
+    if (keyboard) {
+        sc_hid_keyboard_destroy(&s->keyboard);
+    }
+
+    if (aoa_initialized) {
+        sc_aoa_join(&s->aoa);
+        sc_aoa_destroy(&s->aoa);
+    }
+
+    sc_usb_join(&s->usb);
+
+    if (usb_connected) {
+        sc_usb_disconnect(&s->usb);
+    }
+
+    if (usb_device_initialized) {
+        sc_usb_device_destroy(usb_device);
+    }
+
+    sc_usb_destroy(&s->usb);
+
+    return ret;
+}

app/src/usb/scrcpy_otg.h

@@ -0,0 +1,12 @@
+#ifndef SCRCPY_OTG_H
+#define SCRCPY_OTG_H
+
+#include "common.h"
+
+#include <stdbool.h>
+#include "options.h"
+
+bool
+scrcpy_otg(struct scrcpy_options *options);
+
+#endif

app/src/usb/screen_otg.c

@@ -0,0 +1,254 @@
+#include "screen_otg.h"
+
+#include "icon.h"
+#include "options.h"
+#include "util/log.h"
+
+static void
+sc_screen_otg_capture_mouse(struct sc_screen_otg *screen, bool capture) {
+    if (SDL_SetRelativeMouseMode(capture)) {
+        LOGE("Could not set relative mouse mode to %s: %s",
+             capture ? "true" : "false", SDL_GetError());
+        return;
+    }
+
+    screen->mouse_captured = capture;
+}
+
+static void
+sc_screen_otg_render(struct sc_screen_otg *screen) {
+    SDL_RenderClear(screen->renderer);
+    if (screen->texture) {
+        SDL_RenderCopy(screen->renderer, screen->texture, NULL, NULL);
+    }
+    SDL_RenderPresent(screen->renderer);
+}
+
+bool
+sc_screen_otg_init(struct sc_screen_otg *screen,
+                   const struct sc_screen_otg_params *params) {
+    screen->keyboard = params->keyboard;
+    screen->mouse = params->mouse;
+
+    screen->mouse_captured = false;
+    screen->mouse_capture_key_pressed = 0;
+
+    const char *title = params->window_title;
+    assert(title);
+
+    int x = params->window_x != SC_WINDOW_POSITION_UNDEFINED
+          ? params->window_x : (int) SDL_WINDOWPOS_UNDEFINED;
+    int y = params->window_y != SC_WINDOW_POSITION_UNDEFINED
+          ? params->window_y : (int) SDL_WINDOWPOS_UNDEFINED;
+    int width = 256;
+    int height = 256;
+
+    uint32_t window_flags = 0;
+    if (params->always_on_top) {
+        window_flags |= SDL_WINDOW_ALWAYS_ON_TOP;
+    }
+    if (params->window_borderless) {
+        window_flags |= SDL_WINDOW_BORDERLESS;
+    }
+
+    screen->window = SDL_CreateWindow(title, x, y, width, height, window_flags);
+    if (!screen->window) {
+        LOGE("Could not create window: %s", SDL_GetError());
+        return false;
+    }
+
+    screen->renderer = SDL_CreateRenderer(screen->window, -1, 0);
+    if (!screen->renderer) {
+        LOGE("Could not create renderer: %s", SDL_GetError());
+        goto error_destroy_window;
+    }
+
+    SDL_Surface *icon = scrcpy_icon_load();
+
+    if (icon) {
+        SDL_SetWindowIcon(screen->window, icon);
+
+        screen->texture = SDL_CreateTextureFromSurface(screen->renderer, icon);
+        scrcpy_icon_destroy(icon);
+        if (!screen->texture) {
+            goto error_destroy_renderer;
+        }
+    } else {
+        screen->texture = NULL;
+        LOGW("Could not load icon");
+    }
+
+    // Capture mouse on start
+    sc_screen_otg_capture_mouse(screen, true);
+
+    return true;
+
+error_destroy_window:
+    SDL_DestroyWindow(screen->window);
+error_destroy_renderer:
+    SDL_DestroyRenderer(screen->renderer);
+
+    return false;
+}
+
+void
+sc_screen_otg_destroy(struct sc_screen_otg *screen) {
+    if (screen->texture) {
+        SDL_DestroyTexture(screen->texture);
+    }
+    SDL_DestroyRenderer(screen->renderer);
+    SDL_DestroyWindow(screen->window);
+}
+
+static inline bool
+sc_screen_otg_is_mouse_capture_key(SDL_Keycode key) {
+    return key == SDLK_LALT || key == SDLK_LGUI || key == SDLK_RGUI;
+}
+
+static void
+sc_screen_otg_process_key(struct sc_screen_otg *screen,
+                             const SDL_KeyboardEvent *event) {
+    assert(screen->keyboard);
+    struct sc_key_processor *kp = &screen->keyboard->key_processor;
+
+    struct sc_key_event evt = {
+        .action = sc_action_from_sdl_keyboard_type(event->type),
+        .keycode = sc_keycode_from_sdl(event->keysym.sym),
+        .scancode = sc_scancode_from_sdl(event->keysym.scancode),
+        .repeat = event->repeat,
+        .mods_state = sc_mods_state_from_sdl(event->keysym.mod),
+    };
+
+    assert(kp->ops->process_key);
+    kp->ops->process_key(kp, &evt, SC_SEQUENCE_INVALID);
+}
+
+static void
+sc_screen_otg_process_mouse_motion(struct sc_screen_otg *screen,
+                                   const SDL_MouseMotionEvent *event) {
+    assert(screen->mouse);
+    struct sc_mouse_processor *mp = &screen->mouse->mouse_processor;
+
+    struct sc_mouse_motion_event evt = {
+        // .position not used for HID events
+        .xrel = event->xrel,
+        .yrel = event->yrel,
+        .buttons_state = sc_mouse_buttons_state_from_sdl(event->state, true),
+    };
+
+    assert(mp->ops->process_mouse_motion);
+    mp->ops->process_mouse_motion(mp, &evt);
+}
+
+static void
+sc_screen_otg_process_mouse_button(struct sc_screen_otg *screen,
+                                   const SDL_MouseButtonEvent *event) {
+    assert(screen->mouse);
+    struct sc_mouse_processor *mp = &screen->mouse->mouse_processor;
+
+    uint32_t sdl_buttons_state = SDL_GetMouseState(NULL, NULL);
+
+    struct sc_mouse_click_event evt = {
+        // .position not used for HID events
+        .action = sc_action_from_sdl_mousebutton_type(event->type),
+        .button = sc_mouse_button_from_sdl(event->button),
+        .buttons_state =
+            sc_mouse_buttons_state_from_sdl(sdl_buttons_state, true),
+    };
+
+    assert(mp->ops->process_mouse_click);
+    mp->ops->process_mouse_click(mp, &evt);
+}
+
+static void
+sc_screen_otg_process_mouse_wheel(struct sc_screen_otg *screen,
+                                  const SDL_MouseWheelEvent *event) {
+    assert(screen->mouse);
+    struct sc_mouse_processor *mp = &screen->mouse->mouse_processor;
+
+    uint32_t sdl_buttons_state = SDL_GetMouseState(NULL, NULL);
+
+    struct sc_mouse_scroll_event evt = {
+        // .position not used for HID events
+        .hscroll = event->x,
+        .vscroll = event->y,
+        .buttons_state =
+            sc_mouse_buttons_state_from_sdl(sdl_buttons_state, true),
+    };
+
+    assert(mp->ops->process_mouse_scroll);
+    mp->ops->process_mouse_scroll(mp, &evt);
+}
+
+void
+sc_screen_otg_handle_event(struct sc_screen_otg *screen, SDL_Event *event) {
+    switch (event->type) {
+        case SDL_WINDOWEVENT:
+            switch (event->window.event) {
+                case SDL_WINDOWEVENT_EXPOSED:
+                    sc_screen_otg_render(screen);
+                    break;
+                case SDL_WINDOWEVENT_FOCUS_LOST:
+                    sc_screen_otg_capture_mouse(screen, false);
+                    break;
+            }
+            return;
+        case SDL_KEYDOWN: {
+            SDL_Keycode key = event->key.keysym.sym;
+            if (sc_screen_otg_is_mouse_capture_key(key)) {
+                if (!screen->mouse_capture_key_pressed) {
+                    screen->mouse_capture_key_pressed = key;
+                } else {
+                    // Another mouse capture key has been pressed, cancel mouse
+                    // (un)capture
+                    screen->mouse_capture_key_pressed = 0;
+                }
+                // Mouse capture keys are never forwarded to the device
+                return;
+            }
+
+            sc_screen_otg_process_key(screen, &event->key);
+            break;
+        }
+        case SDL_KEYUP: {
+            SDL_Keycode key = event->key.keysym.sym;
+            SDL_Keycode cap = screen->mouse_capture_key_pressed;
+            screen->mouse_capture_key_pressed = 0;
+            if (sc_screen_otg_is_mouse_capture_key(key)) {
+                if (key == cap) {
+                    // A mouse capture key has been pressed then released:
+                    // toggle the capture mouse mode
+                    sc_screen_otg_capture_mouse(screen,
+                                                !screen->mouse_captured);
+                }
+                // Mouse capture keys are never forwarded to the device
+                return;
+            }
+
+            sc_screen_otg_process_key(screen, &event->key);
+            break;
+        }
+        case SDL_MOUSEMOTION:
+            if (screen->mouse_captured) {
+                sc_screen_otg_process_mouse_motion(screen, &event->motion);
+            }
+            break;
+        case SDL_MOUSEBUTTONDOWN:
+            if (screen->mouse_captured) {
+                sc_screen_otg_process_mouse_button(screen, &event->button);
+            }
+            break;
+        case SDL_MOUSEBUTTONUP:
+            if (screen->mouse_captured) {
+                sc_screen_otg_process_mouse_button(screen, &event->button);
+            } else {
+                sc_screen_otg_capture_mouse(screen, true);
+            }
+            break;
+        case SDL_MOUSEWHEEL:
+            if (screen->mouse_captured) {
+                sc_screen_otg_process_mouse_wheel(screen, &event->wheel);
+            }
+            break;
+    }
+}

app/src/usb/screen_otg.h

@@ -0,0 +1,46 @@
+#ifndef SC_SCREEN_OTG_H
+#define SC_SCREEN_OTG_H
+
+#include "common.h"
+
+#include <stdbool.h>
+#include <SDL2/SDL.h>
+
+#include "hid_keyboard.h"
+#include "hid_mouse.h"
+
+struct sc_screen_otg {
+    struct sc_hid_keyboard *keyboard;
+    struct sc_hid_mouse *mouse;
+
+    SDL_Window *window;
+    SDL_Renderer *renderer;
+    SDL_Texture *texture;
+
+    // See equivalent mechanism in screen.h
+    bool mouse_captured;
+    SDL_Keycode mouse_capture_key_pressed;
+};
+
+struct sc_screen_otg_params {
+    struct sc_hid_keyboard *keyboard;
+    struct sc_hid_mouse *mouse;
+
+    const char *window_title;
+    bool always_on_top;
+    int16_t window_x; // accepts SC_WINDOW_POSITION_UNDEFINED
+    int16_t window_y; // accepts SC_WINDOW_POSITION_UNDEFINED
+    bool window_borderless;
+};
+
+bool
+sc_screen_otg_init(struct sc_screen_otg *screen,
+                   const struct sc_screen_otg_params *params);
+
+void
+sc_screen_otg_destroy(struct sc_screen_otg *screen);
+
+void
+sc_screen_otg_handle_event(struct sc_screen_otg *screen, SDL_Event *event);
+
+#endif