From 77f876e29cdd6f78dd9de4be6511b6228d58a1e7 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 15 Sep 2019 16:16:17 +0200 Subject: [PATCH] Add "inject touch" control message Add a control message type in the protocol to forward touch events to the device. --- app/src/control_msg.c | 19 ++++++++++ app/src/control_msg.h | 7 ++++ app/tests/test_control_msg_serialize.c | 36 +++++++++++++++++++ .../com/genymobile/scrcpy/ControlMessage.java | 35 ++++++++++++++---- .../scrcpy/ControlMessageReader.java | 19 ++++++++++ .../scrcpy/ControlMessageReaderTest.java | 31 ++++++++++++++++ 6 files changed, 140 insertions(+), 7 deletions(-) diff --git a/app/src/control_msg.c b/app/src/control_msg.c index fff93592..11e87e40 100644 --- a/app/src/control_msg.c +++ b/app/src/control_msg.c @@ -1,6 +1,7 @@ #include "control_msg.h" #include +#include #include "config.h" #include "buffer_util.h" @@ -24,6 +25,16 @@ write_string(const char *utf8, size_t max_len, unsigned char *buf) { return 2 + len; } +static uint16_t +to_fixed_point_16(float f) { + SDL_assert(f >= 0.0f && f <= 1.0f); + uint32_t u = f * 0x1p16f; // 2^16 + if (u >= 0xffff) { + u = 0xffff; + } + return (uint16_t) u; +} + size_t control_msg_serialize(const struct control_msg *msg, unsigned char *buf) { buf[0] = msg->type; @@ -43,6 +54,14 @@ control_msg_serialize(const struct control_msg *msg, unsigned char *buf) { buffer_write32be(&buf[2], msg->inject_mouse_event.buttons); write_position(&buf[6], &msg->inject_mouse_event.position); return 18; + case CONTROL_MSG_TYPE_INJECT_TOUCH_EVENT: + buf[1] = msg->inject_touch_event.action; + buffer_write64be(&buf[2], msg->inject_touch_event.pointer_id); + write_position(&buf[10], &msg->inject_touch_event.position); + uint16_t pressure = + to_fixed_point_16(msg->inject_touch_event.pressure); + buffer_write16be(&buf[22], pressure); + return 24; case CONTROL_MSG_TYPE_INJECT_SCROLL_EVENT: write_position(&buf[1], &msg->inject_scroll_event.position); buffer_write32be(&buf[13], diff --git a/app/src/control_msg.h b/app/src/control_msg.h index 308d54a3..546564cf 100644 --- a/app/src/control_msg.h +++ b/app/src/control_msg.h @@ -19,6 +19,7 @@ enum control_msg_type { CONTROL_MSG_TYPE_INJECT_KEYCODE, CONTROL_MSG_TYPE_INJECT_TEXT, CONTROL_MSG_TYPE_INJECT_MOUSE_EVENT, + CONTROL_MSG_TYPE_INJECT_TOUCH_EVENT, CONTROL_MSG_TYPE_INJECT_SCROLL_EVENT, CONTROL_MSG_TYPE_BACK_OR_SCREEN_ON, CONTROL_MSG_TYPE_EXPAND_NOTIFICATION_PANEL, @@ -50,6 +51,12 @@ struct control_msg { enum android_motionevent_buttons buttons; struct position position; } inject_mouse_event; + struct { + enum android_motionevent_action action; + uint64_t pointer_id; + struct position position; + float pressure; + } inject_touch_event; struct { struct position position; int32_t hscroll; diff --git a/app/tests/test_control_msg_serialize.c b/app/tests/test_control_msg_serialize.c index c0c501f2..ea06211a 100644 --- a/app/tests/test_control_msg_serialize.c +++ b/app/tests/test_control_msg_serialize.c @@ -100,6 +100,41 @@ static void test_serialize_inject_mouse_event(void) { assert(!memcmp(buf, expected, sizeof(expected))); } +static void test_serialize_inject_touch_event(void) { + struct control_msg msg = { + .type = CONTROL_MSG_TYPE_INJECT_TOUCH_EVENT, + .inject_touch_event = { + .action = AMOTION_EVENT_ACTION_DOWN, + .pointer_id = 0x1234567887654321L, + .position = { + .point = { + .x = 100, + .y = 200, + }, + .screen_size = { + .width = 1080, + .height = 1920, + }, + }, + .pressure = 1.0f, + }, + }; + + unsigned char buf[CONTROL_MSG_SERIALIZED_MAX_SIZE]; + int size = control_msg_serialize(&msg, buf); + assert(size == 24); + + const unsigned char expected[] = { + CONTROL_MSG_TYPE_INJECT_TOUCH_EVENT, + 0x00, // AKEY_EVENT_ACTION_DOWN + 0x12, 0x34, 0x56, 0x78, 0x87, 0x65, 0x43, 0x21, // pointer id + 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0xc8, // 100 200 + 0x04, 0x38, 0x07, 0x80, // 1080 1920 + 0xff, 0xff, // pressure + }; + assert(!memcmp(buf, expected, sizeof(expected))); +} + static void test_serialize_inject_scroll_event(void) { struct control_msg msg = { .type = CONTROL_MSG_TYPE_INJECT_SCROLL_EVENT, @@ -237,6 +272,7 @@ int main(void) { test_serialize_inject_text(); test_serialize_inject_text_long(); test_serialize_inject_mouse_event(); + test_serialize_inject_touch_event(); test_serialize_inject_scroll_event(); test_serialize_back_or_screen_on(); test_serialize_expand_notification_panel(); diff --git a/server/src/main/java/com/genymobile/scrcpy/ControlMessage.java b/server/src/main/java/com/genymobile/scrcpy/ControlMessage.java index a1cd873a..34da7741 100644 --- a/server/src/main/java/com/genymobile/scrcpy/ControlMessage.java +++ b/server/src/main/java/com/genymobile/scrcpy/ControlMessage.java @@ -8,13 +8,14 @@ public final class ControlMessage { public static final int TYPE_INJECT_KEYCODE = 0; public static final int TYPE_INJECT_TEXT = 1; public static final int TYPE_INJECT_MOUSE_EVENT = 2; - public static final int TYPE_INJECT_SCROLL_EVENT = 3; - public static final int TYPE_BACK_OR_SCREEN_ON = 4; - public static final int TYPE_EXPAND_NOTIFICATION_PANEL = 5; - public static final int TYPE_COLLAPSE_NOTIFICATION_PANEL = 6; - public static final int TYPE_GET_CLIPBOARD = 7; - public static final int TYPE_SET_CLIPBOARD = 8; - public static final int TYPE_SET_SCREEN_POWER_MODE = 9; + public static final int TYPE_INJECT_TOUCH_EVENT = 3; + public static final int TYPE_INJECT_SCROLL_EVENT = 4; + public static final int TYPE_BACK_OR_SCREEN_ON = 5; + public static final int TYPE_EXPAND_NOTIFICATION_PANEL = 6; + public static final int TYPE_COLLAPSE_NOTIFICATION_PANEL = 7; + public static final int TYPE_GET_CLIPBOARD = 8; + public static final int TYPE_SET_CLIPBOARD = 9; + public static final int TYPE_SET_SCREEN_POWER_MODE = 10; private int type; private String text; @@ -22,6 +23,8 @@ public final class ControlMessage { private int action; // KeyEvent.ACTION_* or MotionEvent.ACTION_* or POWER_MODE_* private int keycode; // KeyEvent.KEYCODE_* private int buttons; // MotionEvent.BUTTON_* + private long pointerId; + private float pressure; private Position position; private int hScroll; private int vScroll; @@ -54,6 +57,16 @@ public final class ControlMessage { return msg; } + public static ControlMessage createInjectTouchEvent(int action, long pointerId, Position position, float pressure) { + ControlMessage msg = new ControlMessage(); + msg.type = TYPE_INJECT_TOUCH_EVENT; + msg.action = action; + msg.pointerId = pointerId; + msg.pressure = pressure; + msg.position = position; + return msg; + } + public static ControlMessage createInjectScrollEvent(Position position, int hScroll, int vScroll) { ControlMessage msg = new ControlMessage(); msg.type = TYPE_INJECT_SCROLL_EVENT; @@ -110,6 +123,14 @@ public final class ControlMessage { return buttons; } + public long getPointerId() { + return pointerId; + } + + public float getPressure() { + return pressure; + } + public Position getPosition() { return position; } diff --git a/server/src/main/java/com/genymobile/scrcpy/ControlMessageReader.java b/server/src/main/java/com/genymobile/scrcpy/ControlMessageReader.java index 8ced049d..e6a6c905 100644 --- a/server/src/main/java/com/genymobile/scrcpy/ControlMessageReader.java +++ b/server/src/main/java/com/genymobile/scrcpy/ControlMessageReader.java @@ -10,6 +10,7 @@ public class ControlMessageReader { private static final int INJECT_KEYCODE_PAYLOAD_LENGTH = 9; private static final int INJECT_MOUSE_EVENT_PAYLOAD_LENGTH = 17; + private static final int INJECT_TOUCH_EVENT_PAYLOAD_LENGTH = 21; private static final int INJECT_SCROLL_EVENT_PAYLOAD_LENGTH = 20; private static final int SET_SCREEN_POWER_MODE_PAYLOAD_LENGTH = 1; @@ -62,6 +63,9 @@ public class ControlMessageReader { case ControlMessage.TYPE_INJECT_MOUSE_EVENT: msg = parseInjectMouseEvent(); break; + case ControlMessage.TYPE_INJECT_TOUCH_EVENT: + msg = parseInjectTouchEvent(); + break; case ControlMessage.TYPE_INJECT_SCROLL_EVENT: msg = parseInjectScrollEvent(); break; @@ -130,6 +134,21 @@ public class ControlMessageReader { return ControlMessage.createInjectMouseEvent(action, buttons, position); } + @SuppressWarnings("checkstyle:MagicNumber") + private ControlMessage parseInjectTouchEvent() { + if (buffer.remaining() < INJECT_TOUCH_EVENT_PAYLOAD_LENGTH) { + return null; + } + int action = toUnsigned(buffer.get()); + long pointerId = buffer.getLong(); + Position position = readPosition(buffer); + // 16 bits fixed-point + int pressureInt = toUnsigned(buffer.getShort()); + // convert it to a float between 0 and 1 (0x1p16f is 2^16 as float) + float pressure = pressureInt == 0xffff ? 1f : (pressureInt / 0x1p16f); + return ControlMessage.createInjectTouchEvent(action, pointerId, position, pressure); + } + private ControlMessage parseInjectScrollEvent() { if (buffer.remaining() < INJECT_SCROLL_EVENT_PAYLOAD_LENGTH) { return null; diff --git a/server/src/test/java/com/genymobile/scrcpy/ControlMessageReaderTest.java b/server/src/test/java/com/genymobile/scrcpy/ControlMessageReaderTest.java index f0c643d4..33380295 100644 --- a/server/src/test/java/com/genymobile/scrcpy/ControlMessageReaderTest.java +++ b/server/src/test/java/com/genymobile/scrcpy/ControlMessageReaderTest.java @@ -105,6 +105,37 @@ public class ControlMessageReaderTest { Assert.assertEquals(1920, event.getPosition().getScreenSize().getHeight()); } + @Test + @SuppressWarnings("checkstyle:MagicNumber") + public void testParseTouchEvent() throws IOException { + ControlMessageReader reader = new ControlMessageReader(); + + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + DataOutputStream dos = new DataOutputStream(bos); + dos.writeByte(ControlMessage.TYPE_INJECT_TOUCH_EVENT); + dos.writeByte(MotionEvent.ACTION_DOWN); + dos.writeLong(-42); // pointerId + dos.writeInt(100); + dos.writeInt(200); + dos.writeShort(1080); + dos.writeShort(1920); + dos.writeShort(0xffff); // pressure + + byte[] packet = bos.toByteArray(); + + reader.readFrom(new ByteArrayInputStream(packet)); + ControlMessage event = reader.next(); + + Assert.assertEquals(ControlMessage.TYPE_INJECT_TOUCH_EVENT, event.getType()); + Assert.assertEquals(MotionEvent.ACTION_DOWN, event.getAction()); + Assert.assertEquals(-42, event.getPointerId()); + Assert.assertEquals(100, event.getPosition().getPoint().getX()); + Assert.assertEquals(200, event.getPosition().getPoint().getY()); + Assert.assertEquals(1080, event.getPosition().getScreenSize().getWidth()); + Assert.assertEquals(1920, event.getPosition().getScreenSize().getHeight()); + Assert.assertEquals(1f, event.getPressure(), 0f); // must be exact + } + @Test @SuppressWarnings("checkstyle:MagicNumber") public void testParseScrollEvent() throws IOException {