Use two sockets for video and control

The socket used the device-to-computer direction to stream the video and
the computer-to-device direction to send control events.

Some features, like copy-paste from device to computer, require to send
non-video data from the device to the computer.

To make them possible, use two sockets:
 - one for streaming the video from the device to the client;
 - one for control/events in both directions.

app/src/controller.c

@@ -7,7 +7,7 @@
 #include "log.h"
 
 bool
-controller_init(struct controller *controller, socket_t video_socket) {
+controller_init(struct controller *controller, socket_t control_socket) {
     cbuf_init(&controller->queue);
 
     if (!(controller->mutex = SDL_CreateMutex())) {
@@ -19,7 +19,7 @@ controller_init(struct controller *controller, socket_t video_socket) {
         return false;
     }
 
-    controller->video_socket = video_socket;
+    controller->control_socket = control_socket;
     controller->stopped = false;
 
     return true;
@@ -57,7 +57,7 @@ process_event(struct controller *controller,
     if (!length) {
         return false;
     }
-    int w = net_send_all(controller->video_socket, serialized_event, length);
+    int w = net_send_all(controller->control_socket, serialized_event, length);
     return w == length;
 }
 

app/src/controller.h

@@ -12,7 +12,7 @@
 struct control_event_queue CBUF(struct control_event, 64);
 
 struct controller {
-    socket_t video_socket;
+    socket_t control_socket;
     SDL_Thread *thread;
     SDL_mutex *mutex;
     SDL_cond *event_cond;
@@ -21,7 +21,7 @@ struct controller {
 };
 
 bool
-controller_init(struct controller *controller, socket_t video_socket);
+controller_init(struct controller *controller, socket_t control_socket);
 
 void
 controller_destroy(struct controller *controller);

app/src/scrcpy.c

@@ -300,15 +300,13 @@ scrcpy(const struct scrcpy_options *options) {
         goto end;
     }
 
-    socket_t device_socket = server.device_socket;
-
     char device_name[DEVICE_NAME_FIELD_LENGTH];
     struct size frame_size;
 
     // screenrecord does not send frames when the screen content does not
     // change therefore, we transmit the screen size before the video stream,
     // to be able to init the window immediately
-    if (!device_read_info(device_socket, device_name, &frame_size)) {
+    if (!device_read_info(server.video_socket, device_name, &frame_size)) {
         goto end;
     }
 
@@ -344,7 +342,7 @@ scrcpy(const struct scrcpy_options *options) {
 
     av_log_set_callback(av_log_callback);
 
-    stream_init(&stream, device_socket, dec, rec);
+    stream_init(&stream, server.video_socket, dec, rec);
 
     // now we consumed the header values, the socket receives the video stream
     // start the stream
@@ -355,7 +353,7 @@ scrcpy(const struct scrcpy_options *options) {
 
     if (display) {
         if (control) {
-            if (!controller_init(&controller, device_socket)) {
+            if (!controller_init(&controller, server.control_socket)) {
                 goto end;
             }
 

app/src/server.c

@@ -222,8 +222,14 @@ server_start(struct server *server, const char *serial,
 bool
 server_connect_to(struct server *server) {
     if (!server->tunnel_forward) {
-        server->device_socket = net_accept(server->server_socket);
+        server->video_socket = net_accept(server->server_socket);
-        if (server->device_socket == INVALID_SOCKET) {
+        if (server->video_socket == INVALID_SOCKET) {
+            return false;
+        }
+
+        server->control_socket = net_accept(server->server_socket);
+        if (server->control_socket == INVALID_SOCKET) {
+            // the video_socket will be clean up on destroy
             return false;
         }
 
@@ -232,9 +238,16 @@ server_connect_to(struct server *server) {
     } else {
         uint32_t attempts = 100;
         uint32_t delay = 100; // ms
-        server->device_socket = connect_to_server(server->local_port, attempts,
+        server->video_socket =
-                                                  delay);
+            connect_to_server(server->local_port, attempts, delay);
-        if (server->device_socket == INVALID_SOCKET) {
+        if (server->video_socket == INVALID_SOCKET) {
+            return false;
+        }
+
+        // we know that the device is listening, we don't need several attempts
+        server->control_socket =
+            net_connect(IPV4_LOCALHOST, server->local_port);
+        if (server->control_socket == INVALID_SOCKET) {
             return false;
         }
     }
@@ -251,8 +264,11 @@ server_stop(struct server *server) {
     if (server->server_socket != INVALID_SOCKET) {
         close_socket(&server->server_socket);
     }
-    if (server->device_socket != INVALID_SOCKET) {
+    if (server->video_socket != INVALID_SOCKET) {
-        close_socket(&server->device_socket);
+        close_socket(&server->video_socket);
+    }
+    if (server->control_socket != INVALID_SOCKET) {
+        close_socket(&server->control_socket);
     }
 
     SDL_assert(server->process != PROCESS_NONE);

app/src/server.h

@@ -11,7 +11,8 @@ struct server {
     char *serial;
     process_t process;
     socket_t server_socket; // only used if !tunnel_forward
-    socket_t device_socket;
+    socket_t video_socket;
+    socket_t control_socket;
     uint16_t local_port;
     bool tunnel_enabled;
     bool tunnel_forward; // use "adb forward" instead of "adb reverse"
@@ -22,7 +23,8 @@ struct server {
     .serial = NULL,                   \
     .process = PROCESS_NONE,          \
     .server_socket = INVALID_SOCKET,  \
-    .device_socket = INVALID_SOCKET,  \
+    .video_socket = INVALID_SOCKET,   \
+    .control_socket = INVALID_SOCKET, \
     .local_port = 0,                  \
     .tunnel_enabled = false,          \
     .tunnel_forward = false,          \

server/src/main/java/com/genymobile/scrcpy/DesktopConnection.java

@@ -16,16 +16,20 @@ public final class DesktopConnection implements Closeable {
 
     private static final String SOCKET_NAME = "scrcpy";
 
-    private final LocalSocket socket;
+    private final LocalSocket videoSocket;
-    private final InputStream inputStream;
+    private final FileDescriptor videoFd;
-    private final FileDescriptor fd;
+
+    private final LocalSocket controlSocket;
+    private final InputStream controlInputStream;
+
 
     private final ControlEventReader reader = new ControlEventReader();
 
-    private DesktopConnection(LocalSocket socket) throws IOException {
+    private DesktopConnection(LocalSocket videoSocket, LocalSocket controlSocket) throws IOException {
-        this.socket = socket;
+        this.videoSocket = videoSocket;
-        inputStream = socket.getInputStream();
+        this.controlSocket = controlSocket;
-        fd = socket.getFileDescriptor();
+        controlInputStream = controlSocket.getInputStream();
+        videoFd = videoSocket.getFileDescriptor();
     }
 
     private static LocalSocket connect(String abstractName) throws IOException {
@@ -34,35 +38,47 @@ public final class DesktopConnection implements Closeable {
         return localSocket;
     }
 
-    private static LocalSocket listenAndAccept(String abstractName) throws IOException {
+    public static DesktopConnection open(Device device, boolean tunnelForward) throws IOException {
-        LocalServerSocket localServerSocket = new LocalServerSocket(abstractName);
+        LocalSocket videoSocket;
+        LocalSocket controlSocket;
+        if (tunnelForward) {
+            LocalServerSocket localServerSocket = new LocalServerSocket(SOCKET_NAME);
             try {
-            return localServerSocket.accept();
+                videoSocket = localServerSocket.accept();
+                // send one byte so the client may read() to detect a connection error
+                videoSocket.getOutputStream().write(0);
+                try {
+                    controlSocket = localServerSocket.accept();
+                } catch (IOException | RuntimeException e) {
+                    videoSocket.close();
+                    throw e;
+                }
             } finally {
                 localServerSocket.close();
             }
-    }
-
-    public static DesktopConnection open(Device device, boolean tunnelForward) throws IOException {
-        LocalSocket socket;
-        if (tunnelForward) {
-            socket = listenAndAccept(SOCKET_NAME);
-            // send one byte so the client may read() to detect a connection error
-            socket.getOutputStream().write(0);
         } else {
-            socket = connect(SOCKET_NAME);
+            videoSocket = connect(SOCKET_NAME);
+            try {
+                controlSocket = connect(SOCKET_NAME);
+            } catch (IOException | RuntimeException e) {
+                videoSocket.close();
+                throw e;
+            }
         }
 
-        DesktopConnection connection = new DesktopConnection(socket);
+        DesktopConnection connection = new DesktopConnection(videoSocket, controlSocket);
         Size videoSize = device.getScreenInfo().getVideoSize();
         connection.send(Device.getDeviceName(), videoSize.getWidth(), videoSize.getHeight());
         return connection;
     }
 
     public void close() throws IOException {
-        socket.shutdownInput();
+        videoSocket.shutdownInput();
-        socket.shutdownOutput();
+        videoSocket.shutdownOutput();
-        socket.close();
+        videoSocket.close();
+        controlSocket.shutdownInput();
+        controlSocket.shutdownOutput();
+        controlSocket.close();
     }
 
     @SuppressWarnings("checkstyle:MagicNumber")
@@ -78,17 +94,17 @@ public final class DesktopConnection implements Closeable {
         buffer[DEVICE_NAME_FIELD_LENGTH + 1] = (byte) width;
         buffer[DEVICE_NAME_FIELD_LENGTH + 2] = (byte) (height >> 8);
         buffer[DEVICE_NAME_FIELD_LENGTH + 3] = (byte) height;
-        IO.writeFully(fd, buffer, 0, buffer.length);
+        IO.writeFully(videoFd, buffer, 0, buffer.length);
     }
 
-    public FileDescriptor getFd() {
+    public FileDescriptor getVideoFd() {
-        return fd;
+        return videoFd;
     }
 
     public ControlEvent receiveControlEvent() throws IOException {
         ControlEvent event = reader.next();
         while (event == null) {
-            reader.readFrom(inputStream);
+            reader.readFrom(controlInputStream);
             event = reader.next();
         }
         return event;

server/src/main/java/com/genymobile/scrcpy/Server.java

@@ -24,7 +24,7 @@ public final class Server {
 
             try {
                 // synchronous
-                screenEncoder.streamScreen(device, connection.getFd());
+                screenEncoder.streamScreen(device, connection.getVideoFd());
             } catch (IOException e) {
                 // this is expected on close
                 Ln.d("Screen streaming stopped");