Send frame meta only if recording is enabled

The client needs the PTS for each frame only if recording is enabled.
Otherwise, the PTS are not necessary, and the protocol is more
straighforward.

app/src/decoder.c

@@ -20,7 +20,7 @@
 
 #define HEADER_SIZE 12
 
-static int read_packet(void *opaque, uint8_t *buf, int buf_size) {
+static int read_packet_with_meta(void *opaque, uint8_t *buf, int buf_size) {
     struct decoder *decoder = opaque;
     struct receiver_state *state = &decoder->receiver_state;
 
@@ -69,6 +69,11 @@ static int read_packet(void *opaque, uint8_t *buf, int buf_size) {
     return ret;
 }
 
+static int read_raw_packet(void *opaque, uint8_t *buf, int buf_size) {
+    struct decoder *decoder = opaque;
+    return net_recv(decoder->video_socket, buf, buf_size);
+}
+
 // set the decoded frame as ready for rendering, and notify
 static void push_frame(struct decoder *decoder) {
     SDL_bool previous_frame_consumed = frames_offer_decoded_frame(decoder->frames);
@@ -123,7 +128,11 @@ static int run_decoder(void *data) {
     // initialize the receiver state
     decoder->receiver_state.remaining = 0;
 
-    AVIOContext *avio_ctx = avio_alloc_context(buffer, BUFSIZE, 0, decoder, read_packet, NULL, NULL);
+    // if recording is enabled, a "header" is sent between raw packets
+    int (*read_packet)(void *, uint8_t *, int) =
+            decoder->recorder ? read_packet_with_meta : read_raw_packet;
+    AVIOContext *avio_ctx = avio_alloc_context(buffer, BUFSIZE, 0, decoder,
+                                               read_packet, NULL, NULL);
     if (!avio_ctx) {
         LOGC("Could not allocate avio context");
         // avformat_open_input takes ownership of 'buffer'

app/src/scrcpy.c

@@ -140,8 +140,10 @@ static void wait_show_touches(process_t process) {
 }
 
 SDL_bool scrcpy(const struct scrcpy_options *options) {
+    SDL_bool send_frame_meta = !!options->record_filename;
     if (!server_start(&server, options->serial, options->port,
-                      options->max_size, options->bit_rate, options->crop)) {
+                      options->max_size, options->bit_rate, options->crop,
+                      send_frame_meta)) {
         return SDL_FALSE;
     }
 

app/src/server.c

@@ -78,7 +78,8 @@ static SDL_bool disable_tunnel(struct server *server) {
 
 static process_t execute_server(const char *serial,
                                 Uint16 max_size, Uint32 bit_rate,
-                                const char *crop, SDL_bool tunnel_forward) {
+                                SDL_bool tunnel_forward, const char *crop,
+                                SDL_bool send_frame_meta) {
     char max_size_string[6];
     char bit_rate_string[11];
     sprintf(max_size_string, "%"PRIu16, max_size);
@@ -92,7 +93,8 @@ static process_t execute_server(const char *serial,
         max_size_string,
         bit_rate_string,
         tunnel_forward ? "true" : "false",
-        crop ? crop : "",
+        crop ? crop : "''",
+        send_frame_meta ? "true" : "false",
     };
     return adb_execute(serial, cmd, sizeof(cmd) / sizeof(cmd[0]));
 }
@@ -148,8 +150,9 @@ void server_init(struct server *server) {
     *server = (struct server) SERVER_INITIALIZER;
 }
 
-SDL_bool server_start(struct server *server, const char *serial, Uint16 local_port,
+SDL_bool server_start(struct server *server, const char *serial,
-                      Uint16 max_size, Uint32 bit_rate, const char *crop) {
+                      Uint16 local_port, Uint16 max_size, Uint32 bit_rate,
+                      const char *crop, SDL_bool send_frame_meta) {
     server->local_port = local_port;
 
     if (serial) {
@@ -190,8 +193,10 @@ SDL_bool server_start(struct server *server, const char *serial, Uint16 local_po
     }
 
     // server will connect to our server socket
-    server->process = execute_server(serial, max_size, bit_rate, crop,
+    server->process = execute_server(serial, max_size, bit_rate,
-                                     server->tunnel_forward);
+                                     server->tunnel_forward, crop,
+                                     send_frame_meta);
+
     if (server->process == PROCESS_NONE) {
         if (!server->tunnel_forward) {
             close_socket(&server->server_socket);

app/src/server.h

@@ -12,6 +12,7 @@ struct server {
     Uint16 local_port;
     SDL_bool tunnel_enabled;
     SDL_bool tunnel_forward; // use "adb forward" instead of "adb reverse"
+    SDL_bool send_frame_meta; // request frame PTS to be able to record properly
     SDL_bool server_copied_to_device;
 };
 
@@ -23,6 +24,7 @@ struct server {
     .local_port = 0,                      \
     .tunnel_enabled = SDL_FALSE,          \
     .tunnel_forward = SDL_FALSE,          \
+    .send_frame_meta = SDL_FALSE,         \
     .server_copied_to_device = SDL_FALSE, \
 }
 
@@ -30,8 +32,9 @@ struct server {
 void server_init(struct server *server);
 
 // push, enable tunnel et start the server
-SDL_bool server_start(struct server *server, const char *serial, Uint16 local_port,
+SDL_bool server_start(struct server *server, const char *serial,
-                      Uint16 max_size, Uint32 bit_rate, const char *crop);
+                      Uint16 local_port, Uint16 max_size, Uint32 bit_rate,
+                      const char *crop, SDL_bool send_frame_meta);
 
 // block until the communication with the server is established
 socket_t server_connect_to(struct server *server);

server/src/main/java/com/genymobile/scrcpy/Options.java

@@ -7,6 +7,7 @@ public class Options {
     private int bitRate;
     private boolean tunnelForward;
     private Rect crop;
+    private boolean sendFrameMeta; // send PTS so that the client may record properly
 
     public int getMaxSize() {
         return maxSize;
@@ -39,4 +40,12 @@ public class Options {
     public void setCrop(Rect crop) {
         this.crop = crop;
     }
+
+    public boolean getSendFrameMeta() {
+        return sendFrameMeta;
+    }
+
+    public void setSendFrameMeta(boolean sendFrameMeta) {
+        this.sendFrameMeta = sendFrameMeta;
+    }
 }

server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java

@@ -25,20 +25,23 @@ public class ScreenEncoder implements Device.RotationListener {
     private static final int MICROSECONDS_IN_ONE_SECOND = 1_000_000;
 
     private final AtomicBoolean rotationChanged = new AtomicBoolean();
+    private final ByteBuffer headerBuffer = ByteBuffer.allocate(12);
 
     private int bitRate;
     private int frameRate;
     private int iFrameInterval;
+    private boolean sendFrameMeta;
     private long ptsOrigin;
 
-    public ScreenEncoder(int bitRate, int frameRate, int iFrameInterval) {
+    public ScreenEncoder(boolean sendFrameMeta, int bitRate, int frameRate, int iFrameInterval) {
+        this.sendFrameMeta = sendFrameMeta;
         this.bitRate = bitRate;
         this.frameRate = frameRate;
         this.iFrameInterval = iFrameInterval;
     }
 
-    public ScreenEncoder(int bitRate) {
+    public ScreenEncoder(boolean sendFrameMeta, int bitRate) {
-        this(bitRate, DEFAULT_FRAME_RATE, DEFAULT_I_FRAME_INTERVAL);
+        this(sendFrameMeta, bitRate, DEFAULT_FRAME_RATE, DEFAULT_I_FRAME_INTERVAL);
     }
 
     @Override
@@ -82,7 +85,7 @@ public class ScreenEncoder implements Device.RotationListener {
     private boolean encode(MediaCodec codec, FileDescriptor fd) throws IOException {
         boolean eof = false;
         MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();
-        ByteBuffer bBuffer = ByteBuffer.allocate(12);
+
 
         while (!consumeRotationChange() && !eof) {
             int outputBufferId = codec.dequeueOutputBuffer(bufferInfo, -1);
@@ -94,22 +97,11 @@ public class ScreenEncoder implements Device.RotationListener {
                 }
                 if (outputBufferId >= 0) {
                     ByteBuffer codecBuffer = codec.getOutputBuffer(outputBufferId);
-                    bBuffer.clear();
 
-                    long pts;
+                    if (sendFrameMeta) {
-                    if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {
+                        writeFrameMeta(fd, bufferInfo, codecBuffer.remaining());
-                        pts = 0; // non-media data packet
-                    } else {
-                        if (ptsOrigin == 0) {
-                            ptsOrigin = bufferInfo.presentationTimeUs;
-                        }
-                        pts = bufferInfo.presentationTimeUs - ptsOrigin;
                     }
 
-                    bBuffer.putLong(pts);
-                    bBuffer.putInt(codecBuffer.remaining());
-                    bBuffer.flip();
-                    IO.writeFully(fd, bBuffer);
                     IO.writeFully(fd, codecBuffer);
                 }
             } finally {
@@ -122,6 +114,25 @@ public class ScreenEncoder implements Device.RotationListener {
         return !eof;
     }
 
+    private void writeFrameMeta(FileDescriptor fd, MediaCodec.BufferInfo bufferInfo, int packetSize) throws IOException {
+        headerBuffer.clear();
+
+        long pts;
+        if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {
+            pts = 0; // non-media data packet
+        } else {
+            if (ptsOrigin == 0) {
+                ptsOrigin = bufferInfo.presentationTimeUs;
+            }
+            pts = bufferInfo.presentationTimeUs - ptsOrigin;
+        }
+
+        headerBuffer.putLong(pts);
+        headerBuffer.putInt(packetSize);
+        headerBuffer.flip();
+        IO.writeFully(fd, headerBuffer);
+    }
+
     private static MediaCodec createCodec() throws IOException {
         return MediaCodec.createEncoderByType("video/avc");
     }

server/src/main/java/com/genymobile/scrcpy/Server.java

@@ -3,6 +3,7 @@ package com.genymobile.scrcpy;
 import android.graphics.Rect;
 
 import java.io.IOException;
+import java.util.Arrays;
 
 public final class Server {
 
@@ -14,7 +15,7 @@ public final class Server {
         final Device device = new Device(options);
         boolean tunnelForward = options.isTunnelForward();
         try (DesktopConnection connection = DesktopConnection.open(device, tunnelForward)) {
-            ScreenEncoder screenEncoder = new ScreenEncoder(options.getBitRate());
+            ScreenEncoder screenEncoder = new ScreenEncoder(options.getSendFrameMeta(), options.getBitRate());
 
             // asynchronous
             startEventController(device, connection);
@@ -71,6 +72,12 @@ public final class Server {
         Rect crop = parseCrop(args[3]);
         options.setCrop(crop);
 
+        if (args.length < 5) {
+            return options;
+        }
+        boolean sendFrameMeta = Boolean.parseBoolean(args[4]);
+        options.setSendFrameMeta(sendFrameMeta);
+
         return options;
     }