Encode video using MediaCodec API

Replace screenrecord execution by manual screen encoding using the
MediaCodec API.

The "screenrecord" solution had several drawbacks:
 - screenrecord output is buffered, so tiny frames may not be accessible
   immediately;
 - it did not output a frame until the surface changed, leading to a
   black screen on start;
 - it is limited to 3 minutes recording, so it needed to be restarted;
 - screenrecord added black borders in the video when the requested
   dimensions did not preserve aspect-ratio exactly (sometimes
   unavoidable since video dimensions must be multiple of 8);
 - rotation handling was hacky (killing the process and starting a new
   one).

Handling the encoding manually allows to solve all these problems.

server/src/main/java/com/genymobile/scrcpy/DesktopConnection.java

@@ -65,8 +65,8 @@ public class DesktopConnection implements Closeable {
         outputStream.write(buffer, 0, buffer.length);
     }
 
-    public void sendVideoStream(byte[] videoStreamBuffer, int len) throws IOException {
+    public OutputStream getOutputStream() {
-        outputStream.write(videoStreamBuffer, 0, len);
+        return outputStream;
     }
 
     public ControlEvent receiveControlEvent() throws IOException {

server/src/main/java/com/genymobile/scrcpy/Device.java

@@ -45,14 +45,12 @@ public final class Device {
         // Principle:
         // - scale down the great side of the screen to maximumSize (if necessary);
         // - scale down the other side so that the aspect ratio is preserved;
-        // - ceil this value to the next multiple of 8 (H.264 only accepts multiples of 8)
+        // - round this value to the nearest multiple of 8 (H.264 only accepts multiples of 8)
-        // - this may introduce black bands, so store the padding (typically a few pixels)
         DisplayInfo displayInfo = serviceManager.getDisplayManager().getDisplayInfo();
         boolean rotated = (displayInfo.getRotation() & 1) != 0;
         Size deviceSize = displayInfo.getSize();
         int w = deviceSize.getWidth();
         int h = deviceSize.getHeight();
-        int padding = 0;
         if (maximumSize > 0) {
             assert maximumSize % 8 == 0;
             boolean portrait = h > w;
@@ -60,16 +58,15 @@ public final class Device {
             int minor = portrait ? w : h;
             if (major > maximumSize) {
                 int minorExact = minor * maximumSize / major;
-                // +7 to ceil the value on rounding to the next multiple of 8,
+                // +4 to round the value to the nearest multiple of 8
-                // so that any necessary black bands to keep the aspect ratio are added to the smallest dimension
+                minor = (minorExact + 4) & ~7;
-                minor = (minorExact + 7) & ~7;
                 major = maximumSize;
-                padding = minor - minorExact;
             }
             w = portrait ? minor : major;
             h = portrait ? major : minor;
         }
-        return new ScreenInfo(deviceSize, new Size(w, h), padding, rotated);
+        Size videoSize = new Size(w, h);
+        return new ScreenInfo(deviceSize, videoSize, rotated);
     }
 
     public Point getPhysicalPoint(Position position) {
@@ -82,19 +79,9 @@ public final class Device {
             return null;
         }
         Size deviceSize = screenInfo.getDeviceSize();
-        int xPadding = screenInfo.getXPadding();
-        int yPadding = screenInfo.getYPadding();
-        int contentWidth = videoSize.getWidth() - xPadding;
-        int contentHeight = videoSize.getHeight() - yPadding;
         Point point = position.getPoint();
-        int x = point.x - xPadding / 2;
+        int scaledX = point.x * deviceSize.getWidth() / videoSize.getWidth();
-        int y = point.y - yPadding / 2;
+        int scaledY = point.y * deviceSize.getHeight() / videoSize.getHeight();
-        if (x < 0 || x >= contentWidth || y < 0 || y >= contentHeight) {
-            // out of screen
-            return null;
-        }
-        int scaledX = x * deviceSize.getWidth() / videoSize.getWidth();
-        int scaledY = y * deviceSize.getHeight() / videoSize.getHeight();
         return new Point(scaledX, scaledY);
     }
 

server/src/main/java/com/genymobile/scrcpy/ScrCpyServer.java

@@ -4,25 +4,17 @@ import java.io.IOException;
 
 public class ScrCpyServer {
 
-    private static final String TAG = "scrcpy";
-
     private static void scrcpy(Options options) throws IOException {
         final Device device = new Device(options);
         try (DesktopConnection connection = DesktopConnection.open(device)) {
-            final ScreenStreamer streamer = new ScreenStreamer(device, connection);
+            ScreenEncoder screenEncoder = new ScreenEncoder();
-            device.setRotationListener(new Device.RotationListener() {
-                @Override
-                public void onRotationChanged(int rotation) {
-                    streamer.reset();
-                }
-            });
 
             // asynchronous
             startEventController(device, connection);
 
             try {
                 // synchronous
-                streamer.streamScreen();
+                screenEncoder.streamScreen(device, connection.getOutputStream());
             } catch (IOException e) {
                 Ln.e("Screen streaming interrupted", e);
             }

server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java

@@ -0,0 +1,149 @@
+package com.genymobile.scrcpy;
+
+import android.graphics.Rect;
+import android.media.MediaCodec;
+import android.media.MediaCodecInfo;
+import android.media.MediaFormat;
+import android.os.IBinder;
+import android.view.Surface;
+
+import com.genymobile.scrcpy.wrappers.SurfaceControl;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+public class ScreenEncoder implements Device.RotationListener {
+
+    private static final int DEFAULT_BIT_RATE = 4_000_000; // bits per second
+    private static final int DEFAULT_FRAME_RATE = 60; // fps
+    private static final int DEFAULT_I_FRAME_INTERVAL = 10; // seconds
+
+    private static final int REPEAT_FRAME_DELAY = 6; // repeat after 6 frames
+
+    private final AtomicBoolean rotationChanged = new AtomicBoolean();
+
+    private int bitRate;
+    private int frameRate;
+    private int iFrameInterval;
+
+    public ScreenEncoder(int bitRate, int frameRate, int iFrameInterval) {
+        this.bitRate = bitRate;
+        this.frameRate = frameRate;
+        this.iFrameInterval = iFrameInterval;
+    }
+
+    public ScreenEncoder() {
+        this(DEFAULT_BIT_RATE, DEFAULT_FRAME_RATE, DEFAULT_I_FRAME_INTERVAL);
+    }
+
+    @Override
+    public void onRotationChanged(int rotation) {
+        rotationChanged.set(true);
+    }
+
+    public boolean checkRotationChanged() {
+        return rotationChanged.getAndSet(false);
+    }
+
+    public void streamScreen(Device device, OutputStream outputStream) throws IOException {
+        MediaFormat format = createFormat(bitRate, frameRate, iFrameInterval);
+        MediaCodec codec = createCodec();
+        IBinder display = createDisplay();
+        device.setRotationListener(this);
+        boolean alive;
+        try {
+            do {
+                Rect deviceRect = device.getScreenInfo().getDeviceSize().toRect();
+                Rect videoRect = device.getScreenInfo().getVideoSize().toRect();
+                setSize(format, videoRect.width(), videoRect.height());
+                configure(codec, format);
+                Surface surface = codec.createInputSurface();
+                setDisplaySurface(display, surface, deviceRect, videoRect);
+                codec.start();
+                try {
+                    alive = encode(codec, outputStream);
+                } finally {
+                    codec.stop();
+                    surface.release();
+                }
+            } while (alive);
+        } finally {
+            device.setRotationListener(null);
+            destroyDisplay(display);
+            codec.release();
+        }
+    }
+
+    private boolean encode(MediaCodec codec, OutputStream outputStream) throws IOException {
+        byte[] buf = new byte[bitRate / 8]; // may contain up to 1 second of video
+        boolean eof = false;
+        MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();
+        while (!checkRotationChanged() && !eof) {
+            int outputBufferId = codec.dequeueOutputBuffer(bufferInfo, -1);
+            if (checkRotationChanged()) {
+                // must restart encoding with new size
+                break;
+            }
+            if (outputBufferId >= 0) {
+                ByteBuffer outputBuffer = codec.getOutputBuffer(outputBufferId);
+                while (outputBuffer.hasRemaining()) {
+                    int remaining = outputBuffer.remaining();
+                    int len = Math.min(buf.length, remaining);
+                    // the outputBuffer is probably direct (it has no underlying array), and LocalSocket does not expose channels,
+                    // so we must copy the data locally to write them manually to the output stream
+                    outputBuffer.get(buf, 0, len);
+                    outputStream.write(buf, 0, len);
+                }
+                codec.releaseOutputBuffer(outputBufferId, false);
+            }
+        }
+
+        return !eof;
+    }
+
+    private static MediaCodec createCodec() throws IOException {
+        return MediaCodec.createEncoderByType("video/avc");
+    }
+
+    private static MediaFormat createFormat(int bitRate, int frameRate, int iFrameInterval) throws IOException {
+        MediaFormat format = new MediaFormat();
+        format.setString(MediaFormat.KEY_MIME, "video/avc");
+        format.setInteger(MediaFormat.KEY_BIT_RATE, bitRate);
+        format.setInteger(MediaFormat.KEY_FRAME_RATE, frameRate);
+        format.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
+        format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, iFrameInterval);
+        // display the very first frame, and recover from bad quality when no new frames
+        format.setLong(MediaFormat.KEY_REPEAT_PREVIOUS_FRAME_AFTER, 1_000_000 * REPEAT_FRAME_DELAY / frameRate); // µs
+        return format;
+    }
+
+    private static IBinder createDisplay() {
+        return SurfaceControl.createDisplay("scrcpy", false);
+    }
+
+    private static void configure(MediaCodec codec, MediaFormat format) {
+        codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
+    }
+
+    private static void setSize(MediaFormat format, int width, int height) {
+        format.setInteger(MediaFormat.KEY_WIDTH, width);
+        format.setInteger(MediaFormat.KEY_HEIGHT, height);
+    }
+
+    private static void setDisplaySurface(IBinder display, Surface surface, Rect deviceRect, Rect displayRect) {
+        SurfaceControl.openTransaction();
+        try {
+            SurfaceControl.setDisplaySurface(display, surface);
+            SurfaceControl.setDisplayProjection(display, 0, deviceRect, displayRect);
+            SurfaceControl.setDisplayLayerStack(display, 0);
+        } finally {
+            SurfaceControl.closeTransaction();
+        }
+    }
+
+    private static void destroyDisplay(IBinder display) {
+        SurfaceControl.destroyDisplay(display);
+    }
+}

server/src/main/java/com/genymobile/scrcpy/ScreenInfo.java

@@ -3,13 +3,11 @@ package com.genymobile.scrcpy;
 public final class ScreenInfo {
     private final Size deviceSize;
     private final Size videoSize;
-    private final int padding; // padding inside the video stream, along the smallest dimension
     private final boolean rotated;
 
-    public ScreenInfo(Size deviceSize, Size videoSize, int padding, boolean rotated) {
+    public ScreenInfo(Size deviceSize, Size videoSize, boolean rotated) {
         this.deviceSize = deviceSize;
         this.videoSize = videoSize;
-        this.padding = padding;
         this.rotated = rotated;
     }
 
@@ -21,19 +19,11 @@ public final class ScreenInfo {
         return videoSize;
     }
 
-    public int getXPadding() {
-        return videoSize.getWidth() < videoSize.getHeight() ? padding : 0;
-    }
-
-    public int getYPadding() {
-        return videoSize.getHeight() < videoSize.getWidth() ? padding : 0;
-    }
-
     public ScreenInfo withRotation(int rotation) {
         boolean newRotated = (rotation & 1) != 0;
         if (rotated == newRotated) {
             return this;
         }
-        return new ScreenInfo(deviceSize.rotate(), videoSize.rotate(), padding, newRotated);
+        return new ScreenInfo(deviceSize.rotate(), videoSize.rotate(), newRotated);
     }
 }

server/src/main/java/com/genymobile/scrcpy/ScreenStreamer.java

@@ -1,39 +0,0 @@
-package com.genymobile.scrcpy;
-
-import java.io.IOException;
-import java.io.InterruptedIOException;
-
-public class ScreenStreamer {
-
-    private final Device device;
-    private final DesktopConnection connection;
-    private ScreenStreamerSession currentStreamer; // protected by 'this'
-
-    public ScreenStreamer(Device device, DesktopConnection connection) {
-        this.device = device;
-        this.connection = connection;
-    }
-
-    private synchronized ScreenStreamerSession newScreenStreamerSession() {
-        currentStreamer = new ScreenStreamerSession(device, connection);
-        return currentStreamer;
-    }
-
-    public void streamScreen() throws IOException {
-        while (true) {
-            try {
-                ScreenStreamerSession screenStreamer = newScreenStreamerSession();
-                screenStreamer.streamScreen();
-            } catch (InterruptedIOException e) {
-                // the current screenrecord process has probably been killed due to reset(), start a new one without failing
-            }
-        }
-    }
-
-    public synchronized void reset() {
-        if (currentStreamer != null) {
-            // it will stop the blocking call to streamScreen(), so a new streamer will be started
-            currentStreamer.stop();
-        }
-    }
-}

server/src/main/java/com/genymobile/scrcpy/ScreenStreamerSession.java

@@ -1,73 +0,0 @@
-package com.genymobile.scrcpy;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-public class ScreenStreamerSession {
-
-    private final Device device;
-    private final DesktopConnection connection;
-    private Process screenRecordProcess; // protected by 'this'
-    private final AtomicBoolean stopped = new AtomicBoolean();
-    private final byte[] buffer = new byte[0x10000];
-
-    public ScreenStreamerSession(Device device, DesktopConnection connection) {
-        this.device = device;
-        this.connection = connection;
-    }
-
-    public void streamScreen() throws IOException {
-        // screenrecord may not record more than 3 minutes, so restart it on EOF
-        while (!stopped.get() && streamScreenOnce()) ;
-    }
-
-    /**
-     * Starts screenrecord once and relay its output to the desktop connection.
-     *
-     * @return {@code true} if EOF is reached, {@code false} otherwise (i.e. requested to stop).
-     * @throws IOException if an I/O error occurred
-     */
-    private boolean streamScreenOnce() throws IOException {
-        Ln.d("Recording...");
-        Size videoSize = device.getScreenInfo().getVideoSize();
-        Process process = startScreenRecord(videoSize);
-        setCurrentProcess(process);
-        InputStream inputStream = process.getInputStream();
-        int r;
-        while ((r = inputStream.read(buffer)) != -1 && !stopped.get()) {
-            connection.sendVideoStream(buffer, r);
-        }
-        return r != -1;
-    }
-
-    public void stop() {
-        // let the thread stop itself without breaking the video stream
-        stopped.set(true);
-        killCurrentProcess();
-    }
-
-    private static Process startScreenRecord(Size videoSize) throws IOException {
-        List<String> command = new ArrayList<>();
-        command.add("screenrecord");
-        command.add("--output-format=h264");
-        command.add("--size=" + videoSize.getWidth() + "x" + videoSize.getHeight());
-        command.add("-");
-        Process process = new ProcessBuilder(command).start();
-        process.getOutputStream().close();
-        return process;
-    }
-
-    private synchronized void setCurrentProcess(Process screenRecordProcess) {
-        this.screenRecordProcess = screenRecordProcess;
-    }
-
-    private synchronized void killCurrentProcess() {
-        if (screenRecordProcess != null) {
-            screenRecordProcess.destroy();
-            screenRecordProcess = null;
-        }
-    }
-}

server/src/main/java/com/genymobile/scrcpy/Size.java

@@ -1,5 +1,7 @@
 package com.genymobile.scrcpy;
 
+import android.graphics.Rect;
+
 import java.util.Objects;
 
 public final class Size {
@@ -23,6 +25,10 @@ public final class Size {
         return new Size(height, width);
     }
 
+    public Rect toRect() {
+        return new Rect(0, 0, width, height);
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;

server/src/main/java/com/genymobile/scrcpy/wrappers/SurfaceControl.java

@@ -0,0 +1,79 @@
+package com.genymobile.scrcpy.wrappers;
+
+import android.graphics.Rect;
+import android.os.IBinder;
+import android.view.Surface;
+
+public class SurfaceControl {
+
+    private static final Class<?> cls;
+
+    static {
+        try {
+            cls = Class.forName("android.view.SurfaceControl");
+        } catch (ClassNotFoundException e) {
+            throw new AssertionError(e);
+        }
+    }
+
+    private SurfaceControl() {
+        // only static methods
+    }
+
+    public static void openTransaction() {
+        try {
+            cls.getMethod("openTransaction").invoke(null);
+        } catch (Exception e) {
+            throw new AssertionError(e);
+        }
+    }
+
+    public static void closeTransaction() {
+        try {
+            cls.getMethod("closeTransaction").invoke(null);
+        } catch (Exception e) {
+            throw new AssertionError(e);
+        }
+    }
+
+    public static void setDisplayProjection(IBinder displayToken, int orientation, Rect layerStackRect, Rect displayRect) {
+        try {
+            cls.getMethod("setDisplayProjection", IBinder.class, int.class, Rect.class, Rect.class)
+                    .invoke(null, displayToken, orientation, layerStackRect, displayRect);
+        } catch (Exception e) {
+            throw new AssertionError(e);
+        }
+    }
+
+    public static void setDisplayLayerStack(IBinder displayToken, int layerStack) {
+        try {
+            cls.getMethod("setDisplayLayerStack", IBinder.class, int.class).invoke(null, displayToken, layerStack);
+        } catch (Exception e) {
+            throw new AssertionError(e);
+        }
+    }
+
+    public static void setDisplaySurface(IBinder displayToken, Surface surface) {
+        try {
+            cls.getMethod("setDisplaySurface", IBinder.class, Surface.class).invoke(null, displayToken, surface);
+        } catch (Exception e) {
+            throw new AssertionError(e);
+        }
+    }
+
+    public static IBinder createDisplay(String name, boolean secure) {
+        try {
+            return (IBinder) cls.getMethod("createDisplay", String.class, boolean.class).invoke(null, name, secure);
+        } catch (Exception e) {
+            throw new AssertionError(e);
+        }
+    }
+
+    public static void destroyDisplay(IBinder displayToken) {
+        try {
+            cls.getMethod("destroyDisplay", IBinder.class).invoke(null, displayToken);
+        } catch (Exception e) {
+            throw new AssertionError(e);
+        }
+    }
+}