From 865ebb3862398c3ba8c8dcba73189256880372f0 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Wed, 31 Jan 2018 18:37:01 +0100 Subject: [PATCH] Encode video using MediaCodec API Replace screenrecord execution by manual screen encoding using the MediaCodec API. The "screenrecord" solution had several drawbacks: - screenrecord output is buffered, so tiny frames may not be accessible immediately; - it did not output a frame until the surface changed, leading to a black screen on start; - it is limited to 3 minutes recording, so it needed to be restarted; - screenrecord added black borders in the video when the requested dimensions did not preserve aspect-ratio exactly (sometimes unavoidable since video dimensions must be multiple of 8); - rotation handling was hacky (killing the process and starting a new one). Handling the encoding manually allows to solve all these problems. --- .../genymobile/scrcpy/DesktopConnection.java | 4 +- .../java/com/genymobile/scrcpy/Device.java | 27 +--- .../com/genymobile/scrcpy/ScrCpyServer.java | 12 +- .../com/genymobile/scrcpy/ScreenEncoder.java | 149 ++++++++++++++++++ .../com/genymobile/scrcpy/ScreenInfo.java | 14 +- .../com/genymobile/scrcpy/ScreenStreamer.java | 39 ----- .../scrcpy/ScreenStreamerSession.java | 73 --------- .../main/java/com/genymobile/scrcpy/Size.java | 6 + .../scrcpy/wrappers/SurfaceControl.java | 79 ++++++++++ 9 files changed, 247 insertions(+), 156 deletions(-) create mode 100644 server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java delete mode 100644 server/src/main/java/com/genymobile/scrcpy/ScreenStreamer.java delete mode 100644 server/src/main/java/com/genymobile/scrcpy/ScreenStreamerSession.java create mode 100644 server/src/main/java/com/genymobile/scrcpy/wrappers/SurfaceControl.java diff --git a/server/src/main/java/com/genymobile/scrcpy/DesktopConnection.java b/server/src/main/java/com/genymobile/scrcpy/DesktopConnection.java index 6e71c63a..e739abe6 100644 --- a/server/src/main/java/com/genymobile/scrcpy/DesktopConnection.java +++ b/server/src/main/java/com/genymobile/scrcpy/DesktopConnection.java @@ -65,8 +65,8 @@ public class DesktopConnection implements Closeable { outputStream.write(buffer, 0, buffer.length); } - public void sendVideoStream(byte[] videoStreamBuffer, int len) throws IOException { - outputStream.write(videoStreamBuffer, 0, len); + public OutputStream getOutputStream() { + return outputStream; } public ControlEvent receiveControlEvent() throws IOException { diff --git a/server/src/main/java/com/genymobile/scrcpy/Device.java b/server/src/main/java/com/genymobile/scrcpy/Device.java index 4348127e..542f2035 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Device.java +++ b/server/src/main/java/com/genymobile/scrcpy/Device.java @@ -45,14 +45,12 @@ public final class Device { // Principle: // - scale down the great side of the screen to maximumSize (if necessary); // - scale down the other side so that the aspect ratio is preserved; - // - ceil this value to the next multiple of 8 (H.264 only accepts multiples of 8) - // - this may introduce black bands, so store the padding (typically a few pixels) + // - round this value to the nearest multiple of 8 (H.264 only accepts multiples of 8) DisplayInfo displayInfo = serviceManager.getDisplayManager().getDisplayInfo(); boolean rotated = (displayInfo.getRotation() & 1) != 0; Size deviceSize = displayInfo.getSize(); int w = deviceSize.getWidth(); int h = deviceSize.getHeight(); - int padding = 0; if (maximumSize > 0) { assert maximumSize % 8 == 0; boolean portrait = h > w; @@ -60,16 +58,15 @@ public final class Device { int minor = portrait ? w : h; if (major > maximumSize) { int minorExact = minor * maximumSize / major; - // +7 to ceil the value on rounding to the next multiple of 8, - // so that any necessary black bands to keep the aspect ratio are added to the smallest dimension - minor = (minorExact + 7) & ~7; + // +4 to round the value to the nearest multiple of 8 + minor = (minorExact + 4) & ~7; major = maximumSize; - padding = minor - minorExact; } w = portrait ? minor : major; h = portrait ? major : minor; } - return new ScreenInfo(deviceSize, new Size(w, h), padding, rotated); + Size videoSize = new Size(w, h); + return new ScreenInfo(deviceSize, videoSize, rotated); } public Point getPhysicalPoint(Position position) { @@ -82,19 +79,9 @@ public final class Device { return null; } Size deviceSize = screenInfo.getDeviceSize(); - int xPadding = screenInfo.getXPadding(); - int yPadding = screenInfo.getYPadding(); - int contentWidth = videoSize.getWidth() - xPadding; - int contentHeight = videoSize.getHeight() - yPadding; Point point = position.getPoint(); - int x = point.x - xPadding / 2; - int y = point.y - yPadding / 2; - if (x < 0 || x >= contentWidth || y < 0 || y >= contentHeight) { - // out of screen - return null; - } - int scaledX = x * deviceSize.getWidth() / videoSize.getWidth(); - int scaledY = y * deviceSize.getHeight() / videoSize.getHeight(); + int scaledX = point.x * deviceSize.getWidth() / videoSize.getWidth(); + int scaledY = point.y * deviceSize.getHeight() / videoSize.getHeight(); return new Point(scaledX, scaledY); } diff --git a/server/src/main/java/com/genymobile/scrcpy/ScrCpyServer.java b/server/src/main/java/com/genymobile/scrcpy/ScrCpyServer.java index 441fbf8c..bf84f31b 100644 --- a/server/src/main/java/com/genymobile/scrcpy/ScrCpyServer.java +++ b/server/src/main/java/com/genymobile/scrcpy/ScrCpyServer.java @@ -4,25 +4,17 @@ import java.io.IOException; public class ScrCpyServer { - private static final String TAG = "scrcpy"; - private static void scrcpy(Options options) throws IOException { final Device device = new Device(options); try (DesktopConnection connection = DesktopConnection.open(device)) { - final ScreenStreamer streamer = new ScreenStreamer(device, connection); - device.setRotationListener(new Device.RotationListener() { - @Override - public void onRotationChanged(int rotation) { - streamer.reset(); - } - }); + ScreenEncoder screenEncoder = new ScreenEncoder(); // asynchronous startEventController(device, connection); try { // synchronous - streamer.streamScreen(); + screenEncoder.streamScreen(device, connection.getOutputStream()); } catch (IOException e) { Ln.e("Screen streaming interrupted", e); } diff --git a/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java b/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java new file mode 100644 index 00000000..2f33e4e3 --- /dev/null +++ b/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java @@ -0,0 +1,149 @@ +package com.genymobile.scrcpy; + +import android.graphics.Rect; +import android.media.MediaCodec; +import android.media.MediaCodecInfo; +import android.media.MediaFormat; +import android.os.IBinder; +import android.view.Surface; + +import com.genymobile.scrcpy.wrappers.SurfaceControl; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.util.concurrent.atomic.AtomicBoolean; + +public class ScreenEncoder implements Device.RotationListener { + + private static final int DEFAULT_BIT_RATE = 4_000_000; // bits per second + private static final int DEFAULT_FRAME_RATE = 60; // fps + private static final int DEFAULT_I_FRAME_INTERVAL = 10; // seconds + + private static final int REPEAT_FRAME_DELAY = 6; // repeat after 6 frames + + private final AtomicBoolean rotationChanged = new AtomicBoolean(); + + private int bitRate; + private int frameRate; + private int iFrameInterval; + + public ScreenEncoder(int bitRate, int frameRate, int iFrameInterval) { + this.bitRate = bitRate; + this.frameRate = frameRate; + this.iFrameInterval = iFrameInterval; + } + + public ScreenEncoder() { + this(DEFAULT_BIT_RATE, DEFAULT_FRAME_RATE, DEFAULT_I_FRAME_INTERVAL); + } + + @Override + public void onRotationChanged(int rotation) { + rotationChanged.set(true); + } + + public boolean checkRotationChanged() { + return rotationChanged.getAndSet(false); + } + + public void streamScreen(Device device, OutputStream outputStream) throws IOException { + MediaFormat format = createFormat(bitRate, frameRate, iFrameInterval); + MediaCodec codec = createCodec(); + IBinder display = createDisplay(); + device.setRotationListener(this); + boolean alive; + try { + do { + Rect deviceRect = device.getScreenInfo().getDeviceSize().toRect(); + Rect videoRect = device.getScreenInfo().getVideoSize().toRect(); + setSize(format, videoRect.width(), videoRect.height()); + configure(codec, format); + Surface surface = codec.createInputSurface(); + setDisplaySurface(display, surface, deviceRect, videoRect); + codec.start(); + try { + alive = encode(codec, outputStream); + } finally { + codec.stop(); + surface.release(); + } + } while (alive); + } finally { + device.setRotationListener(null); + destroyDisplay(display); + codec.release(); + } + } + + private boolean encode(MediaCodec codec, OutputStream outputStream) throws IOException { + byte[] buf = new byte[bitRate / 8]; // may contain up to 1 second of video + boolean eof = false; + MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo(); + while (!checkRotationChanged() && !eof) { + int outputBufferId = codec.dequeueOutputBuffer(bufferInfo, -1); + if (checkRotationChanged()) { + // must restart encoding with new size + break; + } + if (outputBufferId >= 0) { + ByteBuffer outputBuffer = codec.getOutputBuffer(outputBufferId); + while (outputBuffer.hasRemaining()) { + int remaining = outputBuffer.remaining(); + int len = Math.min(buf.length, remaining); + // the outputBuffer is probably direct (it has no underlying array), and LocalSocket does not expose channels, + // so we must copy the data locally to write them manually to the output stream + outputBuffer.get(buf, 0, len); + outputStream.write(buf, 0, len); + } + codec.releaseOutputBuffer(outputBufferId, false); + } + } + + return !eof; + } + + private static MediaCodec createCodec() throws IOException { + return MediaCodec.createEncoderByType("video/avc"); + } + + private static MediaFormat createFormat(int bitRate, int frameRate, int iFrameInterval) throws IOException { + MediaFormat format = new MediaFormat(); + format.setString(MediaFormat.KEY_MIME, "video/avc"); + format.setInteger(MediaFormat.KEY_BIT_RATE, bitRate); + format.setInteger(MediaFormat.KEY_FRAME_RATE, frameRate); + format.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface); + format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, iFrameInterval); + // display the very first frame, and recover from bad quality when no new frames + format.setLong(MediaFormat.KEY_REPEAT_PREVIOUS_FRAME_AFTER, 1_000_000 * REPEAT_FRAME_DELAY / frameRate); // µs + return format; + } + + private static IBinder createDisplay() { + return SurfaceControl.createDisplay("scrcpy", false); + } + + private static void configure(MediaCodec codec, MediaFormat format) { + codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); + } + + private static void setSize(MediaFormat format, int width, int height) { + format.setInteger(MediaFormat.KEY_WIDTH, width); + format.setInteger(MediaFormat.KEY_HEIGHT, height); + } + + private static void setDisplaySurface(IBinder display, Surface surface, Rect deviceRect, Rect displayRect) { + SurfaceControl.openTransaction(); + try { + SurfaceControl.setDisplaySurface(display, surface); + SurfaceControl.setDisplayProjection(display, 0, deviceRect, displayRect); + SurfaceControl.setDisplayLayerStack(display, 0); + } finally { + SurfaceControl.closeTransaction(); + } + } + + private static void destroyDisplay(IBinder display) { + SurfaceControl.destroyDisplay(display); + } +} diff --git a/server/src/main/java/com/genymobile/scrcpy/ScreenInfo.java b/server/src/main/java/com/genymobile/scrcpy/ScreenInfo.java index 827e9ee8..1f752758 100644 --- a/server/src/main/java/com/genymobile/scrcpy/ScreenInfo.java +++ b/server/src/main/java/com/genymobile/scrcpy/ScreenInfo.java @@ -3,13 +3,11 @@ package com.genymobile.scrcpy; public final class ScreenInfo { private final Size deviceSize; private final Size videoSize; - private final int padding; // padding inside the video stream, along the smallest dimension private final boolean rotated; - public ScreenInfo(Size deviceSize, Size videoSize, int padding, boolean rotated) { + public ScreenInfo(Size deviceSize, Size videoSize, boolean rotated) { this.deviceSize = deviceSize; this.videoSize = videoSize; - this.padding = padding; this.rotated = rotated; } @@ -21,19 +19,11 @@ public final class ScreenInfo { return videoSize; } - public int getXPadding() { - return videoSize.getWidth() < videoSize.getHeight() ? padding : 0; - } - - public int getYPadding() { - return videoSize.getHeight() < videoSize.getWidth() ? padding : 0; - } - public ScreenInfo withRotation(int rotation) { boolean newRotated = (rotation & 1) != 0; if (rotated == newRotated) { return this; } - return new ScreenInfo(deviceSize.rotate(), videoSize.rotate(), padding, newRotated); + return new ScreenInfo(deviceSize.rotate(), videoSize.rotate(), newRotated); } } diff --git a/server/src/main/java/com/genymobile/scrcpy/ScreenStreamer.java b/server/src/main/java/com/genymobile/scrcpy/ScreenStreamer.java deleted file mode 100644 index 38509105..00000000 --- a/server/src/main/java/com/genymobile/scrcpy/ScreenStreamer.java +++ /dev/null @@ -1,39 +0,0 @@ -package com.genymobile.scrcpy; - -import java.io.IOException; -import java.io.InterruptedIOException; - -public class ScreenStreamer { - - private final Device device; - private final DesktopConnection connection; - private ScreenStreamerSession currentStreamer; // protected by 'this' - - public ScreenStreamer(Device device, DesktopConnection connection) { - this.device = device; - this.connection = connection; - } - - private synchronized ScreenStreamerSession newScreenStreamerSession() { - currentStreamer = new ScreenStreamerSession(device, connection); - return currentStreamer; - } - - public void streamScreen() throws IOException { - while (true) { - try { - ScreenStreamerSession screenStreamer = newScreenStreamerSession(); - screenStreamer.streamScreen(); - } catch (InterruptedIOException e) { - // the current screenrecord process has probably been killed due to reset(), start a new one without failing - } - } - } - - public synchronized void reset() { - if (currentStreamer != null) { - // it will stop the blocking call to streamScreen(), so a new streamer will be started - currentStreamer.stop(); - } - } -} diff --git a/server/src/main/java/com/genymobile/scrcpy/ScreenStreamerSession.java b/server/src/main/java/com/genymobile/scrcpy/ScreenStreamerSession.java deleted file mode 100644 index cb3c7bb1..00000000 --- a/server/src/main/java/com/genymobile/scrcpy/ScreenStreamerSession.java +++ /dev/null @@ -1,73 +0,0 @@ -package com.genymobile.scrcpy; - -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.atomic.AtomicBoolean; - -public class ScreenStreamerSession { - - private final Device device; - private final DesktopConnection connection; - private Process screenRecordProcess; // protected by 'this' - private final AtomicBoolean stopped = new AtomicBoolean(); - private final byte[] buffer = new byte[0x10000]; - - public ScreenStreamerSession(Device device, DesktopConnection connection) { - this.device = device; - this.connection = connection; - } - - public void streamScreen() throws IOException { - // screenrecord may not record more than 3 minutes, so restart it on EOF - while (!stopped.get() && streamScreenOnce()) ; - } - - /** - * Starts screenrecord once and relay its output to the desktop connection. - * - * @return {@code true} if EOF is reached, {@code false} otherwise (i.e. requested to stop). - * @throws IOException if an I/O error occurred - */ - private boolean streamScreenOnce() throws IOException { - Ln.d("Recording..."); - Size videoSize = device.getScreenInfo().getVideoSize(); - Process process = startScreenRecord(videoSize); - setCurrentProcess(process); - InputStream inputStream = process.getInputStream(); - int r; - while ((r = inputStream.read(buffer)) != -1 && !stopped.get()) { - connection.sendVideoStream(buffer, r); - } - return r != -1; - } - - public void stop() { - // let the thread stop itself without breaking the video stream - stopped.set(true); - killCurrentProcess(); - } - - private static Process startScreenRecord(Size videoSize) throws IOException { - List command = new ArrayList<>(); - command.add("screenrecord"); - command.add("--output-format=h264"); - command.add("--size=" + videoSize.getWidth() + "x" + videoSize.getHeight()); - command.add("-"); - Process process = new ProcessBuilder(command).start(); - process.getOutputStream().close(); - return process; - } - - private synchronized void setCurrentProcess(Process screenRecordProcess) { - this.screenRecordProcess = screenRecordProcess; - } - - private synchronized void killCurrentProcess() { - if (screenRecordProcess != null) { - screenRecordProcess.destroy(); - screenRecordProcess = null; - } - } -} diff --git a/server/src/main/java/com/genymobile/scrcpy/Size.java b/server/src/main/java/com/genymobile/scrcpy/Size.java index 66d67b37..96026d87 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Size.java +++ b/server/src/main/java/com/genymobile/scrcpy/Size.java @@ -1,5 +1,7 @@ package com.genymobile.scrcpy; +import android.graphics.Rect; + import java.util.Objects; public final class Size { @@ -23,6 +25,10 @@ public final class Size { return new Size(height, width); } + public Rect toRect() { + return new Rect(0, 0, width, height); + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/server/src/main/java/com/genymobile/scrcpy/wrappers/SurfaceControl.java b/server/src/main/java/com/genymobile/scrcpy/wrappers/SurfaceControl.java new file mode 100644 index 00000000..35efbc27 --- /dev/null +++ b/server/src/main/java/com/genymobile/scrcpy/wrappers/SurfaceControl.java @@ -0,0 +1,79 @@ +package com.genymobile.scrcpy.wrappers; + +import android.graphics.Rect; +import android.os.IBinder; +import android.view.Surface; + +public class SurfaceControl { + + private static final Class cls; + + static { + try { + cls = Class.forName("android.view.SurfaceControl"); + } catch (ClassNotFoundException e) { + throw new AssertionError(e); + } + } + + private SurfaceControl() { + // only static methods + } + + public static void openTransaction() { + try { + cls.getMethod("openTransaction").invoke(null); + } catch (Exception e) { + throw new AssertionError(e); + } + } + + public static void closeTransaction() { + try { + cls.getMethod("closeTransaction").invoke(null); + } catch (Exception e) { + throw new AssertionError(e); + } + } + + public static void setDisplayProjection(IBinder displayToken, int orientation, Rect layerStackRect, Rect displayRect) { + try { + cls.getMethod("setDisplayProjection", IBinder.class, int.class, Rect.class, Rect.class) + .invoke(null, displayToken, orientation, layerStackRect, displayRect); + } catch (Exception e) { + throw new AssertionError(e); + } + } + + public static void setDisplayLayerStack(IBinder displayToken, int layerStack) { + try { + cls.getMethod("setDisplayLayerStack", IBinder.class, int.class).invoke(null, displayToken, layerStack); + } catch (Exception e) { + throw new AssertionError(e); + } + } + + public static void setDisplaySurface(IBinder displayToken, Surface surface) { + try { + cls.getMethod("setDisplaySurface", IBinder.class, Surface.class).invoke(null, displayToken, surface); + } catch (Exception e) { + throw new AssertionError(e); + } + } + + public static IBinder createDisplay(String name, boolean secure) { + try { + return (IBinder) cls.getMethod("createDisplay", String.class, boolean.class).invoke(null, name, secure); + } catch (Exception e) { + throw new AssertionError(e); + } + } + + public static void destroyDisplay(IBinder displayToken) { + try { + cls.getMethod("destroyDisplay", IBinder.class).invoke(null, displayToken); + } catch (Exception e) { + throw new AssertionError(e); + } + } +}