Android/scrcpy
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity

Encode video using MediaCodec API

Browse source 
Replace screenrecord execution by manual screen encoding using the
MediaCodec API.

The "screenrecord" solution had several drawbacks:
 - screenrecord output is buffered, so tiny frames may not be accessible
   immediately;
 - it did not output a frame until the surface changed, leading to a
   black screen on start;
 - it is limited to 3 minutes recording, so it needed to be restarted;
 - screenrecord added black borders in the video when the requested
   dimensions did not preserve aspect-ratio exactly (sometimes
   unavoidable since video dimensions must be multiple of 8);
 - rotation handling was hacky (killing the process and starting a new
   one).

Handling the encoding manually allows to solve all these problems.
This commit is contained in:
Romain Vimont 2018-01-31 18:37:01 +01:00
parent 7ed334915e
commit 865ebb3862
9 changed files with 247 additions and 156 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
server/src/main/java/com/genymobile/scrcpy/DesktopConnection.java
View file

@ -65,8 +65,8 @@ public class DesktopConnection implements Closeable {
outputStream.write(buffer, 0, buffer.length);
}
public void sendVideoStream(byte[] videoStreamBuffer, int len) throws IOException {
outputStream.write(videoStreamBuffer, 0, len);
public OutputStream getOutputStream() {
return outputStream;
}
public ControlEvent receiveControlEvent() throws IOException {

27
server/src/main/java/com/genymobile/scrcpy/Device.java
View file

@ -45,14 +45,12 @@ public final class Device {
// Principle:
// - scale down the great side of the screen to maximumSize (if necessary);
// - scale down the other side so that the aspect ratio is preserved;
// - ceil this value to the next multiple of 8 (H.264 only accepts multiples of 8)
// - this may introduce black bands, so store the padding (typically a few pixels)
// - round this value to the nearest multiple of 8 (H.264 only accepts multiples of 8)
DisplayInfo displayInfo = serviceManager.getDisplayManager().getDisplayInfo();
boolean rotated = (displayInfo.getRotation() & 1) != 0;
Size deviceSize = displayInfo.getSize();
int w = deviceSize.getWidth();
int h = deviceSize.getHeight();
int padding = 0;
if (maximumSize > 0) {
assert maximumSize % 8 == 0;
boolean portrait = h > w;
@ -60,16 +58,15 @@ public final class Device {
int minor = portrait ? w : h;
if (major > maximumSize) {
int minorExact = minor * maximumSize / major;
// +7 to ceil the value on rounding to the next multiple of 8,
// so that any necessary black bands to keep the aspect ratio are added to the smallest dimension
minor = (minorExact + 7) & ~7;
// +4 to round the value to the nearest multiple of 8
minor = (minorExact + 4) & ~7;
major = maximumSize;
padding = minor - minorExact;
}
w = portrait ? minor : major;
h = portrait ? major : minor;
}
return new ScreenInfo(deviceSize, new Size(w, h), padding, rotated);
Size videoSize = new Size(w, h);
return new ScreenInfo(deviceSize, videoSize, rotated);
}
public Point getPhysicalPoint(Position position) {
@ -82,19 +79,9 @@ public final class Device {
return null;
}
Size deviceSize = screenInfo.getDeviceSize();
int xPadding = screenInfo.getXPadding();
int yPadding = screenInfo.getYPadding();
int contentWidth = videoSize.getWidth() - xPadding;
int contentHeight = videoSize.getHeight() - yPadding;
Point point = position.getPoint();
int x = point.x - xPadding / 2;
int y = point.y - yPadding / 2;
if (x < 0 || x >= contentWidth || y < 0 || y >= contentHeight) {
// out of screen
return null;
}
int scaledX = x * deviceSize.getWidth() / videoSize.getWidth();
int scaledY = y * deviceSize.getHeight() / videoSize.getHeight();
int scaledX = point.x * deviceSize.getWidth() / videoSize.getWidth();
int scaledY = point.y * deviceSize.getHeight() / videoSize.getHeight();
return new Point(scaledX, scaledY);
}

12
server/src/main/java/com/genymobile/scrcpy/ScrCpyServer.java
View file

@ -4,25 +4,17 @@ import java.io.IOException;
public class ScrCpyServer {
private static final String TAG = "scrcpy";
private static void scrcpy(Options options) throws IOException {
final Device device = new Device(options);
try (DesktopConnection connection = DesktopConnection.open(device)) {
final ScreenStreamer streamer = new ScreenStreamer(device, connection);
device.setRotationListener(new Device.RotationListener() {
@Override
public void onRotationChanged(int rotation) {
streamer.reset();
}
});
ScreenEncoder screenEncoder = new ScreenEncoder();
// asynchronous
startEventController(device, connection);
try {
// synchronous
streamer.streamScreen();
screenEncoder.streamScreen(device, connection.getOutputStream());
} catch (IOException e) {
Ln.e("Screen streaming interrupted", e);
}

149
server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java Normal file
View file

@ -0,0 +1,149 @@
package com.genymobile.scrcpy;
import android.graphics.Rect;
import android.media.MediaCodec;
import android.media.MediaCodecInfo;
import android.media.MediaFormat;
import android.os.IBinder;
import android.view.Surface;
import com.genymobile.scrcpy.wrappers.SurfaceControl;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.util.concurrent.atomic.AtomicBoolean;
public class ScreenEncoder implements Device.RotationListener {
private static final int DEFAULT_BIT_RATE = 4_000_000; // bits per second
private static final int DEFAULT_FRAME_RATE = 60; // fps
private static final int DEFAULT_I_FRAME_INTERVAL = 10; // seconds
private static final int REPEAT_FRAME_DELAY = 6; // repeat after 6 frames
private final AtomicBoolean rotationChanged = new AtomicBoolean();
private int bitRate;
private int frameRate;
private int iFrameInterval;
public ScreenEncoder(int bitRate, int frameRate, int iFrameInterval) {
this.bitRate = bitRate;
this.frameRate = frameRate;
this.iFrameInterval = iFrameInterval;
}
public ScreenEncoder() {
this(DEFAULT_BIT_RATE, DEFAULT_FRAME_RATE, DEFAULT_I_FRAME_INTERVAL);
}
@Override
public void onRotationChanged(int rotation) {
rotationChanged.set(true);
}
public boolean checkRotationChanged() {
return rotationChanged.getAndSet(false);
}
public void streamScreen(Device device, OutputStream outputStream) throws IOException {
MediaFormat format = createFormat(bitRate, frameRate, iFrameInterval);
MediaCodec codec = createCodec();
IBinder display = createDisplay();
device.setRotationListener(this);
boolean alive;
try {
do {
Rect deviceRect = device.getScreenInfo().getDeviceSize().toRect();
Rect videoRect = device.getScreenInfo().getVideoSize().toRect();
setSize(format, videoRect.width(), videoRect.height());
configure(codec, format);
Surface surface = codec.createInputSurface();
setDisplaySurface(display, surface, deviceRect, videoRect);
codec.start();
try {
alive = encode(codec, outputStream);
} finally {
codec.stop();
surface.release();
}
} while (alive);
} finally {
device.setRotationListener(null);
destroyDisplay(display);
codec.release();
}
}
private boolean encode(MediaCodec codec, OutputStream outputStream) throws IOException {
byte[] buf = new byte[bitRate / 8]; // may contain up to 1 second of video
boolean eof = false;
MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();
while (!checkRotationChanged() && !eof) {
int outputBufferId = codec.dequeueOutputBuffer(bufferInfo, -1);
if (checkRotationChanged()) {
// must restart encoding with new size
break;
}
if (outputBufferId >= 0) {
ByteBuffer outputBuffer = codec.getOutputBuffer(outputBufferId);
while (outputBuffer.hasRemaining()) {
int remaining = outputBuffer.remaining();
int len = Math.min(buf.length, remaining);
// the outputBuffer is probably direct (it has no underlying array), and LocalSocket does not expose channels,
// so we must copy the data locally to write them manually to the output stream
outputBuffer.get(buf, 0, len);
outputStream.write(buf, 0, len);
}
codec.releaseOutputBuffer(outputBufferId, false);
}
}
return !eof;
}
private static MediaCodec createCodec() throws IOException {
return MediaCodec.createEncoderByType("video/avc");
}
private static MediaFormat createFormat(int bitRate, int frameRate, int iFrameInterval) throws IOException {
MediaFormat format = new MediaFormat();
format.setString(MediaFormat.KEY_MIME, "video/avc");
format.setInteger(MediaFormat.KEY_BIT_RATE, bitRate);
format.setInteger(MediaFormat.KEY_FRAME_RATE, frameRate);
format.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, iFrameInterval);
// display the very first frame, and recover from bad quality when no new frames
format.setLong(MediaFormat.KEY_REPEAT_PREVIOUS_FRAME_AFTER, 1_000_000 * REPEAT_FRAME_DELAY / frameRate); // µs
return format;
}
private static IBinder createDisplay() {
return SurfaceControl.createDisplay("scrcpy", false);
}
private static void configure(MediaCodec codec, MediaFormat format) {
codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
}
private static void setSize(MediaFormat format, int width, int height) {
format.setInteger(MediaFormat.KEY_WIDTH, width);
format.setInteger(MediaFormat.KEY_HEIGHT, height);
}
private static void setDisplaySurface(IBinder display, Surface surface, Rect deviceRect, Rect displayRect) {
SurfaceControl.openTransaction();
try {
SurfaceControl.setDisplaySurface(display, surface);
SurfaceControl.setDisplayProjection(display, 0, deviceRect, displayRect);
SurfaceControl.setDisplayLayerStack(display, 0);
} finally {
SurfaceControl.closeTransaction();
}
}
private static void destroyDisplay(IBinder display) {
SurfaceControl.destroyDisplay(display);
}
}

14
server/src/main/java/com/genymobile/scrcpy/ScreenInfo.java
View file

@ -3,13 +3,11 @@ package com.genymobile.scrcpy;
public final class ScreenInfo {
private final Size deviceSize;
private final Size videoSize;
private final int padding; // padding inside the video stream, along the smallest dimension
private final boolean rotated;
public ScreenInfo(Size deviceSize, Size videoSize, int padding, boolean rotated) {
public ScreenInfo(Size deviceSize, Size videoSize, boolean rotated) {
this.deviceSize = deviceSize;
this.videoSize = videoSize;
this.padding = padding;
this.rotated = rotated;
}
@ -21,19 +19,11 @@ public final class ScreenInfo {
return videoSize;
}
public int getXPadding() {
return videoSize.getWidth() < videoSize.getHeight() ? padding : 0;
}
public int getYPadding() {
return videoSize.getHeight() < videoSize.getWidth() ? padding : 0;
}
public ScreenInfo withRotation(int rotation) {
boolean newRotated = (rotation & 1) != 0;
if (rotated == newRotated) {
return this;
}
return new ScreenInfo(deviceSize.rotate(), videoSize.rotate(), padding, newRotated);
return new ScreenInfo(deviceSize.rotate(), videoSize.rotate(), newRotated);
}
}

39
server/src/main/java/com/genymobile/scrcpy/ScreenStreamer.java
View file

@ -1,39 +0,0 @@
package com.genymobile.scrcpy;
import java.io.IOException;
import java.io.InterruptedIOException;
public class ScreenStreamer {
private final Device device;
private final DesktopConnection connection;
private ScreenStreamerSession currentStreamer; // protected by 'this'
public ScreenStreamer(Device device, DesktopConnection connection) {
this.device = device;
this.connection = connection;
}
private synchronized ScreenStreamerSession newScreenStreamerSession() {
currentStreamer = new ScreenStreamerSession(device, connection);
return currentStreamer;
}
public void streamScreen() throws IOException {
while (true) {
try {
ScreenStreamerSession screenStreamer = newScreenStreamerSession();
screenStreamer.streamScreen();
} catch (InterruptedIOException e) {
// the current screenrecord process has probably been killed due to reset(), start a new one without failing
}
}
}
public synchronized void reset() {
if (currentStreamer != null) {
// it will stop the blocking call to streamScreen(), so a new streamer will be started
currentStreamer.stop();
}
}
}

73
server/src/main/java/com/genymobile/scrcpy/ScreenStreamerSession.java
View file

@ -1,73 +0,0 @@
package com.genymobile.scrcpy;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
public class ScreenStreamerSession {
private final Device device;
private final DesktopConnection connection;
private Process screenRecordProcess; // protected by 'this'
private final AtomicBoolean stopped = new AtomicBoolean();
private final byte[] buffer = new byte[0x10000];
public ScreenStreamerSession(Device device, DesktopConnection connection) {
this.device = device;
this.connection = connection;
}
public void streamScreen() throws IOException {
// screenrecord may not record more than 3 minutes, so restart it on EOF
while (!stopped.get() && streamScreenOnce()) ;
}
/**
* Starts screenrecord once and relay its output to the desktop connection.
*
* @return {@code true} if EOF is reached, {@code false} otherwise (i.e. requested to stop).
* @throws IOException if an I/O error occurred
*/
private boolean streamScreenOnce() throws IOException {
Ln.d("Recording...");
Size videoSize = device.getScreenInfo().getVideoSize();
Process process = startScreenRecord(videoSize);
setCurrentProcess(process);
InputStream inputStream = process.getInputStream();
int r;
while ((r = inputStream.read(buffer)) != -1 && !stopped.get()) {
connection.sendVideoStream(buffer, r);
}
return r != -1;
}
public void stop() {
// let the thread stop itself without breaking the video stream
stopped.set(true);
killCurrentProcess();
}
private static Process startScreenRecord(Size videoSize) throws IOException {
List<String> command = new ArrayList<>();
command.add("screenrecord");
command.add("--output-format=h264");
command.add("--size=" + videoSize.getWidth() + "x" + videoSize.getHeight());
command.add("-");
Process process = new ProcessBuilder(command).start();
process.getOutputStream().close();
return process;
}
private synchronized void setCurrentProcess(Process screenRecordProcess) {
this.screenRecordProcess = screenRecordProcess;
}
private synchronized void killCurrentProcess() {
if (screenRecordProcess != null) {
screenRecordProcess.destroy();
screenRecordProcess = null;
}
}
}

6
server/src/main/java/com/genymobile/scrcpy/Size.java
View file

@ -1,5 +1,7 @@
package com.genymobile.scrcpy;
import android.graphics.Rect;
import java.util.Objects;
public final class Size {
@ -23,6 +25,10 @@ public final class Size {
return new Size(height, width);
}
public Rect toRect() {
return new Rect(0, 0, width, height);
}
@Override
public boolean equals(Object o) {
if (this == o) return true;

79
server/src/main/java/com/genymobile/scrcpy/wrappers/SurfaceControl.java Normal file
View file

@ -0,0 +1,79 @@
package com.genymobile.scrcpy.wrappers;
import android.graphics.Rect;
import android.os.IBinder;
import android.view.Surface;
public class SurfaceControl {
private static final Class<?> cls;
static {
try {
cls = Class.forName("android.view.SurfaceControl");
} catch (ClassNotFoundException e) {
throw new AssertionError(e);
}
}
private SurfaceControl() {
// only static methods
}
public static void openTransaction() {
try {
cls.getMethod("openTransaction").invoke(null);
} catch (Exception e) {
throw new AssertionError(e);
}
}
public static void closeTransaction() {
try {
cls.getMethod("closeTransaction").invoke(null);
} catch (Exception e) {
throw new AssertionError(e);
}
}
public static void setDisplayProjection(IBinder displayToken, int orientation, Rect layerStackRect, Rect displayRect) {
try {
cls.getMethod("setDisplayProjection", IBinder.class, int.class, Rect.class, Rect.class)
.invoke(null, displayToken, orientation, layerStackRect, displayRect);
} catch (Exception e) {
throw new AssertionError(e);
}
}
public static void setDisplayLayerStack(IBinder displayToken, int layerStack) {
try {
cls.getMethod("setDisplayLayerStack", IBinder.class, int.class).invoke(null, displayToken, layerStack);
} catch (Exception e) {
throw new AssertionError(e);
}
}
public static void setDisplaySurface(IBinder displayToken, Surface surface) {
try {
cls.getMethod("setDisplaySurface", IBinder.class, Surface.class).invoke(null, displayToken, surface);
} catch (Exception e) {
throw new AssertionError(e);
}
}
public static IBinder createDisplay(String name, boolean secure) {
try {
return (IBinder) cls.getMethod("createDisplay", String.class, boolean.class).invoke(null, name, secure);
} catch (Exception e) {
throw new AssertionError(e);
}
}
public static void destroyDisplay(IBinder displayToken) {
try {
cls.getMethod("destroyDisplay", IBinder.class).invoke(null, displayToken);
} catch (Exception e) {
throw new AssertionError(e);
}
}
}