Android/scrcpy
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity

Extract audio capture

Browse source 
The audio capture was implemented in AudioEncoder.

In order to reuse it without encoding, extract it to a separate class.

PR #3757 <https://github.com/Genymobile/scrcpy/pull/3757>
This commit is contained in:
Romain Vimont 2023-03-03 18:46:59 +01:00
parent 02dd1be4a1
commit 65cc9d765d
2 changed files with 161 additions and 123 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

148
server/src/main/java/com/genymobile/scrcpy/AudioCapture.java Normal file
View file

@ -0,0 +1,148 @@
package com.genymobile.scrcpy;
import com.genymobile.scrcpy.wrappers.ServiceManager;
import android.annotation.SuppressLint;
import android.annotation.TargetApi;
import android.content.ComponentName;
import android.content.Intent;
import android.media.AudioFormat;
import android.media.AudioRecord;
import android.media.AudioTimestamp;
import android.media.MediaCodec;
import android.media.MediaRecorder;
import android.os.Build;
import android.os.SystemClock;
import java.io.IOException;
import java.nio.ByteBuffer;
public final class AudioCapture {
public static final int SAMPLE_RATE = 48000;
public static final int CHANNEL_CONFIG = AudioFormat.CHANNEL_IN_STEREO;
public static final int CHANNELS = 2;
public static final int FORMAT = AudioFormat.ENCODING_PCM_16BIT;
public static final int BYTES_PER_SAMPLE = 2;
private AudioRecord recorder;
private final AudioTimestamp timestamp = new AudioTimestamp();
private long previousPts = 0;
private long nextPts = 0;
public static int millisToBytes(int millis) {
return SAMPLE_RATE * CHANNELS * BYTES_PER_SAMPLE * millis / 1000;
}
private static AudioFormat createAudioFormat() {
AudioFormat.Builder builder = new AudioFormat.Builder();
builder.setEncoding(FORMAT);
builder.setSampleRate(SAMPLE_RATE);
builder.setChannelMask(CHANNEL_CONFIG);
return builder.build();
}
@TargetApi(Build.VERSION_CODES.M)
@SuppressLint({"WrongConstant", "MissingPermission"})
private static AudioRecord createAudioRecord() {
AudioRecord.Builder builder = new AudioRecord.Builder();
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
// On older APIs, Workarounds.fillAppInfo() must be called beforehand
builder.setContext(FakeContext.get());
}
builder.setAudioSource(MediaRecorder.AudioSource.REMOTE_SUBMIX);
builder.setAudioFormat(createAudioFormat());
int minBufferSize = AudioRecord.getMinBufferSize(SAMPLE_RATE, CHANNEL_CONFIG, FORMAT);
// This buffer size does not impact latency
builder.setBufferSizeInBytes(8 * minBufferSize);
return builder.build();
}
private static void startWorkaroundAndroid11() {
if (Build.VERSION.SDK_INT == Build.VERSION_CODES.R) {
// Android 11 requires Apps to be at foreground to record audio.
// Normally, each App has its own user ID, so Android checks whether the requesting App has the user ID that's at the foreground.
// But scrcpy server is NOT an App, it's a Java application started from Android shell, so it has the same user ID (2000) with Android
// shell ("com.android.shell").
// If there is an Activity from Android shell running at foreground, then the permission system will believe scrcpy is also in the
// foreground.
if (Build.VERSION.SDK_INT == Build.VERSION_CODES.R) {
Intent intent = new Intent(Intent.ACTION_MAIN);
intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK);
intent.addCategory(Intent.CATEGORY_LAUNCHER);
intent.setComponent(new ComponentName(FakeContext.PACKAGE_NAME, "com.android.shell.HeapDumpActivity"));
ServiceManager.getActivityManager().startActivityAsUserWithFeature(intent);
// Wait for activity to start
SystemClock.sleep(150);
}
}
}
private static void stopWorkaroundAndroid11() {
if (Build.VERSION.SDK_INT == Build.VERSION_CODES.R) {
ServiceManager.getActivityManager().forceStopPackage(FakeContext.PACKAGE_NAME);
}
}
public void start() throws AudioCaptureForegroundException {
startWorkaroundAndroid11();
try {
recorder = createAudioRecord();
recorder.startRecording();
} catch (UnsupportedOperationException e) {
if (Build.VERSION.SDK_INT == Build.VERSION_CODES.R) {
Ln.e("Failed to start audio capture");
Ln.e("On Android 11, it is only possible to capture in foreground, make sure that the device is unlocked when starting scrcpy.");
throw new AudioCaptureForegroundException();
}
throw e;
} finally {
stopWorkaroundAndroid11();
}
}
public void stop() {
if (recorder != null) {
// Will call .stop() if necessary, without throwing an IllegalStateException
recorder.release();
}
}
@TargetApi(Build.VERSION_CODES.N)
public int read(ByteBuffer directBuffer, int size, MediaCodec.BufferInfo outBufferInfo) throws IOException {
int r = recorder.read(directBuffer, size);
if (r < 0) {
return r;
}
long pts;
int ret = recorder.getTimestamp(timestamp, AudioTimestamp.TIMEBASE_MONOTONIC);
if (ret == AudioRecord.SUCCESS) {
pts = timestamp.nanoTime / 1000;
} else {
if (nextPts == 0) {
Ln.w("Could not get any audio timestamp");
}
// compute from previous timestamp and packet size
pts = nextPts;
}
long durationUs = r * 1000000 / (CHANNELS * BYTES_PER_SAMPLE * SAMPLE_RATE);
nextPts = pts + durationUs;
if (previousPts != 0 && pts < previousPts) {
// Audio PTS may come from two sources:
// - recorder.getTimestamp() if the call works;
// - an estimation from the previous PTS and the packet size as a fallback.
//
// Therefore, the property that PTS are monotonically increasing is no guaranteed in corner cases, so enforce it.
pts = previousPts + 1;
}
previousPts = pts;
outBufferInfo.set(0, r, pts, 0);
return r;
}
}

136
server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java
View file

@ -1,22 +1,12 @@
package com.genymobile.scrcpy;
import com.genymobile.scrcpy.wrappers.ServiceManager;
import android.annotation.SuppressLint;
import android.annotation.TargetApi;
import android.content.ComponentName;
import android.content.Intent;
import android.media.AudioFormat;
import android.media.AudioRecord;
import android.media.AudioTimestamp;
import android.media.MediaCodec;
import android.media.MediaFormat;
import android.media.MediaRecorder;
import android.os.Build;
import android.os.Handler;
import android.os.HandlerThread;
import android.os.Looper;
import android.os.SystemClock;
import java.io.IOException;
import java.nio.ByteBuffer;
@ -44,14 +34,11 @@ public final class AudioEncoder {
}
}
private static final int SAMPLE_RATE = 48000;
private static final int CHANNEL_CONFIG = AudioFormat.CHANNEL_IN_STEREO;
private static final int CHANNELS = 2;
private static final int FORMAT = AudioFormat.ENCODING_PCM_16BIT;
private static final int BYTES_PER_SAMPLE = 2;
private static final int SAMPLE_RATE = AudioCapture.SAMPLE_RATE;
private static final int CHANNELS = AudioCapture.CHANNELS;
private static final int READ_MS = 5; // milliseconds
private static final int READ_SIZE = SAMPLE_RATE * CHANNELS * BYTES_PER_SAMPLE * READ_MS / 1000;
private static final int READ_SIZE = AudioCapture.millisToBytes(READ_MS);
private final Streamer streamer;
private final int bitRate;
@ -78,30 +65,6 @@ public final class AudioEncoder {
this.encoderName = encoderName;
}
private static AudioFormat createAudioFormat() {
AudioFormat.Builder builder = new AudioFormat.Builder();
builder.setEncoding(FORMAT);
builder.setSampleRate(SAMPLE_RATE);
builder.setChannelMask(CHANNEL_CONFIG);
return builder.build();
}
@TargetApi(Build.VERSION_CODES.M)
@SuppressLint({"WrongConstant", "MissingPermission"})
private static AudioRecord createAudioRecord() {
AudioRecord.Builder builder = new AudioRecord.Builder();
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
// On older APIs, Workarounds.fillAppInfo() must be called beforehand
builder.setContext(FakeContext.get());
}
builder.setAudioSource(MediaRecorder.AudioSource.REMOTE_SUBMIX);
builder.setAudioFormat(createAudioFormat());
int minBufferSize = AudioRecord.getMinBufferSize(SAMPLE_RATE, CHANNEL_CONFIG, FORMAT);
// This buffer size does not impact latency
builder.setBufferSizeInBytes(8 * minBufferSize);
return builder.build();
}
private static MediaFormat createFormat(String mimeType, int bitRate, List<CodecOption> codecOptions) {
MediaFormat format = new MediaFormat();
format.setString(MediaFormat.KEY_MIME, mimeType);
@ -122,47 +85,18 @@ public final class AudioEncoder {
}
@TargetApi(Build.VERSION_CODES.N)
private void inputThread(MediaCodec mediaCodec, AudioRecord recorder) throws IOException, InterruptedException {
final AudioTimestamp timestamp = new AudioTimestamp();
long previousPts = 0;
long nextPts = 0;
private void inputThread(MediaCodec mediaCodec, AudioCapture capture) throws IOException, InterruptedException {
final MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();
while (!Thread.currentThread().isInterrupted()) {
InputTask task = inputTasks.take();
ByteBuffer buffer = mediaCodec.getInputBuffer(task.index);
int r = recorder.read(buffer, READ_SIZE);
int r = capture.read(buffer, READ_SIZE, bufferInfo);
if (r < 0) {
throw new IOException("Could not read audio: " + r);
}
long pts;
int ret = recorder.getTimestamp(timestamp, AudioTimestamp.TIMEBASE_MONOTONIC);
if (ret == AudioRecord.SUCCESS) {
pts = timestamp.nanoTime / 1000;
} else {
if (nextPts == 0) {
Ln.w("Could not get any audio timestamp");
}
// compute from previous timestamp and packet size
pts = nextPts;
}
long durationUs = r * 1000000 / (CHANNELS * BYTES_PER_SAMPLE * SAMPLE_RATE);
nextPts = pts + durationUs;
if (previousPts != 0 && pts < previousPts) {
// Audio PTS may come from two sources:
// - recorder.getTimestamp() if the call works;
// - an estimation from the previous PTS and the packet size as a fallback.
//
// Therefore, the property that PTS are monotonically increasing is no guaranteed in corner cases, so enforce it.
pts = previousPts + 1;
}
previousPts = pts;
mediaCodec.queueInputBuffer(task.index, 0, r, pts, 0);
mediaCodec.queueInputBuffer(task.index, bufferInfo.offset, bufferInfo.size, bufferInfo.presentationTimeUs, bufferInfo.flags);
}
}
@ -223,32 +157,6 @@ public final class AudioEncoder {
}
}
private static void startWorkaroundAndroid11() {
if (Build.VERSION.SDK_INT == Build.VERSION_CODES.R) {
// Android 11 requires Apps to be at foreground to record audio.
// Normally, each App has its own user ID, so Android checks whether the requesting App has the user ID that's at the foreground.
// But scrcpy server is NOT an App, it's a Java application started from Android shell, so it has the same user ID (2000) with Android
// shell ("com.android.shell").
// If there is an Activity from Android shell running at foreground, then the permission system will believe scrcpy is also in the
// foreground.
if (Build.VERSION.SDK_INT == Build.VERSION_CODES.R) {
Intent intent = new Intent(Intent.ACTION_MAIN);
intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK);
intent.addCategory(Intent.CATEGORY_LAUNCHER);
intent.setComponent(new ComponentName(FakeContext.PACKAGE_NAME, "com.android.shell.HeapDumpActivity"));
ServiceManager.getActivityManager().startActivityAsUserWithFeature(intent);
// Wait for activity to start
SystemClock.sleep(150);
}
}
}
private static void stopWorkaroundAndroid11() {
if (Build.VERSION.SDK_INT == Build.VERSION_CODES.R) {
ServiceManager.getActivityManager().forceStopPackage(FakeContext.PACKAGE_NAME);
}
}
@TargetApi(Build.VERSION_CODES.M)
public void encode() throws IOException, ConfigurationException, AudioCaptureForegroundException {
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) {
@ -258,10 +166,9 @@ public final class AudioEncoder {
}
MediaCodec mediaCodec = null;
AudioRecord recorder = null;
AudioCapture capture = new AudioCapture();
boolean mediaCodecStarted = false;
boolean recorderStarted = false;
try {
Codec codec = streamer.getCodec();
mediaCodec = createMediaCodec(codec, encoderName);
@ -273,27 +180,13 @@ public final class AudioEncoder {
mediaCodec.setCallback(new EncoderCallback(), new Handler(mediaCodecThread.getLooper()));
mediaCodec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
startWorkaroundAndroid11();
try {
recorder = createAudioRecord();
recorder.startRecording();
} catch (UnsupportedOperationException e) {
if (Build.VERSION.SDK_INT == Build.VERSION_CODES.R) {
Ln.e("Failed to start audio capture");
Ln.e("On Android 11, it is only possible to capture in foreground, make sure that the device is unlocked when starting scrcpy.");
throw new AudioCaptureForegroundException();
}
throw e;
} finally {
stopWorkaroundAndroid11();
}
recorderStarted = true;
capture.start();
final MediaCodec mediaCodecRef = mediaCodec;
final AudioRecord recorderRef = recorder;
final AudioCapture captureRef = capture;
inputThread = new Thread(() -> {
try {
inputThread(mediaCodecRef, recorderRef);
inputThread(mediaCodecRef, captureRef);
} catch (IOException | InterruptedException e) {
Ln.e("Audio capture error", e);
} finally {
@ -366,11 +259,8 @@ public final class AudioEncoder {
}
mediaCodec.release();
}
if (recorder != null) {
if (recorderStarted) {
recorder.stop();
}
recorder.release();
if (capture != null) {
capture.stop();
}
}
}