Android/scrcpy
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity

Encode recorded audio on the device

Browse source 
For now, the encoded packets are just logged into the console.

PR #3757 <https://github.com/Genymobile/scrcpy/pull/3757>
This commit is contained in:
Romain Vimont 2023-02-07 23:08:57 +01:00
parent 464a35b05e
commit 5eed2c52c2
1 changed files with 261 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

270
server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java
View file

@ -4,21 +4,63 @@ import android.annotation.SuppressLint;
import android.annotation.TargetApi;
import android.media.AudioFormat;
import android.media.AudioRecord;
import android.media.AudioTimestamp;
import android.media.MediaCodec;
import android.media.MediaFormat;
import android.media.MediaRecorder;
import android.os.Build;
import android.os.Handler;
import android.os.HandlerThread;
import android.os.Looper;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
public final class AudioEncoder {
private static class InputTask {
private final int index;
InputTask(int index) {
this.index = index;
}
}
private static class OutputTask {
private final int index;
private final MediaCodec.BufferInfo bufferInfo;
OutputTask(int index, MediaCodec.BufferInfo bufferInfo) {
this.index = index;
this.bufferInfo = bufferInfo;
}
}
private static final String MIMETYPE = MediaFormat.MIMETYPE_AUDIO_OPUS;
private static final int SAMPLE_RATE = 48000;
private static final int CHANNEL_CONFIG = AudioFormat.CHANNEL_IN_STEREO;
private static final int CHANNELS = 2;
private static final int FORMAT = AudioFormat.ENCODING_PCM_16BIT;
private static final int BYTES_PER_SAMPLE = 2;
private static final int BIT_RATE = 128000;
private static final int READ_MS = 5; // milliseconds
private static final int READ_SIZE = SAMPLE_RATE * CHANNELS * BYTES_PER_SAMPLE * READ_MS / 1000;
// Capacity of 64 is in practice "infinite" (it is limited by the number of available MediaCodec buffers, typically 4).
// So many pending tasks would lead to an unacceptable delay anyway.
private final BlockingQueue<InputTask> inputTasks = new ArrayBlockingQueue<>(64);
private final BlockingQueue<OutputTask> outputTasks = new ArrayBlockingQueue<>(64);
private Thread thread;
private HandlerThread mediaCodecThread;
private Thread inputThread;
private Thread outputThread;
private boolean ended;
private static AudioFormat createAudioFormat() {
AudioFormat.Builder builder = new AudioFormat.Builder();
@ -44,23 +86,80 @@ public final class AudioEncoder {
return builder.build();
}
public void start() {
AudioRecord recorder = createAudioRecord();
private static MediaFormat createFormat() {
MediaFormat format = new MediaFormat();
format.setString(MediaFormat.KEY_MIME, MIMETYPE);
format.setInteger(MediaFormat.KEY_BIT_RATE, BIT_RATE);
format.setInteger(MediaFormat.KEY_CHANNEL_COUNT, CHANNELS);
format.setInteger(MediaFormat.KEY_SAMPLE_RATE, SAMPLE_RATE);
return format;
}
@TargetApi(Build.VERSION_CODES.N)
private void inputThread(MediaCodec mediaCodec, AudioRecord recorder) throws IOException, InterruptedException {
final AudioTimestamp timestamp = new AudioTimestamp();
long previousPts = 0;
long nextPts = 0;
thread = new Thread(() -> {
recorder.startRecording();
try {
byte[] buf = new byte[READ_SIZE];
while (!Thread.currentThread().isInterrupted()) {
int r = recorder.read(buf, 0, READ_SIZE);
if (r > 0) {
Ln.i("Audio captured: " + r + " bytes");
InputTask task = inputTasks.take();
ByteBuffer buffer = mediaCodec.getInputBuffer(task.index);
int r = recorder.read(buffer, READ_SIZE);
if (r < 0) {
throw new IOException("Could not read audio: " + r);
}
long pts;
int ret = recorder.getTimestamp(timestamp, AudioTimestamp.TIMEBASE_MONOTONIC);
if (ret == AudioRecord.SUCCESS) {
pts = timestamp.nanoTime / 1000;
} else {
Ln.e("Audio capture error: " + r);
if (nextPts == 0) {
Ln.w("Could not get any audio timestamp");
}
// compute from previous timestamp and packet size
pts = nextPts;
}
long durationUs = r * 1000000 / (CHANNELS * BYTES_PER_SAMPLE * SAMPLE_RATE);
nextPts = pts + durationUs;
if (previousPts != 0 && pts < previousPts) {
// Audio PTS may come from two sources:
// - recorder.getTimestamp() if the call works;
// - an estimation from the previous PTS and the packet size as a fallback.
//
// Therefore, the property that PTS are monotonically increasing is no guaranteed in corner cases, so enforce it.
pts = previousPts + 1;
}
previousPts = pts;
mediaCodec.queueInputBuffer(task.index, 0, r, pts, 0);
}
}
private void outputThread(MediaCodec mediaCodec) throws IOException, InterruptedException {
while (!Thread.currentThread().isInterrupted()) {
OutputTask task = outputTasks.take();
ByteBuffer buffer = mediaCodec.getOutputBuffer(task.index);
try {
Ln.i("Audio packet [pts=" + task.bufferInfo.presentationTimeUs + "] " + buffer.remaining() + " bytes");
} finally {
recorder.stop();
mediaCodec.releaseOutputBuffer(task.index, false);
}
}
}
public void start() {
thread = new Thread(() -> {
try {
encode();
} catch (IOException e) {
Ln.e("Audio encoding error", e);
} finally {
Ln.d("Audio encoder stopped");
}
});
thread.start();
@ -68,7 +167,8 @@ public final class AudioEncoder {
public void stop() {
if (thread != null) {
thread.interrupt();
// Just wake up the blocking wait from the thread, so that it properly releases all its resources and terminates
end();
}
}
@ -77,4 +177,150 @@ public final class AudioEncoder {
thread.join();
}
}
private synchronized void end() {
ended = true;
notify();
}
private synchronized void waitEnded() {
try {
while (!ended) {
wait();
}
} catch (InterruptedException e) {
// ignore
}
}
@TargetApi(Build.VERSION_CODES.M)
public void encode() throws IOException {
MediaCodec mediaCodec = null;
AudioRecord recorder = null;
boolean mediaCodecStarted = false;
boolean recorderStarted = false;
try {
mediaCodec = MediaCodec.createEncoderByType(MIMETYPE); // may throw IOException
mediaCodecThread = new HandlerThread("AudioEncoder");
mediaCodecThread.start();
MediaFormat format = createFormat();
mediaCodec.setCallback(new EncoderCallback(), new Handler(mediaCodecThread.getLooper()));
mediaCodec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
recorder = createAudioRecord();
recorder.startRecording();
recorderStarted = true;
final MediaCodec mediaCodecRef = mediaCodec;
final AudioRecord recorderRef = recorder;
inputThread = new Thread(() -> {
try {
inputThread(mediaCodecRef, recorderRef);
} catch (IOException | InterruptedException e) {
Ln.e("Audio capture error", e);
} finally {
end();
}
});
outputThread = new Thread(() -> {
try {
outputThread(mediaCodecRef);
} catch (InterruptedException e) {
// this is expected on close
} catch (IOException e) {
// Broken pipe is expected on close, because the socket is closed by the client
if (!IO.isBrokenPipe(e)) {
Ln.e("Audio encoding error", e);
}
} finally {
end();
}
});
mediaCodec.start();
mediaCodecStarted = true;
inputThread.start();
outputThread.start();
waitEnded();
} finally {
// Cleanup everything (either at the end or on error at any step of the initialization)
if (mediaCodecThread != null) {
Looper looper = mediaCodecThread.getLooper();
if (looper != null) {
looper.quitSafely();
}
}
if (inputThread != null) {
inputThread.interrupt();
}
if (outputThread != null) {
outputThread.interrupt();
}
try {
if (mediaCodecThread != null) {
mediaCodecThread.join();
}
if (inputThread != null) {
inputThread.join();
}
if (outputThread != null) {
outputThread.join();
}
} catch (InterruptedException e) {
// Should never happen
throw new AssertionError(e);
}
if (mediaCodec != null) {
if (mediaCodecStarted) {
mediaCodec.stop();
}
mediaCodec.release();
}
if (recorder != null) {
if (recorderStarted) {
recorder.stop();
}
recorder.release();
}
}
}
private class EncoderCallback extends MediaCodec.Callback {
@TargetApi(Build.VERSION_CODES.N)
@Override
public void onInputBufferAvailable(MediaCodec codec, int index) {
try {
inputTasks.put(new InputTask(index));
} catch (InterruptedException e) {
end();
}
}
@Override
public void onOutputBufferAvailable(MediaCodec codec, int index, MediaCodec.BufferInfo bufferInfo) {
try {
outputTasks.put(new OutputTask(index, bufferInfo));
} catch (InterruptedException e) {
end();
}
}
@Override
public void onError(MediaCodec codec, MediaCodec.CodecException e) {
Ln.e("MediaCodec error", e);
end();
}
@Override
public void onOutputFormatChanged(MediaCodec codec, MediaFormat format) {
// ignore
}
}
}