diff --git a/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java b/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java index 45e217fc..7b571126 100644 --- a/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java +++ b/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java @@ -4,21 +4,63 @@ import android.annotation.SuppressLint; import android.annotation.TargetApi; import android.media.AudioFormat; import android.media.AudioRecord; +import android.media.AudioTimestamp; +import android.media.MediaCodec; +import android.media.MediaFormat; import android.media.MediaRecorder; import android.os.Build; +import android.os.Handler; +import android.os.HandlerThread; +import android.os.Looper; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; public final class AudioEncoder { + private static class InputTask { + private final int index; + + InputTask(int index) { + this.index = index; + } + } + + private static class OutputTask { + private final int index; + private final MediaCodec.BufferInfo bufferInfo; + + OutputTask(int index, MediaCodec.BufferInfo bufferInfo) { + this.index = index; + this.bufferInfo = bufferInfo; + } + } + + private static final String MIMETYPE = MediaFormat.MIMETYPE_AUDIO_OPUS; private static final int SAMPLE_RATE = 48000; private static final int CHANNEL_CONFIG = AudioFormat.CHANNEL_IN_STEREO; private static final int CHANNELS = 2; private static final int FORMAT = AudioFormat.ENCODING_PCM_16BIT; private static final int BYTES_PER_SAMPLE = 2; + private static final int BIT_RATE = 128000; private static final int READ_MS = 5; // milliseconds private static final int READ_SIZE = SAMPLE_RATE * CHANNELS * BYTES_PER_SAMPLE * READ_MS / 1000; + // Capacity of 64 is in practice "infinite" (it is limited by the number of available MediaCodec buffers, typically 4). + // So many pending tasks would lead to an unacceptable delay anyway. + private final BlockingQueue inputTasks = new ArrayBlockingQueue<>(64); + private final BlockingQueue outputTasks = new ArrayBlockingQueue<>(64); + private Thread thread; + private HandlerThread mediaCodecThread; + + private Thread inputThread; + private Thread outputThread; + + private boolean ended; private static AudioFormat createAudioFormat() { AudioFormat.Builder builder = new AudioFormat.Builder(); @@ -44,23 +86,80 @@ public final class AudioEncoder { return builder.build(); } - public void start() { - AudioRecord recorder = createAudioRecord(); + private static MediaFormat createFormat() { + MediaFormat format = new MediaFormat(); + format.setString(MediaFormat.KEY_MIME, MIMETYPE); + format.setInteger(MediaFormat.KEY_BIT_RATE, BIT_RATE); + format.setInteger(MediaFormat.KEY_CHANNEL_COUNT, CHANNELS); + format.setInteger(MediaFormat.KEY_SAMPLE_RATE, SAMPLE_RATE); + return format; + } - thread = new Thread(() -> { - recorder.startRecording(); - try { - byte[] buf = new byte[READ_SIZE]; - while (!Thread.currentThread().isInterrupted()) { - int r = recorder.read(buf, 0, READ_SIZE); - if (r > 0) { - Ln.i("Audio captured: " + r + " bytes"); - } else { - Ln.e("Audio capture error: " + r); - } + @TargetApi(Build.VERSION_CODES.N) + private void inputThread(MediaCodec mediaCodec, AudioRecord recorder) throws IOException, InterruptedException { + final AudioTimestamp timestamp = new AudioTimestamp(); + long previousPts = 0; + long nextPts = 0; + + while (!Thread.currentThread().isInterrupted()) { + InputTask task = inputTasks.take(); + ByteBuffer buffer = mediaCodec.getInputBuffer(task.index); + int r = recorder.read(buffer, READ_SIZE); + if (r < 0) { + throw new IOException("Could not read audio: " + r); + } + + long pts; + + int ret = recorder.getTimestamp(timestamp, AudioTimestamp.TIMEBASE_MONOTONIC); + if (ret == AudioRecord.SUCCESS) { + pts = timestamp.nanoTime / 1000; + } else { + if (nextPts == 0) { + Ln.w("Could not get any audio timestamp"); } + // compute from previous timestamp and packet size + pts = nextPts; + } + + long durationUs = r * 1000000 / (CHANNELS * BYTES_PER_SAMPLE * SAMPLE_RATE); + nextPts = pts + durationUs; + + if (previousPts != 0 && pts < previousPts) { + // Audio PTS may come from two sources: + // - recorder.getTimestamp() if the call works; + // - an estimation from the previous PTS and the packet size as a fallback. + // + // Therefore, the property that PTS are monotonically increasing is no guaranteed in corner cases, so enforce it. + pts = previousPts + 1; + } + + previousPts = pts; + + mediaCodec.queueInputBuffer(task.index, 0, r, pts, 0); + } + } + + private void outputThread(MediaCodec mediaCodec) throws IOException, InterruptedException { + while (!Thread.currentThread().isInterrupted()) { + OutputTask task = outputTasks.take(); + ByteBuffer buffer = mediaCodec.getOutputBuffer(task.index); + try { + Ln.i("Audio packet [pts=" + task.bufferInfo.presentationTimeUs + "] " + buffer.remaining() + " bytes"); } finally { - recorder.stop(); + mediaCodec.releaseOutputBuffer(task.index, false); + } + } + } + + public void start() { + thread = new Thread(() -> { + try { + encode(); + } catch (IOException e) { + Ln.e("Audio encoding error", e); + } finally { + Ln.d("Audio encoder stopped"); } }); thread.start(); @@ -68,7 +167,8 @@ public final class AudioEncoder { public void stop() { if (thread != null) { - thread.interrupt(); + // Just wake up the blocking wait from the thread, so that it properly releases all its resources and terminates + end(); } } @@ -77,4 +177,150 @@ public final class AudioEncoder { thread.join(); } } + + private synchronized void end() { + ended = true; + notify(); + } + + private synchronized void waitEnded() { + try { + while (!ended) { + wait(); + } + } catch (InterruptedException e) { + // ignore + } + } + + @TargetApi(Build.VERSION_CODES.M) + public void encode() throws IOException { + MediaCodec mediaCodec = null; + AudioRecord recorder = null; + + boolean mediaCodecStarted = false; + boolean recorderStarted = false; + try { + mediaCodec = MediaCodec.createEncoderByType(MIMETYPE); // may throw IOException + + mediaCodecThread = new HandlerThread("AudioEncoder"); + mediaCodecThread.start(); + + MediaFormat format = createFormat(); + mediaCodec.setCallback(new EncoderCallback(), new Handler(mediaCodecThread.getLooper())); + mediaCodec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); + + recorder = createAudioRecord(); + recorder.startRecording(); + recorderStarted = true; + + final MediaCodec mediaCodecRef = mediaCodec; + final AudioRecord recorderRef = recorder; + inputThread = new Thread(() -> { + try { + inputThread(mediaCodecRef, recorderRef); + } catch (IOException | InterruptedException e) { + Ln.e("Audio capture error", e); + } finally { + end(); + } + }); + + outputThread = new Thread(() -> { + try { + outputThread(mediaCodecRef); + } catch (InterruptedException e) { + // this is expected on close + } catch (IOException e) { + // Broken pipe is expected on close, because the socket is closed by the client + if (!IO.isBrokenPipe(e)) { + Ln.e("Audio encoding error", e); + } + } finally { + end(); + } + }); + + mediaCodec.start(); + mediaCodecStarted = true; + inputThread.start(); + outputThread.start(); + + waitEnded(); + } finally { + // Cleanup everything (either at the end or on error at any step of the initialization) + if (mediaCodecThread != null) { + Looper looper = mediaCodecThread.getLooper(); + if (looper != null) { + looper.quitSafely(); + } + } + if (inputThread != null) { + inputThread.interrupt(); + } + if (outputThread != null) { + outputThread.interrupt(); + } + + try { + if (mediaCodecThread != null) { + mediaCodecThread.join(); + } + if (inputThread != null) { + inputThread.join(); + } + if (outputThread != null) { + outputThread.join(); + } + } catch (InterruptedException e) { + // Should never happen + throw new AssertionError(e); + } + + if (mediaCodec != null) { + if (mediaCodecStarted) { + mediaCodec.stop(); + } + mediaCodec.release(); + } + if (recorder != null) { + if (recorderStarted) { + recorder.stop(); + } + recorder.release(); + } + } + } + + private class EncoderCallback extends MediaCodec.Callback { + @TargetApi(Build.VERSION_CODES.N) + @Override + public void onInputBufferAvailable(MediaCodec codec, int index) { + try { + inputTasks.put(new InputTask(index)); + } catch (InterruptedException e) { + end(); + } + } + + @Override + public void onOutputBufferAvailable(MediaCodec codec, int index, MediaCodec.BufferInfo bufferInfo) { + try { + outputTasks.put(new OutputTask(index, bufferInfo)); + } catch (InterruptedException e) { + end(); + } + } + + @Override + public void onError(MediaCodec codec, MediaCodec.CodecException e) { + Ln.e("MediaCodec error", e); + end(); + } + + @Override + public void onOutputFormatChanged(MediaCodec codec, MediaFormat format) { + // ignore + } + } }