Encode recorded audio on the device

For now, the encoded packets are just logged into the console.

PR #3757 <https://github.com/Genymobile/scrcpy/pull/3757>

server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java

@@ -4,21 +4,63 @@ import android.annotation.SuppressLint;
 import android.annotation.TargetApi;
 import android.media.AudioFormat;
 import android.media.AudioRecord;
+import android.media.AudioTimestamp;
+import android.media.MediaCodec;
+import android.media.MediaFormat;
 import android.media.MediaRecorder;
 import android.os.Build;
+import android.os.Handler;
+import android.os.HandlerThread;
+import android.os.Looper;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
 
 public final class AudioEncoder {
 
+    private static class InputTask {
+        private final int index;
+
+        InputTask(int index) {
+            this.index = index;
+        }
+    }
+
+    private static class OutputTask {
+        private final int index;
+        private final MediaCodec.BufferInfo bufferInfo;
+
+        OutputTask(int index, MediaCodec.BufferInfo bufferInfo) {
+            this.index = index;
+            this.bufferInfo = bufferInfo;
+        }
+    }
+
+    private static final String MIMETYPE = MediaFormat.MIMETYPE_AUDIO_OPUS;
     private static final int SAMPLE_RATE = 48000;
     private static final int CHANNEL_CONFIG = AudioFormat.CHANNEL_IN_STEREO;
     private static final int CHANNELS = 2;
     private static final int FORMAT = AudioFormat.ENCODING_PCM_16BIT;
     private static final int BYTES_PER_SAMPLE = 2;
+    private static final int BIT_RATE = 128000;
 
     private static final int READ_MS = 5; // milliseconds
     private static final int READ_SIZE = SAMPLE_RATE * CHANNELS * BYTES_PER_SAMPLE * READ_MS / 1000;
 
+    // Capacity of 64 is in practice "infinite" (it is limited by the number of available MediaCodec buffers, typically 4).
+    // So many pending tasks would lead to an unacceptable delay anyway.
+    private final BlockingQueue<InputTask> inputTasks = new ArrayBlockingQueue<>(64);
+    private final BlockingQueue<OutputTask> outputTasks = new ArrayBlockingQueue<>(64);
+
     private Thread thread;
+    private HandlerThread mediaCodecThread;
+
+    private Thread inputThread;
+    private Thread outputThread;
+
+    private boolean ended;
 
     private static AudioFormat createAudioFormat() {
         AudioFormat.Builder builder = new AudioFormat.Builder();
@@ -44,23 +86,80 @@ public final class AudioEncoder {
         return builder.build();
     }
 
-    public void start() {
+    private static MediaFormat createFormat() {
-        AudioRecord recorder = createAudioRecord();
+        MediaFormat format = new MediaFormat();
+        format.setString(MediaFormat.KEY_MIME, MIMETYPE);
+        format.setInteger(MediaFormat.KEY_BIT_RATE, BIT_RATE);
+        format.setInteger(MediaFormat.KEY_CHANNEL_COUNT, CHANNELS);
+        format.setInteger(MediaFormat.KEY_SAMPLE_RATE, SAMPLE_RATE);
+        return format;
+    }
 
-        thread = new Thread(() -> {
+    @TargetApi(Build.VERSION_CODES.N)
-            recorder.startRecording();
+    private void inputThread(MediaCodec mediaCodec, AudioRecord recorder) throws IOException, InterruptedException {
-            try {
+        final AudioTimestamp timestamp = new AudioTimestamp();
-                byte[] buf = new byte[READ_SIZE];
+        long previousPts = 0;
-                while (!Thread.currentThread().isInterrupted()) {
+        long nextPts = 0;
-                    int r = recorder.read(buf, 0, READ_SIZE);
+
-                    if (r > 0) {
+        while (!Thread.currentThread().isInterrupted()) {
-                        Ln.i("Audio captured: " + r + " bytes");
+            InputTask task = inputTasks.take();
-                    } else {
+            ByteBuffer buffer = mediaCodec.getInputBuffer(task.index);
-                        Ln.e("Audio capture error: " + r);
+            int r = recorder.read(buffer, READ_SIZE);
-                    }
+            if (r < 0) {
+                throw new IOException("Could not read audio: " + r);
+            }
+
+            long pts;
+
+            int ret = recorder.getTimestamp(timestamp, AudioTimestamp.TIMEBASE_MONOTONIC);
+            if (ret == AudioRecord.SUCCESS) {
+                pts = timestamp.nanoTime / 1000;
+            } else {
+                if (nextPts == 0) {
+                    Ln.w("Could not get any audio timestamp");
                 }
+                // compute from previous timestamp and packet size
+                pts = nextPts;
+            }
+
+            long durationUs = r * 1000000 / (CHANNELS * BYTES_PER_SAMPLE * SAMPLE_RATE);
+            nextPts = pts + durationUs;
+
+            if (previousPts != 0 && pts < previousPts) {
+                // Audio PTS may come from two sources:
+                //  - recorder.getTimestamp() if the call works;
+                //  - an estimation from the previous PTS and the packet size as a fallback.
+                //
+                // Therefore, the property that PTS are monotonically increasing is no guaranteed in corner cases, so enforce it.
+                pts = previousPts + 1;
+            }
+
+            previousPts = pts;
+
+            mediaCodec.queueInputBuffer(task.index, 0, r, pts, 0);
+        }
+    }
+
+    private void outputThread(MediaCodec mediaCodec) throws IOException, InterruptedException {
+        while (!Thread.currentThread().isInterrupted()) {
+            OutputTask task = outputTasks.take();
+            ByteBuffer buffer = mediaCodec.getOutputBuffer(task.index);
+            try {
+                Ln.i("Audio packet [pts=" + task.bufferInfo.presentationTimeUs + "] " + buffer.remaining() + " bytes");
             } finally {
-                recorder.stop();
+                mediaCodec.releaseOutputBuffer(task.index, false);
+            }
+        }
+    }
+
+    public void start() {
+        thread = new Thread(() -> {
+            try {
+                encode();
+            } catch (IOException e) {
+                Ln.e("Audio encoding error", e);
+            } finally {
+                Ln.d("Audio encoder stopped");
             }
         });
         thread.start();
@@ -68,7 +167,8 @@ public final class AudioEncoder {
 
     public void stop() {
         if (thread != null) {
-            thread.interrupt();
+            // Just wake up the blocking wait from the thread, so that it properly releases all its resources and terminates
+            end();
         }
     }
 
@@ -77,4 +177,150 @@ public final class AudioEncoder {
             thread.join();
         }
     }
+
+    private synchronized void end() {
+        ended = true;
+        notify();
+    }
+
+    private synchronized void waitEnded() {
+        try {
+            while (!ended) {
+                wait();
+            }
+        } catch (InterruptedException e) {
+            // ignore
+        }
+    }
+
+    @TargetApi(Build.VERSION_CODES.M)
+    public void encode() throws IOException {
+        MediaCodec mediaCodec = null;
+        AudioRecord recorder = null;
+
+        boolean mediaCodecStarted = false;
+        boolean recorderStarted = false;
+        try {
+            mediaCodec = MediaCodec.createEncoderByType(MIMETYPE); // may throw IOException
+
+            mediaCodecThread = new HandlerThread("AudioEncoder");
+            mediaCodecThread.start();
+
+            MediaFormat format = createFormat();
+            mediaCodec.setCallback(new EncoderCallback(), new Handler(mediaCodecThread.getLooper()));
+            mediaCodec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
+
+            recorder = createAudioRecord();
+            recorder.startRecording();
+            recorderStarted = true;
+
+            final MediaCodec mediaCodecRef = mediaCodec;
+            final AudioRecord recorderRef = recorder;
+            inputThread = new Thread(() -> {
+                try {
+                    inputThread(mediaCodecRef, recorderRef);
+                } catch (IOException | InterruptedException e) {
+                    Ln.e("Audio capture error", e);
+                } finally {
+                    end();
+                }
+            });
+
+            outputThread = new Thread(() -> {
+                try {
+                    outputThread(mediaCodecRef);
+                } catch (InterruptedException e) {
+                    // this is expected on close
+                } catch (IOException e) {
+                    // Broken pipe is expected on close, because the socket is closed by the client
+                    if (!IO.isBrokenPipe(e)) {
+                        Ln.e("Audio encoding error", e);
+                    }
+                } finally {
+                    end();
+                }
+            });
+
+            mediaCodec.start();
+            mediaCodecStarted = true;
+            inputThread.start();
+            outputThread.start();
+
+            waitEnded();
+        } finally {
+            // Cleanup everything (either at the end or on error at any step of the initialization)
+            if (mediaCodecThread != null) {
+                Looper looper = mediaCodecThread.getLooper();
+                if (looper != null) {
+                    looper.quitSafely();
+                }
+            }
+            if (inputThread != null) {
+                inputThread.interrupt();
+            }
+            if (outputThread != null) {
+                outputThread.interrupt();
+            }
+
+            try {
+                if (mediaCodecThread != null) {
+                    mediaCodecThread.join();
+                }
+                if (inputThread != null) {
+                    inputThread.join();
+                }
+                if (outputThread != null) {
+                    outputThread.join();
+                }
+            } catch (InterruptedException e) {
+                // Should never happen
+                throw new AssertionError(e);
+            }
+
+            if (mediaCodec != null) {
+                if (mediaCodecStarted) {
+                    mediaCodec.stop();
+                }
+                mediaCodec.release();
+            }
+            if (recorder != null) {
+                if (recorderStarted) {
+                    recorder.stop();
+                }
+                recorder.release();
+            }
+        }
+    }
+
+    private class EncoderCallback extends MediaCodec.Callback {
+        @TargetApi(Build.VERSION_CODES.N)
+        @Override
+        public void onInputBufferAvailable(MediaCodec codec, int index) {
+            try {
+                inputTasks.put(new InputTask(index));
+            } catch (InterruptedException e) {
+                end();
+            }
+        }
+
+        @Override
+        public void onOutputBufferAvailable(MediaCodec codec, int index, MediaCodec.BufferInfo bufferInfo) {
+            try {
+                outputTasks.put(new OutputTask(index, bufferInfo));
+            } catch (InterruptedException e) {
+                end();
+            }
+        }
+
+        @Override
+        public void onError(MediaCodec codec, MediaCodec.CodecException e) {
+            Ln.e("MediaCodec error", e);
+            end();
+        }
+
+        @Override
+        public void onOutputFormatChanged(MediaCodec codec, MediaFormat format) {
+            // ignore
+        }
+    }
 }