Extract audio capture

The audio capture was implemented in AudioEncoder.

In order to reuse it without encoding, extract it to a separate class.

PR #3757 <https://github.com/Genymobile/scrcpy/pull/3757>

server/src/main/java/com/genymobile/scrcpy/AudioCapture.java

@@ -0,0 +1,148 @@
+package com.genymobile.scrcpy;
+
+import com.genymobile.scrcpy.wrappers.ServiceManager;
+
+import android.annotation.SuppressLint;
+import android.annotation.TargetApi;
+import android.content.ComponentName;
+import android.content.Intent;
+import android.media.AudioFormat;
+import android.media.AudioRecord;
+import android.media.AudioTimestamp;
+import android.media.MediaCodec;
+import android.media.MediaRecorder;
+import android.os.Build;
+import android.os.SystemClock;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+public final class AudioCapture {
+
+    public static final int SAMPLE_RATE = 48000;
+    public static final int CHANNEL_CONFIG = AudioFormat.CHANNEL_IN_STEREO;
+    public static final int CHANNELS = 2;
+    public static final int FORMAT = AudioFormat.ENCODING_PCM_16BIT;
+    public static final int BYTES_PER_SAMPLE = 2;
+
+    private AudioRecord recorder;
+
+    private final AudioTimestamp timestamp = new AudioTimestamp();
+    private long previousPts = 0;
+    private long nextPts = 0;
+
+    public static int millisToBytes(int millis) {
+        return SAMPLE_RATE * CHANNELS * BYTES_PER_SAMPLE * millis / 1000;
+    }
+
+    private static AudioFormat createAudioFormat() {
+        AudioFormat.Builder builder = new AudioFormat.Builder();
+        builder.setEncoding(FORMAT);
+        builder.setSampleRate(SAMPLE_RATE);
+        builder.setChannelMask(CHANNEL_CONFIG);
+        return builder.build();
+    }
+
+    @TargetApi(Build.VERSION_CODES.M)
+    @SuppressLint({"WrongConstant", "MissingPermission"})
+    private static AudioRecord createAudioRecord() {
+        AudioRecord.Builder builder = new AudioRecord.Builder();
+        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
+            // On older APIs, Workarounds.fillAppInfo() must be called beforehand
+            builder.setContext(FakeContext.get());
+        }
+        builder.setAudioSource(MediaRecorder.AudioSource.REMOTE_SUBMIX);
+        builder.setAudioFormat(createAudioFormat());
+        int minBufferSize = AudioRecord.getMinBufferSize(SAMPLE_RATE, CHANNEL_CONFIG, FORMAT);
+        // This buffer size does not impact latency
+        builder.setBufferSizeInBytes(8 * minBufferSize);
+        return builder.build();
+    }
+
+    private static void startWorkaroundAndroid11() {
+        if (Build.VERSION.SDK_INT == Build.VERSION_CODES.R) {
+            // Android 11 requires Apps to be at foreground to record audio.
+            // Normally, each App has its own user ID, so Android checks whether the requesting App has the user ID that's at the foreground.
+            // But scrcpy server is NOT an App, it's a Java application started from Android shell, so it has the same user ID (2000) with Android
+            // shell ("com.android.shell").
+            // If there is an Activity from Android shell running at foreground, then the permission system will believe scrcpy is also in the
+            // foreground.
+            if (Build.VERSION.SDK_INT == Build.VERSION_CODES.R) {
+                Intent intent = new Intent(Intent.ACTION_MAIN);
+                intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK);
+                intent.addCategory(Intent.CATEGORY_LAUNCHER);
+                intent.setComponent(new ComponentName(FakeContext.PACKAGE_NAME, "com.android.shell.HeapDumpActivity"));
+                ServiceManager.getActivityManager().startActivityAsUserWithFeature(intent);
+                // Wait for activity to start
+                SystemClock.sleep(150);
+            }
+        }
+    }
+
+    private static void stopWorkaroundAndroid11() {
+        if (Build.VERSION.SDK_INT == Build.VERSION_CODES.R) {
+            ServiceManager.getActivityManager().forceStopPackage(FakeContext.PACKAGE_NAME);
+        }
+    }
+
+    public void start() throws AudioCaptureForegroundException {
+        startWorkaroundAndroid11();
+        try {
+            recorder = createAudioRecord();
+            recorder.startRecording();
+        } catch (UnsupportedOperationException e) {
+            if (Build.VERSION.SDK_INT == Build.VERSION_CODES.R) {
+                Ln.e("Failed to start audio capture");
+                Ln.e("On Android 11, it is only possible to capture in foreground, make sure that the device is unlocked when starting scrcpy.");
+                throw new AudioCaptureForegroundException();
+            }
+            throw e;
+        } finally {
+            stopWorkaroundAndroid11();
+        }
+    }
+
+    public void stop() {
+        if (recorder != null) {
+            // Will call .stop() if necessary, without throwing an IllegalStateException
+            recorder.release();
+        }
+    }
+
+    @TargetApi(Build.VERSION_CODES.N)
+    public int read(ByteBuffer directBuffer, int size, MediaCodec.BufferInfo outBufferInfo) throws IOException {
+        int r = recorder.read(directBuffer, size);
+        if (r < 0) {
+            return r;
+        }
+
+        long pts;
+
+        int ret = recorder.getTimestamp(timestamp, AudioTimestamp.TIMEBASE_MONOTONIC);
+        if (ret == AudioRecord.SUCCESS) {
+            pts = timestamp.nanoTime / 1000;
+        } else {
+            if (nextPts == 0) {
+                Ln.w("Could not get any audio timestamp");
+            }
+            // compute from previous timestamp and packet size
+            pts = nextPts;
+        }
+
+        long durationUs = r * 1000000 / (CHANNELS * BYTES_PER_SAMPLE * SAMPLE_RATE);
+        nextPts = pts + durationUs;
+
+        if (previousPts != 0 && pts < previousPts) {
+            // Audio PTS may come from two sources:
+            //  - recorder.getTimestamp() if the call works;
+            //  - an estimation from the previous PTS and the packet size as a fallback.
+            //
+            // Therefore, the property that PTS are monotonically increasing is no guaranteed in corner cases, so enforce it.
+            pts = previousPts + 1;
+        }
+        previousPts = pts;
+
+        outBufferInfo.set(0, r, pts, 0);
+        return r;
+    }
+}

server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java

@@ -1,22 +1,12 @@
 package com.genymobile.scrcpy;
 
-import com.genymobile.scrcpy.wrappers.ServiceManager;
-
-import android.annotation.SuppressLint;
 import android.annotation.TargetApi;
-import android.content.ComponentName;
-import android.content.Intent;
-import android.media.AudioFormat;
-import android.media.AudioRecord;
-import android.media.AudioTimestamp;
 import android.media.MediaCodec;
 import android.media.MediaFormat;
-import android.media.MediaRecorder;
 import android.os.Build;
 import android.os.Handler;
 import android.os.HandlerThread;
 import android.os.Looper;
-import android.os.SystemClock;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
@@ -44,14 +34,11 @@ public final class AudioEncoder {
         }
     }
 
-    private static final int SAMPLE_RATE = 48000;
+    private static final int SAMPLE_RATE = AudioCapture.SAMPLE_RATE;
-    private static final int CHANNEL_CONFIG = AudioFormat.CHANNEL_IN_STEREO;
+    private static final int CHANNELS = AudioCapture.CHANNELS;
-    private static final int CHANNELS = 2;
-    private static final int FORMAT = AudioFormat.ENCODING_PCM_16BIT;
-    private static final int BYTES_PER_SAMPLE = 2;
 
     private static final int READ_MS = 5; // milliseconds
-    private static final int READ_SIZE = SAMPLE_RATE * CHANNELS * BYTES_PER_SAMPLE * READ_MS / 1000;
+    private static final int READ_SIZE = AudioCapture.millisToBytes(READ_MS);
 
     private final Streamer streamer;
     private final int bitRate;
@@ -78,30 +65,6 @@ public final class AudioEncoder {
         this.encoderName = encoderName;
     }
 
-    private static AudioFormat createAudioFormat() {
-        AudioFormat.Builder builder = new AudioFormat.Builder();
-        builder.setEncoding(FORMAT);
-        builder.setSampleRate(SAMPLE_RATE);
-        builder.setChannelMask(CHANNEL_CONFIG);
-        return builder.build();
-    }
-
-    @TargetApi(Build.VERSION_CODES.M)
-    @SuppressLint({"WrongConstant", "MissingPermission"})
-    private static AudioRecord createAudioRecord() {
-        AudioRecord.Builder builder = new AudioRecord.Builder();
-        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
-            // On older APIs, Workarounds.fillAppInfo() must be called beforehand
-            builder.setContext(FakeContext.get());
-        }
-        builder.setAudioSource(MediaRecorder.AudioSource.REMOTE_SUBMIX);
-        builder.setAudioFormat(createAudioFormat());
-        int minBufferSize = AudioRecord.getMinBufferSize(SAMPLE_RATE, CHANNEL_CONFIG, FORMAT);
-        // This buffer size does not impact latency
-        builder.setBufferSizeInBytes(8 * minBufferSize);
-        return builder.build();
-    }
-
     private static MediaFormat createFormat(String mimeType, int bitRate, List<CodecOption> codecOptions) {
         MediaFormat format = new MediaFormat();
         format.setString(MediaFormat.KEY_MIME, mimeType);
@@ -122,47 +85,18 @@ public final class AudioEncoder {
     }
 
     @TargetApi(Build.VERSION_CODES.N)
-    private void inputThread(MediaCodec mediaCodec, AudioRecord recorder) throws IOException, InterruptedException {
+    private void inputThread(MediaCodec mediaCodec, AudioCapture capture) throws IOException, InterruptedException {
-        final AudioTimestamp timestamp = new AudioTimestamp();
+        final MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();
-        long previousPts = 0;
-        long nextPts = 0;
 
         while (!Thread.currentThread().isInterrupted()) {
             InputTask task = inputTasks.take();
             ByteBuffer buffer = mediaCodec.getInputBuffer(task.index);
-            int r = recorder.read(buffer, READ_SIZE);
+            int r = capture.read(buffer, READ_SIZE, bufferInfo);
             if (r < 0) {
                 throw new IOException("Could not read audio: " + r);
             }
 
-            long pts;
+            mediaCodec.queueInputBuffer(task.index, bufferInfo.offset, bufferInfo.size, bufferInfo.presentationTimeUs, bufferInfo.flags);
-
-            int ret = recorder.getTimestamp(timestamp, AudioTimestamp.TIMEBASE_MONOTONIC);
-            if (ret == AudioRecord.SUCCESS) {
-                pts = timestamp.nanoTime / 1000;
-            } else {
-                if (nextPts == 0) {
-                    Ln.w("Could not get any audio timestamp");
-                }
-                // compute from previous timestamp and packet size
-                pts = nextPts;
-            }
-
-            long durationUs = r * 1000000 / (CHANNELS * BYTES_PER_SAMPLE * SAMPLE_RATE);
-            nextPts = pts + durationUs;
-
-            if (previousPts != 0 && pts < previousPts) {
-                // Audio PTS may come from two sources:
-                //  - recorder.getTimestamp() if the call works;
-                //  - an estimation from the previous PTS and the packet size as a fallback.
-                //
-                // Therefore, the property that PTS are monotonically increasing is no guaranteed in corner cases, so enforce it.
-                pts = previousPts + 1;
-            }
-
-            previousPts = pts;
-
-            mediaCodec.queueInputBuffer(task.index, 0, r, pts, 0);
         }
     }
 
@@ -223,32 +157,6 @@ public final class AudioEncoder {
         }
     }
 
-    private static void startWorkaroundAndroid11() {
-        if (Build.VERSION.SDK_INT == Build.VERSION_CODES.R) {
-            // Android 11 requires Apps to be at foreground to record audio.
-            // Normally, each App has its own user ID, so Android checks whether the requesting App has the user ID that's at the foreground.
-            // But scrcpy server is NOT an App, it's a Java application started from Android shell, so it has the same user ID (2000) with Android
-            // shell ("com.android.shell").
-            // If there is an Activity from Android shell running at foreground, then the permission system will believe scrcpy is also in the
-            // foreground.
-            if (Build.VERSION.SDK_INT == Build.VERSION_CODES.R) {
-                Intent intent = new Intent(Intent.ACTION_MAIN);
-                intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK);
-                intent.addCategory(Intent.CATEGORY_LAUNCHER);
-                intent.setComponent(new ComponentName(FakeContext.PACKAGE_NAME, "com.android.shell.HeapDumpActivity"));
-                ServiceManager.getActivityManager().startActivityAsUserWithFeature(intent);
-                // Wait for activity to start
-                SystemClock.sleep(150);
-            }
-        }
-    }
-
-    private static void stopWorkaroundAndroid11() {
-        if (Build.VERSION.SDK_INT == Build.VERSION_CODES.R) {
-            ServiceManager.getActivityManager().forceStopPackage(FakeContext.PACKAGE_NAME);
-        }
-    }
-
     @TargetApi(Build.VERSION_CODES.M)
     public void encode() throws IOException, ConfigurationException, AudioCaptureForegroundException {
         if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) {
@@ -258,10 +166,9 @@ public final class AudioEncoder {
         }
 
         MediaCodec mediaCodec = null;
-        AudioRecord recorder = null;
+        AudioCapture capture = new AudioCapture();
 
         boolean mediaCodecStarted = false;
-        boolean recorderStarted = false;
         try {
             Codec codec = streamer.getCodec();
             mediaCodec = createMediaCodec(codec, encoderName);
@@ -273,27 +180,13 @@ public final class AudioEncoder {
             mediaCodec.setCallback(new EncoderCallback(), new Handler(mediaCodecThread.getLooper()));
             mediaCodec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
 
-            startWorkaroundAndroid11();
+            capture.start();
-            try {
-                recorder = createAudioRecord();
-                recorder.startRecording();
-            } catch (UnsupportedOperationException e) {
-                if (Build.VERSION.SDK_INT == Build.VERSION_CODES.R) {
-                    Ln.e("Failed to start audio capture");
-                    Ln.e("On Android 11, it is only possible to capture in foreground, make sure that the device is unlocked when starting scrcpy.");
-                    throw new AudioCaptureForegroundException();
-                }
-                throw e;
-            } finally {
-                stopWorkaroundAndroid11();
-            }
-            recorderStarted = true;
 
             final MediaCodec mediaCodecRef = mediaCodec;
-            final AudioRecord recorderRef = recorder;
+            final AudioCapture captureRef = capture;
             inputThread = new Thread(() -> {
                 try {
-                    inputThread(mediaCodecRef, recorderRef);
+                    inputThread(mediaCodecRef, captureRef);
                 } catch (IOException | InterruptedException e) {
                     Ln.e("Audio capture error", e);
                 } finally {
@@ -366,11 +259,8 @@ public final class AudioEncoder {
                 }
                 mediaCodec.release();
             }
-            if (recorder != null) {
+            if (capture != null) {
-                if (recorderStarted) {
+                capture.stop();
-                    recorder.stop();
-                }
-                recorder.release();
             }
         }
     }