Android/scrcpy
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity

Pass video size as codec metadata

Browse source 
On initial connection, scrcpy sent some device metadata:
 - the device name (to be used as window title)
 - the initial video size (before any frame or even SPS/PPS)

But it is better to provide the initial video size as part as the video
stream, so that it can be demuxed and exposed via AVCodecContext to
sinks.

This avoids to pass an explicit "initial frame size" for the screen, the
recorder and the v4l2 sink.
This commit is contained in:
Romain Vimont 2023-03-11 09:21:49 +01:00
parent 3a72f3fb4d
commit 238ab872ba
17 changed files with 104 additions and 69 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
app/src/demuxer.c
View file

@ -57,6 +57,20 @@ sc_demuxer_recv_codec_id(struct sc_demuxer *demuxer, uint32_t *codec_id) {
return true;
}
static bool
sc_demuxer_recv_video_size(struct sc_demuxer *demuxer, uint32_t *width,
uint32_t *height) {
uint8_t data[8];
ssize_t r = net_recv_all(demuxer->socket, data, 8);
if (r < 8) {
return false;
}
*width = sc_read32be(data);
*height = sc_read32be(data + 4);
return true;
}
static bool
sc_demuxer_recv_packet(struct sc_demuxer *demuxer, AVPacket *packet) {
// The video stream contains raw packets, without time information. When we
@ -169,7 +183,15 @@ run_demuxer(void *data) {
codec_ctx->flags |= AV_CODEC_FLAG_LOW_DELAY;
if (codec->type == AVMEDIA_TYPE_VIDEO) {
// Hardcoded video properties
uint32_t width;
uint32_t height;
ok = sc_demuxer_recv_video_size(demuxer, &width, &height);
if (!ok) {
goto finally_free_context;
}
codec_ctx->width = width;
codec_ctx->height = height;
codec_ctx->pix_fmt = AV_PIX_FMT_YUV420P;
} else {
// Hardcoded audio properties

1
app/src/events.h
View file

@ -5,3 +5,4 @@
#define SC_EVENT_USB_DEVICE_DISCONNECTED (SDL_USEREVENT + 4)
#define SC_EVENT_DEMUXER_ERROR (SDL_USEREVENT + 5)
#define SC_EVENT_RECORDER_ERROR (SDL_USEREVENT + 6)
#define SC_EVENT_SCREEN_INIT_SIZE (SDL_USEREVENT + 7)

5
app/src/recorder.c
View file

@ -479,9 +479,6 @@ sc_recorder_video_packet_sink_open(struct sc_packet_sink *sink,
return false;
}
stream->codecpar->width = recorder->declared_frame_size.width;
stream->codecpar->height = recorder->declared_frame_size.height;
recorder->video_stream_index = stream->index;
recorder->video_init = true;
@ -643,7 +640,6 @@ sc_recorder_audio_packet_sink_disable(struct sc_packet_sink *sink) {
bool
sc_recorder_init(struct sc_recorder *recorder, const char *filename,
enum sc_record_format format, bool audio,
struct sc_size declared_frame_size,
const struct sc_recorder_callbacks *cbs, void *cbs_userdata) {
recorder->filename = strdup(filename);
if (!recorder->filename) {
@ -679,7 +675,6 @@ sc_recorder_init(struct sc_recorder *recorder, const char *filename,
recorder->audio_stream_index = -1;
recorder->format = format;
recorder->declared_frame_size = declared_frame_size;
assert(cbs && cbs->on_ended);
recorder->cbs = cbs;

2
app/src/recorder.h
View file

@ -31,7 +31,6 @@ struct sc_recorder {
char *filename;
enum sc_record_format format;
AVFormatContext *ctx;
struct sc_size declared_frame_size;
sc_thread thread;
sc_mutex mutex;
@ -61,7 +60,6 @@ struct sc_recorder_callbacks {
bool
sc_recorder_init(struct sc_recorder *recorder, const char *filename,
enum sc_record_format format, bool audio,
struct sc_size declared_frame_size,
const struct sc_recorder_callbacks *cbs, void *cbs_userdata);
bool

6
app/src/scrcpy.c
View file

@ -473,7 +473,7 @@ scrcpy(struct scrcpy_options *options) {
};
if (!sc_recorder_init(&s->recorder, options->record_filename,
options->record_format, options->audio,
info->frame_size, &recorder_cbs, NULL)) {
&recorder_cbs, NULL)) {
goto end;
}
recorder_initialized = true;
@ -660,7 +660,6 @@ aoa_hid_end:
.clipboard_autosync = options->clipboard_autosync,
.shortcut_mods = &options->shortcut_mods,
.window_title = window_title,
.frame_size = info->frame_size,
.always_on_top = options->always_on_top,
.window_x = options->window_x,
.window_y = options->window_y,
@ -697,8 +696,7 @@ aoa_hid_end:
#ifdef HAVE_V4L2
if (options->v4l2_device) {
if (!sc_v4l2_sink_init(&s->v4l2_sink, options->v4l2_device,
info->frame_size)) {
if (!sc_v4l2_sink_init(&s->v4l2_sink, options->v4l2_device)) {
goto end;
}

79
app/src/screen.c
View file

@ -239,7 +239,7 @@ sc_screen_update_content_rect(struct sc_screen *screen) {
}
}
static inline SDL_Texture *
static bool
create_texture(struct sc_screen *screen) {
SDL_Renderer *renderer = screen->renderer;
struct sc_size size = screen->frame_size;
@ -247,7 +247,8 @@ create_texture(struct sc_screen *screen) {
SDL_TEXTUREACCESS_STREAMING,
size.width, size.height);
if (!texture) {
return NULL;
LOGE("Could not create texture: %s", SDL_GetError());
return false;
}
if (screen->mipmaps) {
@ -263,7 +264,8 @@ create_texture(struct sc_screen *screen) {
SDL_GL_UnbindTexture(texture);
}
return texture;
screen->texture = texture;
return true;
}
// render the texture to the renderer
@ -335,7 +337,25 @@ sc_screen_frame_sink_open(struct sc_frame_sink *sink,
(void) ctx;
struct sc_screen *screen = DOWNCAST(sink);
(void) screen;
assert(ctx->width > 0 && ctx->width <= 0xFFFF);
assert(ctx->height > 0 && ctx->height <= 0xFFFF);
// screen->frame_size is never used before the event is pushed, and the
// event acts as a memory barrier so it is safe without mutex
screen->frame_size.width = ctx->width;
screen->frame_size.height = ctx->height;
static SDL_Event event = {
.type = SC_EVENT_SCREEN_INIT_SIZE,
};
// Post the event on the UI thread (the texture must be created from there)
int ret = SDL_PushEvent(&event);
if (ret < 0) {
LOGW("Could not post init size event: %s", SDL_GetError());
return false;
}
#ifndef NDEBUG
screen->open = true;
#endif
@ -410,14 +430,10 @@ sc_screen_init(struct sc_screen *screen,
goto error_destroy_frame_buffer;
}
screen->frame_size = params->frame_size;
screen->rotation = params->rotation;
if (screen->rotation) {
LOGI("Initial display rotation set to %u", screen->rotation);
}
struct sc_size content_size =
get_rotated_size(screen->frame_size, screen->rotation);
screen->content_size = content_size;
uint32_t window_flags = SDL_WINDOW_HIDDEN
| SDL_WINDOW_RESIZABLE
@ -485,18 +501,10 @@ sc_screen_init(struct sc_screen *screen,
LOGW("Could not load icon");
}
LOGI("Initial texture: %" PRIu16 "x%" PRIu16, params->frame_size.width,
params->frame_size.height);
screen->texture = create_texture(screen);
if (!screen->texture) {
LOGE("Could not create texture: %s", SDL_GetError());
goto error_destroy_renderer;
}
screen->frame = av_frame_alloc();
if (!screen->frame) {
LOG_OOM();
goto error_destroy_texture;
goto error_destroy_renderer;
}
struct sc_input_manager_params im_params = {
@ -531,8 +539,6 @@ sc_screen_init(struct sc_screen *screen,
return true;
error_destroy_texture:
SDL_DestroyTexture(screen->texture);
error_destroy_renderer:
SDL_DestroyRenderer(screen->renderer);
error_destroy_window:
@ -591,7 +597,9 @@ sc_screen_destroy(struct sc_screen *screen) {
assert(!screen->open);
#endif
av_frame_free(&screen->frame);
SDL_DestroyTexture(screen->texture);
if (screen->texture) {
SDL_DestroyTexture(screen->texture);
}
SDL_DestroyRenderer(screen->renderer);
SDL_DestroyWindow(screen->window);
sc_fps_counter_destroy(&screen->fps_counter);
@ -655,6 +663,23 @@ sc_screen_set_rotation(struct sc_screen *screen, unsigned rotation) {
sc_screen_render(screen, true);
}
static bool
sc_screen_init_size(struct sc_screen *screen) {
// Before first frame
assert(!screen->has_frame);
assert(!screen->texture);
// The requested size is passed via screen->frame_size
struct sc_size content_size =
get_rotated_size(screen->frame_size, screen->rotation);
screen->content_size = content_size;
LOGI("Initial texture: %" PRIu16 "x%" PRIu16,
screen->frame_size.width, screen->frame_size.height);
return create_texture(screen);
}
// recreate the texture and resize the window if the frame size has changed
static bool
prepare_for_frame(struct sc_screen *screen, struct sc_size new_frame_size) {
@ -673,11 +698,7 @@ prepare_for_frame(struct sc_screen *screen, struct sc_size new_frame_size) {
LOGI("New texture: %" PRIu16 "x%" PRIu16,
screen->frame_size.width, screen->frame_size.height);
screen->texture = create_texture(screen);
if (!screen->texture) {
LOGE("Could not create texture: %s", SDL_GetError());
return false;
}
return create_texture(screen);
}
return true;
@ -795,6 +816,14 @@ sc_screen_handle_event(struct sc_screen *screen, SDL_Event *event) {
bool relative_mode = sc_screen_is_relative_mode(screen);
switch (event->type) {
case SC_EVENT_SCREEN_INIT_SIZE:
// The initial size is passed via screen->frame_size
bool ok = sc_screen_init_size(screen);
if (!ok) {
LOGE("Could not initialize screen size");
return false;
}
return true;
case SC_EVENT_NEW_FRAME: {
bool ok = sc_screen_update_frame(screen);
if (!ok) {

1
app/src/screen.h
View file

@ -78,7 +78,6 @@ struct sc_screen_params {
const struct sc_shortcut_mods *shortcut_mods;
const char *window_title;
struct sc_size frame_size;
bool always_on_top;
int16_t window_x; // accepts SC_WINDOW_POSITION_UNDEFINED

7
app/src/server.c
View file

@ -441,9 +441,9 @@ sc_server_init(struct sc_server *server, const struct sc_server_params *params,
static bool
device_read_info(struct sc_intr *intr, sc_socket device_socket,
struct sc_server_info *info) {
unsigned char buf[SC_DEVICE_NAME_FIELD_LENGTH + 4];
unsigned char buf[SC_DEVICE_NAME_FIELD_LENGTH];
ssize_t r = net_recv_all_intr(intr, device_socket, buf, sizeof(buf));
if (r < SC_DEVICE_NAME_FIELD_LENGTH + 4) {
if (r < SC_DEVICE_NAME_FIELD_LENGTH) {
LOGE("Could not retrieve device information");
return false;
}
@ -451,9 +451,6 @@ device_read_info(struct sc_intr *intr, sc_socket device_socket,
buf[SC_DEVICE_NAME_FIELD_LENGTH - 1] = '\0';
memcpy(info->device_name, (char *) buf, sizeof(info->device_name));
unsigned char *fields = &buf[SC_DEVICE_NAME_FIELD_LENGTH];
info->frame_size.width = sc_read16be(fields);
info->frame_size.height = sc_read16be(&fields[2]);
return true;
}

1
app/src/server.h
View file

@ -18,7 +18,6 @@
#define SC_DEVICE_NAME_FIELD_LENGTH 64
struct sc_server_info {
char device_name[SC_DEVICE_NAME_FIELD_LENGTH];
struct sc_size frame_size;
};
struct sc_server_params {

12
app/src/v4l2_sink.c
View file

@ -210,9 +210,6 @@ sc_v4l2_sink_open(struct sc_v4l2_sink *vs, const AVCodecContext *ctx) {
goto error_avformat_free_context;
}
ostream->codecpar->width = vs->frame_size.width;
ostream->codecpar->height = vs->frame_size.height;
int ret = avio_open(&vs->format_ctx->pb, vs->device_name, AVIO_FLAG_WRITE);
if (ret < 0) {
LOGE("Failed to open output device: %s", vs->device_name);
@ -226,8 +223,8 @@ sc_v4l2_sink_open(struct sc_v4l2_sink *vs, const AVCodecContext *ctx) {
goto error_avio_close;
}
vs->encoder_ctx->width = vs->frame_size.width;
vs->encoder_ctx->height = vs->frame_size.height;
vs->encoder_ctx->width = ctx->width;
vs->encoder_ctx->height = ctx->height;
vs->encoder_ctx->pix_fmt = AV_PIX_FMT_YUV420P;
vs->encoder_ctx->time_base.num = 1;
vs->encoder_ctx->time_base.den = 1;
@ -343,16 +340,13 @@ sc_v4l2_frame_sink_push(struct sc_frame_sink *sink, const AVFrame *frame) {
}
bool
sc_v4l2_sink_init(struct sc_v4l2_sink *vs, const char *device_name,
struct sc_size frame_size) {
sc_v4l2_sink_init(struct sc_v4l2_sink *vs, const char *device_name) {
vs->device_name = strdup(device_name);
if (!vs->device_name) {
LOGE("Could not strdup v4l2 device name");
return false;
}
vs->frame_size = frame_size;
static const struct sc_frame_sink_ops ops = {
.open = sc_v4l2_frame_sink_open,
.close = sc_v4l2_frame_sink_close,

4
app/src/v4l2_sink.h
View file

@ -19,7 +19,6 @@ struct sc_v4l2_sink {
AVCodecContext *encoder_ctx;
char *device_name;
struct sc_size frame_size;
sc_thread thread;
sc_mutex mutex;
@ -33,8 +32,7 @@ struct sc_v4l2_sink {
};
bool
sc_v4l2_sink_init(struct sc_v4l2_sink *vs, const char *device_name,
struct sc_size frame_size);
sc_v4l2_sink_init(struct sc_v4l2_sink *vs, const char *device_name);
void
sc_v4l2_sink_destroy(struct sc_v4l2_sink *vs);

2
server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java
View file

@ -101,7 +101,7 @@ public final class AudioEncoder implements AsyncProcessor {
}
private void outputThread(MediaCodec mediaCodec) throws IOException, InterruptedException {
streamer.writeHeader();
streamer.writeAudioHeader();
while (!Thread.currentThread().isInterrupted()) {
OutputTask task = outputTasks.take();

2
server/src/main/java/com/genymobile/scrcpy/AudioRawRecorder.java
View file

@ -26,7 +26,7 @@ public final class AudioRawRecorder implements AsyncProcessor {
try {
capture.start();
streamer.writeHeader();
streamer.writeAudioHeader();
while (!Thread.currentThread().isInterrupted()) {
buffer.position(0);
int r = capture.read(buffer, READ_SIZE, bufferInfo);

8
server/src/main/java/com/genymobile/scrcpy/DesktopConnection.java
View file

@ -122,18 +122,14 @@ public final class DesktopConnection implements Closeable {
}
}
public void sendDeviceMeta(String deviceName, int width, int height) throws IOException {
byte[] buffer = new byte[DEVICE_NAME_FIELD_LENGTH + 4];
public void sendDeviceMeta(String deviceName) throws IOException {
byte[] buffer = new byte[DEVICE_NAME_FIELD_LENGTH];
byte[] deviceNameBytes = deviceName.getBytes(StandardCharsets.UTF_8);
int len = StringUtils.getUtf8TruncationIndex(deviceNameBytes, DEVICE_NAME_FIELD_LENGTH - 1);
System.arraycopy(deviceNameBytes, 0, buffer, 0, len);
// byte[] are always 0-initialized in java, no need to set '\0' explicitly
buffer[DEVICE_NAME_FIELD_LENGTH] = (byte) (width >> 8);
buffer[DEVICE_NAME_FIELD_LENGTH + 1] = (byte) width;
buffer[DEVICE_NAME_FIELD_LENGTH + 2] = (byte) (height >> 8);
buffer[DEVICE_NAME_FIELD_LENGTH + 3] = (byte) height;
IO.writeFully(videoFd, buffer, 0, buffer.length);
}

2
server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java
View file

@ -66,7 +66,7 @@ public class ScreenEncoder implements Device.RotationListener {
IBinder display = createDisplay();
device.setRotationListener(this);
streamer.writeHeader();
streamer.writeVideoHeader(device.getScreenInfo().getVideoSize());
boolean alive;
try {

3
server/src/main/java/com/genymobile/scrcpy/Server.java
View file

@ -96,8 +96,7 @@ public final class Server {
try (DesktopConnection connection = DesktopConnection.open(scid, tunnelForward, audio, control, sendDummyByte)) {
if (options.getSendDeviceMeta()) {
Size videoSize = device.getScreenInfo().getVideoSize();
connection.sendDeviceMeta(Device.getDeviceName(), videoSize.getWidth(), videoSize.getHeight());
connection.sendDeviceMeta(Device.getDeviceName());
}
if (control) {

14
server/src/main/java/com/genymobile/scrcpy/Streamer.java
View file

@ -30,8 +30,7 @@ public final class Streamer {
public Codec getCodec() {
return codec;
}
public void writeHeader() throws IOException {
public void writeAudioHeader() throws IOException {
if (sendCodecMeta) {
ByteBuffer buffer = ByteBuffer.allocate(4);
buffer.putInt(codec.getId());
@ -40,6 +39,17 @@ public final class Streamer {
}
}
public void writeVideoHeader(Size videoSize) throws IOException {
if (sendCodecMeta) {
ByteBuffer buffer = ByteBuffer.allocate(12);
buffer.putInt(codec.getId());
buffer.putInt(videoSize.getWidth());
buffer.putInt(videoSize.getHeight());
buffer.flip();
IO.writeFully(fd, buffer);
}
}
public void writeDisableStream(boolean error) throws IOException {
// Writing a specific code as codec-id means that the device disables the stream
// code 0: it explicitly disables the stream (because it could not capture audio), scrcpy should continue mirroring video only