Use delay buffer as a frame source/sink

The components needing delayed frames (sc_screen and sc_v4l2_sink)
managed a sc_video_buffer instance, which itself embedded a
sc_frame_buffer instance (to keep only the most recent frame).

In theory, these components should not be aware of delaying: they should
just receive AVFrames later, and only handle a sc_frame_buffer.

Therefore, refactor sc_delay_buffer as a frame source (it consumes)
frames) and a frame sink (it produces frames, after some delay), and
plug an instance in the pipeline only when a delay is requested.

This also removes the need for a specific sc_video_buffer.

PR #3757 <https://github.com/Genymobile/scrcpy/pull/3757>
This commit is contained in:
Romain Vimont 2023-03-02 21:30:24 +01:00
parent 974227a3fc
commit 1230149fdd
11 changed files with 150 additions and 320 deletions

View file

@ -29,7 +29,6 @@ src = [
'src/screen.c',
'src/server.c',
'src/version.c',
'src/video_buffer.c',
'src/trait/frame_source.c',
'src/trait/packet_source.c',
'src/util/acksync.c',

View file

@ -4,7 +4,6 @@
#include <libavformat/avformat.h>
#include "events.h"
#include "video_buffer.h"
#include "trait/frame_sink.h"
#include "util/log.h"

View file

@ -10,6 +10,9 @@
#define SC_BUFFERING_NDEBUG // comment to debug
/** Downcast frame_sink to sc_delay_buffer */
#define DOWNCAST(SINK) container_of(SINK, struct sc_delay_buffer, frame_sink)
static bool
sc_delayed_frame_init(struct sc_delayed_frame *dframe, const AVFrame *frame) {
dframe->frame = av_frame_alloc();
@ -33,11 +36,6 @@ sc_delayed_frame_destroy(struct sc_delayed_frame *dframe) {
av_frame_free(&dframe->frame);
}
static bool
sc_delay_buffer_offer(struct sc_delay_buffer *db, const AVFrame *frame) {
return db->cbs->on_new_frame(db, frame, db->cbs_userdata);
}
static int
run_buffering(void *data) {
struct sc_delay_buffer *db = data;
@ -87,12 +85,12 @@ run_buffering(void *data) {
pts, dframe.push_date, sc_tick_now());
#endif
bool ok = sc_delay_buffer_offer(db, dframe.frame);
bool ok = sc_frame_source_sinks_push(&db->frame_source, dframe.frame);
sc_delayed_frame_destroy(&dframe);
if (!ok) {
LOGE("Delayed frame could not be pushed, stopping");
sc_mutex_lock(&db->b.mutex);
// Prevent to push any new packet
// Prevent to push any new frame
db->b.stopped = true;
sc_mutex_unlock(&db->b.mutex);
goto stopped;
@ -113,13 +111,12 @@ stopped:
return 0;
}
bool
sc_delay_buffer_init(struct sc_delay_buffer *db, sc_tick delay,
const struct sc_delay_buffer_callbacks *cbs,
void *cbs_userdata) {
assert(delay >= 0);
static bool
sc_delay_buffer_frame_sink_open(struct sc_frame_sink *sink,
const AVCodecContext *ctx) {
struct sc_delay_buffer *db = DOWNCAST(sink);
(void) ctx;
if (delay) {
bool ok = sc_mutex_init(&db->b.mutex);
if (!ok) {
return false;
@ -127,78 +124,64 @@ sc_delay_buffer_init(struct sc_delay_buffer *db, sc_tick delay,
ok = sc_cond_init(&db->b.queue_cond);
if (!ok) {
sc_mutex_destroy(&db->b.mutex);
return false;
goto error_destroy_mutex;
}
ok = sc_cond_init(&db->b.wait_cond);
if (!ok) {
sc_cond_destroy(&db->b.queue_cond);
sc_mutex_destroy(&db->b.mutex);
return false;
goto error_destroy_queue_cond;
}
sc_clock_init(&db->b.clock);
sc_vecdeque_init(&db->b.queue);
if (!sc_frame_source_sinks_open(&db->frame_source, ctx)) {
goto error_destroy_wait_cond;
}
assert(cbs);
assert(cbs->on_new_frame);
db->delay = delay;
db->cbs = cbs;
db->cbs_userdata = cbs_userdata;
return true;
}
bool
sc_delay_buffer_start(struct sc_delay_buffer *db) {
if (db->delay) {
bool ok =
sc_thread_create(&db->b.thread, run_buffering, "scrcpy-dbuf", db);
ok = sc_thread_create(&db->b.thread, run_buffering, "scrcpy-dbuf", db);
if (!ok) {
LOGE("Could not start buffering thread");
return false;
}
goto error_close_sinks;
}
return true;
error_close_sinks:
sc_frame_source_sinks_close(&db->frame_source);
error_destroy_wait_cond:
sc_cond_destroy(&db->b.wait_cond);
error_destroy_queue_cond:
sc_cond_destroy(&db->b.queue_cond);
error_destroy_mutex:
sc_mutex_destroy(&db->b.mutex);
return false;
}
void
sc_delay_buffer_stop(struct sc_delay_buffer *db) {
if (db->delay) {
static void
sc_delay_buffer_frame_sink_close(struct sc_frame_sink *sink) {
struct sc_delay_buffer *db = DOWNCAST(sink);
sc_mutex_lock(&db->b.mutex);
db->b.stopped = true;
sc_cond_signal(&db->b.queue_cond);
sc_cond_signal(&db->b.wait_cond);
sc_mutex_unlock(&db->b.mutex);
}
}
void
sc_delay_buffer_join(struct sc_delay_buffer *db) {
if (db->delay) {
sc_thread_join(&db->b.thread, NULL);
}
}
void
sc_delay_buffer_destroy(struct sc_delay_buffer *db) {
if (db->delay) {
sc_frame_source_sinks_close(&db->frame_source);
sc_cond_destroy(&db->b.wait_cond);
sc_cond_destroy(&db->b.queue_cond);
sc_mutex_destroy(&db->b.mutex);
}
}
bool
sc_delay_buffer_push(struct sc_delay_buffer *db, const AVFrame *frame) {
if (!db->delay) {
// No buffering
return sc_delay_buffer_offer(db, frame);
}
static bool
sc_delay_buffer_frame_sink_push(struct sc_frame_sink *sink,
const AVFrame *frame) {
struct sc_delay_buffer *db = DOWNCAST(sink);
sc_mutex_lock(&db->b.mutex);
@ -213,11 +196,11 @@ sc_delay_buffer_push(struct sc_delay_buffer *db, const AVFrame *frame) {
if (db->b.clock.count == 1) {
sc_mutex_unlock(&db->b.mutex);
// First frame, offer it immediately, for two reasons:
// First frame, push it immediately, for two reasons:
// - not to delay the opening of the scrcpy window
// - the buffering estimation needs at least two clock points, so it
// could not handle the first frame
return sc_delay_buffer_offer(db, frame);
return sc_frame_source_sinks_push(&db->frame_source, frame);
}
struct sc_delayed_frame dframe;
@ -244,3 +227,20 @@ sc_delay_buffer_push(struct sc_delay_buffer *db, const AVFrame *frame) {
return true;
}
void
sc_delay_buffer_init(struct sc_delay_buffer *db, sc_tick delay) {
assert(delay > 0);
db->delay = delay;
sc_frame_source_init(&db->frame_source);
static const struct sc_frame_sink_ops ops = {
.open = sc_delay_buffer_frame_sink_open,
.close = sc_delay_buffer_frame_sink_close,
.push = sc_delay_buffer_frame_sink_push,
};
db->frame_sink.ops = &ops;
}

View file

@ -6,6 +6,8 @@
#include <stdbool.h>
#include "clock.h"
#include "trait/frame_source.h"
#include "trait/frame_sink.h"
#include "util/thread.h"
#include "util/tick.h"
#include "util/vecdeque.h"
@ -23,9 +25,11 @@ struct sc_delayed_frame {
struct sc_delayed_frame_queue SC_VECDEQUE(struct sc_delayed_frame);
struct sc_delay_buffer {
struct sc_frame_source frame_source; // frame source trait
struct sc_frame_sink frame_sink; // frame sink trait
sc_tick delay;
// only if delay > 0
struct {
sc_thread thread;
sc_mutex mutex;
@ -36,9 +40,6 @@ struct sc_delay_buffer {
struct sc_delayed_frame_queue queue;
bool stopped;
} b; // buffering
const struct sc_delay_buffer_callbacks *cbs;
void *cbs_userdata;
};
struct sc_delay_buffer_callbacks {
@ -46,24 +47,12 @@ struct sc_delay_buffer_callbacks {
void *userdata);
};
bool
sc_delay_buffer_init(struct sc_delay_buffer *db, sc_tick delay,
const struct sc_delay_buffer_callbacks *cbs,
void *cbs_userdata);
bool
sc_delay_buffer_start(struct sc_delay_buffer *db);
/**
* Initialize a delay buffer.
*
* \param delay a (strictly) positive delay
*/
void
sc_delay_buffer_stop(struct sc_delay_buffer *db);
void
sc_delay_buffer_join(struct sc_delay_buffer *db);
void
sc_delay_buffer_destroy(struct sc_delay_buffer *db);
bool
sc_delay_buffer_push(struct sc_delay_buffer *db, const AVFrame *frame);
sc_delay_buffer_init(struct sc_delay_buffer *db, sc_tick delay);
#endif

View file

@ -15,6 +15,7 @@
#include "controller.h"
#include "decoder.h"
#include "delay_buffer.h"
#include "demuxer.h"
#include "events.h"
#include "file_pusher.h"
@ -45,8 +46,10 @@ struct scrcpy {
struct sc_decoder video_decoder;
struct sc_decoder audio_decoder;
struct sc_recorder recorder;
struct sc_delay_buffer display_buffer;
#ifdef HAVE_V4L2
struct sc_v4l2_sink v4l2_sink;
struct sc_delay_buffer v4l2_buffer;
#endif
struct sc_controller controller;
struct sc_file_pusher file_pusher;
@ -657,7 +660,6 @@ aoa_hid_end:
.mipmaps = options->mipmaps,
.fullscreen = options->fullscreen,
.start_fps_counter = options->start_fps_counter,
.buffering_time = options->display_buffer,
};
if (!sc_screen_init(&s->screen, &screen_params)) {
@ -665,19 +667,31 @@ aoa_hid_end:
}
screen_initialized = true;
sc_frame_source_add_sink(&s->video_decoder.frame_source,
&s->screen.frame_sink);
struct sc_frame_source *src = &s->video_decoder.frame_source;
if (options->display_buffer) {
sc_delay_buffer_init(&s->display_buffer, options->display_buffer);
sc_frame_source_add_sink(src, &s->display_buffer.frame_sink);
src = &s->display_buffer.frame_source;
}
sc_frame_source_add_sink(src, &s->screen.frame_sink);
}
#ifdef HAVE_V4L2
if (options->v4l2_device) {
if (!sc_v4l2_sink_init(&s->v4l2_sink, options->v4l2_device,
info->frame_size, options->v4l2_buffer)) {
info->frame_size)) {
goto end;
}
sc_frame_source_add_sink(&s->video_decoder.frame_source,
&s->v4l2_sink.frame_sink);
struct sc_frame_source *src = &s->video_decoder.frame_source;
if (options->v4l2_buffer) {
sc_delay_buffer_init(&s->v4l2_buffer, options->v4l2_buffer);
sc_frame_source_add_sink(src, &s->v4l2_buffer.frame_sink);
src = &s->v4l2_buffer.frame_source;
}
sc_frame_source_add_sink(src, &s->v4l2_sink.frame_sink);
v4l2_sink_initialized = true;
}

View file

@ -7,7 +7,6 @@
#include "events.h"
#include "icon.h"
#include "options.h"
#include "video_buffer.h"
#include "util/log.h"
#define DISPLAY_MARGINS 96
@ -359,14 +358,12 @@ sc_screen_frame_sink_close(struct sc_frame_sink *sink) {
static bool
sc_screen_frame_sink_push(struct sc_frame_sink *sink, const AVFrame *frame) {
struct sc_screen *screen = DOWNCAST(sink);
return sc_video_buffer_push(&screen->vb, frame);
}
static bool
sc_video_buffer_on_new_frame(struct sc_video_buffer *vb, bool previous_skipped,
void *userdata) {
(void) vb;
struct sc_screen *screen = userdata;
bool previous_skipped;
bool ok = sc_frame_buffer_push(&screen->fb, frame, &previous_skipped);
if (!ok) {
return false;
}
if (previous_skipped) {
sc_fps_counter_add_skipped_frame(&screen->fps_counter);
@ -404,23 +401,13 @@ sc_screen_init(struct sc_screen *screen,
screen->req.fullscreen = params->fullscreen;
screen->req.start_fps_counter = params->start_fps_counter;
static const struct sc_video_buffer_callbacks cbs = {
.on_new_frame = sc_video_buffer_on_new_frame,
};
bool ok = sc_video_buffer_init(&screen->vb, params->buffering_time, &cbs,
screen);
bool ok = sc_frame_buffer_init(&screen->fb);
if (!ok) {
return false;
}
ok = sc_video_buffer_start(&screen->vb);
if (!ok) {
goto error_destroy_video_buffer;
}
if (!sc_fps_counter_init(&screen->fps_counter)) {
goto error_stop_and_join_video_buffer;
goto error_destroy_frame_buffer;
}
screen->frame_size = params->frame_size;
@ -552,11 +539,8 @@ error_destroy_window:
SDL_DestroyWindow(screen->window);
error_destroy_fps_counter:
sc_fps_counter_destroy(&screen->fps_counter);
error_stop_and_join_video_buffer:
sc_video_buffer_stop(&screen->vb);
sc_video_buffer_join(&screen->vb);
error_destroy_video_buffer:
sc_video_buffer_destroy(&screen->vb);
error_destroy_frame_buffer:
sc_frame_buffer_destroy(&screen->fb);
return false;
}
@ -593,13 +577,11 @@ sc_screen_hide_window(struct sc_screen *screen) {
void
sc_screen_interrupt(struct sc_screen *screen) {
sc_video_buffer_stop(&screen->vb);
sc_fps_counter_interrupt(&screen->fps_counter);
}
void
sc_screen_join(struct sc_screen *screen) {
sc_video_buffer_join(&screen->vb);
sc_fps_counter_join(&screen->fps_counter);
}
@ -613,7 +595,7 @@ sc_screen_destroy(struct sc_screen *screen) {
SDL_DestroyRenderer(screen->renderer);
SDL_DestroyWindow(screen->window);
sc_fps_counter_destroy(&screen->fps_counter);
sc_video_buffer_destroy(&screen->vb);
sc_frame_buffer_destroy(&screen->fb);
}
static void
@ -719,7 +701,7 @@ update_texture(struct sc_screen *screen, const AVFrame *frame) {
static bool
sc_screen_update_frame(struct sc_screen *screen) {
av_frame_unref(screen->frame);
sc_video_buffer_consume(&screen->vb, screen->frame);
sc_frame_buffer_consume(&screen->fb, screen->frame);
AVFrame *frame = screen->frame;
sc_fps_counter_add_rendered_frame(&screen->fps_counter);

View file

@ -10,12 +10,12 @@
#include "controller.h"
#include "coords.h"
#include "fps_counter.h"
#include "frame_buffer.h"
#include "input_manager.h"
#include "opengl.h"
#include "trait/key_processor.h"
#include "trait/frame_sink.h"
#include "trait/mouse_processor.h"
#include "video_buffer.h"
struct sc_screen {
struct sc_frame_sink frame_sink; // frame sink trait
@ -25,7 +25,7 @@ struct sc_screen {
#endif
struct sc_input_manager im;
struct sc_video_buffer vb;
struct sc_frame_buffer fb;
struct sc_fps_counter fps_counter;
// The initial requested window properties
@ -93,8 +93,6 @@ struct sc_screen_params {
bool fullscreen;
bool start_fps_counter;
sc_tick buffering_time;
};
// initialize screen, create window, renderer and texture (window is hidden)

View file

@ -126,7 +126,7 @@ run_v4l2_sink(void *data) {
vs->has_frame = false;
sc_mutex_unlock(&vs->mutex);
sc_video_buffer_consume(&vs->vb, vs->frame);
sc_frame_buffer_consume(&vs->fb, vs->frame);
bool ok = encode_and_write_frame(vs, vs->frame);
av_frame_unref(vs->frame);
@ -141,44 +141,19 @@ run_v4l2_sink(void *data) {
return 0;
}
static bool
sc_video_buffer_on_new_frame(struct sc_video_buffer *vb, bool previous_skipped,
void *userdata) {
(void) vb;
struct sc_v4l2_sink *vs = userdata;
if (!previous_skipped) {
sc_mutex_lock(&vs->mutex);
vs->has_frame = true;
sc_cond_signal(&vs->cond);
sc_mutex_unlock(&vs->mutex);
}
return true;
}
static bool
sc_v4l2_sink_open(struct sc_v4l2_sink *vs, const AVCodecContext *ctx) {
assert(ctx->pix_fmt == AV_PIX_FMT_YUV420P);
(void) ctx;
static const struct sc_video_buffer_callbacks cbs = {
.on_new_frame = sc_video_buffer_on_new_frame,
};
bool ok = sc_video_buffer_init(&vs->vb, vs->buffering_time, &cbs, vs);
bool ok = sc_frame_buffer_init(&vs->fb);
if (!ok) {
return false;
}
ok = sc_video_buffer_start(&vs->vb);
if (!ok) {
goto error_video_buffer_destroy;
}
ok = sc_mutex_init(&vs->mutex);
if (!ok) {
goto error_video_buffer_stop_and_join;
goto error_frame_buffer_destroy;
}
ok = sc_cond_init(&vs->cond);
@ -303,11 +278,8 @@ error_cond_destroy:
sc_cond_destroy(&vs->cond);
error_mutex_destroy:
sc_mutex_destroy(&vs->mutex);
error_video_buffer_stop_and_join:
sc_video_buffer_stop(&vs->vb);
sc_video_buffer_join(&vs->vb);
error_video_buffer_destroy:
sc_video_buffer_destroy(&vs->vb);
error_frame_buffer_destroy:
sc_frame_buffer_destroy(&vs->fb);
return false;
}
@ -319,10 +291,7 @@ sc_v4l2_sink_close(struct sc_v4l2_sink *vs) {
sc_cond_signal(&vs->cond);
sc_mutex_unlock(&vs->mutex);
sc_video_buffer_stop(&vs->vb);
sc_thread_join(&vs->thread, NULL);
sc_video_buffer_join(&vs->vb);
av_packet_free(&vs->packet);
av_frame_free(&vs->frame);
@ -332,12 +301,25 @@ sc_v4l2_sink_close(struct sc_v4l2_sink *vs) {
avformat_free_context(vs->format_ctx);
sc_cond_destroy(&vs->cond);
sc_mutex_destroy(&vs->mutex);
sc_video_buffer_destroy(&vs->vb);
sc_frame_buffer_destroy(&vs->fb);
}
static bool
sc_v4l2_sink_push(struct sc_v4l2_sink *vs, const AVFrame *frame) {
return sc_video_buffer_push(&vs->vb, frame);
bool previous_skipped;
bool ok = sc_frame_buffer_push(&vs->fb, frame, &previous_skipped);
if (!ok) {
return false;
}
if (!previous_skipped) {
sc_mutex_lock(&vs->mutex);
vs->has_frame = true;
sc_cond_signal(&vs->cond);
sc_mutex_unlock(&vs->mutex);
}
return true;
}
static bool
@ -360,7 +342,7 @@ sc_v4l2_frame_sink_push(struct sc_frame_sink *sink, const AVFrame *frame) {
bool
sc_v4l2_sink_init(struct sc_v4l2_sink *vs, const char *device_name,
struct sc_size frame_size, sc_tick buffering_time) {
struct sc_size frame_size) {
vs->device_name = strdup(device_name);
if (!vs->device_name) {
LOGE("Could not strdup v4l2 device name");
@ -368,7 +350,6 @@ sc_v4l2_sink_init(struct sc_v4l2_sink *vs, const char *device_name,
}
vs->frame_size = frame_size;
vs->buffering_time = buffering_time;
static const struct sc_frame_sink_ops ops = {
.open = sc_v4l2_frame_sink_open,

View file

@ -8,19 +8,18 @@
#include "coords.h"
#include "trait/frame_sink.h"
#include "video_buffer.h"
#include "frame_buffer.h"
#include "util/tick.h"
struct sc_v4l2_sink {
struct sc_frame_sink frame_sink; // frame sink trait
struct sc_video_buffer vb;
struct sc_frame_buffer fb;
AVFormatContext *format_ctx;
AVCodecContext *encoder_ctx;
char *device_name;
struct sc_size frame_size;
sc_tick buffering_time;
sc_thread thread;
sc_mutex mutex;
@ -35,7 +34,7 @@ struct sc_v4l2_sink {
bool
sc_v4l2_sink_init(struct sc_v4l2_sink *vs, const char *device_name,
struct sc_size frame_size, sc_tick buffering_time);
struct sc_size frame_size);
void
sc_v4l2_sink_destroy(struct sc_v4l2_sink *vs);

View file

@ -1,84 +0,0 @@
#include "video_buffer.h"
#include <assert.h>
#include <stdlib.h>
#include <libavutil/avutil.h>
#include <libavformat/avformat.h>
#include "util/log.h"
static bool
sc_delay_buffer_on_new_frame(struct sc_delay_buffer *db, const AVFrame *frame,
void *userdata) {
(void) db;
struct sc_video_buffer *vb = userdata;
bool previous_skipped;
bool ok = sc_frame_buffer_push(&vb->fb, frame, &previous_skipped);
if (!ok) {
return false;
}
return vb->cbs->on_new_frame(vb, previous_skipped, vb->cbs_userdata);
}
bool
sc_video_buffer_init(struct sc_video_buffer *vb, sc_tick delay,
const struct sc_video_buffer_callbacks *cbs,
void *cbs_userdata) {
bool ok = sc_frame_buffer_init(&vb->fb);
if (!ok) {
return false;
}
static const struct sc_delay_buffer_callbacks db_cbs = {
.on_new_frame = sc_delay_buffer_on_new_frame,
};
ok = sc_delay_buffer_init(&vb->db, delay, &db_cbs, vb);
if (!ok) {
sc_frame_buffer_destroy(&vb->fb);
return false;
}
assert(cbs);
assert(cbs->on_new_frame);
vb->cbs = cbs;
vb->cbs_userdata = cbs_userdata;
return true;
}
bool
sc_video_buffer_start(struct sc_video_buffer *vb) {
return sc_delay_buffer_start(&vb->db);
}
void
sc_video_buffer_stop(struct sc_video_buffer *vb) {
return sc_delay_buffer_stop(&vb->db);
}
void
sc_video_buffer_join(struct sc_video_buffer *vb) {
return sc_delay_buffer_join(&vb->db);
}
void
sc_video_buffer_destroy(struct sc_video_buffer *vb) {
sc_frame_buffer_destroy(&vb->fb);
sc_delay_buffer_destroy(&vb->db);
}
bool
sc_video_buffer_push(struct sc_video_buffer *vb, const AVFrame *frame) {
return sc_delay_buffer_push(&vb->db, frame);
}
void
sc_video_buffer_consume(struct sc_video_buffer *vb, AVFrame *dst) {
sc_frame_buffer_consume(&vb->fb, dst);
}

View file

@ -1,47 +0,0 @@
#ifndef SC_VIDEO_BUFFER_H
#define SC_VIDEO_BUFFER_H
#include "common.h"
#include <stdbool.h>
#include "delay_buffer.h"
#include "frame_buffer.h"
struct sc_video_buffer {
struct sc_delay_buffer db;
struct sc_frame_buffer fb;
const struct sc_video_buffer_callbacks *cbs;
void *cbs_userdata;
};
struct sc_video_buffer_callbacks {
bool (*on_new_frame)(struct sc_video_buffer *vb, bool previous_skipped,
void *userdata);
};
bool
sc_video_buffer_init(struct sc_video_buffer *vb, sc_tick delay,
const struct sc_video_buffer_callbacks *cbs,
void *cbs_userdata);
bool
sc_video_buffer_start(struct sc_video_buffer *vb);
void
sc_video_buffer_stop(struct sc_video_buffer *vb);
void
sc_video_buffer_join(struct sc_video_buffer *vb);
void
sc_video_buffer_destroy(struct sc_video_buffer *vb);
bool
sc_video_buffer_push(struct sc_video_buffer *vb, const AVFrame *frame);
void
sc_video_buffer_consume(struct sc_video_buffer *vb, AVFrame *dst);
#endif