Jerry/kmsvnc
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

vaapi fix

Browse source
This commit is contained in:
JerryXiao 2023-04-29 15:17:06 +08:00
parent 6ec6c260d0
commit 6971d7df62
Signed by: Jerry
GPG key ID: 22618F758B5BE2E5
5 changed files with 202 additions and 40 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
drm.c
View file

@ -16,10 +16,6 @@ static void convert_copy(const char *in, int width, int height, char *buff) {
memcpy(buff, in, width * height * 4);
}
static void convert_vaapi(const char *in, int width, int height, char *buff) {
va_hwframe_to_vaapi(buff);
}
static void convert_bgrx_to_rgb(const char *in, int width, int height, char *buff)
{
for (int y = 0; y < height; y++)
@ -37,6 +33,17 @@ static char *kms_convert_buf = NULL;
static size_t kms_convert_buf_len = 0;
static char *kms_cpy_tmp_buf = NULL;
static size_t kms_cpy_tmp_buf_len = 0;
static inline char convert_buf_allocate(size_t len) {
if (kms_convert_buf_len < len)
{
if (kms_convert_buf)
free(kms_convert_buf);
kms_convert_buf = malloc(len);
if (!kms_convert_buf) return 1;
kms_convert_buf_len = len;
}
return 0;
}
static inline void convert_x_tiled(const int tilex, const int tiley, const char *in, int width, int height, char *buff)
{
if (width % tilex)
@ -59,14 +66,7 @@ static inline void convert_x_tiled(const int tilex, const int tiley, const char
memcpy(kms_cpy_tmp_buf, in, max_offset * 4 + 4);
in = (const char *)kms_cpy_tmp_buf;
}
if (kms_convert_buf_len < width * height * 4)
{
if (kms_convert_buf)
free(kms_convert_buf);
kms_convert_buf = malloc(width * height * 4);
if (!kms_convert_buf) return;
kms_convert_buf_len = width * height * 4;
}
if (convert_buf_allocate(width * height * 4)) return;
for (int y = 0; y < height; y++)
{
for (int x = 0; x < width; x++)
@ -89,6 +89,24 @@ void convert_intel_x_tiled_kmsbuf(const char *in, int width, int height, char *b
convert_x_tiled(128, 8, in, width, height, buff);
}
static void convert_vaapi(const char *in, int width, int height, char *buff) {
if (kmsvnc->va->is_xrgb || kmsvnc->va->is_bgr) {
if (convert_buf_allocate(width * height * BYTES_PER_PIXEL)) return;
va_hwframe_to_vaapi(kms_convert_buf);
if (kmsvnc->va->is_xrgb) {
for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) {
*((uint32_t*)(kms_convert_buf + i)) <<= 8;
}
}
if (kmsvnc->va->is_bgr) {
convert_bgrx_to_rgb(kms_convert_buf, width, height, buff);
}
}
else {
va_hwframe_to_vaapi(buff);
}
}
static inline void drm_sync(int drmfd, uint64_t flags)
{
struct dma_buf_sync sync = {
@ -290,7 +308,7 @@ static int drm_kmsbuf_prime_vaapi() {
if (va_init()) return 1;
drm->mmap_fd = drm->prime_fd;
drm->mapped = kmsvnc->va->imgbuf;
drm->skip_map = 1;
return 0;
}
@ -392,7 +410,7 @@ int drm_vendors() {
if (drm_kmsbuf_dumb()) return 1;
}
if (!drm->mapped)
if (!drm->skip_map && !drm->mapped)
{
printf("mapping with size = %d, offset = %d, fd = %d\n", drm->mmap_size, drm->mmap_offset, drm->mmap_fd);
drm->mapped = mmap(NULL, drm->mmap_size, PROT_READ, MAP_SHARED, drm->mmap_fd, drm->mmap_offset);

9
kmsvnc.c
View file

@ -143,6 +143,7 @@ static struct argp_option kmsvnc_main_options[] = {
{"disable-always-shared", 0xff01, 0, OPTION_ARG_OPTIONAL, "Do not always treat incoming connections as shared"},
{"disable-compare-fb", 0xff02, 0, OPTION_ARG_OPTIONAL, "Do not compare pixels"},
{"capture-raw-fb", 0xff03, "/tmp/rawfb.bin", 0, "Capture RAW framebuffer instead of starting the vnc server (for debugging)"},
{"va-derive", 0xff04, "off", 0, "Enable derive with vaapi"},
{"disable-input", 'i', 0, OPTION_ARG_OPTIONAL, "Disable uinput"},
{"desktop-name", 'n', "kmsvnc", 0, "Specify vnc desktop name"},
{0}
@ -203,6 +204,13 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
kmsvnc->debug_capture_fb = arg;
kmsvnc->disable_input = 1;
break;
case 0xff04:
if (!strcmp("on", arg) || !strcmp("y", arg) || !strcmp("yes", arg) || !strcmp("1", arg)) {
kmsvnc->va_derive_enabled = 1;
}
else {
kmsvnc->va_derive_enabled = 0;
}
case 'i':
kmsvnc->disable_input = 1;
break;
@ -233,6 +241,7 @@ int main(int argc, char **argv)
kmsvnc->vnc_opt = vncopt;
kmsvnc->card = "/dev/dri/card0";
kmsvnc->va_derive_enabled = -1;
kmsvnc->vnc_opt->bind = &(struct in_addr){0};
kmsvnc->vnc_opt->always_shared = 1;
kmsvnc->vnc_opt->port = 5900;

5
kmsvnc.h
View file

@ -32,6 +32,7 @@ struct kmsvnc_data
char *force_driver;
struct vnc_opt *vnc_opt;
char disable_input;
int va_derive_enabled;
int source_plane;
int source_crtc;
struct kmsvnc_drm_data *drm;
@ -87,6 +88,7 @@ struct kmsvnc_drm_data
size_t mmap_size;
off_t mmap_offset;
char *mapped;
char skip_map;
struct kmsvnc_drm_funcs *funcs;
char *pixfmt_name;
char *mod_vendor;
@ -100,6 +102,9 @@ struct kmsvnc_va_data
VASurfaceID surface_id;
VAImage *image;
char *imgbuf;
char is_bgr; // bgr -> rgb
char is_xrgb; // shift 8
char derive_enabled;
};
#define KMSVNC_FATAL(...) do{ fprintf(stderr, __VA_ARGS__); return 1; } while(0)

178
va.c
View file

@ -44,6 +44,28 @@ static void va_error_callback(void *user_context, const char *message) {
printf("va error: %s");
}
static char* fourcc_to_str(int fourcc) {
static char ret[5];
ret[4] = 0;
for (int i = 0; i < 4; i++) {
ret[i] = fourcc >> 8*i & 0xff;
}
return ret;
}
static void print_va_image_fmt(VAImageFormat *fmt) {
printf("image fmt: fourcc %d, %s, byte_order %s, bpp %d, depth %d, blue_mask %#x, green_mask %#x, red_mask %#x, reserved %#x\n", fmt->fourcc,
fourcc_to_str(fmt->fourcc),
fmt->byte_order == 1 ? "VA_LSB_FIRST" : "VA_MSB_FIRST",
fmt->bits_per_pixel,
fmt->depth,
fmt->blue_mask,
fmt->green_mask,
fmt->red_mask,
fmt->va_reserved
);
}
int va_init() {
if (!kmsvnc->drm || !kmsvnc->drm->drm_fd || !kmsvnc->drm->prime_fd) {
KMSVNC_FATAL("drm is not initialized\n");
@ -104,9 +126,10 @@ int va_init() {
}
};
prime_desc.fourcc = kmsvnc->drm->mfb->pixel_format == KMSVNC_FOURCC_TO_INT('X', 'R', '2', '4') ?
KMSVNC_FOURCC_TO_INT('B', 'G', 'R', 'X') :
KMSVNC_FOURCC_TO_INT('B', 'G', 'R', 'A') ;
char is_alpha = kmsvnc->drm->mfb->pixel_format != KMSVNC_FOURCC_TO_INT('X', 'R', '2', '4');
prime_desc.fourcc = kmsvnc->drm->mfb->pixel_format == is_alpha ?
KMSVNC_FOURCC_TO_INT('B', 'G', 'R', 'A') :
KMSVNC_FOURCC_TO_INT('B', 'G', 'R', 'X') ;
prime_desc.width = kmsvnc->drm->mfb->width;
prime_desc.height = kmsvnc->drm->mfb->height;
@ -195,44 +218,151 @@ int va_init() {
}
for (int i = 0; i < img_fmt_count; i++) {
printf("fmt %d: fourcc %d, %c%c%c%c, byte_order %s, bpp %d, depth %d, blue_mask %#x, green_mask %#x, red_mask %#x, reserved %#x\n", i, img_fmts[i].fourcc,
img_fmts[i].fourcc & 0xff,
img_fmts[i].fourcc >> 8 & 0xff,
img_fmts[i].fourcc >> 16 & 0xff,
img_fmts[i].fourcc >> 24 & 0xff,
img_fmts[i].byte_order - 1 ? "VA_LSB_FIRST" : "VA_MSB_FIRST",
img_fmts[i].bits_per_pixel,
img_fmts[i].depth,
img_fmts[i].blue_mask,
img_fmts[i].green_mask,
img_fmts[i].red_mask,
img_fmts[i].va_reserved
);
print_va_image_fmt(img_fmts + i);
}
#endif
VAImageFormat format = {
VAImageFormat fmt_rgbx = {
.fourcc = KMSVNC_FOURCC_TO_INT('R','G','B','X'),
.byte_order = VA_LSB_FIRST,
.bits_per_pixel = 32,
.depth = 24,
.blue_mask = 0x0000ff00,
.green_mask = 0x00ff0000,
.red_mask = 0xff000000,
.va_reserved = 0x00000000,
};
VAImageFormat fmt_bgrx = {
.fourcc = KMSVNC_FOURCC_TO_INT('B','G','R','X'),
.byte_order = VA_LSB_FIRST,
.bits_per_pixel = 32,
.depth = 24,
.blue_mask = 0xff000000,
.green_mask = 0x00ff0000,
.red_mask = 0x0000ff00,
.va_reserved = 0x00000000,
};
VAImageFormat fmt_xrgb = {
.fourcc = KMSVNC_FOURCC_TO_INT('X','R','G','B'),
.byte_order = VA_LSB_FIRST,
.bits_per_pixel = 32,
.depth = 24,
.blue_mask = 0x000000ff,
.green_mask = 0x0000ff00,
.red_mask = 0x00ff0000,
.va_reserved = 0x00000000,
};
VAImageFormat fmt_xbgr = {
.fourcc = KMSVNC_FOURCC_TO_INT('X','B','G','R'),
.byte_order = VA_LSB_FIRST,
.bits_per_pixel = 32,
.depth = 24,
.blue_mask = 0x00ff0000,
.green_mask = 0x0000ff00,
.red_mask = 0x000000ff,
.va_reserved = 0x00000000,
};
va->image = malloc(sizeof(VAImage));
if (!va->image) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
if ((s = vaCreateImage(va->dpy, &format, kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, va->image)) != VA_STATUS_SUCCESS) {
free(va->image);
va->image = NULL;
VA_MUST(s);
struct fourcc_data {
VAImageFormat *fmt;
char is_alpha;
int fourcc;
char is_bgr;
char is_xrgb;
};
struct fourcc_data format_to_try[] = {
{&fmt_rgbx, 0, KMSVNC_FOURCC_TO_INT('R','G','B','X'), 0, 0},
{&fmt_rgbx, 1, KMSVNC_FOURCC_TO_INT('R','G','B','A'), 0, 0},
{&fmt_xrgb, 0, KMSVNC_FOURCC_TO_INT('X','R','G','B'), 0, 1},
{&fmt_xrgb, 1, KMSVNC_FOURCC_TO_INT('A','R','G','B'), 0, 1},
{&fmt_bgrx, 0, KMSVNC_FOURCC_TO_INT('B','G','R','X'), 1, 0},
{&fmt_bgrx, 1, KMSVNC_FOURCC_TO_INT('B','G','R','A'), 1, 0},
{&fmt_xbgr, 0, KMSVNC_FOURCC_TO_INT('X','B','G','R'), 1, 1},
{&fmt_xbgr, 1, KMSVNC_FOURCC_TO_INT('A','B','G','R'), 1, 1},
};
va->derive_enabled = strcmp(kmsvnc->drm->drm_ver->name, "i915") != 0;
va->derive_enabled = kmsvnc->va_derive_enabled < 0 ? va->derive_enabled : kmsvnc->va_derive_enabled != 0;
if (va->derive_enabled) {
if ((s = vaDeriveImage(va->dpy, va->surface_id, va->image)) == VA_STATUS_SUCCESS) {
switch (va->image->format.fourcc) {
case KMSVNC_FOURCC_TO_INT('B','G','R','X'):
case KMSVNC_FOURCC_TO_INT('B','G','R','A'):
va->is_bgr = 1;
break;
case KMSVNC_FOURCC_TO_INT('R','G','B','X'):
case KMSVNC_FOURCC_TO_INT('R','G','B','A'):
break;
case KMSVNC_FOURCC_TO_INT('X','R','G','B'):
va->is_xrgb = 1;
break;
case KMSVNC_FOURCC_TO_INT('X','B','G','R'):
va->is_bgr = 1;
va->is_xrgb = 1;
break;
default:
va->derive_enabled = 0;
printf("vaDeriveImage returned unknown fourcc %d %s\n", va->image->format.fourcc, fourcc_to_str(va->image->format.fourcc));
VA_MUST(vaDestroyImage(kmsvnc->va->dpy, kmsvnc->va->image->image_id));
}
}
VA_MAY(s);
}
VA_MUST(vaMapBuffer(va->dpy, va->image->buf, (void**)&va->imgbuf));
if (va->derive_enabled) {
if ((s = vaMapBuffer(va->dpy, va->image->buf, (void**)&va->imgbuf)) != VA_STATUS_SUCCESS) {
VA_MAY(s);
VA_MAY(vaDestroyImage(kmsvnc->va->dpy, kmsvnc->va->image->image_id));
va->derive_enabled = 0;
}
}
if (!va->derive_enabled) {
char success = 0;
for (int i = 0; i < KMSVNC_ARRAY_ELEMENTS(format_to_try); i++) {
if (is_alpha != format_to_try[i].is_alpha) continue;
VAImageFormat *fmt = format_to_try[i].fmt;
va->is_bgr = format_to_try[i].is_bgr;
va->is_xrgb = format_to_try[i].is_xrgb;
fmt->fourcc = format_to_try[i].fourcc;
if ((s = vaCreateImage(va->dpy, fmt, kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, va->image)) != VA_STATUS_SUCCESS) {
VA_MAY(s);
continue;
}
if ((s = vaMapBuffer(va->dpy, va->image->buf, (void**)&va->imgbuf)) != VA_STATUS_SUCCESS) {
VA_MAY(s);
VA_MAY(vaDestroyImage(kmsvnc->va->dpy, kmsvnc->va->image->image_id));
continue;
}
if ((s = vaGetImage(kmsvnc->va->dpy, kmsvnc->va->surface_id, 0, 0,
kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height,
kmsvnc->va->image->image_id)) != VA_STATUS_SUCCESS)
{
VA_MAY(s);
VA_MAY(vaUnmapBuffer(kmsvnc->va->dpy, kmsvnc->va->image->buf));
VA_MAY(vaDestroyImage(kmsvnc->va->dpy, kmsvnc->va->image->image_id));
continue;
}
else {
success = 1;
break;
}
}
if (!success) {
va->imgbuf = NULL;
KMSVNC_FATAL("failed to get vaapi image\n");
}
}
printf("vaapi %simage fourcc isbgr %hd isxrgb %hd\n", va->derive_enabled ? "derive " : "", va->is_bgr, va->is_xrgb);
print_va_image_fmt(&va->image->format);
return 0;
}
int va_hwframe_to_vaapi(char *out) {
VA_MUST(vaGetImage(kmsvnc->va->dpy, kmsvnc->va->surface_id, 0, 0,
kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, kmsvnc->va->image->image_id));
if (!kmsvnc->va->derive_enabled) {
VA_MUST(vaGetImage(kmsvnc->va->dpy, kmsvnc->va->surface_id, 0, 0,
kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, kmsvnc->va->image->image_id));
}
memcpy(out, kmsvnc->va->imgbuf, kmsvnc->drm->mfb->width * kmsvnc->drm->mfb->height * BYTES_PER_PIXEL);
}

4
va.h
View file

@ -1,7 +1,7 @@
#pragma once
#define VA_MUST(x) do{VAStatus s; if ((s = (x)) != VA_STATUS_SUCCESS) KMSVNC_FATAL("va operation error %#x %s on line %d\n", s, vaErrorStr(s), __LINE__); } while (0)
#define VA_MAY(x) do{VAStatus s; if ((s = (x)) != VA_STATUS_SUCCESS) fprintf(stderr, "va operation error %#x %s on line %d\n", s, vaErrorStr(s), __LINE__); } while (0)
#define VA_MUST(x) do{VAStatus _s; if ((_s = (x)) != VA_STATUS_SUCCESS) KMSVNC_FATAL("va operation error %#x %s on line %d\n", _s, vaErrorStr(_s), __LINE__); } while (0)
#define VA_MAY(x) do{VAStatus _s; if ((_s = (x)) != VA_STATUS_SUCCESS) fprintf(stderr, "va operation error %#x %s on line %d\n", _s, vaErrorStr(_s), __LINE__); } while (0)
void va_cleanup();
int va_init();