Jerry/kmsvnc
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

support RGB32_10 pixfmt

Browse source
This commit is contained in:
JerryXiao 2023-05-07 17:57:53 +08:00
parent b592832c0a
commit aff000aa05
Signed by: Jerry
GPG key ID: 22618F758B5BE2E5
4 changed files with 134 additions and 111 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

52
drm.c
View file

@ -12,6 +12,17 @@
extern struct kmsvnc_data *kmsvnc; extern struct kmsvnc_data *kmsvnc;
static int check_pixfmt_non_vaapi() {
if (
kmsvnc->drm->mfb->pixel_format != KMSVNC_FOURCC_TO_INT('X', 'R', '2', '4') &&
kmsvnc->drm->mfb->pixel_format != KMSVNC_FOURCC_TO_INT('A', 'R', '2', '4')
)
{
KMSVNC_FATAL("Unsupported pixfmt %s, please create an issue with your pixfmt.\n", kmsvnc->drm->pixfmt_name);
}
return 0;
}
static void convert_copy(const char *in, int width, int height, char *buff) { static void convert_copy(const char *in, int width, int height, char *buff) {
memcpy(buff, in, width * height * 4); memcpy(buff, in, width * height * 4);
} }
@ -90,20 +101,34 @@ void convert_intel_x_tiled_kmsbuf(const char *in, int width, int height, char *b
} }
static void convert_vaapi(const char *in, int width, int height, char *buff) { static void convert_vaapi(const char *in, int width, int height, char *buff) {
if (kmsvnc->va->is_xrgb || kmsvnc->va->is_bgr) { if (KMSVNC_FOURCC_TO_INT('R','G','B', 0) & kmsvnc->va->image->format.fourcc == KMSVNC_FOURCC_TO_INT('R','G','B', 0)) {
va_hwframe_to_vaapi(buff);
}
else {
if (convert_buf_allocate(width * height * BYTES_PER_PIXEL)) return; if (convert_buf_allocate(width * height * BYTES_PER_PIXEL)) return;
va_hwframe_to_vaapi(kms_convert_buf); va_hwframe_to_vaapi(kms_convert_buf);
if (kmsvnc->va->is_xrgb) { // is 30 depth?
if (kmsvnc->va->image->format.depth == 30) {
for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) {
uint32_t pixdata = *((uint32_t*)(kms_convert_buf + i));
kms_convert_buf[i] = (pixdata & 0x3ff00000) >> 20 >> 2;
kms_convert_buf[i+1] = (pixdata & 0xffc00) >> 10 >> 2;
kms_convert_buf[i+2] = (pixdata & 0x3ff) >> 2;
}
}
// is xrgb?
if ((kmsvnc->va->image->format.blue_mask | kmsvnc->va->image->format.red_mask) < 0x1000000) {
for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) { for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) {
*((uint32_t*)(kms_convert_buf + i)) <<= 8; *((uint32_t*)(kms_convert_buf + i)) <<= 8;
} }
} }
if (kmsvnc->va->is_bgr) { // is bgr?
if (kmsvnc->va->image->format.blue_mask > kmsvnc->va->image->format.red_mask) {
convert_bgrx_to_rgb(kms_convert_buf, width, height, buff); convert_bgrx_to_rgb(kms_convert_buf, width, height, buff);
} }
} else {
else { memcpy(buff, kms_convert_buf, width * height * BYTES_PER_PIXEL);
va_hwframe_to_vaapi(buff); }
} }
} }
@ -255,14 +280,6 @@ int drm_open() {
printf("pitches %u %u %u %u\n", drm->mfb->pitches[0], drm->mfb->pitches[1], drm->mfb->pitches[2], drm->mfb->pitches[3]); printf("pitches %u %u %u %u\n", drm->mfb->pitches[0], drm->mfb->pitches[1], drm->mfb->pitches[2], drm->mfb->pitches[3]);
printf("format %s, modifier %s:%s\n", drm->pixfmt_name, drm->mod_vendor, drm->mod_name); printf("format %s, modifier %s:%s\n", drm->pixfmt_name, drm->mod_vendor, drm->mod_name);
if (
drm->mfb->pixel_format != KMSVNC_FOURCC_TO_INT('X', 'R', '2', '4') &&
drm->mfb->pixel_format != KMSVNC_FOURCC_TO_INT('A', 'R', '2', '4')
)
{
KMSVNC_FATAL("Unsupported pixfmt %s, please create an issue with your pixfmt.\n", drm->pixfmt_name);
}
if (!drm->mfb->handles[0]) if (!drm->mfb->handles[0])
{ {
KMSVNC_FATAL("No handle set on framebuffer: maybe you need some additional capabilities?\n"); KMSVNC_FATAL("No handle set on framebuffer: maybe you need some additional capabilities?\n");
@ -357,6 +374,7 @@ int drm_vendors() {
} }
else if (strcmp(driver_name, "nvidia-drm") == 0) else if (strcmp(driver_name, "nvidia-drm") == 0)
{ {
if (check_pixfmt_non_vaapi()) return 1;
printf("warn: nvidia card detected. Currently only x-tiled framebuffer is supported. Performance may suffer.\n"); printf("warn: nvidia card detected. Currently only x-tiled framebuffer is supported. Performance may suffer.\n");
drm->funcs->convert = &convert_nvidia_x_tiled_kmsbuf; drm->funcs->convert = &convert_nvidia_x_tiled_kmsbuf;
if (drm_kmsbuf_dumb()) return 1; if (drm_kmsbuf_dumb()) return 1;
@ -366,6 +384,7 @@ int drm_vendors() {
strcmp(driver_name, "virtio_gpu") == 0 strcmp(driver_name, "virtio_gpu") == 0
) )
{ {
if (check_pixfmt_non_vaapi()) return 1;
if (drm->mfb->modifier != DRM_FORMAT_MOD_NONE && drm->mfb->modifier != DRM_FORMAT_MOD_LINEAR) { if (drm->mfb->modifier != DRM_FORMAT_MOD_NONE && drm->mfb->modifier != DRM_FORMAT_MOD_LINEAR) {
printf("warn: modifier is not LINEAR, please create an issue with your modifier.\n"); printf("warn: modifier is not LINEAR, please create an issue with your modifier.\n");
} }
@ -374,14 +393,17 @@ int drm_vendors() {
} }
else if (strcmp(driver_name, "test-prime") == 0) else if (strcmp(driver_name, "test-prime") == 0)
{ {
if (check_pixfmt_non_vaapi()) return 1;
if (drm_kmsbuf_prime()) return 1; if (drm_kmsbuf_prime()) return 1;
} }
else if (strcmp(driver_name, "test-map-dumb") == 0) else if (strcmp(driver_name, "test-map-dumb") == 0)
{ {
if (check_pixfmt_non_vaapi()) return 1;
if (drm_kmsbuf_dumb()) return 1; if (drm_kmsbuf_dumb()) return 1;
} }
else if (strcmp(driver_name, "test-i915-gem") == 0) else if (strcmp(driver_name, "test-i915-gem") == 0)
{ {
if (check_pixfmt_non_vaapi()) return 1;
struct drm_gem_flink flink; struct drm_gem_flink flink;
flink.handle = drm->mfb->handles[0]; flink.handle = drm->mfb->handles[0];
DRM_IOCTL_MUST(drm->drm_fd, DRM_IOCTL_GEM_FLINK, &flink); DRM_IOCTL_MUST(drm->drm_fd, DRM_IOCTL_GEM_FLINK, &flink);
@ -398,11 +420,13 @@ int drm_vendors() {
} }
else if (strcmp(driver_name, "test-i915-prime-xtiled") == 0) else if (strcmp(driver_name, "test-i915-prime-xtiled") == 0)
{ {
if (check_pixfmt_non_vaapi()) return 1;
drm->funcs->convert = &convert_intel_x_tiled_kmsbuf; drm->funcs->convert = &convert_intel_x_tiled_kmsbuf;
if (drm_kmsbuf_prime()) return 1; if (drm_kmsbuf_prime()) return 1;
} }
else else
{ {
if (check_pixfmt_non_vaapi()) return 1;
fprintf(stderr, "Untested drm driver, use at your own risk!\n"); fprintf(stderr, "Untested drm driver, use at your own risk!\n");
if (drm->mfb->modifier != DRM_FORMAT_MOD_NONE && drm->mfb->modifier != DRM_FORMAT_MOD_LINEAR) { if (drm->mfb->modifier != DRM_FORMAT_MOD_NONE && drm->mfb->modifier != DRM_FORMAT_MOD_LINEAR) {
printf("warn: modifier is not LINEAR, please create an issue with your driver and modifier.\n"); printf("warn: modifier is not LINEAR, please create an issue with your driver and modifier.\n");

4
kmsvnc.c
View file

@ -144,6 +144,7 @@ static struct argp_option kmsvnc_main_options[] = {
{"disable-compare-fb", 0xff02, 0, OPTION_ARG_OPTIONAL, "Do not compare pixels"}, {"disable-compare-fb", 0xff02, 0, OPTION_ARG_OPTIONAL, "Do not compare pixels"},
{"capture-raw-fb", 0xff03, "/tmp/rawfb.bin", 0, "Capture RAW framebuffer instead of starting the vnc server (for debugging)"}, {"capture-raw-fb", 0xff03, "/tmp/rawfb.bin", 0, "Capture RAW framebuffer instead of starting the vnc server (for debugging)"},
{"va-derive", 0xff04, "off", 0, "Enable derive with vaapi"}, {"va-derive", 0xff04, "off", 0, "Enable derive with vaapi"},
{"va-print-format", 0xff05, 0, OPTION_ARG_OPTIONAL, "Print supported vaImage format"},
{"wakeup", 'w', 0, OPTION_ARG_OPTIONAL, "Move mouse to wake the system up before start"}, {"wakeup", 'w', 0, OPTION_ARG_OPTIONAL, "Move mouse to wake the system up before start"},
{"disable-input", 'i', 0, OPTION_ARG_OPTIONAL, "Disable uinput"}, {"disable-input", 'i', 0, OPTION_ARG_OPTIONAL, "Disable uinput"},
{"desktop-name", 'n', "kmsvnc", 0, "Specify vnc desktop name"}, {"desktop-name", 'n', "kmsvnc", 0, "Specify vnc desktop name"},
@ -213,6 +214,9 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
kmsvnc->va_derive_enabled = 0; kmsvnc->va_derive_enabled = 0;
} }
break; break;
case 0xff05:
kmsvnc->va_print_fmt = 1;
break;
case 'w': case 'w':
kmsvnc->input_wakeup = 1; kmsvnc->input_wakeup = 1;
break; break;

5
kmsvnc.h
View file

@ -34,6 +34,7 @@ struct kmsvnc_data
char input_wakeup; char input_wakeup;
char disable_input; char disable_input;
int va_derive_enabled; int va_derive_enabled;
int va_print_fmt;
int source_plane; int source_plane;
int source_crtc; int source_crtc;
struct kmsvnc_drm_data *drm; struct kmsvnc_drm_data *drm;
@ -103,9 +104,9 @@ struct kmsvnc_va_data
VASurfaceID surface_id; VASurfaceID surface_id;
VAImage *image; VAImage *image;
char *imgbuf; char *imgbuf;
char is_bgr; // bgr -> rgb
char is_xrgb; // shift 8
char derive_enabled; char derive_enabled;
VAImageFormat* img_fmts;
int img_fmt_count;
}; };
#define KMSVNC_FATAL(...) do{ fprintf(stderr, __VA_ARGS__); return 1; } while(0) #define KMSVNC_FATAL(...) do{ fprintf(stderr, __VA_ARGS__); return 1; } while(0)

184
va.c
View file

@ -10,6 +10,10 @@ extern struct kmsvnc_data *kmsvnc;
void va_cleanup() { void va_cleanup() {
VAStatus s; VAStatus s;
if (kmsvnc->va) { if (kmsvnc->va) {
if (kmsvnc->va->img_fmts) {
free(kmsvnc->va->img_fmts);
kmsvnc->va->img_fmts = NULL;
}
if (kmsvnc->va->imgbuf) { if (kmsvnc->va->imgbuf) {
VA_MAY(vaUnmapBuffer(kmsvnc->va->dpy, kmsvnc->va->image->buf)); VA_MAY(vaUnmapBuffer(kmsvnc->va->dpy, kmsvnc->va->image->buf));
kmsvnc->va->imgbuf = NULL; kmsvnc->va->imgbuf = NULL;
@ -35,9 +39,9 @@ void va_cleanup() {
} }
static void va_msg_callback(void *user_context, const char *message) { static void va_msg_callback(void *user_context, const char *message) {
#ifdef KMSVNC_VA_DEBUG if (kmsvnc->va_print_fmt) {
printf("va msg: %s", message); printf("va msg: %s", message);
#endif }
} }
static void va_error_callback(void *user_context, const char *message) { static void va_error_callback(void *user_context, const char *message) {
@ -53,6 +57,18 @@ static char* fourcc_to_str(int fourcc) {
return ret; return ret;
} }
static const struct {
uint32_t drm_fourcc;
uint32_t va_fourcc;
uint32_t va_rt_format;
char alpha;
} va_format_map[] = {
{KMSVNC_FOURCC_TO_INT('X', 'R', '2', '4'), KMSVNC_FOURCC_TO_INT('B', 'G', 'R', 'X'), VA_RT_FORMAT_RGB32, 0},
{KMSVNC_FOURCC_TO_INT('A', 'R', '2', '4'), KMSVNC_FOURCC_TO_INT('B', 'G', 'R', 'A'), VA_RT_FORMAT_RGB32, 1},
{KMSVNC_FOURCC_TO_INT('X', 'R', '3', '0'), KMSVNC_FOURCC_TO_INT('X', 'R', '3', '0'), VA_RT_FORMAT_RGB32_10, 0},
{KMSVNC_FOURCC_TO_INT('A', 'R', '3', '0'), KMSVNC_FOURCC_TO_INT('A', 'R', '3', '0'), VA_RT_FORMAT_RGB32_10, 1},
};
static void print_va_image_fmt(VAImageFormat *fmt) { static void print_va_image_fmt(VAImageFormat *fmt) {
printf("image fmt: fourcc %d, %s, byte_order %s, bpp %d, depth %d, blue_mask %#x, green_mask %#x, red_mask %#x, reserved %#x %#x %#x %#x\n", fmt->fourcc, printf("image fmt: fourcc %d, %s, byte_order %s, bpp %d, depth %d, blue_mask %#x, green_mask %#x, red_mask %#x, reserved %#x %#x %#x %#x\n", fmt->fourcc,
fourcc_to_str(fmt->fourcc), fourcc_to_str(fmt->fourcc),
@ -129,10 +145,22 @@ int va_init() {
} }
}; };
char is_alpha = kmsvnc->drm->mfb->pixel_format != KMSVNC_FOURCC_TO_INT('X', 'R', '2', '4'); uint32_t rt_format = 0;
prime_desc.fourcc = kmsvnc->drm->mfb->pixel_format == is_alpha ? char is_alpha = 0;
KMSVNC_FOURCC_TO_INT('B', 'G', 'R', 'A') : for (int i = 0; i < KMSVNC_ARRAY_ELEMENTS(va_format_map); i++) {
KMSVNC_FOURCC_TO_INT('B', 'G', 'R', 'X') ; if (kmsvnc->drm->mfb->pixel_format == va_format_map[i].drm_fourcc) {
prime_desc.fourcc = va_format_map[i].va_fourcc;
rt_format = va_format_map[i].va_rt_format;
is_alpha = va_format_map[i].alpha;
break;
}
}
if (!rt_format) {
KMSVNC_FATAL("Unsupported pixfmt %s for vaapi, please create an issue with your pixfmt.", kmsvnc->drm->pixfmt_name);
}
if (kmsvnc->va_print_fmt) {
printf("selected rt_format %u, alpha %d\n", rt_format, is_alpha);
}
prime_desc.width = kmsvnc->drm->mfb->width; prime_desc.width = kmsvnc->drm->mfb->width;
prime_desc.height = kmsvnc->drm->mfb->height; prime_desc.height = kmsvnc->drm->mfb->height;
@ -164,7 +192,7 @@ int va_init() {
prime_desc.num_objects = 1; prime_desc.num_objects = 1;
VAStatus s; VAStatus s;
if ((s = vaCreateSurfaces(va->dpy, VA_RT_FORMAT_RGB32, if ((s = vaCreateSurfaces(va->dpy, rt_format,
kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, &va->surface_id, 1, kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, &va->surface_id, 1,
prime_attrs, KMSVNC_ARRAY_ELEMENTS(prime_attrs))) != VA_STATUS_SUCCESS) prime_attrs, KMSVNC_ARRAY_ELEMENTS(prime_attrs))) != VA_STATUS_SUCCESS)
{ {
@ -203,113 +231,78 @@ int va_init() {
buffer_desc.num_planes = prime_desc.layers[0].num_planes; buffer_desc.num_planes = prime_desc.layers[0].num_planes;
VA_MUST(vaCreateSurfaces(va->dpy, VA_RT_FORMAT_RGB32, VA_MUST(vaCreateSurfaces(va->dpy, rt_format,
kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, &va->surface_id, 1, kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, &va->surface_id, 1,
buffer_attrs, KMSVNC_ARRAY_ELEMENTS(buffer_attrs))); buffer_attrs, KMSVNC_ARRAY_ELEMENTS(buffer_attrs)));
} }
#ifdef KMSVNC_VA_DEBUG
int img_fmt_count = vaMaxNumImageFormats(va->dpy); va->img_fmt_count = vaMaxNumImageFormats(va->dpy);
VAImageFormat *img_fmts = malloc(sizeof(VAImageFormat) * img_fmt_count); va->img_fmts = malloc(sizeof(VAImageFormat) * va->img_fmt_count);
if (!img_fmts) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__); if (!va->img_fmts) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
{ {
int got; int got;
vaQueryImageFormats(va->dpy, img_fmts, &got); vaQueryImageFormats(va->dpy, va->img_fmts, &got);
if (got != img_fmt_count) { if (got != va->img_fmt_count) {
KMSVNC_FATAL("got less VAImageFormats, %d instead of %d\n", got, img_fmt_count); printf("got less VAImageFormats, %d instead of %d\n", got, va->img_fmt_count);
va->img_fmt_count = got;
} }
} }
for (int i = 0; i < img_fmt_count; i++) { if (kmsvnc->va_print_fmt) {
print_va_image_fmt(img_fmts + i); for (int i = 0; i < va->img_fmt_count; i++) {
print_va_image_fmt(va->img_fmts + i);
}
} }
#endif
VAImageFormat fmt_rgbx = {
.fourcc = KMSVNC_FOURCC_TO_INT('R','G','B','X'),
.byte_order = VA_LSB_FIRST,
.bits_per_pixel = 32,
.depth = 24,
.blue_mask = 0x0000ff00,
.green_mask = 0x00ff0000,
.red_mask = 0xff000000,
.va_reserved = {0,0,0,0},
};
VAImageFormat fmt_bgrx = {
.fourcc = KMSVNC_FOURCC_TO_INT('B','G','R','X'),
.byte_order = VA_LSB_FIRST,
.bits_per_pixel = 32,
.depth = 24,
.blue_mask = 0xff000000,
.green_mask = 0x00ff0000,
.red_mask = 0x0000ff00,
.va_reserved = {0,0,0,0},
};
VAImageFormat fmt_xrgb = {
.fourcc = KMSVNC_FOURCC_TO_INT('X','R','G','B'),
.byte_order = VA_LSB_FIRST,
.bits_per_pixel = 32,
.depth = 24,
.blue_mask = 0x000000ff,
.green_mask = 0x0000ff00,
.red_mask = 0x00ff0000,
.va_reserved = {0,0,0,0},
};
VAImageFormat fmt_xbgr = {
.fourcc = KMSVNC_FOURCC_TO_INT('X','B','G','R'),
.byte_order = VA_LSB_FIRST,
.bits_per_pixel = 32,
.depth = 24,
.blue_mask = 0x00ff0000,
.green_mask = 0x0000ff00,
.red_mask = 0x000000ff,
.va_reserved = {0,0,0,0},
};
va->image = malloc(sizeof(VAImage));
if (!va->image) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
struct fourcc_data { struct fourcc_data {
uint32_t va_fourcc;
VAImageFormat *fmt; VAImageFormat *fmt;
char is_alpha; char is_alpha;
int fourcc; uint32_t va_rt_format;
char is_bgr;
char is_xrgb;
}; };
struct fourcc_data format_to_try[] = { struct fourcc_data format_to_try[] = {
{&fmt_rgbx, 0, KMSVNC_FOURCC_TO_INT('R','G','B','X'), 0, 0}, {KMSVNC_FOURCC_TO_INT('R','G','B','X'), NULL, 0, VA_RT_FORMAT_RGB32},
{&fmt_rgbx, 1, KMSVNC_FOURCC_TO_INT('R','G','B','A'), 0, 0}, {KMSVNC_FOURCC_TO_INT('R','G','B','A'), NULL, 1, VA_RT_FORMAT_RGB32},
{&fmt_xrgb, 0, KMSVNC_FOURCC_TO_INT('X','R','G','B'), 0, 1}, {KMSVNC_FOURCC_TO_INT('X','R','G','B'), NULL, 0, VA_RT_FORMAT_RGB32},
{&fmt_xrgb, 1, KMSVNC_FOURCC_TO_INT('A','R','G','B'), 0, 1}, {KMSVNC_FOURCC_TO_INT('A','R','G','B'), NULL, 1, VA_RT_FORMAT_RGB32},
{&fmt_bgrx, 0, KMSVNC_FOURCC_TO_INT('B','G','R','X'), 1, 0}, {KMSVNC_FOURCC_TO_INT('B','G','R','X'), NULL, 0, VA_RT_FORMAT_RGB32},
{&fmt_bgrx, 1, KMSVNC_FOURCC_TO_INT('B','G','R','A'), 1, 0}, {KMSVNC_FOURCC_TO_INT('B','G','R','A'), NULL, 1, VA_RT_FORMAT_RGB32},
{&fmt_xbgr, 0, KMSVNC_FOURCC_TO_INT('X','B','G','R'), 1, 1}, {KMSVNC_FOURCC_TO_INT('X','B','G','R'), NULL, 0, VA_RT_FORMAT_RGB32},
{&fmt_xbgr, 1, KMSVNC_FOURCC_TO_INT('A','B','G','R'), 1, 1}, {KMSVNC_FOURCC_TO_INT('A','B','G','R'), NULL, 1, VA_RT_FORMAT_RGB32},
{KMSVNC_FOURCC_TO_INT('X','R','3','0'), NULL, 0, VA_RT_FORMAT_RGB32_10},
{KMSVNC_FOURCC_TO_INT('A','R','3','0'), NULL, 1, VA_RT_FORMAT_RGB32_10},
{KMSVNC_FOURCC_TO_INT('X','B','3','0'), NULL, 0, VA_RT_FORMAT_RGB32_10},
{KMSVNC_FOURCC_TO_INT('A','B','3','0'), NULL, 1, VA_RT_FORMAT_RGB32_10},
}; };
for (int i = 0; i < va->img_fmt_count; i++) {
for (int j = 0; j < KMSVNC_ARRAY_ELEMENTS(format_to_try); j++) {
if (va->img_fmts[i].fourcc == format_to_try[j].va_fourcc) {
format_to_try[j].fmt = va->img_fmts + i;
}
}
}
va->image = malloc(sizeof(VAImage));
if (!va->image) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
va->derive_enabled = 0; va->derive_enabled = 0;
va->derive_enabled = kmsvnc->va_derive_enabled < 0 ? va->derive_enabled : kmsvnc->va_derive_enabled != 0; va->derive_enabled = kmsvnc->va_derive_enabled < 0 ? va->derive_enabled : kmsvnc->va_derive_enabled != 0;
if (va->derive_enabled) { if (va->derive_enabled) {
if ((s = vaDeriveImage(va->dpy, va->surface_id, va->image)) == VA_STATUS_SUCCESS) { if ((s = vaDeriveImage(va->dpy, va->surface_id, va->image)) == VA_STATUS_SUCCESS) {
switch (va->image->format.fourcc) { char found = 0;
case KMSVNC_FOURCC_TO_INT('B','G','R','X'): for (int i = 0; i < KMSVNC_ARRAY_ELEMENTS(format_to_try); i++) {
case KMSVNC_FOURCC_TO_INT('B','G','R','A'): if (va->image->format.fourcc == format_to_try[i].fmt->fourcc) {
va->is_bgr = 1; found = 1;
break; break;
case KMSVNC_FOURCC_TO_INT('R','G','B','X'): }
case KMSVNC_FOURCC_TO_INT('R','G','B','A'): }
break; if (!found) {
case KMSVNC_FOURCC_TO_INT('X','R','G','B'): va->derive_enabled = 0;
va->is_xrgb = 1; printf("vaDeriveImage returned unknown fourcc %d %s\n", va->image->format.fourcc, fourcc_to_str(va->image->format.fourcc));
break; VA_MAY(vaDestroyImage(kmsvnc->va->dpy, kmsvnc->va->image->image_id));
case KMSVNC_FOURCC_TO_INT('X','B','G','R'):
va->is_bgr = 1;
va->is_xrgb = 1;
break;
default:
va->derive_enabled = 0;
printf("vaDeriveImage returned unknown fourcc %d %s\n", va->image->format.fourcc, fourcc_to_str(va->image->format.fourcc));
VA_MUST(vaDestroyImage(kmsvnc->va->dpy, kmsvnc->va->image->image_id));
} }
} }
VA_MAY(s); VA_MAY(s);
@ -324,11 +317,11 @@ int va_init() {
if (!va->derive_enabled) { if (!va->derive_enabled) {
char success = 0; char success = 0;
for (int i = 0; i < KMSVNC_ARRAY_ELEMENTS(format_to_try); i++) { for (int i = 0; i < KMSVNC_ARRAY_ELEMENTS(format_to_try); i++) {
if (format_to_try[i].fmt == NULL) continue;
if (!kmsvnc->va_print_fmt && rt_format != format_to_try[i].va_rt_format) continue;
if (is_alpha != format_to_try[i].is_alpha) continue; if (is_alpha != format_to_try[i].is_alpha) continue;
VAImageFormat *fmt = format_to_try[i].fmt; VAImageFormat *fmt = format_to_try[i].fmt;
va->is_bgr = format_to_try[i].is_bgr;
va->is_xrgb = format_to_try[i].is_xrgb;
fmt->fourcc = format_to_try[i].fourcc;
if ((s = vaCreateImage(va->dpy, fmt, kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, va->image)) != VA_STATUS_SUCCESS) { if ((s = vaCreateImage(va->dpy, fmt, kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, va->image)) != VA_STATUS_SUCCESS) {
VA_MAY(s); VA_MAY(s);
continue; continue;
@ -357,7 +350,7 @@ int va_init() {
KMSVNC_FATAL("failed to get vaapi image\n"); KMSVNC_FATAL("failed to get vaapi image\n");
} }
} }
printf("vaapi %simage fourcc isbgr %hd isxrgb %hd\n", va->derive_enabled ? "derive " : "", va->is_bgr, va->is_xrgb); printf("got vaapi %simage:\n", va->derive_enabled ? "derive " : "");
print_va_image_fmt(&va->image->format); print_va_image_fmt(&va->image->format);
return 0; return 0;
} }
@ -368,4 +361,5 @@ int va_hwframe_to_vaapi(char *out) {
kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, kmsvnc->va->image->image_id)); kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, kmsvnc->va->image->image_id));
} }
memcpy(out, kmsvnc->va->imgbuf, kmsvnc->drm->mfb->width * kmsvnc->drm->mfb->height * BYTES_PER_PIXEL); memcpy(out, kmsvnc->va->imgbuf, kmsvnc->drm->mfb->width * kmsvnc->drm->mfb->height * BYTES_PER_PIXEL);
return 0;
} }