Jerry/kmsvnc
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

use hardcoded image format by default (github #7)

Browse source
This commit is contained in:
JerryXiao 2023-09-21 16:06:31 +08:00
parent 586d14e848
commit c9b056a3f5
Signed by: Jerry
GPG key ID: 22618F758B5BE2E5
6 changed files with 146 additions and 73 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
.drone.yml
View file

@ -6,11 +6,12 @@ steps:
- name: build - name: build
image: archlinux:latest image: archlinux:latest
commands: commands:
- pacman -Syu --noconfirm --needed base-devel libvncserver libxkbcommon libdrm libva git cmake - pacman -Syu --noconfirm --needed base-devel libvncserver libxkbcommon libdrm libva git cmake clang
- mkdir build - export CFLAGS="-pipe -fno-plt -fexceptions -fstack-clash-protection -fcf-protection -Wp,-D_FORTIFY_SOURCE=2 -Wformat -Werror=format-security"
- cd build - CC=gcc cmake -B gcc-out
- cmake .. - cmake --build gcc-out
- make - CC=clang cmake -B clang-out
- cmake --build clang-out
trigger: trigger:
branch: branch:

8
drm.c
View file

@ -97,14 +97,14 @@ void convert_intel_x_tiled_kmsbuf(const char *in, int width, int height, char *b
} }
static void convert_vaapi(const char *in, int width, int height, char *buff) { static void convert_vaapi(const char *in, int width, int height, char *buff) {
if (KMSVNC_FOURCC_TO_INT('R','G','B', 0) & kmsvnc->va->image->format.fourcc == KMSVNC_FOURCC_TO_INT('R','G','B', 0)) { if ((KMSVNC_FOURCC_TO_INT('R','G','B', 0) & kmsvnc->va->selected_fmt->fourcc) == KMSVNC_FOURCC_TO_INT('R','G','B', 0)) {
va_hwframe_to_vaapi(buff); va_hwframe_to_vaapi(buff);
} }
else { else {
if (convert_buf_allocate(width * height * BYTES_PER_PIXEL)) return; if (convert_buf_allocate(width * height * BYTES_PER_PIXEL)) return;
va_hwframe_to_vaapi(kmsvnc->drm->kms_convert_buf); va_hwframe_to_vaapi(kmsvnc->drm->kms_convert_buf);
// is 30 depth? // is 30 depth?
if (kmsvnc->va->image->format.depth == 30) { if (kmsvnc->va->selected_fmt->depth == 30) {
for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) { for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) {
// ensure little endianess // ensure little endianess
uint32_t pixdata = __builtin_bswap32(htonl(*((uint32_t*)(kmsvnc->drm->kms_convert_buf + i)))); uint32_t pixdata = __builtin_bswap32(htonl(*((uint32_t*)(kmsvnc->drm->kms_convert_buf + i))));
@ -114,14 +114,14 @@ static void convert_vaapi(const char *in, int width, int height, char *buff) {
} }
} }
// is xrgb? // is xrgb?
if ((kmsvnc->va->image->format.blue_mask | kmsvnc->va->image->format.red_mask) < 0x1000000) { if ((kmsvnc->va->selected_fmt->blue_mask | kmsvnc->va->selected_fmt->red_mask) < 0x1000000) {
for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) { for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) {
uint32_t *pixdata = (uint32_t*)(kmsvnc->drm->kms_convert_buf + i); uint32_t *pixdata = (uint32_t*)(kmsvnc->drm->kms_convert_buf + i);
*pixdata = ntohl(htonl(*pixdata) << 8); *pixdata = ntohl(htonl(*pixdata) << 8);
} }
} }
// is bgrx? // is bgrx?
if (kmsvnc->va->image->format.blue_mask > kmsvnc->va->image->format.red_mask) { if (kmsvnc->va->selected_fmt->blue_mask > kmsvnc->va->selected_fmt->red_mask) {
for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) { for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) {
uint32_t pixdata = htonl(*((uint32_t*)(kmsvnc->drm->kms_convert_buf + i))); uint32_t pixdata = htonl(*((uint32_t*)(kmsvnc->drm->kms_convert_buf + i)));
buff[i+0] = (pixdata & 0x0000ff00) >> 8; buff[i+0] = (pixdata & 0x0000ff00) >> 8;

10
drm_master.c
View file

@ -1,4 +1,5 @@
#define _GNU_SOURCE #define _GNU_SOURCE
#include <stdio.h> #include <stdio.h>
#include <errno.h> #include <errno.h>
#include <sys/types.h> #include <sys/types.h>
@ -38,9 +39,15 @@ static inline int cmp_fds(pid_t pid, const char *drm_pth) {
if (ret == -1 && fdlist[n]->d_type == DT_LNK) { if (ret == -1 && fdlist[n]->d_type == DT_LNK) {
char link_pth[PATH_MAX+1]; char link_pth[PATH_MAX+1];
char real_pth[PATH_MAX+1]; char real_pth[PATH_MAX+1];
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wformat-truncation"
snprintf(link_pth, PATH_MAX+1, "%s/%s", path, fdlist[n]->d_name); snprintf(link_pth, PATH_MAX+1, "%s/%s", path, fdlist[n]->d_name);
#pragma GCC diagnostic pop
memset(real_pth, 0, PATH_MAX+1); memset(real_pth, 0, PATH_MAX+1);
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-result"
realpath(link_pth, real_pth); realpath(link_pth, real_pth);
#pragma GCC diagnostic pop
if (!strncmp(real_pth, drm_pth, PATH_MAX)) { if (!strncmp(real_pth, drm_pth, PATH_MAX)) {
int fd = atoi(fdlist[n]->d_name); int fd = atoi(fdlist[n]->d_name);
if (fd > 0) { if (fd > 0) {
@ -69,7 +76,10 @@ static inline int cmp_fds(pid_t pid, const char *drm_pth) {
int drm_get_master_fd() { int drm_get_master_fd() {
char drm_pth[PATH_MAX+1]; char drm_pth[PATH_MAX+1];
memset(drm_pth, 0, PATH_MAX+1); memset(drm_pth, 0, PATH_MAX+1);
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-result"
realpath(kmsvnc->card, drm_pth); realpath(kmsvnc->card, drm_pth);
#pragma GCC diagnostic pop
struct dirent **proclist; struct dirent **proclist;
int count = scandir("/proc", &proclist, NULL, versionsort); int count = scandir("/proc", &proclist, NULL, versionsort);

68
kmsvnc.c
View file

@ -1,3 +1,5 @@
#define _GNU_SOURCE
#include <fcntl.h> #include <fcntl.h>
#include <stdio.h> #include <stdio.h>
#include <string.h> #include <string.h>
@ -127,7 +129,7 @@ static inline void update_vnc_cursor(char *data, int width, int height) {
if (!kmsvnc->cursor_bitmap) return; if (!kmsvnc->cursor_bitmap) return;
kmsvnc->cursor_bitmap_len = rwidth * rheight * BYTES_PER_PIXEL; kmsvnc->cursor_bitmap_len = rwidth * rheight * BYTES_PER_PIXEL;
} }
char *rich_source = malloc(rwidth * rheight * BYTES_PER_PIXEL); unsigned char *rich_source = malloc(rwidth * rheight * BYTES_PER_PIXEL);
if (!rich_source) return; if (!rich_source) return;
char *maskString = malloc(rwidth * rheight); char *maskString = malloc(rwidth * rheight);
if (!maskString) { if (!maskString) {
@ -241,6 +243,7 @@ static struct argp_option kmsvnc_main_options[] = {
{"input-offy", 0xff09, "0", 0, "Set input offset of y axis on a multi display system"}, {"input-offy", 0xff09, "0", 0, "Set input offset of y axis on a multi display system"},
{"screen-blank", 0xff0a, 0, OPTION_ARG_OPTIONAL, "Blank screen with gamma set on crtc"}, {"screen-blank", 0xff0a, 0, OPTION_ARG_OPTIONAL, "Blank screen with gamma set on crtc"},
{"screen-blank-restore-linear", 0xff0b, 0, OPTION_ARG_OPTIONAL, "Restore linear values on exit in case of messed up gamma"}, {"screen-blank-restore-linear", 0xff0b, 0, OPTION_ARG_OPTIONAL, "Restore linear values on exit in case of messed up gamma"},
{"trust-va-format", 0xff0c, 0, OPTION_ARG_OPTIONAL, "trust VAImageFormat returned by vaapi implementation unconditionally"},
{"wakeup", 'w', 0, OPTION_ARG_OPTIONAL, "Move mouse to wake the system up before start"}, {"wakeup", 'w', 0, OPTION_ARG_OPTIONAL, "Move mouse to wake the system up before start"},
{"disable-input", 'i', 0, OPTION_ARG_OPTIONAL, "Disable uinput"}, {"disable-input", 'i', 0, OPTION_ARG_OPTIONAL, "Disable uinput"},
{"desktop-name", 'n', "kmsvnc", 0, "Specify vnc desktop name"}, {"desktop-name", 'n', "kmsvnc", 0, "Specify vnc desktop name"},
@ -272,24 +275,28 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
kmsvnc->vnc_opt->bind6 = arg; kmsvnc->vnc_opt->bind6 = arg;
break; break;
case 'p': case 'p':
int port = atoi(arg); {
if (port > 0 && port < 65536) { int port = atoi(arg);
kmsvnc->vnc_opt->port = port; if (port > 0 && port < 65536) {
} kmsvnc->vnc_opt->port = port;
else { }
argp_error(state, "invalid port %s", arg); else {
argp_error(state, "invalid port %s", arg);
}
} }
break; break;
case '4': case '4':
kmsvnc->vnc_opt->disable_ipv6 = 1; kmsvnc->vnc_opt->disable_ipv6 = 1;
break; break;
case 0xff00: case 0xff00:
int fps = atoi(arg); {
if (fps > 0 && fps < 1000) { int fps = atoi(arg);
kmsvnc->vnc_opt->sleep_ns = NS_IN_S / fps; if (fps > 0 && fps < 1000) {
} kmsvnc->vnc_opt->sleep_ns = NS_IN_S / fps;
else { }
argp_error(state, "invalid fps %s", arg); else {
argp_error(state, "invalid fps %s", arg);
}
} }
break; break;
case 0xff01: case 0xff01:
@ -317,27 +324,35 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
kmsvnc->debug_enabled = 1; kmsvnc->debug_enabled = 1;
break; break;
case 0xff06: case 0xff06:
int width = atoi(arg); {
if (width > 0) { int width = atoi(arg);
kmsvnc->input_width = width; if (width > 0) {
kmsvnc->input_width = width;
}
} }
break; break;
case 0xff07: case 0xff07:
int height = atoi(arg); {
if (height > 0) { int height = atoi(arg);
kmsvnc->input_height = height; if (height > 0) {
kmsvnc->input_height = height;
}
} }
break; break;
case 0xff08: case 0xff08:
int offset_x = atoi(arg); {
if (offset_x > 0) { int offset_x = atoi(arg);
kmsvnc->input_offx = offset_x; if (offset_x > 0) {
kmsvnc->input_offx = offset_x;
}
} }
break; break;
case 0xff09: case 0xff09:
int offset_y = atoi(arg); {
if (offset_y > 0) { int offset_y = atoi(arg);
kmsvnc->input_offy = offset_y; if (offset_y > 0) {
kmsvnc->input_offy = offset_y;
}
} }
break; break;
case 0xff0a: case 0xff0a:
@ -346,6 +361,9 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
case 0xff0b: case 0xff0b:
kmsvnc->screen_blank_restore = 1; kmsvnc->screen_blank_restore = 1;
break; break;
case 0xff0c:
kmsvnc->trust_va_format = 1;
break;
case 'w': case 'w':
kmsvnc->input_wakeup = 1; kmsvnc->input_wakeup = 1;
break; break;

3
kmsvnc.h
View file

@ -1,6 +1,7 @@
#pragma once #pragma once
#include <rfb/rfb.h> #include <rfb/rfb.h>
#include <stdint.h>
#include <xkbcommon/xkbcommon.h> #include <xkbcommon/xkbcommon.h>
#include <xf86drm.h> #include <xf86drm.h>
@ -44,6 +45,7 @@ struct kmsvnc_data
int input_offy; int input_offy;
char screen_blank; char screen_blank;
char screen_blank_restore; char screen_blank_restore;
char trust_va_format;
struct kmsvnc_drm_data *drm; struct kmsvnc_drm_data *drm;
struct kmsvnc_input_data *input; struct kmsvnc_input_data *input;
struct kmsvnc_keymap_data *keymap; struct kmsvnc_keymap_data *keymap;
@ -137,6 +139,7 @@ struct kmsvnc_va_data
char derive_enabled; char derive_enabled;
VAImageFormat* img_fmts; VAImageFormat* img_fmts;
int img_fmt_count; int img_fmt_count;
VAImageFormat* selected_fmt;
}; };
#define KMSVNC_FATAL(...) do{ fprintf(stderr, __VA_ARGS__); return 1; } while(0) #define KMSVNC_FATAL(...) do{ fprintf(stderr, __VA_ARGS__); return 1; } while(0)

119
va.c
View file

@ -1,3 +1,8 @@
#define _GNU_SOURCE
#include <stdint.h>
#include <stdio.h>
#include <va/va.h>
#include <va/va_drm.h> #include <va/va_drm.h>
#include <va/va_drmcommon.h> #include <va/va_drmcommon.h>
#include <fcntl.h> #include <fcntl.h>
@ -69,20 +74,49 @@ static const struct {
{KMSVNC_FOURCC_TO_INT('A', 'R', '3', '0'), KMSVNC_FOURCC_TO_INT('A', 'R', '3', '0'), VA_RT_FORMAT_RGB32_10, 1}, {KMSVNC_FOURCC_TO_INT('A', 'R', '3', '0'), KMSVNC_FOURCC_TO_INT('A', 'R', '3', '0'), VA_RT_FORMAT_RGB32_10, 1},
}; };
struct va_fmt_data {
uint32_t va_fourcc;
VAImageFormat *fmt;
char is_alpha;
uint32_t va_rt_format;
uint32_t depth;
uint32_t blue_mask;
uint32_t green_mask;
uint32_t red_mask;
};
static VAImageFormat* vaImgFmt_from_vaFmtData(struct va_fmt_data* data) {
static VAImageFormat ret = {0};
VAImageFormat fmt = {
.fourcc = data->va_fourcc,
.byte_order = VA_LSB_FIRST,
.bits_per_pixel = 32,
.depth = data->depth,
.blue_mask = data->blue_mask,
.green_mask = data->green_mask,
.red_mask = data->red_mask,
.alpha_mask = 0,
.va_reserved = {0}
};
memcpy(&ret, &fmt, sizeof(VAImageFormat));
return &ret;
}
static void print_va_image_fmt(VAImageFormat *fmt) { static void print_va_image_fmt(VAImageFormat *fmt) {
printf("image fmt: fourcc %d, %s, byte_order %s, bpp %d, depth %d, blue_mask %#x, green_mask %#x, red_mask %#x, reserved %#x %#x %#x %#x\n", fmt->fourcc, printf("image fmt: fourcc %d, %s, byte_order %s, bpp %d, depth %d, blue_mask %#x, green_mask %#x, red_mask %#x, alpha_mask %#x, reserved %#x %#x %#x %#x\n", fmt->fourcc,
fourcc_to_str(fmt->fourcc), fourcc_to_str(fmt->fourcc),
fmt->byte_order == 1 ? "VA_LSB_FIRST" : "VA_MSB_FIRST", fmt->byte_order == 1 ? "VA_LSB_FIRST" : "VA_MSB_FIRST",
fmt->bits_per_pixel, fmt->bits_per_pixel,
fmt->depth, fmt->depth,
fmt->blue_mask, fmt->blue_mask,
fmt->green_mask, fmt->green_mask,
fmt->red_mask, fmt->red_mask,
fmt->va_reserved[0], fmt->alpha_mask,
fmt->va_reserved[1], fmt->va_reserved[0],
fmt->va_reserved[2], fmt->va_reserved[1],
fmt->va_reserved[3] fmt->va_reserved[2],
); fmt->va_reserved[3]
);
} }
int va_init() { int va_init() {
@ -99,7 +133,7 @@ int va_init() {
char* render_node; char* render_node;
int effective_fd = 0; int effective_fd = 0;
if (render_node = drmGetRenderDeviceNameFromFd(kmsvnc->drm->drm_fd)) { if ((render_node = drmGetRenderDeviceNameFromFd(kmsvnc->drm->drm_fd))) {
va->render_node_fd = open(render_node, O_RDWR); va->render_node_fd = open(render_node, O_RDWR);
free(render_node); free(render_node);
} }
@ -255,28 +289,23 @@ int va_init() {
} }
} }
struct fourcc_data { struct va_fmt_data format_to_try[] = {
uint32_t va_fourcc; {KMSVNC_FOURCC_TO_INT('R','G','B','X'), NULL, 0, VA_RT_FORMAT_RGB32, 24, 0xff00, 0xff0000, 0xff000000},
VAImageFormat *fmt; {KMSVNC_FOURCC_TO_INT('R','G','B','A'), NULL, 1, VA_RT_FORMAT_RGB32, 32, 0xff00, 0xff0000, 0xff000000},
char is_alpha; {KMSVNC_FOURCC_TO_INT('X','R','G','B'), NULL, 0, VA_RT_FORMAT_RGB32, 24, 0xff, 0xff00, 0xff0000},
uint32_t va_rt_format; {KMSVNC_FOURCC_TO_INT('A','R','G','B'), NULL, 1, VA_RT_FORMAT_RGB32, 32, 0xff, 0xff00, 0xff0000},
};
struct fourcc_data format_to_try[] = {
{KMSVNC_FOURCC_TO_INT('R','G','B','X'), NULL, 0, VA_RT_FORMAT_RGB32},
{KMSVNC_FOURCC_TO_INT('R','G','B','A'), NULL, 1, VA_RT_FORMAT_RGB32},
{KMSVNC_FOURCC_TO_INT('X','R','G','B'), NULL, 0, VA_RT_FORMAT_RGB32},
{KMSVNC_FOURCC_TO_INT('A','R','G','B'), NULL, 1, VA_RT_FORMAT_RGB32},
{KMSVNC_FOURCC_TO_INT('B','G','R','X'), NULL, 0, VA_RT_FORMAT_RGB32}, {KMSVNC_FOURCC_TO_INT('B','G','R','X'), NULL, 0, VA_RT_FORMAT_RGB32, 24, 0xff000000, 0xff0000, 0xff00},
{KMSVNC_FOURCC_TO_INT('B','G','R','A'), NULL, 1, VA_RT_FORMAT_RGB32}, {KMSVNC_FOURCC_TO_INT('B','G','R','A'), NULL, 1, VA_RT_FORMAT_RGB32, 32, 0xff000000, 0xff0000, 0xff00},
{KMSVNC_FOURCC_TO_INT('X','B','G','R'), NULL, 0, VA_RT_FORMAT_RGB32}, {KMSVNC_FOURCC_TO_INT('X','B','G','R'), NULL, 0, VA_RT_FORMAT_RGB32, 24, 0xff0000, 0xff00, 0xff},
{KMSVNC_FOURCC_TO_INT('A','B','G','R'), NULL, 1, VA_RT_FORMAT_RGB32}, {KMSVNC_FOURCC_TO_INT('A','B','G','R'), NULL, 1, VA_RT_FORMAT_RGB32, 32, 0xff0000, 0xff00, 0xff},
{KMSVNC_FOURCC_TO_INT('X','R','3','0'), NULL, 0, VA_RT_FORMAT_RGB32_10}, {KMSVNC_FOURCC_TO_INT('X','R','3','0'), NULL, 0, VA_RT_FORMAT_RGB32_10, 30, 0x3ff, 0xffc00, 0x3ff00000},
{KMSVNC_FOURCC_TO_INT('A','R','3','0'), NULL, 1, VA_RT_FORMAT_RGB32_10}, {KMSVNC_FOURCC_TO_INT('A','R','3','0'), NULL, 1, VA_RT_FORMAT_RGB32_10, 30, 0x3ff, 0xffc00, 0x3ff00000},
{KMSVNC_FOURCC_TO_INT('X','B','3','0'), NULL, 0, VA_RT_FORMAT_RGB32_10}, {KMSVNC_FOURCC_TO_INT('X','B','3','0'), NULL, 0, VA_RT_FORMAT_RGB32_10, 30, 0x3ff00000, 0xffc00, 0x3ff},
{KMSVNC_FOURCC_TO_INT('A','B','3','0'), NULL, 1, VA_RT_FORMAT_RGB32_10}, {KMSVNC_FOURCC_TO_INT('A','B','3','0'), NULL, 1, VA_RT_FORMAT_RGB32_10, 30, 0x3ff00000, 0xffc00, 0x3ff},
}; };
for (int i = 0; i < va->img_fmt_count; i++) { for (int i = 0; i < va->img_fmt_count; i++) {
for (int j = 0; j < KMSVNC_ARRAY_ELEMENTS(format_to_try); j++) { for (int j = 0; j < KMSVNC_ARRAY_ELEMENTS(format_to_try); j++) {
if (va->img_fmts[i].fourcc == format_to_try[j].va_fourcc) { if (va->img_fmts[i].fourcc == format_to_try[j].va_fourcc) {
@ -292,15 +321,14 @@ int va_init() {
va->derive_enabled = kmsvnc->va_derive_enabled < 0 ? va->derive_enabled : kmsvnc->va_derive_enabled != 0; va->derive_enabled = kmsvnc->va_derive_enabled < 0 ? va->derive_enabled : kmsvnc->va_derive_enabled != 0;
if (va->derive_enabled) { if (va->derive_enabled) {
if ((s = vaDeriveImage(va->dpy, va->surface_id, va->image)) == VA_STATUS_SUCCESS) { if ((s = vaDeriveImage(va->dpy, va->surface_id, va->image)) == VA_STATUS_SUCCESS) {
char found = 0;
for (int i = 0; i < KMSVNC_ARRAY_ELEMENTS(format_to_try); i++) { for (int i = 0; i < KMSVNC_ARRAY_ELEMENTS(format_to_try); i++) {
if (format_to_try[i].fmt == NULL) continue; if (format_to_try[i].fmt == NULL) continue;
if (va->image->format.fourcc == format_to_try[i].fmt->fourcc) { if (va->image->format.fourcc == format_to_try[i].fmt->fourcc) {
found = 1; va->selected_fmt = vaImgFmt_from_vaFmtData(format_to_try + i);
break; break;
} }
} }
if (!found) { if (!va->selected_fmt) {
va->derive_enabled = 0; va->derive_enabled = 0;
printf("vaDeriveImage returned unknown fourcc %d %s\n", va->image->format.fourcc, fourcc_to_str(va->image->format.fourcc)); printf("vaDeriveImage returned unknown fourcc %d %s\n", va->image->format.fourcc, fourcc_to_str(va->image->format.fourcc));
VA_MAY(vaDestroyImage(kmsvnc->va->dpy, kmsvnc->va->image->image_id)); VA_MAY(vaDestroyImage(kmsvnc->va->dpy, kmsvnc->va->image->image_id));
@ -316,7 +344,6 @@ int va_init() {
} }
} }
if (!va->derive_enabled) { if (!va->derive_enabled) {
char success = 0;
for (int i = 0; i < KMSVNC_ARRAY_ELEMENTS(format_to_try); i++) { for (int i = 0; i < KMSVNC_ARRAY_ELEMENTS(format_to_try); i++) {
if (format_to_try[i].fmt == NULL) continue; if (format_to_try[i].fmt == NULL) continue;
if (!kmsvnc->debug_enabled && rt_format != format_to_try[i].va_rt_format) continue; if (!kmsvnc->debug_enabled && rt_format != format_to_try[i].va_rt_format) continue;
@ -342,17 +369,31 @@ int va_init() {
continue; continue;
} }
else { else {
success = 1; va->selected_fmt = vaImgFmt_from_vaFmtData(format_to_try + i);
break; break;
} }
} }
if (!success) { if (!va->selected_fmt) {
va->imgbuf = NULL; va->imgbuf = NULL;
KMSVNC_FATAL("failed to get vaapi image\n"); KMSVNC_FATAL("failed to get vaapi image\n");
} }
} }
printf("got vaapi %simage:\n", va->derive_enabled ? "derive " : ""); printf("got vaapi %simage:\n", va->derive_enabled ? "derive " : "");
print_va_image_fmt(&va->image->format); print_va_image_fmt(&va->image->format);
if (
va->selected_fmt->depth != va->image->format.depth ||
va->selected_fmt->blue_mask != va->image->format.blue_mask ||
va->selected_fmt->green_mask != va->image->format.green_mask ||
va->selected_fmt->red_mask != va->image->format.red_mask ||
va->selected_fmt->fourcc != va->image->format.fourcc
) {
fprintf(stderr, "differs from selected image format\n");
print_va_image_fmt(va->selected_fmt);
if (kmsvnc->trust_va_format) {
fprintf(stderr, "trust VAImageFormat returned by vaapi implementation unconditionally as requested\n");
va->selected_fmt = &va->image->format;
}
}
return 0; return 0;
} }