Jerry/kmsvnc
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

proper byte order quirk handling

Browse source
This commit is contained in:
JerryXiao 2023-09-21 20:46:57 +08:00
parent c9b056a3f5
commit 4ba2de050e
Signed by: Jerry
GPG key ID: 22618F758B5BE2E5
7 changed files with 56 additions and 37 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
drm.c
View file

@ -97,7 +97,10 @@ void convert_intel_x_tiled_kmsbuf(const char *in, int width, int height, char *b
} }
static void convert_vaapi(const char *in, int width, int height, char *buff) { static void convert_vaapi(const char *in, int width, int height, char *buff) {
if ((KMSVNC_FOURCC_TO_INT('R','G','B', 0) & kmsvnc->va->selected_fmt->fourcc) == KMSVNC_FOURCC_TO_INT('R','G','B', 0)) { if (
(!kmsvnc->va->selected_fmt->byte_order && (KMSVNC_FOURCC_TO_INT('R','G','B',0) & kmsvnc->va->selected_fmt->fourcc) == KMSVNC_FOURCC_TO_INT('R','G','B',0)) ||
(kmsvnc->va->selected_fmt->byte_order && (KMSVNC_FOURCC_TO_INT(0,'B','G','R') & kmsvnc->va->selected_fmt->fourcc) == KMSVNC_FOURCC_TO_INT(0,'B','G','R'))
) {
va_hwframe_to_vaapi(buff); va_hwframe_to_vaapi(buff);
} }
else { else {
@ -113,6 +116,15 @@ static void convert_vaapi(const char *in, int width, int height, char *buff) {
kmsvnc->drm->kms_convert_buf[i+2] = (pixdata & 0x3ff) >> 2; kmsvnc->drm->kms_convert_buf[i+2] = (pixdata & 0x3ff) >> 2;
} }
} }
else {
// handle ihd and mesa byte order quirk
if (kmsvnc->va->selected_fmt->byte_order) {
for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) {
uint32_t *pixdata = (uint32_t*)(kmsvnc->drm->kms_convert_buf + i);
*pixdata = __builtin_bswap32(*pixdata);
}
}
}
// is xrgb? // is xrgb?
if ((kmsvnc->va->selected_fmt->blue_mask | kmsvnc->va->selected_fmt->red_mask) < 0x1000000) { if ((kmsvnc->va->selected_fmt->blue_mask | kmsvnc->va->selected_fmt->red_mask) < 0x1000000) {
for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) { for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) {

6
drm.h
View file

@ -2,9 +2,9 @@
#include "kmsvnc.h" #include "kmsvnc.h"
#define DRM_IOCTL_MUST(...) do{ int e; if (e = drmIoctl(__VA_ARGS__)) KMSVNC_FATAL("DRM ioctl error %d on line %d\n", e, __LINE__); } while(0) #define DRM_IOCTL_MUST(...) do{ int e; if ((e = drmIoctl(__VA_ARGS__))) KMSVNC_FATAL("DRM ioctl error %d on line %d\n", e, __LINE__); } while(0)
#define DRM_IOCTL_MAY(...) do{ int e; if (e = drmIoctl(__VA_ARGS__)) fprintf(stderr, "DRM ioctl error %d on line %d\n", e, __LINE__); } while(0) #define DRM_IOCTL_MAY(...) do{ int e; if ((e = drmIoctl(__VA_ARGS__))) fprintf(stderr, "DRM ioctl error %d on line %d\n", e, __LINE__); } while(0)
#define DRM_R_IOCTL_MAY(...) do{ int e; if (e = ioctl(__VA_ARGS__)) fprintf(stderr, "DRM ioctl error %d on line %d\n", e, __LINE__); } while(0) #define DRM_R_IOCTL_MAY(...) do{ int e; if ((e = ioctl(__VA_ARGS__))) fprintf(stderr, "DRM ioctl error %d on line %d\n", e, __LINE__); } while(0)
void drm_cleanup(); void drm_cleanup();

2
drm_master.c
View file

@ -40,6 +40,8 @@ static inline int cmp_fds(pid_t pid, const char *drm_pth) {
char link_pth[PATH_MAX+1]; char link_pth[PATH_MAX+1];
char real_pth[PATH_MAX+1]; char real_pth[PATH_MAX+1];
#pragma GCC diagnostic push #pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wpragmas"
#pragma GCC diagnostic ignored "-Wunknown-warning-option"
#pragma GCC diagnostic ignored "-Wformat-truncation" #pragma GCC diagnostic ignored "-Wformat-truncation"
snprintf(link_pth, PATH_MAX+1, "%s/%s", path, fdlist[n]->d_name); snprintf(link_pth, PATH_MAX+1, "%s/%s", path, fdlist[n]->d_name);
#pragma GCC diagnostic pop #pragma GCC diagnostic pop

4
input.h
View file

@ -7,8 +7,8 @@
#define UINPUT_ABS_MAX INT16_MAX #define UINPUT_ABS_MAX INT16_MAX
#define UINPUT_MAX_KEY 256 #define UINPUT_MAX_KEY 256
#define INP_IOCTL_MUST(...) do{ int e; if (e = ioctl(__VA_ARGS__)) KMSVNC_FATAL("uinput ioctl error %d on line %d\n", e, __LINE__); } while(0) #define INP_IOCTL_MUST(...) do{ int e; if ((e = ioctl(__VA_ARGS__))) KMSVNC_FATAL("uinput ioctl error %d on line %d\n", e, __LINE__); } while(0)
#define INP_IOCTL_MAY(...) do{ int e; if (e = ioctl(__VA_ARGS__)) fprintf(stderr, "uinput ioctl error %d on line %d\n", e, __LINE__); } while(0) #define INP_IOCTL_MAY(...) do{ int e; if ((e = ioctl(__VA_ARGS__))) fprintf(stderr, "uinput ioctl error %d on line %d\n", e, __LINE__); } while(0)
void uinput_cleanup(); void uinput_cleanup();
int uinput_init(); int uinput_init();

4
kmsvnc.c
View file

@ -243,7 +243,6 @@ static struct argp_option kmsvnc_main_options[] = {
{"input-offy", 0xff09, "0", 0, "Set input offset of y axis on a multi display system"}, {"input-offy", 0xff09, "0", 0, "Set input offset of y axis on a multi display system"},
{"screen-blank", 0xff0a, 0, OPTION_ARG_OPTIONAL, "Blank screen with gamma set on crtc"}, {"screen-blank", 0xff0a, 0, OPTION_ARG_OPTIONAL, "Blank screen with gamma set on crtc"},
{"screen-blank-restore-linear", 0xff0b, 0, OPTION_ARG_OPTIONAL, "Restore linear values on exit in case of messed up gamma"}, {"screen-blank-restore-linear", 0xff0b, 0, OPTION_ARG_OPTIONAL, "Restore linear values on exit in case of messed up gamma"},
{"trust-va-format", 0xff0c, 0, OPTION_ARG_OPTIONAL, "trust VAImageFormat returned by vaapi implementation unconditionally"},
{"wakeup", 'w', 0, OPTION_ARG_OPTIONAL, "Move mouse to wake the system up before start"}, {"wakeup", 'w', 0, OPTION_ARG_OPTIONAL, "Move mouse to wake the system up before start"},
{"disable-input", 'i', 0, OPTION_ARG_OPTIONAL, "Disable uinput"}, {"disable-input", 'i', 0, OPTION_ARG_OPTIONAL, "Disable uinput"},
{"desktop-name", 'n', "kmsvnc", 0, "Specify vnc desktop name"}, {"desktop-name", 'n', "kmsvnc", 0, "Specify vnc desktop name"},
@ -361,9 +360,6 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
case 0xff0b: case 0xff0b:
kmsvnc->screen_blank_restore = 1; kmsvnc->screen_blank_restore = 1;
break; break;
case 0xff0c:
kmsvnc->trust_va_format = 1;
break;
case 'w': case 'w':
kmsvnc->input_wakeup = 1; kmsvnc->input_wakeup = 1;
break; break;

1
kmsvnc.h
View file

@ -45,7 +45,6 @@ struct kmsvnc_data
int input_offy; int input_offy;
char screen_blank; char screen_blank;
char screen_blank_restore; char screen_blank_restore;
char trust_va_format;
struct kmsvnc_drm_data *drm; struct kmsvnc_drm_data *drm;
struct kmsvnc_input_data *input; struct kmsvnc_input_data *input;
struct kmsvnc_keymap_data *keymap; struct kmsvnc_keymap_data *keymap;

62
va.c
View file

@ -83,13 +83,14 @@ struct va_fmt_data {
uint32_t blue_mask; uint32_t blue_mask;
uint32_t green_mask; uint32_t green_mask;
uint32_t red_mask; uint32_t red_mask;
uint32_t byte_order;
}; };
static VAImageFormat* vaImgFmt_from_vaFmtData(struct va_fmt_data* data) { static VAImageFormat* vaImgFmt_from_vaFmtData(struct va_fmt_data* data) {
static VAImageFormat ret = {0}; static VAImageFormat ret = {0};
VAImageFormat fmt = { VAImageFormat fmt = {
.fourcc = data->va_fourcc, .fourcc = data->va_fourcc,
.byte_order = VA_LSB_FIRST, .byte_order = data->byte_order,
.bits_per_pixel = 32, .bits_per_pixel = 32,
.depth = data->depth, .depth = data->depth,
.blue_mask = data->blue_mask, .blue_mask = data->blue_mask,
@ -290,26 +291,45 @@ int va_init() {
} }
struct va_fmt_data format_to_try[] = { struct va_fmt_data format_to_try[] = {
{KMSVNC_FOURCC_TO_INT('R','G','B','X'), NULL, 0, VA_RT_FORMAT_RGB32, 24, 0xff00, 0xff0000, 0xff000000}, {KMSVNC_FOURCC_TO_INT('R','G','B','X'), NULL, 0, VA_RT_FORMAT_RGB32, 24, 0xff00, 0xff0000, 0xff000000, 0},
{KMSVNC_FOURCC_TO_INT('R','G','B','A'), NULL, 1, VA_RT_FORMAT_RGB32, 32, 0xff00, 0xff0000, 0xff000000}, {KMSVNC_FOURCC_TO_INT('R','G','B','A'), NULL, 1, VA_RT_FORMAT_RGB32, 32, 0xff00, 0xff0000, 0xff000000, 0},
{KMSVNC_FOURCC_TO_INT('X','R','G','B'), NULL, 0, VA_RT_FORMAT_RGB32, 24, 0xff, 0xff00, 0xff0000},
{KMSVNC_FOURCC_TO_INT('A','R','G','B'), NULL, 1, VA_RT_FORMAT_RGB32, 32, 0xff, 0xff00, 0xff0000},
{KMSVNC_FOURCC_TO_INT('B','G','R','X'), NULL, 0, VA_RT_FORMAT_RGB32, 24, 0xff000000, 0xff0000, 0xff00}, {KMSVNC_FOURCC_TO_INT('X','B','G','R'), NULL, 0, VA_RT_FORMAT_RGB32, 24, 0xff0000, 0xff00, 0xff, 0},
{KMSVNC_FOURCC_TO_INT('B','G','R','A'), NULL, 1, VA_RT_FORMAT_RGB32, 32, 0xff000000, 0xff0000, 0xff00}, {KMSVNC_FOURCC_TO_INT('A','B','G','R'), NULL, 1, VA_RT_FORMAT_RGB32, 32, 0xff0000, 0xff00, 0xff, 0},
{KMSVNC_FOURCC_TO_INT('X','B','G','R'), NULL, 0, VA_RT_FORMAT_RGB32, 24, 0xff0000, 0xff00, 0xff},
{KMSVNC_FOURCC_TO_INT('A','B','G','R'), NULL, 1, VA_RT_FORMAT_RGB32, 32, 0xff0000, 0xff00, 0xff},
{KMSVNC_FOURCC_TO_INT('X','R','3','0'), NULL, 0, VA_RT_FORMAT_RGB32_10, 30, 0x3ff, 0xffc00, 0x3ff00000}, {KMSVNC_FOURCC_TO_INT('X','R','G','B'), NULL, 0, VA_RT_FORMAT_RGB32, 24, 0xff, 0xff00, 0xff0000, 0},
{KMSVNC_FOURCC_TO_INT('A','R','3','0'), NULL, 1, VA_RT_FORMAT_RGB32_10, 30, 0x3ff, 0xffc00, 0x3ff00000}, {KMSVNC_FOURCC_TO_INT('A','R','G','B'), NULL, 1, VA_RT_FORMAT_RGB32, 32, 0xff, 0xff00, 0xff0000, 0},
{KMSVNC_FOURCC_TO_INT('X','B','3','0'), NULL, 0, VA_RT_FORMAT_RGB32_10, 30, 0x3ff00000, 0xffc00, 0x3ff},
{KMSVNC_FOURCC_TO_INT('A','B','3','0'), NULL, 1, VA_RT_FORMAT_RGB32_10, 30, 0x3ff00000, 0xffc00, 0x3ff}, {KMSVNC_FOURCC_TO_INT('B','G','R','X'), NULL, 0, VA_RT_FORMAT_RGB32, 24, 0xff000000, 0xff0000, 0xff00, 0},
{KMSVNC_FOURCC_TO_INT('B','G','R','A'), NULL, 1, VA_RT_FORMAT_RGB32, 32, 0xff000000, 0xff0000, 0xff00, 0},
{KMSVNC_FOURCC_TO_INT('X','R','3','0'), NULL, 0, VA_RT_FORMAT_RGB32_10, 30, 0x3ff, 0xffc00, 0x3ff00000, 0},
{KMSVNC_FOURCC_TO_INT('A','R','3','0'), NULL, 1, VA_RT_FORMAT_RGB32_10, 30, 0x3ff, 0xffc00, 0x3ff00000, 0},
{KMSVNC_FOURCC_TO_INT('X','B','3','0'), NULL, 0, VA_RT_FORMAT_RGB32_10, 30, 0x3ff00000, 0xffc00, 0x3ff, 0},
{KMSVNC_FOURCC_TO_INT('A','B','3','0'), NULL, 1, VA_RT_FORMAT_RGB32_10, 30, 0x3ff00000, 0xffc00, 0x3ff, 0},
}; };
for (int i = 0; i < va->img_fmt_count; i++) { for (int i = 0; i < va->img_fmt_count; i++) {
for (int j = 0; j < KMSVNC_ARRAY_ELEMENTS(format_to_try); j++) { for (int j = 0; j < KMSVNC_ARRAY_ELEMENTS(format_to_try); j++) {
if (va->img_fmts[i].fourcc == format_to_try[j].va_fourcc) { if (va->img_fmts[i].fourcc == format_to_try[j].va_fourcc) {
format_to_try[j].fmt = va->img_fmts + i; if (
va->img_fmts[i].blue_mask == format_to_try[j].blue_mask &&
va->img_fmts[i].green_mask == format_to_try[j].green_mask &&
va->img_fmts[i].red_mask == format_to_try[j].red_mask
) {
format_to_try[j].fmt = va->img_fmts + i;
}
else if (
format_to_try[j].depth != 30 &&
va->img_fmts[i].blue_mask == __builtin_bswap32(format_to_try[j].blue_mask) &&
va->img_fmts[i].green_mask == __builtin_bswap32(format_to_try[j].green_mask) &&
va->img_fmts[i].red_mask == __builtin_bswap32(format_to_try[j].red_mask)
) {
// mesa quirk: mesa fourcc and pixel data is lsb_first for RGB32 formats, msb_first for RGB32_10 formats
format_to_try[j].byte_order = 1u;
format_to_try[j].fmt = va->img_fmts + i;
}
} }
} }
} }
@ -380,19 +400,9 @@ int va_init() {
} }
printf("got vaapi %simage:\n", va->derive_enabled ? "derive " : ""); printf("got vaapi %simage:\n", va->derive_enabled ? "derive " : "");
print_va_image_fmt(&va->image->format); print_va_image_fmt(&va->image->format);
if ( if (kmsvnc->debug_enabled) {
va->selected_fmt->depth != va->image->format.depth || fprintf(stderr, "selected image format:\n");
va->selected_fmt->blue_mask != va->image->format.blue_mask ||
va->selected_fmt->green_mask != va->image->format.green_mask ||
va->selected_fmt->red_mask != va->image->format.red_mask ||
va->selected_fmt->fourcc != va->image->format.fourcc
) {
fprintf(stderr, "differs from selected image format\n");
print_va_image_fmt(va->selected_fmt); print_va_image_fmt(va->selected_fmt);
if (kmsvnc->trust_va_format) {
fprintf(stderr, "trust VAImageFormat returned by vaapi implementation unconditionally as requested\n");
va->selected_fmt = &va->image->format;
}
} }
return 0; return 0;
} }