From 19e048b56a6bab948f620537d312471d8d600f0f Mon Sep 17 00:00:00 2001 From: Jerry Date: Tue, 23 Jan 2024 10:40:35 +0800 Subject: [PATCH] finally fixed color issue, github #7 --- drm.c | 14 ++-------- kmsvnc.c | 4 +++ kmsvnc.h | 2 ++ va.c | 80 +++++++++++++++++++++----------------------------------- 4 files changed, 38 insertions(+), 62 deletions(-) diff --git a/drm.c b/drm.c index 4e22eaa..d472633 100644 --- a/drm.c +++ b/drm.c @@ -103,18 +103,8 @@ void convert_intel_x_tiled_kmsbuf(const char *in, int width, int height, char *b static void convert_vaapi(const char *in, int width, int height, char *buff) { va_hwframe_to_vaapi(buff); if ( - !kmsvnc->va->selected_fmt->byte_order && (KMSVNC_FOURCC_TO_INT('R','G','B',0) & kmsvnc->va->selected_fmt->fourcc) == KMSVNC_FOURCC_TO_INT('R','G','B',0) ) {} - else if ( - kmsvnc->va->selected_fmt->byte_order && - (KMSVNC_FOURCC_TO_INT(0,'B','G','R') & kmsvnc->va->selected_fmt->fourcc) == KMSVNC_FOURCC_TO_INT(0,'B','G','R') - ) { - for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) { - uint32_t *pixdata = (uint32_t*)(buff + i); - *pixdata = __builtin_bswap32(*pixdata); - } - } else { // is 30 depth? if (kmsvnc->va->selected_fmt->depth == 30) { @@ -127,8 +117,8 @@ static void convert_vaapi(const char *in, int width, int height, char *buff) { } } else { - // handle ihd and mesa byte order quirk - if (kmsvnc->va->selected_fmt->byte_order) { + // actually, does anyone use this? + if (!kmsvnc->va->selected_fmt->byte_order) { for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) { uint32_t *pixdata = (uint32_t*)(buff + i); *pixdata = __builtin_bswap32(*pixdata); diff --git a/kmsvnc.c b/kmsvnc.c index 5b738b3..8d3ee7d 100644 --- a/kmsvnc.c +++ b/kmsvnc.c @@ -243,6 +243,7 @@ static struct argp_option kmsvnc_main_options[] = { {"input-offy", 0xff09, "0", 0, "Set input offset of y axis on a multi display system"}, {"screen-blank", 0xff0a, 0, OPTION_ARG_OPTIONAL, "Blank screen with gamma set on crtc"}, {"screen-blank-restore-linear", 0xff0b, 0, OPTION_ARG_OPTIONAL, "Restore linear values on exit in case of messed up gamma"}, + {"va-byteorder-swap", 0xff0c, 0, OPTION_ARG_OPTIONAL, "Force swap vaapi image rgb byteorder"}, {"wakeup", 'w', 0, OPTION_ARG_OPTIONAL, "Move mouse to wake the system up before start"}, {"disable-input", 'i', 0, OPTION_ARG_OPTIONAL, "Disable uinput"}, {"desktop-name", 'n', "kmsvnc", 0, "Specify vnc desktop name"}, @@ -360,6 +361,9 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { case 0xff0b: kmsvnc->screen_blank_restore = 1; break; + case 0xff0c: + kmsvnc->va_byteorder_swap = 1; + break; case 'w': kmsvnc->input_wakeup = 1; break; diff --git a/kmsvnc.h b/kmsvnc.h index 25f135e..14f7db8 100644 --- a/kmsvnc.h +++ b/kmsvnc.h @@ -45,6 +45,7 @@ struct kmsvnc_data int input_offy; char screen_blank; char screen_blank_restore; + char va_byteorder_swap; struct kmsvnc_drm_data *drm; struct kmsvnc_input_data *input; struct kmsvnc_keymap_data *keymap; @@ -139,6 +140,7 @@ struct kmsvnc_va_data VAImageFormat* img_fmts; int img_fmt_count; VAImageFormat* selected_fmt; + const char *vendor_string; }; #define KMSVNC_FATAL(...) do{ fprintf(stderr, __VA_ARGS__); return 1; } while(0) diff --git a/va.c b/va.c index fed9b57..55b1d48 100644 --- a/va.c +++ b/va.c @@ -6,6 +6,7 @@ #include #include #include +#include #include "va.h" #include "kmsvnc.h" @@ -38,6 +39,9 @@ void va_cleanup() { VA_MAY(vaTerminate(kmsvnc->va->dpy)); kmsvnc->va->dpy = NULL; } + if (kmsvnc->va->vendor_string) { + kmsvnc->va->vendor_string = NULL; + } free(kmsvnc->va); kmsvnc->va = NULL; } @@ -80,26 +84,17 @@ struct va_fmt_data { char is_alpha; uint32_t va_rt_format; uint32_t depth; - uint32_t blue_mask; - uint32_t green_mask; - uint32_t red_mask; - uint32_t byte_order; }; -static VAImageFormat* vaImgFmt_from_vaFmtData(struct va_fmt_data* data) { +static VAImageFormat* vaImgFmt_apply_quirks(struct va_fmt_data* data) { static VAImageFormat ret = {0}; - VAImageFormat fmt = { - .fourcc = data->va_fourcc, - .byte_order = data->byte_order, - .bits_per_pixel = 32, - .depth = data->depth, - .blue_mask = data->blue_mask, - .green_mask = data->green_mask, - .red_mask = data->red_mask, - .alpha_mask = 0, - .va_reserved = {0} - }; - memcpy(&ret, &fmt, sizeof(VAImageFormat)); + memcpy(&ret, data->fmt, sizeof(VAImageFormat)); + if ((kmsvnc->va_byteorder_swap ^ !strncmp(kmsvnc->va->vendor_string, "Mesa", 4)) && data->depth != 30) { + printf("applying rgb mask byte order swap\n"); + ret.blue_mask = __builtin_bswap32(data->fmt->blue_mask); + ret.green_mask = __builtin_bswap32(data->fmt->green_mask); + ret.red_mask = __builtin_bswap32(data->fmt->red_mask); + } return &ret; } @@ -126,6 +121,7 @@ int va_init() { } setenv("DISPLAY", "", 1); + setenv("WAYLAND_DISPLAY", "", 1); struct kmsvnc_va_data *va = malloc(sizeof(struct kmsvnc_va_data)); if (!va) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__); @@ -161,8 +157,8 @@ int va_init() { VAStatus status; VA_MUST(vaInitialize(va->dpy, &major, &minor)); - const char *vendor_string = vaQueryVendorString(va->dpy); - printf("vaapi vendor %s\n", vendor_string); + va->vendor_string = vaQueryVendorString(va->dpy); + printf("vaapi vendor %s\n", va->vendor_string); VADRMPRIMESurfaceDescriptor prime_desc; VASurfaceAttrib prime_attrs[2] = { @@ -291,45 +287,29 @@ int va_init() { } struct va_fmt_data format_to_try[] = { - {KMSVNC_FOURCC_TO_INT('R','G','B','X'), NULL, 0, VA_RT_FORMAT_RGB32, 24, 0xff00, 0xff0000, 0xff000000, 0}, - {KMSVNC_FOURCC_TO_INT('R','G','B','A'), NULL, 1, VA_RT_FORMAT_RGB32, 32, 0xff00, 0xff0000, 0xff000000, 0}, + {KMSVNC_FOURCC_TO_INT('R','G','B','X'), NULL, 0, VA_RT_FORMAT_RGB32, 24}, + {KMSVNC_FOURCC_TO_INT('R','G','B','A'), NULL, 1, VA_RT_FORMAT_RGB32, 32}, - {KMSVNC_FOURCC_TO_INT('X','B','G','R'), NULL, 0, VA_RT_FORMAT_RGB32, 24, 0xff0000, 0xff00, 0xff, 0}, - {KMSVNC_FOURCC_TO_INT('A','B','G','R'), NULL, 1, VA_RT_FORMAT_RGB32, 32, 0xff0000, 0xff00, 0xff, 0}, + {KMSVNC_FOURCC_TO_INT('X','B','G','R'), NULL, 0, VA_RT_FORMAT_RGB32, 24}, + {KMSVNC_FOURCC_TO_INT('A','B','G','R'), NULL, 1, VA_RT_FORMAT_RGB32, 32}, - {KMSVNC_FOURCC_TO_INT('X','R','G','B'), NULL, 0, VA_RT_FORMAT_RGB32, 24, 0xff, 0xff00, 0xff0000, 0}, - {KMSVNC_FOURCC_TO_INT('A','R','G','B'), NULL, 1, VA_RT_FORMAT_RGB32, 32, 0xff, 0xff00, 0xff0000, 0}, + {KMSVNC_FOURCC_TO_INT('X','R','G','B'), NULL, 0, VA_RT_FORMAT_RGB32, 24}, + {KMSVNC_FOURCC_TO_INT('A','R','G','B'), NULL, 1, VA_RT_FORMAT_RGB32, 32}, - {KMSVNC_FOURCC_TO_INT('B','G','R','X'), NULL, 0, VA_RT_FORMAT_RGB32, 24, 0xff000000, 0xff0000, 0xff00, 0}, - {KMSVNC_FOURCC_TO_INT('B','G','R','A'), NULL, 1, VA_RT_FORMAT_RGB32, 32, 0xff000000, 0xff0000, 0xff00, 0}, + {KMSVNC_FOURCC_TO_INT('B','G','R','X'), NULL, 0, VA_RT_FORMAT_RGB32, 24}, + {KMSVNC_FOURCC_TO_INT('B','G','R','A'), NULL, 1, VA_RT_FORMAT_RGB32, 32}, - {KMSVNC_FOURCC_TO_INT('X','R','3','0'), NULL, 0, VA_RT_FORMAT_RGB32_10, 30, 0x3ff, 0xffc00, 0x3ff00000, 0}, - {KMSVNC_FOURCC_TO_INT('A','R','3','0'), NULL, 1, VA_RT_FORMAT_RGB32_10, 30, 0x3ff, 0xffc00, 0x3ff00000, 0}, - {KMSVNC_FOURCC_TO_INT('X','B','3','0'), NULL, 0, VA_RT_FORMAT_RGB32_10, 30, 0x3ff00000, 0xffc00, 0x3ff, 0}, - {KMSVNC_FOURCC_TO_INT('A','B','3','0'), NULL, 1, VA_RT_FORMAT_RGB32_10, 30, 0x3ff00000, 0xffc00, 0x3ff, 0}, + {KMSVNC_FOURCC_TO_INT('X','R','3','0'), NULL, 0, VA_RT_FORMAT_RGB32_10, 30}, + {KMSVNC_FOURCC_TO_INT('A','R','3','0'), NULL, 1, VA_RT_FORMAT_RGB32_10, 30}, + {KMSVNC_FOURCC_TO_INT('X','B','3','0'), NULL, 0, VA_RT_FORMAT_RGB32_10, 30}, + {KMSVNC_FOURCC_TO_INT('A','B','3','0'), NULL, 1, VA_RT_FORMAT_RGB32_10, 30}, }; for (int i = 0; i < va->img_fmt_count; i++) { for (int j = 0; j < KMSVNC_ARRAY_ELEMENTS(format_to_try); j++) { if (va->img_fmts[i].fourcc == format_to_try[j].va_fourcc) { - if ( - va->img_fmts[i].blue_mask == format_to_try[j].blue_mask && - va->img_fmts[i].green_mask == format_to_try[j].green_mask && - va->img_fmts[i].red_mask == format_to_try[j].red_mask - ) { - format_to_try[j].fmt = va->img_fmts + i; - } - else if ( - format_to_try[j].depth != 30 && - va->img_fmts[i].blue_mask == __builtin_bswap32(format_to_try[j].blue_mask) && - va->img_fmts[i].green_mask == __builtin_bswap32(format_to_try[j].green_mask) && - va->img_fmts[i].red_mask == __builtin_bswap32(format_to_try[j].red_mask) - ) { - // mesa quirk: mesa fourcc and pixel data is lsb_first for RGB32 formats, msb_first for RGB32_10 formats - format_to_try[j].byte_order = 1u; - format_to_try[j].fmt = va->img_fmts + i; - } + format_to_try[j].fmt = va->img_fmts + i; } } } @@ -344,7 +324,7 @@ int va_init() { for (int i = 0; i < KMSVNC_ARRAY_ELEMENTS(format_to_try); i++) { if (format_to_try[i].fmt == NULL) continue; if (va->image->format.fourcc == format_to_try[i].fmt->fourcc) { - va->selected_fmt = vaImgFmt_from_vaFmtData(format_to_try + i); + va->selected_fmt = vaImgFmt_apply_quirks(format_to_try + i); break; } } @@ -389,7 +369,7 @@ int va_init() { continue; } else { - va->selected_fmt = vaImgFmt_from_vaFmtData(format_to_try + i); + va->selected_fmt = vaImgFmt_apply_quirks(format_to_try + i); break; } }