From 4ba2de050e787134872f02bdf8740ff6e0b5d2b2 Mon Sep 17 00:00:00 2001 From: Jerry Date: Thu, 21 Sep 2023 20:46:57 +0800 Subject: [PATCH] proper byte order quirk handling --- drm.c | 14 +++++++++++- drm.h | 6 ++--- drm_master.c | 2 ++ input.h | 4 ++-- kmsvnc.c | 4 ---- kmsvnc.h | 1 - va.c | 62 ++++++++++++++++++++++++++++++---------------------- 7 files changed, 56 insertions(+), 37 deletions(-) diff --git a/drm.c b/drm.c index fd17b5f..90607fd 100644 --- a/drm.c +++ b/drm.c @@ -97,7 +97,10 @@ void convert_intel_x_tiled_kmsbuf(const char *in, int width, int height, char *b } static void convert_vaapi(const char *in, int width, int height, char *buff) { - if ((KMSVNC_FOURCC_TO_INT('R','G','B', 0) & kmsvnc->va->selected_fmt->fourcc) == KMSVNC_FOURCC_TO_INT('R','G','B', 0)) { + if ( + (!kmsvnc->va->selected_fmt->byte_order && (KMSVNC_FOURCC_TO_INT('R','G','B',0) & kmsvnc->va->selected_fmt->fourcc) == KMSVNC_FOURCC_TO_INT('R','G','B',0)) || + (kmsvnc->va->selected_fmt->byte_order && (KMSVNC_FOURCC_TO_INT(0,'B','G','R') & kmsvnc->va->selected_fmt->fourcc) == KMSVNC_FOURCC_TO_INT(0,'B','G','R')) + ) { va_hwframe_to_vaapi(buff); } else { @@ -113,6 +116,15 @@ static void convert_vaapi(const char *in, int width, int height, char *buff) { kmsvnc->drm->kms_convert_buf[i+2] = (pixdata & 0x3ff) >> 2; } } + else { + // handle ihd and mesa byte order quirk + if (kmsvnc->va->selected_fmt->byte_order) { + for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) { + uint32_t *pixdata = (uint32_t*)(kmsvnc->drm->kms_convert_buf + i); + *pixdata = __builtin_bswap32(*pixdata); + } + } + } // is xrgb? if ((kmsvnc->va->selected_fmt->blue_mask | kmsvnc->va->selected_fmt->red_mask) < 0x1000000) { for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) { diff --git a/drm.h b/drm.h index a2ba9b2..483677f 100644 --- a/drm.h +++ b/drm.h @@ -2,9 +2,9 @@ #include "kmsvnc.h" -#define DRM_IOCTL_MUST(...) do{ int e; if (e = drmIoctl(__VA_ARGS__)) KMSVNC_FATAL("DRM ioctl error %d on line %d\n", e, __LINE__); } while(0) -#define DRM_IOCTL_MAY(...) do{ int e; if (e = drmIoctl(__VA_ARGS__)) fprintf(stderr, "DRM ioctl error %d on line %d\n", e, __LINE__); } while(0) -#define DRM_R_IOCTL_MAY(...) do{ int e; if (e = ioctl(__VA_ARGS__)) fprintf(stderr, "DRM ioctl error %d on line %d\n", e, __LINE__); } while(0) +#define DRM_IOCTL_MUST(...) do{ int e; if ((e = drmIoctl(__VA_ARGS__))) KMSVNC_FATAL("DRM ioctl error %d on line %d\n", e, __LINE__); } while(0) +#define DRM_IOCTL_MAY(...) do{ int e; if ((e = drmIoctl(__VA_ARGS__))) fprintf(stderr, "DRM ioctl error %d on line %d\n", e, __LINE__); } while(0) +#define DRM_R_IOCTL_MAY(...) do{ int e; if ((e = ioctl(__VA_ARGS__))) fprintf(stderr, "DRM ioctl error %d on line %d\n", e, __LINE__); } while(0) void drm_cleanup(); diff --git a/drm_master.c b/drm_master.c index b48a85e..dc08586 100644 --- a/drm_master.c +++ b/drm_master.c @@ -40,6 +40,8 @@ static inline int cmp_fds(pid_t pid, const char *drm_pth) { char link_pth[PATH_MAX+1]; char real_pth[PATH_MAX+1]; #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wpragmas" + #pragma GCC diagnostic ignored "-Wunknown-warning-option" #pragma GCC diagnostic ignored "-Wformat-truncation" snprintf(link_pth, PATH_MAX+1, "%s/%s", path, fdlist[n]->d_name); #pragma GCC diagnostic pop diff --git a/input.h b/input.h index 5877bcf..212f342 100644 --- a/input.h +++ b/input.h @@ -7,8 +7,8 @@ #define UINPUT_ABS_MAX INT16_MAX #define UINPUT_MAX_KEY 256 -#define INP_IOCTL_MUST(...) do{ int e; if (e = ioctl(__VA_ARGS__)) KMSVNC_FATAL("uinput ioctl error %d on line %d\n", e, __LINE__); } while(0) -#define INP_IOCTL_MAY(...) do{ int e; if (e = ioctl(__VA_ARGS__)) fprintf(stderr, "uinput ioctl error %d on line %d\n", e, __LINE__); } while(0) +#define INP_IOCTL_MUST(...) do{ int e; if ((e = ioctl(__VA_ARGS__))) KMSVNC_FATAL("uinput ioctl error %d on line %d\n", e, __LINE__); } while(0) +#define INP_IOCTL_MAY(...) do{ int e; if ((e = ioctl(__VA_ARGS__))) fprintf(stderr, "uinput ioctl error %d on line %d\n", e, __LINE__); } while(0) void uinput_cleanup(); int uinput_init(); diff --git a/kmsvnc.c b/kmsvnc.c index 47b528e..5b738b3 100644 --- a/kmsvnc.c +++ b/kmsvnc.c @@ -243,7 +243,6 @@ static struct argp_option kmsvnc_main_options[] = { {"input-offy", 0xff09, "0", 0, "Set input offset of y axis on a multi display system"}, {"screen-blank", 0xff0a, 0, OPTION_ARG_OPTIONAL, "Blank screen with gamma set on crtc"}, {"screen-blank-restore-linear", 0xff0b, 0, OPTION_ARG_OPTIONAL, "Restore linear values on exit in case of messed up gamma"}, - {"trust-va-format", 0xff0c, 0, OPTION_ARG_OPTIONAL, "trust VAImageFormat returned by vaapi implementation unconditionally"}, {"wakeup", 'w', 0, OPTION_ARG_OPTIONAL, "Move mouse to wake the system up before start"}, {"disable-input", 'i', 0, OPTION_ARG_OPTIONAL, "Disable uinput"}, {"desktop-name", 'n', "kmsvnc", 0, "Specify vnc desktop name"}, @@ -361,9 +360,6 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { case 0xff0b: kmsvnc->screen_blank_restore = 1; break; - case 0xff0c: - kmsvnc->trust_va_format = 1; - break; case 'w': kmsvnc->input_wakeup = 1; break; diff --git a/kmsvnc.h b/kmsvnc.h index 1ac04a4..327c9f6 100644 --- a/kmsvnc.h +++ b/kmsvnc.h @@ -45,7 +45,6 @@ struct kmsvnc_data int input_offy; char screen_blank; char screen_blank_restore; - char trust_va_format; struct kmsvnc_drm_data *drm; struct kmsvnc_input_data *input; struct kmsvnc_keymap_data *keymap; diff --git a/va.c b/va.c index f67232d..fed9b57 100644 --- a/va.c +++ b/va.c @@ -83,13 +83,14 @@ struct va_fmt_data { uint32_t blue_mask; uint32_t green_mask; uint32_t red_mask; + uint32_t byte_order; }; static VAImageFormat* vaImgFmt_from_vaFmtData(struct va_fmt_data* data) { static VAImageFormat ret = {0}; VAImageFormat fmt = { .fourcc = data->va_fourcc, - .byte_order = VA_LSB_FIRST, + .byte_order = data->byte_order, .bits_per_pixel = 32, .depth = data->depth, .blue_mask = data->blue_mask, @@ -290,26 +291,45 @@ int va_init() { } struct va_fmt_data format_to_try[] = { - {KMSVNC_FOURCC_TO_INT('R','G','B','X'), NULL, 0, VA_RT_FORMAT_RGB32, 24, 0xff00, 0xff0000, 0xff000000}, - {KMSVNC_FOURCC_TO_INT('R','G','B','A'), NULL, 1, VA_RT_FORMAT_RGB32, 32, 0xff00, 0xff0000, 0xff000000}, - {KMSVNC_FOURCC_TO_INT('X','R','G','B'), NULL, 0, VA_RT_FORMAT_RGB32, 24, 0xff, 0xff00, 0xff0000}, - {KMSVNC_FOURCC_TO_INT('A','R','G','B'), NULL, 1, VA_RT_FORMAT_RGB32, 32, 0xff, 0xff00, 0xff0000}, + {KMSVNC_FOURCC_TO_INT('R','G','B','X'), NULL, 0, VA_RT_FORMAT_RGB32, 24, 0xff00, 0xff0000, 0xff000000, 0}, + {KMSVNC_FOURCC_TO_INT('R','G','B','A'), NULL, 1, VA_RT_FORMAT_RGB32, 32, 0xff00, 0xff0000, 0xff000000, 0}, - {KMSVNC_FOURCC_TO_INT('B','G','R','X'), NULL, 0, VA_RT_FORMAT_RGB32, 24, 0xff000000, 0xff0000, 0xff00}, - {KMSVNC_FOURCC_TO_INT('B','G','R','A'), NULL, 1, VA_RT_FORMAT_RGB32, 32, 0xff000000, 0xff0000, 0xff00}, - {KMSVNC_FOURCC_TO_INT('X','B','G','R'), NULL, 0, VA_RT_FORMAT_RGB32, 24, 0xff0000, 0xff00, 0xff}, - {KMSVNC_FOURCC_TO_INT('A','B','G','R'), NULL, 1, VA_RT_FORMAT_RGB32, 32, 0xff0000, 0xff00, 0xff}, + {KMSVNC_FOURCC_TO_INT('X','B','G','R'), NULL, 0, VA_RT_FORMAT_RGB32, 24, 0xff0000, 0xff00, 0xff, 0}, + {KMSVNC_FOURCC_TO_INT('A','B','G','R'), NULL, 1, VA_RT_FORMAT_RGB32, 32, 0xff0000, 0xff00, 0xff, 0}, - {KMSVNC_FOURCC_TO_INT('X','R','3','0'), NULL, 0, VA_RT_FORMAT_RGB32_10, 30, 0x3ff, 0xffc00, 0x3ff00000}, - {KMSVNC_FOURCC_TO_INT('A','R','3','0'), NULL, 1, VA_RT_FORMAT_RGB32_10, 30, 0x3ff, 0xffc00, 0x3ff00000}, - {KMSVNC_FOURCC_TO_INT('X','B','3','0'), NULL, 0, VA_RT_FORMAT_RGB32_10, 30, 0x3ff00000, 0xffc00, 0x3ff}, - {KMSVNC_FOURCC_TO_INT('A','B','3','0'), NULL, 1, VA_RT_FORMAT_RGB32_10, 30, 0x3ff00000, 0xffc00, 0x3ff}, + {KMSVNC_FOURCC_TO_INT('X','R','G','B'), NULL, 0, VA_RT_FORMAT_RGB32, 24, 0xff, 0xff00, 0xff0000, 0}, + {KMSVNC_FOURCC_TO_INT('A','R','G','B'), NULL, 1, VA_RT_FORMAT_RGB32, 32, 0xff, 0xff00, 0xff0000, 0}, + + {KMSVNC_FOURCC_TO_INT('B','G','R','X'), NULL, 0, VA_RT_FORMAT_RGB32, 24, 0xff000000, 0xff0000, 0xff00, 0}, + {KMSVNC_FOURCC_TO_INT('B','G','R','A'), NULL, 1, VA_RT_FORMAT_RGB32, 32, 0xff000000, 0xff0000, 0xff00, 0}, + + + {KMSVNC_FOURCC_TO_INT('X','R','3','0'), NULL, 0, VA_RT_FORMAT_RGB32_10, 30, 0x3ff, 0xffc00, 0x3ff00000, 0}, + {KMSVNC_FOURCC_TO_INT('A','R','3','0'), NULL, 1, VA_RT_FORMAT_RGB32_10, 30, 0x3ff, 0xffc00, 0x3ff00000, 0}, + {KMSVNC_FOURCC_TO_INT('X','B','3','0'), NULL, 0, VA_RT_FORMAT_RGB32_10, 30, 0x3ff00000, 0xffc00, 0x3ff, 0}, + {KMSVNC_FOURCC_TO_INT('A','B','3','0'), NULL, 1, VA_RT_FORMAT_RGB32_10, 30, 0x3ff00000, 0xffc00, 0x3ff, 0}, }; for (int i = 0; i < va->img_fmt_count; i++) { for (int j = 0; j < KMSVNC_ARRAY_ELEMENTS(format_to_try); j++) { if (va->img_fmts[i].fourcc == format_to_try[j].va_fourcc) { - format_to_try[j].fmt = va->img_fmts + i; + if ( + va->img_fmts[i].blue_mask == format_to_try[j].blue_mask && + va->img_fmts[i].green_mask == format_to_try[j].green_mask && + va->img_fmts[i].red_mask == format_to_try[j].red_mask + ) { + format_to_try[j].fmt = va->img_fmts + i; + } + else if ( + format_to_try[j].depth != 30 && + va->img_fmts[i].blue_mask == __builtin_bswap32(format_to_try[j].blue_mask) && + va->img_fmts[i].green_mask == __builtin_bswap32(format_to_try[j].green_mask) && + va->img_fmts[i].red_mask == __builtin_bswap32(format_to_try[j].red_mask) + ) { + // mesa quirk: mesa fourcc and pixel data is lsb_first for RGB32 formats, msb_first for RGB32_10 formats + format_to_try[j].byte_order = 1u; + format_to_try[j].fmt = va->img_fmts + i; + } } } } @@ -380,19 +400,9 @@ int va_init() { } printf("got vaapi %simage:\n", va->derive_enabled ? "derive " : ""); print_va_image_fmt(&va->image->format); - if ( - va->selected_fmt->depth != va->image->format.depth || - va->selected_fmt->blue_mask != va->image->format.blue_mask || - va->selected_fmt->green_mask != va->image->format.green_mask || - va->selected_fmt->red_mask != va->image->format.red_mask || - va->selected_fmt->fourcc != va->image->format.fourcc - ) { - fprintf(stderr, "differs from selected image format\n"); + if (kmsvnc->debug_enabled) { + fprintf(stderr, "selected image format:\n"); print_va_image_fmt(va->selected_fmt); - if (kmsvnc->trust_va_format) { - fprintf(stderr, "trust VAImageFormat returned by vaapi implementation unconditionally as requested\n"); - va->selected_fmt = &va->image->format; - } } return 0; }