From c9b056a3f5671eedde9e04268f9b4cbed10e2785 Mon Sep 17 00:00:00 2001 From: Jerry Date: Thu, 21 Sep 2023 16:06:31 +0800 Subject: [PATCH] use hardcoded image format by default (github #7) --- .drone.yml | 11 ++--- drm.c | 8 ++-- drm_master.c | 10 +++++ kmsvnc.c | 68 ++++++++++++++++++----------- kmsvnc.h | 3 ++ va.c | 119 ++++++++++++++++++++++++++++++++++----------------- 6 files changed, 146 insertions(+), 73 deletions(-) diff --git a/.drone.yml b/.drone.yml index 352fef6..cc2f441 100644 --- a/.drone.yml +++ b/.drone.yml @@ -6,11 +6,12 @@ steps: - name: build image: archlinux:latest commands: - - pacman -Syu --noconfirm --needed base-devel libvncserver libxkbcommon libdrm libva git cmake - - mkdir build - - cd build - - cmake .. - - make + - pacman -Syu --noconfirm --needed base-devel libvncserver libxkbcommon libdrm libva git cmake clang + - export CFLAGS="-pipe -fno-plt -fexceptions -fstack-clash-protection -fcf-protection -Wp,-D_FORTIFY_SOURCE=2 -Wformat -Werror=format-security" + - CC=gcc cmake -B gcc-out + - cmake --build gcc-out + - CC=clang cmake -B clang-out + - cmake --build clang-out trigger: branch: diff --git a/drm.c b/drm.c index 6deaef3..fd17b5f 100644 --- a/drm.c +++ b/drm.c @@ -97,14 +97,14 @@ void convert_intel_x_tiled_kmsbuf(const char *in, int width, int height, char *b } static void convert_vaapi(const char *in, int width, int height, char *buff) { - if (KMSVNC_FOURCC_TO_INT('R','G','B', 0) & kmsvnc->va->image->format.fourcc == KMSVNC_FOURCC_TO_INT('R','G','B', 0)) { + if ((KMSVNC_FOURCC_TO_INT('R','G','B', 0) & kmsvnc->va->selected_fmt->fourcc) == KMSVNC_FOURCC_TO_INT('R','G','B', 0)) { va_hwframe_to_vaapi(buff); } else { if (convert_buf_allocate(width * height * BYTES_PER_PIXEL)) return; va_hwframe_to_vaapi(kmsvnc->drm->kms_convert_buf); // is 30 depth? - if (kmsvnc->va->image->format.depth == 30) { + if (kmsvnc->va->selected_fmt->depth == 30) { for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) { // ensure little endianess uint32_t pixdata = __builtin_bswap32(htonl(*((uint32_t*)(kmsvnc->drm->kms_convert_buf + i)))); @@ -114,14 +114,14 @@ static void convert_vaapi(const char *in, int width, int height, char *buff) { } } // is xrgb? - if ((kmsvnc->va->image->format.blue_mask | kmsvnc->va->image->format.red_mask) < 0x1000000) { + if ((kmsvnc->va->selected_fmt->blue_mask | kmsvnc->va->selected_fmt->red_mask) < 0x1000000) { for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) { uint32_t *pixdata = (uint32_t*)(kmsvnc->drm->kms_convert_buf + i); *pixdata = ntohl(htonl(*pixdata) << 8); } } // is bgrx? - if (kmsvnc->va->image->format.blue_mask > kmsvnc->va->image->format.red_mask) { + if (kmsvnc->va->selected_fmt->blue_mask > kmsvnc->va->selected_fmt->red_mask) { for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) { uint32_t pixdata = htonl(*((uint32_t*)(kmsvnc->drm->kms_convert_buf + i))); buff[i+0] = (pixdata & 0x0000ff00) >> 8; diff --git a/drm_master.c b/drm_master.c index 8d762ce..b48a85e 100644 --- a/drm_master.c +++ b/drm_master.c @@ -1,4 +1,5 @@ #define _GNU_SOURCE + #include #include #include @@ -38,9 +39,15 @@ static inline int cmp_fds(pid_t pid, const char *drm_pth) { if (ret == -1 && fdlist[n]->d_type == DT_LNK) { char link_pth[PATH_MAX+1]; char real_pth[PATH_MAX+1]; + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wformat-truncation" snprintf(link_pth, PATH_MAX+1, "%s/%s", path, fdlist[n]->d_name); + #pragma GCC diagnostic pop memset(real_pth, 0, PATH_MAX+1); + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wunused-result" realpath(link_pth, real_pth); + #pragma GCC diagnostic pop if (!strncmp(real_pth, drm_pth, PATH_MAX)) { int fd = atoi(fdlist[n]->d_name); if (fd > 0) { @@ -69,7 +76,10 @@ static inline int cmp_fds(pid_t pid, const char *drm_pth) { int drm_get_master_fd() { char drm_pth[PATH_MAX+1]; memset(drm_pth, 0, PATH_MAX+1); + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wunused-result" realpath(kmsvnc->card, drm_pth); + #pragma GCC diagnostic pop struct dirent **proclist; int count = scandir("/proc", &proclist, NULL, versionsort); diff --git a/kmsvnc.c b/kmsvnc.c index 2e4eb32..47b528e 100644 --- a/kmsvnc.c +++ b/kmsvnc.c @@ -1,3 +1,5 @@ +#define _GNU_SOURCE + #include #include #include @@ -127,7 +129,7 @@ static inline void update_vnc_cursor(char *data, int width, int height) { if (!kmsvnc->cursor_bitmap) return; kmsvnc->cursor_bitmap_len = rwidth * rheight * BYTES_PER_PIXEL; } - char *rich_source = malloc(rwidth * rheight * BYTES_PER_PIXEL); + unsigned char *rich_source = malloc(rwidth * rheight * BYTES_PER_PIXEL); if (!rich_source) return; char *maskString = malloc(rwidth * rheight); if (!maskString) { @@ -241,6 +243,7 @@ static struct argp_option kmsvnc_main_options[] = { {"input-offy", 0xff09, "0", 0, "Set input offset of y axis on a multi display system"}, {"screen-blank", 0xff0a, 0, OPTION_ARG_OPTIONAL, "Blank screen with gamma set on crtc"}, {"screen-blank-restore-linear", 0xff0b, 0, OPTION_ARG_OPTIONAL, "Restore linear values on exit in case of messed up gamma"}, + {"trust-va-format", 0xff0c, 0, OPTION_ARG_OPTIONAL, "trust VAImageFormat returned by vaapi implementation unconditionally"}, {"wakeup", 'w', 0, OPTION_ARG_OPTIONAL, "Move mouse to wake the system up before start"}, {"disable-input", 'i', 0, OPTION_ARG_OPTIONAL, "Disable uinput"}, {"desktop-name", 'n', "kmsvnc", 0, "Specify vnc desktop name"}, @@ -272,24 +275,28 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { kmsvnc->vnc_opt->bind6 = arg; break; case 'p': - int port = atoi(arg); - if (port > 0 && port < 65536) { - kmsvnc->vnc_opt->port = port; - } - else { - argp_error(state, "invalid port %s", arg); + { + int port = atoi(arg); + if (port > 0 && port < 65536) { + kmsvnc->vnc_opt->port = port; + } + else { + argp_error(state, "invalid port %s", arg); + } } break; case '4': kmsvnc->vnc_opt->disable_ipv6 = 1; break; case 0xff00: - int fps = atoi(arg); - if (fps > 0 && fps < 1000) { - kmsvnc->vnc_opt->sleep_ns = NS_IN_S / fps; - } - else { - argp_error(state, "invalid fps %s", arg); + { + int fps = atoi(arg); + if (fps > 0 && fps < 1000) { + kmsvnc->vnc_opt->sleep_ns = NS_IN_S / fps; + } + else { + argp_error(state, "invalid fps %s", arg); + } } break; case 0xff01: @@ -317,27 +324,35 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { kmsvnc->debug_enabled = 1; break; case 0xff06: - int width = atoi(arg); - if (width > 0) { - kmsvnc->input_width = width; + { + int width = atoi(arg); + if (width > 0) { + kmsvnc->input_width = width; + } } break; case 0xff07: - int height = atoi(arg); - if (height > 0) { - kmsvnc->input_height = height; + { + int height = atoi(arg); + if (height > 0) { + kmsvnc->input_height = height; + } } break; case 0xff08: - int offset_x = atoi(arg); - if (offset_x > 0) { - kmsvnc->input_offx = offset_x; + { + int offset_x = atoi(arg); + if (offset_x > 0) { + kmsvnc->input_offx = offset_x; + } } break; case 0xff09: - int offset_y = atoi(arg); - if (offset_y > 0) { - kmsvnc->input_offy = offset_y; + { + int offset_y = atoi(arg); + if (offset_y > 0) { + kmsvnc->input_offy = offset_y; + } } break; case 0xff0a: @@ -346,6 +361,9 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { case 0xff0b: kmsvnc->screen_blank_restore = 1; break; + case 0xff0c: + kmsvnc->trust_va_format = 1; + break; case 'w': kmsvnc->input_wakeup = 1; break; diff --git a/kmsvnc.h b/kmsvnc.h index 1a5c0d1..1ac04a4 100644 --- a/kmsvnc.h +++ b/kmsvnc.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -44,6 +45,7 @@ struct kmsvnc_data int input_offy; char screen_blank; char screen_blank_restore; + char trust_va_format; struct kmsvnc_drm_data *drm; struct kmsvnc_input_data *input; struct kmsvnc_keymap_data *keymap; @@ -137,6 +139,7 @@ struct kmsvnc_va_data char derive_enabled; VAImageFormat* img_fmts; int img_fmt_count; + VAImageFormat* selected_fmt; }; #define KMSVNC_FATAL(...) do{ fprintf(stderr, __VA_ARGS__); return 1; } while(0) diff --git a/va.c b/va.c index 85eb559..f67232d 100644 --- a/va.c +++ b/va.c @@ -1,3 +1,8 @@ +#define _GNU_SOURCE + +#include +#include +#include #include #include #include @@ -69,20 +74,49 @@ static const struct { {KMSVNC_FOURCC_TO_INT('A', 'R', '3', '0'), KMSVNC_FOURCC_TO_INT('A', 'R', '3', '0'), VA_RT_FORMAT_RGB32_10, 1}, }; +struct va_fmt_data { + uint32_t va_fourcc; + VAImageFormat *fmt; + char is_alpha; + uint32_t va_rt_format; + uint32_t depth; + uint32_t blue_mask; + uint32_t green_mask; + uint32_t red_mask; +}; + +static VAImageFormat* vaImgFmt_from_vaFmtData(struct va_fmt_data* data) { + static VAImageFormat ret = {0}; + VAImageFormat fmt = { + .fourcc = data->va_fourcc, + .byte_order = VA_LSB_FIRST, + .bits_per_pixel = 32, + .depth = data->depth, + .blue_mask = data->blue_mask, + .green_mask = data->green_mask, + .red_mask = data->red_mask, + .alpha_mask = 0, + .va_reserved = {0} + }; + memcpy(&ret, &fmt, sizeof(VAImageFormat)); + return &ret; +} + static void print_va_image_fmt(VAImageFormat *fmt) { - printf("image fmt: fourcc %d, %s, byte_order %s, bpp %d, depth %d, blue_mask %#x, green_mask %#x, red_mask %#x, reserved %#x %#x %#x %#x\n", fmt->fourcc, - fourcc_to_str(fmt->fourcc), - fmt->byte_order == 1 ? "VA_LSB_FIRST" : "VA_MSB_FIRST", - fmt->bits_per_pixel, - fmt->depth, - fmt->blue_mask, - fmt->green_mask, - fmt->red_mask, - fmt->va_reserved[0], - fmt->va_reserved[1], - fmt->va_reserved[2], - fmt->va_reserved[3] - ); + printf("image fmt: fourcc %d, %s, byte_order %s, bpp %d, depth %d, blue_mask %#x, green_mask %#x, red_mask %#x, alpha_mask %#x, reserved %#x %#x %#x %#x\n", fmt->fourcc, + fourcc_to_str(fmt->fourcc), + fmt->byte_order == 1 ? "VA_LSB_FIRST" : "VA_MSB_FIRST", + fmt->bits_per_pixel, + fmt->depth, + fmt->blue_mask, + fmt->green_mask, + fmt->red_mask, + fmt->alpha_mask, + fmt->va_reserved[0], + fmt->va_reserved[1], + fmt->va_reserved[2], + fmt->va_reserved[3] + ); } int va_init() { @@ -99,7 +133,7 @@ int va_init() { char* render_node; int effective_fd = 0; - if (render_node = drmGetRenderDeviceNameFromFd(kmsvnc->drm->drm_fd)) { + if ((render_node = drmGetRenderDeviceNameFromFd(kmsvnc->drm->drm_fd))) { va->render_node_fd = open(render_node, O_RDWR); free(render_node); } @@ -255,28 +289,23 @@ int va_init() { } } - struct fourcc_data { - uint32_t va_fourcc; - VAImageFormat *fmt; - char is_alpha; - uint32_t va_rt_format; - }; - struct fourcc_data format_to_try[] = { - {KMSVNC_FOURCC_TO_INT('R','G','B','X'), NULL, 0, VA_RT_FORMAT_RGB32}, - {KMSVNC_FOURCC_TO_INT('R','G','B','A'), NULL, 1, VA_RT_FORMAT_RGB32}, - {KMSVNC_FOURCC_TO_INT('X','R','G','B'), NULL, 0, VA_RT_FORMAT_RGB32}, - {KMSVNC_FOURCC_TO_INT('A','R','G','B'), NULL, 1, VA_RT_FORMAT_RGB32}, + struct va_fmt_data format_to_try[] = { + {KMSVNC_FOURCC_TO_INT('R','G','B','X'), NULL, 0, VA_RT_FORMAT_RGB32, 24, 0xff00, 0xff0000, 0xff000000}, + {KMSVNC_FOURCC_TO_INT('R','G','B','A'), NULL, 1, VA_RT_FORMAT_RGB32, 32, 0xff00, 0xff0000, 0xff000000}, + {KMSVNC_FOURCC_TO_INT('X','R','G','B'), NULL, 0, VA_RT_FORMAT_RGB32, 24, 0xff, 0xff00, 0xff0000}, + {KMSVNC_FOURCC_TO_INT('A','R','G','B'), NULL, 1, VA_RT_FORMAT_RGB32, 32, 0xff, 0xff00, 0xff0000}, - {KMSVNC_FOURCC_TO_INT('B','G','R','X'), NULL, 0, VA_RT_FORMAT_RGB32}, - {KMSVNC_FOURCC_TO_INT('B','G','R','A'), NULL, 1, VA_RT_FORMAT_RGB32}, - {KMSVNC_FOURCC_TO_INT('X','B','G','R'), NULL, 0, VA_RT_FORMAT_RGB32}, - {KMSVNC_FOURCC_TO_INT('A','B','G','R'), NULL, 1, VA_RT_FORMAT_RGB32}, + {KMSVNC_FOURCC_TO_INT('B','G','R','X'), NULL, 0, VA_RT_FORMAT_RGB32, 24, 0xff000000, 0xff0000, 0xff00}, + {KMSVNC_FOURCC_TO_INT('B','G','R','A'), NULL, 1, VA_RT_FORMAT_RGB32, 32, 0xff000000, 0xff0000, 0xff00}, + {KMSVNC_FOURCC_TO_INT('X','B','G','R'), NULL, 0, VA_RT_FORMAT_RGB32, 24, 0xff0000, 0xff00, 0xff}, + {KMSVNC_FOURCC_TO_INT('A','B','G','R'), NULL, 1, VA_RT_FORMAT_RGB32, 32, 0xff0000, 0xff00, 0xff}, - {KMSVNC_FOURCC_TO_INT('X','R','3','0'), NULL, 0, VA_RT_FORMAT_RGB32_10}, - {KMSVNC_FOURCC_TO_INT('A','R','3','0'), NULL, 1, VA_RT_FORMAT_RGB32_10}, - {KMSVNC_FOURCC_TO_INT('X','B','3','0'), NULL, 0, VA_RT_FORMAT_RGB32_10}, - {KMSVNC_FOURCC_TO_INT('A','B','3','0'), NULL, 1, VA_RT_FORMAT_RGB32_10}, + {KMSVNC_FOURCC_TO_INT('X','R','3','0'), NULL, 0, VA_RT_FORMAT_RGB32_10, 30, 0x3ff, 0xffc00, 0x3ff00000}, + {KMSVNC_FOURCC_TO_INT('A','R','3','0'), NULL, 1, VA_RT_FORMAT_RGB32_10, 30, 0x3ff, 0xffc00, 0x3ff00000}, + {KMSVNC_FOURCC_TO_INT('X','B','3','0'), NULL, 0, VA_RT_FORMAT_RGB32_10, 30, 0x3ff00000, 0xffc00, 0x3ff}, + {KMSVNC_FOURCC_TO_INT('A','B','3','0'), NULL, 1, VA_RT_FORMAT_RGB32_10, 30, 0x3ff00000, 0xffc00, 0x3ff}, }; + for (int i = 0; i < va->img_fmt_count; i++) { for (int j = 0; j < KMSVNC_ARRAY_ELEMENTS(format_to_try); j++) { if (va->img_fmts[i].fourcc == format_to_try[j].va_fourcc) { @@ -292,15 +321,14 @@ int va_init() { va->derive_enabled = kmsvnc->va_derive_enabled < 0 ? va->derive_enabled : kmsvnc->va_derive_enabled != 0; if (va->derive_enabled) { if ((s = vaDeriveImage(va->dpy, va->surface_id, va->image)) == VA_STATUS_SUCCESS) { - char found = 0; for (int i = 0; i < KMSVNC_ARRAY_ELEMENTS(format_to_try); i++) { if (format_to_try[i].fmt == NULL) continue; if (va->image->format.fourcc == format_to_try[i].fmt->fourcc) { - found = 1; + va->selected_fmt = vaImgFmt_from_vaFmtData(format_to_try + i); break; } } - if (!found) { + if (!va->selected_fmt) { va->derive_enabled = 0; printf("vaDeriveImage returned unknown fourcc %d %s\n", va->image->format.fourcc, fourcc_to_str(va->image->format.fourcc)); VA_MAY(vaDestroyImage(kmsvnc->va->dpy, kmsvnc->va->image->image_id)); @@ -316,7 +344,6 @@ int va_init() { } } if (!va->derive_enabled) { - char success = 0; for (int i = 0; i < KMSVNC_ARRAY_ELEMENTS(format_to_try); i++) { if (format_to_try[i].fmt == NULL) continue; if (!kmsvnc->debug_enabled && rt_format != format_to_try[i].va_rt_format) continue; @@ -342,17 +369,31 @@ int va_init() { continue; } else { - success = 1; + va->selected_fmt = vaImgFmt_from_vaFmtData(format_to_try + i); break; } } - if (!success) { + if (!va->selected_fmt) { va->imgbuf = NULL; KMSVNC_FATAL("failed to get vaapi image\n"); } } printf("got vaapi %simage:\n", va->derive_enabled ? "derive " : ""); print_va_image_fmt(&va->image->format); + if ( + va->selected_fmt->depth != va->image->format.depth || + va->selected_fmt->blue_mask != va->image->format.blue_mask || + va->selected_fmt->green_mask != va->image->format.green_mask || + va->selected_fmt->red_mask != va->image->format.red_mask || + va->selected_fmt->fourcc != va->image->format.fourcc + ) { + fprintf(stderr, "differs from selected image format\n"); + print_va_image_fmt(va->selected_fmt); + if (kmsvnc->trust_va_format) { + fprintf(stderr, "trust VAImageFormat returned by vaapi implementation unconditionally as requested\n"); + va->selected_fmt = &va->image->format; + } + } return 0; }