diff --git a/drm.c b/drm.c index e0c6903..d270cd9 100644 --- a/drm.c +++ b/drm.c @@ -12,6 +12,17 @@ extern struct kmsvnc_data *kmsvnc; +static int check_pixfmt_non_vaapi() { + if ( + kmsvnc->drm->mfb->pixel_format != KMSVNC_FOURCC_TO_INT('X', 'R', '2', '4') && + kmsvnc->drm->mfb->pixel_format != KMSVNC_FOURCC_TO_INT('A', 'R', '2', '4') + ) + { + KMSVNC_FATAL("Unsupported pixfmt %s, please create an issue with your pixfmt.\n", kmsvnc->drm->pixfmt_name); + } + return 0; +} + static void convert_copy(const char *in, int width, int height, char *buff) { memcpy(buff, in, width * height * 4); } @@ -90,20 +101,34 @@ void convert_intel_x_tiled_kmsbuf(const char *in, int width, int height, char *b } static void convert_vaapi(const char *in, int width, int height, char *buff) { - if (kmsvnc->va->is_xrgb || kmsvnc->va->is_bgr) { + if (KMSVNC_FOURCC_TO_INT('R','G','B', 0) & kmsvnc->va->image->format.fourcc == KMSVNC_FOURCC_TO_INT('R','G','B', 0)) { + va_hwframe_to_vaapi(buff); + } + else { if (convert_buf_allocate(width * height * BYTES_PER_PIXEL)) return; va_hwframe_to_vaapi(kms_convert_buf); - if (kmsvnc->va->is_xrgb) { + // is 30 depth? + if (kmsvnc->va->image->format.depth == 30) { + for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) { + uint32_t pixdata = *((uint32_t*)(kms_convert_buf + i)); + kms_convert_buf[i] = (pixdata & 0x3ff00000) >> 20 >> 2; + kms_convert_buf[i+1] = (pixdata & 0xffc00) >> 10 >> 2; + kms_convert_buf[i+2] = (pixdata & 0x3ff) >> 2; + } + } + // is xrgb? + if ((kmsvnc->va->image->format.blue_mask | kmsvnc->va->image->format.red_mask) < 0x1000000) { for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) { *((uint32_t*)(kms_convert_buf + i)) <<= 8; } } - if (kmsvnc->va->is_bgr) { + // is bgr? + if (kmsvnc->va->image->format.blue_mask > kmsvnc->va->image->format.red_mask) { convert_bgrx_to_rgb(kms_convert_buf, width, height, buff); } - } - else { - va_hwframe_to_vaapi(buff); + else { + memcpy(buff, kms_convert_buf, width * height * BYTES_PER_PIXEL); + } } } @@ -255,14 +280,6 @@ int drm_open() { printf("pitches %u %u %u %u\n", drm->mfb->pitches[0], drm->mfb->pitches[1], drm->mfb->pitches[2], drm->mfb->pitches[3]); printf("format %s, modifier %s:%s\n", drm->pixfmt_name, drm->mod_vendor, drm->mod_name); - if ( - drm->mfb->pixel_format != KMSVNC_FOURCC_TO_INT('X', 'R', '2', '4') && - drm->mfb->pixel_format != KMSVNC_FOURCC_TO_INT('A', 'R', '2', '4') - ) - { - KMSVNC_FATAL("Unsupported pixfmt %s, please create an issue with your pixfmt.\n", drm->pixfmt_name); - } - if (!drm->mfb->handles[0]) { KMSVNC_FATAL("No handle set on framebuffer: maybe you need some additional capabilities?\n"); @@ -357,6 +374,7 @@ int drm_vendors() { } else if (strcmp(driver_name, "nvidia-drm") == 0) { + if (check_pixfmt_non_vaapi()) return 1; printf("warn: nvidia card detected. Currently only x-tiled framebuffer is supported. Performance may suffer.\n"); drm->funcs->convert = &convert_nvidia_x_tiled_kmsbuf; if (drm_kmsbuf_dumb()) return 1; @@ -366,6 +384,7 @@ int drm_vendors() { strcmp(driver_name, "virtio_gpu") == 0 ) { + if (check_pixfmt_non_vaapi()) return 1; if (drm->mfb->modifier != DRM_FORMAT_MOD_NONE && drm->mfb->modifier != DRM_FORMAT_MOD_LINEAR) { printf("warn: modifier is not LINEAR, please create an issue with your modifier.\n"); } @@ -374,14 +393,17 @@ int drm_vendors() { } else if (strcmp(driver_name, "test-prime") == 0) { + if (check_pixfmt_non_vaapi()) return 1; if (drm_kmsbuf_prime()) return 1; } else if (strcmp(driver_name, "test-map-dumb") == 0) { + if (check_pixfmt_non_vaapi()) return 1; if (drm_kmsbuf_dumb()) return 1; } else if (strcmp(driver_name, "test-i915-gem") == 0) { + if (check_pixfmt_non_vaapi()) return 1; struct drm_gem_flink flink; flink.handle = drm->mfb->handles[0]; DRM_IOCTL_MUST(drm->drm_fd, DRM_IOCTL_GEM_FLINK, &flink); @@ -398,11 +420,13 @@ int drm_vendors() { } else if (strcmp(driver_name, "test-i915-prime-xtiled") == 0) { + if (check_pixfmt_non_vaapi()) return 1; drm->funcs->convert = &convert_intel_x_tiled_kmsbuf; if (drm_kmsbuf_prime()) return 1; } else { + if (check_pixfmt_non_vaapi()) return 1; fprintf(stderr, "Untested drm driver, use at your own risk!\n"); if (drm->mfb->modifier != DRM_FORMAT_MOD_NONE && drm->mfb->modifier != DRM_FORMAT_MOD_LINEAR) { printf("warn: modifier is not LINEAR, please create an issue with your driver and modifier.\n"); diff --git a/kmsvnc.c b/kmsvnc.c index 5c1e93d..04aa87c 100644 --- a/kmsvnc.c +++ b/kmsvnc.c @@ -144,6 +144,7 @@ static struct argp_option kmsvnc_main_options[] = { {"disable-compare-fb", 0xff02, 0, OPTION_ARG_OPTIONAL, "Do not compare pixels"}, {"capture-raw-fb", 0xff03, "/tmp/rawfb.bin", 0, "Capture RAW framebuffer instead of starting the vnc server (for debugging)"}, {"va-derive", 0xff04, "off", 0, "Enable derive with vaapi"}, + {"va-print-format", 0xff05, 0, OPTION_ARG_OPTIONAL, "Print supported vaImage format"}, {"wakeup", 'w', 0, OPTION_ARG_OPTIONAL, "Move mouse to wake the system up before start"}, {"disable-input", 'i', 0, OPTION_ARG_OPTIONAL, "Disable uinput"}, {"desktop-name", 'n', "kmsvnc", 0, "Specify vnc desktop name"}, @@ -213,6 +214,9 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { kmsvnc->va_derive_enabled = 0; } break; + case 0xff05: + kmsvnc->va_print_fmt = 1; + break; case 'w': kmsvnc->input_wakeup = 1; break; diff --git a/kmsvnc.h b/kmsvnc.h index 497c620..4dae072 100644 --- a/kmsvnc.h +++ b/kmsvnc.h @@ -34,6 +34,7 @@ struct kmsvnc_data char input_wakeup; char disable_input; int va_derive_enabled; + int va_print_fmt; int source_plane; int source_crtc; struct kmsvnc_drm_data *drm; @@ -103,9 +104,9 @@ struct kmsvnc_va_data VASurfaceID surface_id; VAImage *image; char *imgbuf; - char is_bgr; // bgr -> rgb - char is_xrgb; // shift 8 char derive_enabled; + VAImageFormat* img_fmts; + int img_fmt_count; }; #define KMSVNC_FATAL(...) do{ fprintf(stderr, __VA_ARGS__); return 1; } while(0) diff --git a/va.c b/va.c index 6385585..4543798 100644 --- a/va.c +++ b/va.c @@ -10,6 +10,10 @@ extern struct kmsvnc_data *kmsvnc; void va_cleanup() { VAStatus s; if (kmsvnc->va) { + if (kmsvnc->va->img_fmts) { + free(kmsvnc->va->img_fmts); + kmsvnc->va->img_fmts = NULL; + } if (kmsvnc->va->imgbuf) { VA_MAY(vaUnmapBuffer(kmsvnc->va->dpy, kmsvnc->va->image->buf)); kmsvnc->va->imgbuf = NULL; @@ -35,9 +39,9 @@ void va_cleanup() { } static void va_msg_callback(void *user_context, const char *message) { - #ifdef KMSVNC_VA_DEBUG - printf("va msg: %s", message); - #endif + if (kmsvnc->va_print_fmt) { + printf("va msg: %s", message); + } } static void va_error_callback(void *user_context, const char *message) { @@ -53,6 +57,18 @@ static char* fourcc_to_str(int fourcc) { return ret; } +static const struct { + uint32_t drm_fourcc; + uint32_t va_fourcc; + uint32_t va_rt_format; + char alpha; +} va_format_map[] = { + {KMSVNC_FOURCC_TO_INT('X', 'R', '2', '4'), KMSVNC_FOURCC_TO_INT('B', 'G', 'R', 'X'), VA_RT_FORMAT_RGB32, 0}, + {KMSVNC_FOURCC_TO_INT('A', 'R', '2', '4'), KMSVNC_FOURCC_TO_INT('B', 'G', 'R', 'A'), VA_RT_FORMAT_RGB32, 1}, + {KMSVNC_FOURCC_TO_INT('X', 'R', '3', '0'), KMSVNC_FOURCC_TO_INT('X', 'R', '3', '0'), VA_RT_FORMAT_RGB32_10, 0}, + {KMSVNC_FOURCC_TO_INT('A', 'R', '3', '0'), KMSVNC_FOURCC_TO_INT('A', 'R', '3', '0'), VA_RT_FORMAT_RGB32_10, 1}, +}; + static void print_va_image_fmt(VAImageFormat *fmt) { printf("image fmt: fourcc %d, %s, byte_order %s, bpp %d, depth %d, blue_mask %#x, green_mask %#x, red_mask %#x, reserved %#x %#x %#x %#x\n", fmt->fourcc, fourcc_to_str(fmt->fourcc), @@ -129,10 +145,22 @@ int va_init() { } }; - char is_alpha = kmsvnc->drm->mfb->pixel_format != KMSVNC_FOURCC_TO_INT('X', 'R', '2', '4'); - prime_desc.fourcc = kmsvnc->drm->mfb->pixel_format == is_alpha ? - KMSVNC_FOURCC_TO_INT('B', 'G', 'R', 'A') : - KMSVNC_FOURCC_TO_INT('B', 'G', 'R', 'X') ; + uint32_t rt_format = 0; + char is_alpha = 0; + for (int i = 0; i < KMSVNC_ARRAY_ELEMENTS(va_format_map); i++) { + if (kmsvnc->drm->mfb->pixel_format == va_format_map[i].drm_fourcc) { + prime_desc.fourcc = va_format_map[i].va_fourcc; + rt_format = va_format_map[i].va_rt_format; + is_alpha = va_format_map[i].alpha; + break; + } + } + if (!rt_format) { + KMSVNC_FATAL("Unsupported pixfmt %s for vaapi, please create an issue with your pixfmt.", kmsvnc->drm->pixfmt_name); + } + if (kmsvnc->va_print_fmt) { + printf("selected rt_format %u, alpha %d\n", rt_format, is_alpha); + } prime_desc.width = kmsvnc->drm->mfb->width; prime_desc.height = kmsvnc->drm->mfb->height; @@ -164,7 +192,7 @@ int va_init() { prime_desc.num_objects = 1; VAStatus s; - if ((s = vaCreateSurfaces(va->dpy, VA_RT_FORMAT_RGB32, + if ((s = vaCreateSurfaces(va->dpy, rt_format, kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, &va->surface_id, 1, prime_attrs, KMSVNC_ARRAY_ELEMENTS(prime_attrs))) != VA_STATUS_SUCCESS) { @@ -203,113 +231,78 @@ int va_init() { buffer_desc.num_planes = prime_desc.layers[0].num_planes; - VA_MUST(vaCreateSurfaces(va->dpy, VA_RT_FORMAT_RGB32, + VA_MUST(vaCreateSurfaces(va->dpy, rt_format, kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, &va->surface_id, 1, buffer_attrs, KMSVNC_ARRAY_ELEMENTS(buffer_attrs))); } - #ifdef KMSVNC_VA_DEBUG - int img_fmt_count = vaMaxNumImageFormats(va->dpy); - VAImageFormat *img_fmts = malloc(sizeof(VAImageFormat) * img_fmt_count); - if (!img_fmts) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__); + + va->img_fmt_count = vaMaxNumImageFormats(va->dpy); + va->img_fmts = malloc(sizeof(VAImageFormat) * va->img_fmt_count); + if (!va->img_fmts) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__); { int got; - vaQueryImageFormats(va->dpy, img_fmts, &got); - if (got != img_fmt_count) { - KMSVNC_FATAL("got less VAImageFormats, %d instead of %d\n", got, img_fmt_count); + vaQueryImageFormats(va->dpy, va->img_fmts, &got); + if (got != va->img_fmt_count) { + printf("got less VAImageFormats, %d instead of %d\n", got, va->img_fmt_count); + va->img_fmt_count = got; } } - for (int i = 0; i < img_fmt_count; i++) { - print_va_image_fmt(img_fmts + i); + if (kmsvnc->va_print_fmt) { + for (int i = 0; i < va->img_fmt_count; i++) { + print_va_image_fmt(va->img_fmts + i); + } } - #endif - - VAImageFormat fmt_rgbx = { - .fourcc = KMSVNC_FOURCC_TO_INT('R','G','B','X'), - .byte_order = VA_LSB_FIRST, - .bits_per_pixel = 32, - .depth = 24, - .blue_mask = 0x0000ff00, - .green_mask = 0x00ff0000, - .red_mask = 0xff000000, - .va_reserved = {0,0,0,0}, - }; - VAImageFormat fmt_bgrx = { - .fourcc = KMSVNC_FOURCC_TO_INT('B','G','R','X'), - .byte_order = VA_LSB_FIRST, - .bits_per_pixel = 32, - .depth = 24, - .blue_mask = 0xff000000, - .green_mask = 0x00ff0000, - .red_mask = 0x0000ff00, - .va_reserved = {0,0,0,0}, - }; - VAImageFormat fmt_xrgb = { - .fourcc = KMSVNC_FOURCC_TO_INT('X','R','G','B'), - .byte_order = VA_LSB_FIRST, - .bits_per_pixel = 32, - .depth = 24, - .blue_mask = 0x000000ff, - .green_mask = 0x0000ff00, - .red_mask = 0x00ff0000, - .va_reserved = {0,0,0,0}, - }; - VAImageFormat fmt_xbgr = { - .fourcc = KMSVNC_FOURCC_TO_INT('X','B','G','R'), - .byte_order = VA_LSB_FIRST, - .bits_per_pixel = 32, - .depth = 24, - .blue_mask = 0x00ff0000, - .green_mask = 0x0000ff00, - .red_mask = 0x000000ff, - .va_reserved = {0,0,0,0}, - }; - va->image = malloc(sizeof(VAImage)); - if (!va->image) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__); struct fourcc_data { + uint32_t va_fourcc; VAImageFormat *fmt; char is_alpha; - int fourcc; - char is_bgr; - char is_xrgb; + uint32_t va_rt_format; }; struct fourcc_data format_to_try[] = { - {&fmt_rgbx, 0, KMSVNC_FOURCC_TO_INT('R','G','B','X'), 0, 0}, - {&fmt_rgbx, 1, KMSVNC_FOURCC_TO_INT('R','G','B','A'), 0, 0}, - {&fmt_xrgb, 0, KMSVNC_FOURCC_TO_INT('X','R','G','B'), 0, 1}, - {&fmt_xrgb, 1, KMSVNC_FOURCC_TO_INT('A','R','G','B'), 0, 1}, + {KMSVNC_FOURCC_TO_INT('R','G','B','X'), NULL, 0, VA_RT_FORMAT_RGB32}, + {KMSVNC_FOURCC_TO_INT('R','G','B','A'), NULL, 1, VA_RT_FORMAT_RGB32}, + {KMSVNC_FOURCC_TO_INT('X','R','G','B'), NULL, 0, VA_RT_FORMAT_RGB32}, + {KMSVNC_FOURCC_TO_INT('A','R','G','B'), NULL, 1, VA_RT_FORMAT_RGB32}, - {&fmt_bgrx, 0, KMSVNC_FOURCC_TO_INT('B','G','R','X'), 1, 0}, - {&fmt_bgrx, 1, KMSVNC_FOURCC_TO_INT('B','G','R','A'), 1, 0}, - {&fmt_xbgr, 0, KMSVNC_FOURCC_TO_INT('X','B','G','R'), 1, 1}, - {&fmt_xbgr, 1, KMSVNC_FOURCC_TO_INT('A','B','G','R'), 1, 1}, + {KMSVNC_FOURCC_TO_INT('B','G','R','X'), NULL, 0, VA_RT_FORMAT_RGB32}, + {KMSVNC_FOURCC_TO_INT('B','G','R','A'), NULL, 1, VA_RT_FORMAT_RGB32}, + {KMSVNC_FOURCC_TO_INT('X','B','G','R'), NULL, 0, VA_RT_FORMAT_RGB32}, + {KMSVNC_FOURCC_TO_INT('A','B','G','R'), NULL, 1, VA_RT_FORMAT_RGB32}, + + {KMSVNC_FOURCC_TO_INT('X','R','3','0'), NULL, 0, VA_RT_FORMAT_RGB32_10}, + {KMSVNC_FOURCC_TO_INT('A','R','3','0'), NULL, 1, VA_RT_FORMAT_RGB32_10}, + {KMSVNC_FOURCC_TO_INT('X','B','3','0'), NULL, 0, VA_RT_FORMAT_RGB32_10}, + {KMSVNC_FOURCC_TO_INT('A','B','3','0'), NULL, 1, VA_RT_FORMAT_RGB32_10}, }; + for (int i = 0; i < va->img_fmt_count; i++) { + for (int j = 0; j < KMSVNC_ARRAY_ELEMENTS(format_to_try); j++) { + if (va->img_fmts[i].fourcc == format_to_try[j].va_fourcc) { + format_to_try[j].fmt = va->img_fmts + i; + } + } + } + + va->image = malloc(sizeof(VAImage)); + if (!va->image) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__); va->derive_enabled = 0; va->derive_enabled = kmsvnc->va_derive_enabled < 0 ? va->derive_enabled : kmsvnc->va_derive_enabled != 0; if (va->derive_enabled) { if ((s = vaDeriveImage(va->dpy, va->surface_id, va->image)) == VA_STATUS_SUCCESS) { - switch (va->image->format.fourcc) { - case KMSVNC_FOURCC_TO_INT('B','G','R','X'): - case KMSVNC_FOURCC_TO_INT('B','G','R','A'): - va->is_bgr = 1; + char found = 0; + for (int i = 0; i < KMSVNC_ARRAY_ELEMENTS(format_to_try); i++) { + if (va->image->format.fourcc == format_to_try[i].fmt->fourcc) { + found = 1; break; - case KMSVNC_FOURCC_TO_INT('R','G','B','X'): - case KMSVNC_FOURCC_TO_INT('R','G','B','A'): - break; - case KMSVNC_FOURCC_TO_INT('X','R','G','B'): - va->is_xrgb = 1; - break; - case KMSVNC_FOURCC_TO_INT('X','B','G','R'): - va->is_bgr = 1; - va->is_xrgb = 1; - break; - default: - va->derive_enabled = 0; - printf("vaDeriveImage returned unknown fourcc %d %s\n", va->image->format.fourcc, fourcc_to_str(va->image->format.fourcc)); - VA_MUST(vaDestroyImage(kmsvnc->va->dpy, kmsvnc->va->image->image_id)); + } + } + if (!found) { + va->derive_enabled = 0; + printf("vaDeriveImage returned unknown fourcc %d %s\n", va->image->format.fourcc, fourcc_to_str(va->image->format.fourcc)); + VA_MAY(vaDestroyImage(kmsvnc->va->dpy, kmsvnc->va->image->image_id)); } } VA_MAY(s); @@ -324,11 +317,11 @@ int va_init() { if (!va->derive_enabled) { char success = 0; for (int i = 0; i < KMSVNC_ARRAY_ELEMENTS(format_to_try); i++) { + if (format_to_try[i].fmt == NULL) continue; + if (!kmsvnc->va_print_fmt && rt_format != format_to_try[i].va_rt_format) continue; if (is_alpha != format_to_try[i].is_alpha) continue; + VAImageFormat *fmt = format_to_try[i].fmt; - va->is_bgr = format_to_try[i].is_bgr; - va->is_xrgb = format_to_try[i].is_xrgb; - fmt->fourcc = format_to_try[i].fourcc; if ((s = vaCreateImage(va->dpy, fmt, kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, va->image)) != VA_STATUS_SUCCESS) { VA_MAY(s); continue; @@ -357,7 +350,7 @@ int va_init() { KMSVNC_FATAL("failed to get vaapi image\n"); } } - printf("vaapi %simage fourcc isbgr %hd isxrgb %hd\n", va->derive_enabled ? "derive " : "", va->is_bgr, va->is_xrgb); + printf("got vaapi %simage:\n", va->derive_enabled ? "derive " : ""); print_va_image_fmt(&va->image->format); return 0; } @@ -368,4 +361,5 @@ int va_hwframe_to_vaapi(char *out) { kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, kmsvnc->va->image->image_id)); } memcpy(out, kmsvnc->va->imgbuf, kmsvnc->drm->mfb->width * kmsvnc->drm->mfb->height * BYTES_PER_PIXEL); + return 0; }