diff --git a/drm.c b/drm.c index 71ce110..bfd6fe2 100644 --- a/drm.c +++ b/drm.c @@ -16,10 +16,6 @@ static void convert_copy(const char *in, int width, int height, char *buff) { memcpy(buff, in, width * height * 4); } -static void convert_vaapi(const char *in, int width, int height, char *buff) { - va_hwframe_to_vaapi(buff); -} - static void convert_bgrx_to_rgb(const char *in, int width, int height, char *buff) { for (int y = 0; y < height; y++) @@ -37,6 +33,17 @@ static char *kms_convert_buf = NULL; static size_t kms_convert_buf_len = 0; static char *kms_cpy_tmp_buf = NULL; static size_t kms_cpy_tmp_buf_len = 0; +static inline char convert_buf_allocate(size_t len) { + if (kms_convert_buf_len < len) + { + if (kms_convert_buf) + free(kms_convert_buf); + kms_convert_buf = malloc(len); + if (!kms_convert_buf) return 1; + kms_convert_buf_len = len; + } + return 0; +} static inline void convert_x_tiled(const int tilex, const int tiley, const char *in, int width, int height, char *buff) { if (width % tilex) @@ -59,14 +66,7 @@ static inline void convert_x_tiled(const int tilex, const int tiley, const char memcpy(kms_cpy_tmp_buf, in, max_offset * 4 + 4); in = (const char *)kms_cpy_tmp_buf; } - if (kms_convert_buf_len < width * height * 4) - { - if (kms_convert_buf) - free(kms_convert_buf); - kms_convert_buf = malloc(width * height * 4); - if (!kms_convert_buf) return; - kms_convert_buf_len = width * height * 4; - } + if (convert_buf_allocate(width * height * 4)) return; for (int y = 0; y < height; y++) { for (int x = 0; x < width; x++) @@ -89,6 +89,24 @@ void convert_intel_x_tiled_kmsbuf(const char *in, int width, int height, char *b convert_x_tiled(128, 8, in, width, height, buff); } +static void convert_vaapi(const char *in, int width, int height, char *buff) { + if (kmsvnc->va->is_xrgb || kmsvnc->va->is_bgr) { + if (convert_buf_allocate(width * height * BYTES_PER_PIXEL)) return; + va_hwframe_to_vaapi(kms_convert_buf); + if (kmsvnc->va->is_xrgb) { + for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) { + *((uint32_t*)(kms_convert_buf + i)) <<= 8; + } + } + if (kmsvnc->va->is_bgr) { + convert_bgrx_to_rgb(kms_convert_buf, width, height, buff); + } + } + else { + va_hwframe_to_vaapi(buff); + } +} + static inline void drm_sync(int drmfd, uint64_t flags) { struct dma_buf_sync sync = { @@ -290,7 +308,7 @@ static int drm_kmsbuf_prime_vaapi() { if (va_init()) return 1; drm->mmap_fd = drm->prime_fd; - drm->mapped = kmsvnc->va->imgbuf; + drm->skip_map = 1; return 0; } @@ -392,7 +410,7 @@ int drm_vendors() { if (drm_kmsbuf_dumb()) return 1; } - if (!drm->mapped) + if (!drm->skip_map && !drm->mapped) { printf("mapping with size = %d, offset = %d, fd = %d\n", drm->mmap_size, drm->mmap_offset, drm->mmap_fd); drm->mapped = mmap(NULL, drm->mmap_size, PROT_READ, MAP_SHARED, drm->mmap_fd, drm->mmap_offset); diff --git a/kmsvnc.c b/kmsvnc.c index 452e576..26c2a62 100644 --- a/kmsvnc.c +++ b/kmsvnc.c @@ -143,6 +143,7 @@ static struct argp_option kmsvnc_main_options[] = { {"disable-always-shared", 0xff01, 0, OPTION_ARG_OPTIONAL, "Do not always treat incoming connections as shared"}, {"disable-compare-fb", 0xff02, 0, OPTION_ARG_OPTIONAL, "Do not compare pixels"}, {"capture-raw-fb", 0xff03, "/tmp/rawfb.bin", 0, "Capture RAW framebuffer instead of starting the vnc server (for debugging)"}, + {"va-derive", 0xff04, "off", 0, "Enable derive with vaapi"}, {"disable-input", 'i', 0, OPTION_ARG_OPTIONAL, "Disable uinput"}, {"desktop-name", 'n', "kmsvnc", 0, "Specify vnc desktop name"}, {0} @@ -203,6 +204,13 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { kmsvnc->debug_capture_fb = arg; kmsvnc->disable_input = 1; break; + case 0xff04: + if (!strcmp("on", arg) || !strcmp("y", arg) || !strcmp("yes", arg) || !strcmp("1", arg)) { + kmsvnc->va_derive_enabled = 1; + } + else { + kmsvnc->va_derive_enabled = 0; + } case 'i': kmsvnc->disable_input = 1; break; @@ -233,6 +241,7 @@ int main(int argc, char **argv) kmsvnc->vnc_opt = vncopt; kmsvnc->card = "/dev/dri/card0"; + kmsvnc->va_derive_enabled = -1; kmsvnc->vnc_opt->bind = &(struct in_addr){0}; kmsvnc->vnc_opt->always_shared = 1; kmsvnc->vnc_opt->port = 5900; diff --git a/kmsvnc.h b/kmsvnc.h index b8b5204..541754a 100644 --- a/kmsvnc.h +++ b/kmsvnc.h @@ -32,6 +32,7 @@ struct kmsvnc_data char *force_driver; struct vnc_opt *vnc_opt; char disable_input; + int va_derive_enabled; int source_plane; int source_crtc; struct kmsvnc_drm_data *drm; @@ -87,6 +88,7 @@ struct kmsvnc_drm_data size_t mmap_size; off_t mmap_offset; char *mapped; + char skip_map; struct kmsvnc_drm_funcs *funcs; char *pixfmt_name; char *mod_vendor; @@ -100,6 +102,9 @@ struct kmsvnc_va_data VASurfaceID surface_id; VAImage *image; char *imgbuf; + char is_bgr; // bgr -> rgb + char is_xrgb; // shift 8 + char derive_enabled; }; #define KMSVNC_FATAL(...) do{ fprintf(stderr, __VA_ARGS__); return 1; } while(0) diff --git a/va.c b/va.c index be54c8a..b1ef0f6 100644 --- a/va.c +++ b/va.c @@ -44,6 +44,28 @@ static void va_error_callback(void *user_context, const char *message) { printf("va error: %s"); } +static char* fourcc_to_str(int fourcc) { + static char ret[5]; + ret[4] = 0; + for (int i = 0; i < 4; i++) { + ret[i] = fourcc >> 8*i & 0xff; + } + return ret; +} + +static void print_va_image_fmt(VAImageFormat *fmt) { + printf("image fmt: fourcc %d, %s, byte_order %s, bpp %d, depth %d, blue_mask %#x, green_mask %#x, red_mask %#x, reserved %#x\n", fmt->fourcc, + fourcc_to_str(fmt->fourcc), + fmt->byte_order == 1 ? "VA_LSB_FIRST" : "VA_MSB_FIRST", + fmt->bits_per_pixel, + fmt->depth, + fmt->blue_mask, + fmt->green_mask, + fmt->red_mask, + fmt->va_reserved + ); +} + int va_init() { if (!kmsvnc->drm || !kmsvnc->drm->drm_fd || !kmsvnc->drm->prime_fd) { KMSVNC_FATAL("drm is not initialized\n"); @@ -104,9 +126,10 @@ int va_init() { } }; - prime_desc.fourcc = kmsvnc->drm->mfb->pixel_format == KMSVNC_FOURCC_TO_INT('X', 'R', '2', '4') ? - KMSVNC_FOURCC_TO_INT('B', 'G', 'R', 'X') : - KMSVNC_FOURCC_TO_INT('B', 'G', 'R', 'A') ; + char is_alpha = kmsvnc->drm->mfb->pixel_format != KMSVNC_FOURCC_TO_INT('X', 'R', '2', '4'); + prime_desc.fourcc = kmsvnc->drm->mfb->pixel_format == is_alpha ? + KMSVNC_FOURCC_TO_INT('B', 'G', 'R', 'A') : + KMSVNC_FOURCC_TO_INT('B', 'G', 'R', 'X') ; prime_desc.width = kmsvnc->drm->mfb->width; prime_desc.height = kmsvnc->drm->mfb->height; @@ -195,44 +218,151 @@ int va_init() { } for (int i = 0; i < img_fmt_count; i++) { - printf("fmt %d: fourcc %d, %c%c%c%c, byte_order %s, bpp %d, depth %d, blue_mask %#x, green_mask %#x, red_mask %#x, reserved %#x\n", i, img_fmts[i].fourcc, - img_fmts[i].fourcc & 0xff, - img_fmts[i].fourcc >> 8 & 0xff, - img_fmts[i].fourcc >> 16 & 0xff, - img_fmts[i].fourcc >> 24 & 0xff, - img_fmts[i].byte_order - 1 ? "VA_LSB_FIRST" : "VA_MSB_FIRST", - img_fmts[i].bits_per_pixel, - img_fmts[i].depth, - img_fmts[i].blue_mask, - img_fmts[i].green_mask, - img_fmts[i].red_mask, - img_fmts[i].va_reserved - ); + print_va_image_fmt(img_fmts + i); } #endif - VAImageFormat format = { + VAImageFormat fmt_rgbx = { .fourcc = KMSVNC_FOURCC_TO_INT('R','G','B','X'), .byte_order = VA_LSB_FIRST, .bits_per_pixel = 32, .depth = 24, + .blue_mask = 0x0000ff00, + .green_mask = 0x00ff0000, + .red_mask = 0xff000000, + .va_reserved = 0x00000000, + }; + VAImageFormat fmt_bgrx = { + .fourcc = KMSVNC_FOURCC_TO_INT('B','G','R','X'), + .byte_order = VA_LSB_FIRST, + .bits_per_pixel = 32, + .depth = 24, + .blue_mask = 0xff000000, + .green_mask = 0x00ff0000, + .red_mask = 0x0000ff00, + .va_reserved = 0x00000000, + }; + VAImageFormat fmt_xrgb = { + .fourcc = KMSVNC_FOURCC_TO_INT('X','R','G','B'), + .byte_order = VA_LSB_FIRST, + .bits_per_pixel = 32, + .depth = 24, .blue_mask = 0x000000ff, .green_mask = 0x0000ff00, .red_mask = 0x00ff0000, .va_reserved = 0x00000000, }; + VAImageFormat fmt_xbgr = { + .fourcc = KMSVNC_FOURCC_TO_INT('X','B','G','R'), + .byte_order = VA_LSB_FIRST, + .bits_per_pixel = 32, + .depth = 24, + .blue_mask = 0x00ff0000, + .green_mask = 0x0000ff00, + .red_mask = 0x000000ff, + .va_reserved = 0x00000000, + }; va->image = malloc(sizeof(VAImage)); if (!va->image) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__); - if ((s = vaCreateImage(va->dpy, &format, kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, va->image)) != VA_STATUS_SUCCESS) { - free(va->image); - va->image = NULL; - VA_MUST(s); + + struct fourcc_data { + VAImageFormat *fmt; + char is_alpha; + int fourcc; + char is_bgr; + char is_xrgb; + }; + struct fourcc_data format_to_try[] = { + {&fmt_rgbx, 0, KMSVNC_FOURCC_TO_INT('R','G','B','X'), 0, 0}, + {&fmt_rgbx, 1, KMSVNC_FOURCC_TO_INT('R','G','B','A'), 0, 0}, + {&fmt_xrgb, 0, KMSVNC_FOURCC_TO_INT('X','R','G','B'), 0, 1}, + {&fmt_xrgb, 1, KMSVNC_FOURCC_TO_INT('A','R','G','B'), 0, 1}, + + {&fmt_bgrx, 0, KMSVNC_FOURCC_TO_INT('B','G','R','X'), 1, 0}, + {&fmt_bgrx, 1, KMSVNC_FOURCC_TO_INT('B','G','R','A'), 1, 0}, + {&fmt_xbgr, 0, KMSVNC_FOURCC_TO_INT('X','B','G','R'), 1, 1}, + {&fmt_xbgr, 1, KMSVNC_FOURCC_TO_INT('A','B','G','R'), 1, 1}, + }; + + va->derive_enabled = strcmp(kmsvnc->drm->drm_ver->name, "i915") != 0; + va->derive_enabled = kmsvnc->va_derive_enabled < 0 ? va->derive_enabled : kmsvnc->va_derive_enabled != 0; + if (va->derive_enabled) { + if ((s = vaDeriveImage(va->dpy, va->surface_id, va->image)) == VA_STATUS_SUCCESS) { + switch (va->image->format.fourcc) { + case KMSVNC_FOURCC_TO_INT('B','G','R','X'): + case KMSVNC_FOURCC_TO_INT('B','G','R','A'): + va->is_bgr = 1; + break; + case KMSVNC_FOURCC_TO_INT('R','G','B','X'): + case KMSVNC_FOURCC_TO_INT('R','G','B','A'): + break; + case KMSVNC_FOURCC_TO_INT('X','R','G','B'): + va->is_xrgb = 1; + break; + case KMSVNC_FOURCC_TO_INT('X','B','G','R'): + va->is_bgr = 1; + va->is_xrgb = 1; + break; + default: + va->derive_enabled = 0; + printf("vaDeriveImage returned unknown fourcc %d %s\n", va->image->format.fourcc, fourcc_to_str(va->image->format.fourcc)); + VA_MUST(vaDestroyImage(kmsvnc->va->dpy, kmsvnc->va->image->image_id)); + } + } + VA_MAY(s); } - VA_MUST(vaMapBuffer(va->dpy, va->image->buf, (void**)&va->imgbuf)); + if (va->derive_enabled) { + if ((s = vaMapBuffer(va->dpy, va->image->buf, (void**)&va->imgbuf)) != VA_STATUS_SUCCESS) { + VA_MAY(s); + VA_MAY(vaDestroyImage(kmsvnc->va->dpy, kmsvnc->va->image->image_id)); + va->derive_enabled = 0; + } + } + if (!va->derive_enabled) { + char success = 0; + for (int i = 0; i < KMSVNC_ARRAY_ELEMENTS(format_to_try); i++) { + if (is_alpha != format_to_try[i].is_alpha) continue; + VAImageFormat *fmt = format_to_try[i].fmt; + va->is_bgr = format_to_try[i].is_bgr; + va->is_xrgb = format_to_try[i].is_xrgb; + fmt->fourcc = format_to_try[i].fourcc; + if ((s = vaCreateImage(va->dpy, fmt, kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, va->image)) != VA_STATUS_SUCCESS) { + VA_MAY(s); + continue; + } + if ((s = vaMapBuffer(va->dpy, va->image->buf, (void**)&va->imgbuf)) != VA_STATUS_SUCCESS) { + VA_MAY(s); + VA_MAY(vaDestroyImage(kmsvnc->va->dpy, kmsvnc->va->image->image_id)); + continue; + } + if ((s = vaGetImage(kmsvnc->va->dpy, kmsvnc->va->surface_id, 0, 0, + kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, + kmsvnc->va->image->image_id)) != VA_STATUS_SUCCESS) + { + VA_MAY(s); + VA_MAY(vaUnmapBuffer(kmsvnc->va->dpy, kmsvnc->va->image->buf)); + VA_MAY(vaDestroyImage(kmsvnc->va->dpy, kmsvnc->va->image->image_id)); + continue; + } + else { + success = 1; + break; + } + } + if (!success) { + va->imgbuf = NULL; + KMSVNC_FATAL("failed to get vaapi image\n"); + } + } + printf("vaapi %simage fourcc isbgr %hd isxrgb %hd\n", va->derive_enabled ? "derive " : "", va->is_bgr, va->is_xrgb); + print_va_image_fmt(&va->image->format); + return 0; } int va_hwframe_to_vaapi(char *out) { - VA_MUST(vaGetImage(kmsvnc->va->dpy, kmsvnc->va->surface_id, 0, 0, - kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, kmsvnc->va->image->image_id)); + if (!kmsvnc->va->derive_enabled) { + VA_MUST(vaGetImage(kmsvnc->va->dpy, kmsvnc->va->surface_id, 0, 0, + kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, kmsvnc->va->image->image_id)); + } memcpy(out, kmsvnc->va->imgbuf, kmsvnc->drm->mfb->width * kmsvnc->drm->mfb->height * BYTES_PER_PIXEL); } diff --git a/va.h b/va.h index 0e1e67d..3747a50 100644 --- a/va.h +++ b/va.h @@ -1,7 +1,7 @@ #pragma once -#define VA_MUST(x) do{VAStatus s; if ((s = (x)) != VA_STATUS_SUCCESS) KMSVNC_FATAL("va operation error %#x %s on line %d\n", s, vaErrorStr(s), __LINE__); } while (0) -#define VA_MAY(x) do{VAStatus s; if ((s = (x)) != VA_STATUS_SUCCESS) fprintf(stderr, "va operation error %#x %s on line %d\n", s, vaErrorStr(s), __LINE__); } while (0) +#define VA_MUST(x) do{VAStatus _s; if ((_s = (x)) != VA_STATUS_SUCCESS) KMSVNC_FATAL("va operation error %#x %s on line %d\n", _s, vaErrorStr(_s), __LINE__); } while (0) +#define VA_MAY(x) do{VAStatus _s; if ((_s = (x)) != VA_STATUS_SUCCESS) fprintf(stderr, "va operation error %#x %s on line %d\n", _s, vaErrorStr(_s), __LINE__); } while (0) void va_cleanup(); int va_init();