vaapi fix

drm.c

@@ -16,10 +16,6 @@ static void convert_copy(const char *in, int width, int height, char *buff) {
     memcpy(buff, in, width * height * 4);
 }
 
-static void convert_vaapi(const char *in, int width, int height, char *buff) {
-    va_hwframe_to_vaapi(buff);
-}
-
 static void convert_bgrx_to_rgb(const char *in, int width, int height, char *buff)
 {
     for (int y = 0; y < height; y++)
@@ -37,6 +33,17 @@ static char *kms_convert_buf = NULL;
 static size_t kms_convert_buf_len = 0;
 static char *kms_cpy_tmp_buf = NULL;
 static size_t kms_cpy_tmp_buf_len = 0;
+static inline char convert_buf_allocate(size_t len) {
+    if (kms_convert_buf_len < len)
+    {
+        if (kms_convert_buf)
+            free(kms_convert_buf);
+        kms_convert_buf = malloc(len);
+        if (!kms_convert_buf) return 1;
+        kms_convert_buf_len = len;
+    }
+    return 0;
+}
 static inline void convert_x_tiled(const int tilex, const int tiley, const char *in, int width, int height, char *buff)
 {
     if (width % tilex)
@@ -59,14 +66,7 @@ static inline void convert_x_tiled(const int tilex, const int tiley, const char
         memcpy(kms_cpy_tmp_buf, in, max_offset * 4 + 4);
         in = (const char *)kms_cpy_tmp_buf;
     }
-    if (kms_convert_buf_len < width * height * 4)
+    if (convert_buf_allocate(width * height * 4)) return;
-    {
-        if (kms_convert_buf)
-            free(kms_convert_buf);
-        kms_convert_buf = malloc(width * height * 4);
-        if (!kms_convert_buf) return;
-        kms_convert_buf_len = width * height * 4;
-    }
     for (int y = 0; y < height; y++)
     {
         for (int x = 0; x < width; x++)
@@ -89,6 +89,24 @@ void convert_intel_x_tiled_kmsbuf(const char *in, int width, int height, char *b
     convert_x_tiled(128, 8, in, width, height, buff);
 }
 
+static void convert_vaapi(const char *in, int width, int height, char *buff) {
+    if (kmsvnc->va->is_xrgb || kmsvnc->va->is_bgr) {
+        if (convert_buf_allocate(width * height * BYTES_PER_PIXEL)) return;
+        va_hwframe_to_vaapi(kms_convert_buf);
+        if (kmsvnc->va->is_xrgb) {
+            for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) {
+                *((uint32_t*)(kms_convert_buf + i)) <<= 8;
+            }
+        }
+        if (kmsvnc->va->is_bgr) {
+            convert_bgrx_to_rgb(kms_convert_buf, width, height, buff);
+        }
+    }
+    else {
+        va_hwframe_to_vaapi(buff);
+    }
+}
+
 static inline void drm_sync(int drmfd, uint64_t flags)
 {
     struct dma_buf_sync sync = {
@@ -290,7 +308,7 @@ static int drm_kmsbuf_prime_vaapi() {
     if (va_init()) return 1;
 
     drm->mmap_fd = drm->prime_fd;
-    drm->mapped = kmsvnc->va->imgbuf;
+    drm->skip_map = 1;
     return 0;
 }
 
@@ -392,7 +410,7 @@ int drm_vendors() {
         if (drm_kmsbuf_dumb()) return 1;
     }
 
-    if (!drm->mapped)
+    if (!drm->skip_map && !drm->mapped)
     {
         printf("mapping with size = %d, offset = %d, fd = %d\n", drm->mmap_size, drm->mmap_offset, drm->mmap_fd);
         drm->mapped = mmap(NULL, drm->mmap_size, PROT_READ, MAP_SHARED, drm->mmap_fd, drm->mmap_offset);

kmsvnc.c

@@ -143,6 +143,7 @@ static struct argp_option kmsvnc_main_options[] = {
     {"disable-always-shared", 0xff01, 0, OPTION_ARG_OPTIONAL, "Do not always treat incoming connections as shared"},
     {"disable-compare-fb", 0xff02, 0, OPTION_ARG_OPTIONAL, "Do not compare pixels"},
     {"capture-raw-fb", 0xff03, "/tmp/rawfb.bin", 0, "Capture RAW framebuffer instead of starting the vnc server (for debugging)"},
+    {"va-derive", 0xff04, "off", 0, "Enable derive with vaapi"},
     {"disable-input", 'i', 0, OPTION_ARG_OPTIONAL, "Disable uinput"},
     {"desktop-name", 'n', "kmsvnc", 0, "Specify vnc desktop name"},
     {0}
@@ -203,6 +204,13 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
             kmsvnc->debug_capture_fb = arg;
             kmsvnc->disable_input = 1;
             break;
+        case 0xff04:
+            if (!strcmp("on", arg) || !strcmp("y", arg) || !strcmp("yes", arg) || !strcmp("1", arg)) {
+                kmsvnc->va_derive_enabled = 1;
+            }
+            else {
+                kmsvnc->va_derive_enabled = 0;
+            }
         case 'i':
             kmsvnc->disable_input = 1;
             break;
@@ -233,6 +241,7 @@ int main(int argc, char **argv)
     kmsvnc->vnc_opt = vncopt;
 
     kmsvnc->card = "/dev/dri/card0";
+    kmsvnc->va_derive_enabled = -1;
     kmsvnc->vnc_opt->bind = &(struct in_addr){0};
     kmsvnc->vnc_opt->always_shared = 1;
     kmsvnc->vnc_opt->port = 5900;

kmsvnc.h

@@ -32,6 +32,7 @@ struct kmsvnc_data
     char *force_driver;
     struct vnc_opt *vnc_opt;
     char disable_input;
+    int va_derive_enabled;
     int source_plane;
     int source_crtc;
     struct kmsvnc_drm_data *drm;
@@ -87,6 +88,7 @@ struct kmsvnc_drm_data
     size_t mmap_size;
     off_t mmap_offset;
     char *mapped;
+    char skip_map;
     struct kmsvnc_drm_funcs *funcs;
     char *pixfmt_name;
     char *mod_vendor;
@@ -100,6 +102,9 @@ struct kmsvnc_va_data
     VASurfaceID surface_id;
     VAImage *image;
     char *imgbuf;
+    char is_bgr;  // bgr -> rgb
+    char is_xrgb; // shift 8
+    char derive_enabled;
 };
 
 #define KMSVNC_FATAL(...) do{ fprintf(stderr, __VA_ARGS__); return 1; } while(0)

va.c

@@ -44,6 +44,28 @@ static void va_error_callback(void *user_context, const char *message) {
     printf("va error: %s");
 }
 
+static char* fourcc_to_str(int fourcc) {
+    static char ret[5];
+    ret[4] = 0;
+    for (int i = 0; i < 4; i++) {
+        ret[i] = fourcc >> 8*i & 0xff;
+    }
+    return ret;
+}
+
+static void print_va_image_fmt(VAImageFormat *fmt) {
+        printf("image fmt: fourcc %d, %s, byte_order %s, bpp %d, depth %d, blue_mask %#x, green_mask %#x, red_mask %#x, reserved %#x\n", fmt->fourcc,
+            fourcc_to_str(fmt->fourcc),
+            fmt->byte_order == 1 ? "VA_LSB_FIRST" : "VA_MSB_FIRST",
+            fmt->bits_per_pixel,
+            fmt->depth,
+            fmt->blue_mask,
+            fmt->green_mask,
+            fmt->red_mask,
+            fmt->va_reserved
+        );
+}
+
 int va_init() {
     if (!kmsvnc->drm || !kmsvnc->drm->drm_fd || !kmsvnc->drm->prime_fd) {
         KMSVNC_FATAL("drm is not initialized\n");
@@ -104,9 +126,10 @@ int va_init() {
         }
     };
 
-    prime_desc.fourcc = kmsvnc->drm->mfb->pixel_format == KMSVNC_FOURCC_TO_INT('X', 'R', '2', '4') ?
+    char is_alpha = kmsvnc->drm->mfb->pixel_format != KMSVNC_FOURCC_TO_INT('X', 'R', '2', '4');
-        KMSVNC_FOURCC_TO_INT('B', 'G', 'R', 'X') :
+    prime_desc.fourcc = kmsvnc->drm->mfb->pixel_format == is_alpha ?
-        KMSVNC_FOURCC_TO_INT('B', 'G', 'R', 'A') ;
+        KMSVNC_FOURCC_TO_INT('B', 'G', 'R', 'A') :
+        KMSVNC_FOURCC_TO_INT('B', 'G', 'R', 'X') ;
     prime_desc.width = kmsvnc->drm->mfb->width;
     prime_desc.height = kmsvnc->drm->mfb->height;
 
@@ -195,44 +218,151 @@ int va_init() {
     }
 
     for (int i = 0; i < img_fmt_count; i++) {
-        printf("fmt %d: fourcc %d, %c%c%c%c, byte_order %s, bpp %d, depth %d, blue_mask %#x, green_mask %#x, red_mask %#x, reserved %#x\n", i, img_fmts[i].fourcc,
+        print_va_image_fmt(img_fmts + i);
-            img_fmts[i].fourcc & 0xff,
-            img_fmts[i].fourcc >> 8 & 0xff,
-            img_fmts[i].fourcc >> 16 & 0xff,
-            img_fmts[i].fourcc >> 24 & 0xff,
-            img_fmts[i].byte_order - 1 ? "VA_LSB_FIRST" : "VA_MSB_FIRST",
-            img_fmts[i].bits_per_pixel,
-            img_fmts[i].depth,
-            img_fmts[i].blue_mask,
-            img_fmts[i].green_mask,
-            img_fmts[i].red_mask,
-            img_fmts[i].va_reserved
-        );
     }
     #endif
 
-    VAImageFormat format = {
+    VAImageFormat fmt_rgbx = {
         .fourcc = KMSVNC_FOURCC_TO_INT('R','G','B','X'),
         .byte_order = VA_LSB_FIRST,
         .bits_per_pixel = 32,
         .depth = 24,
+        .blue_mask = 0x0000ff00,
+        .green_mask = 0x00ff0000,
+        .red_mask = 0xff000000,
+        .va_reserved = 0x00000000,
+    };
+    VAImageFormat fmt_bgrx = {
+        .fourcc = KMSVNC_FOURCC_TO_INT('B','G','R','X'),
+        .byte_order = VA_LSB_FIRST,
+        .bits_per_pixel = 32,
+        .depth = 24,
+        .blue_mask = 0xff000000,
+        .green_mask = 0x00ff0000,
+        .red_mask = 0x0000ff00,
+        .va_reserved = 0x00000000,
+    };
+    VAImageFormat fmt_xrgb = {
+        .fourcc = KMSVNC_FOURCC_TO_INT('X','R','G','B'),
+        .byte_order = VA_LSB_FIRST,
+        .bits_per_pixel = 32,
+        .depth = 24,
         .blue_mask = 0x000000ff,
         .green_mask = 0x0000ff00,
         .red_mask = 0x00ff0000,
         .va_reserved = 0x00000000,
     };
+    VAImageFormat fmt_xbgr = {
+        .fourcc = KMSVNC_FOURCC_TO_INT('X','B','G','R'),
+        .byte_order = VA_LSB_FIRST,
+        .bits_per_pixel = 32,
+        .depth = 24,
+        .blue_mask = 0x00ff0000,
+        .green_mask = 0x0000ff00,
+        .red_mask = 0x000000ff,
+        .va_reserved = 0x00000000,
+    };
     va->image = malloc(sizeof(VAImage));
     if (!va->image) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
-    if ((s = vaCreateImage(va->dpy, &format, kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, va->image)) != VA_STATUS_SUCCESS) {
+
-        free(va->image);
+    struct fourcc_data {
-        va->image = NULL;
+        VAImageFormat *fmt;
-        VA_MUST(s);
+        char is_alpha;
+        int fourcc;
+        char is_bgr;
+        char is_xrgb;
+    };
+    struct fourcc_data format_to_try[] = {
+        {&fmt_rgbx, 0, KMSVNC_FOURCC_TO_INT('R','G','B','X'), 0, 0},
+        {&fmt_rgbx, 1, KMSVNC_FOURCC_TO_INT('R','G','B','A'), 0, 0},
+        {&fmt_xrgb, 0, KMSVNC_FOURCC_TO_INT('X','R','G','B'), 0, 1},
+        {&fmt_xrgb, 1, KMSVNC_FOURCC_TO_INT('A','R','G','B'), 0, 1},
+
+        {&fmt_bgrx, 0, KMSVNC_FOURCC_TO_INT('B','G','R','X'), 1, 0},
+        {&fmt_bgrx, 1, KMSVNC_FOURCC_TO_INT('B','G','R','A'), 1, 0},
+        {&fmt_xbgr, 0, KMSVNC_FOURCC_TO_INT('X','B','G','R'), 1, 1},
+        {&fmt_xbgr, 1, KMSVNC_FOURCC_TO_INT('A','B','G','R'), 1, 1},
+    };
+
+    va->derive_enabled = strcmp(kmsvnc->drm->drm_ver->name, "i915") != 0;
+    va->derive_enabled = kmsvnc->va_derive_enabled < 0 ? va->derive_enabled : kmsvnc->va_derive_enabled != 0;
+    if (va->derive_enabled) {
+        if ((s = vaDeriveImage(va->dpy, va->surface_id, va->image)) == VA_STATUS_SUCCESS) {
+            switch (va->image->format.fourcc) {
+                case KMSVNC_FOURCC_TO_INT('B','G','R','X'):
+                case KMSVNC_FOURCC_TO_INT('B','G','R','A'):
+                    va->is_bgr = 1;
+                    break;
+                case KMSVNC_FOURCC_TO_INT('R','G','B','X'):
+                case KMSVNC_FOURCC_TO_INT('R','G','B','A'):
+                    break;
+                case KMSVNC_FOURCC_TO_INT('X','R','G','B'):
+                    va->is_xrgb = 1;
+                    break;
+                case KMSVNC_FOURCC_TO_INT('X','B','G','R'):
+                    va->is_bgr = 1;
+                    va->is_xrgb = 1;
+                    break;
+                default:
+                    va->derive_enabled = 0;
+                    printf("vaDeriveImage returned unknown fourcc %d %s\n", va->image->format.fourcc, fourcc_to_str(va->image->format.fourcc));
+                    VA_MUST(vaDestroyImage(kmsvnc->va->dpy, kmsvnc->va->image->image_id));
+            }
+        }
+        VA_MAY(s);
     }
-    VA_MUST(vaMapBuffer(va->dpy, va->image->buf, (void**)&va->imgbuf));
+    if (va->derive_enabled) {
+        if ((s = vaMapBuffer(va->dpy, va->image->buf, (void**)&va->imgbuf)) != VA_STATUS_SUCCESS) {
+            VA_MAY(s);
+            VA_MAY(vaDestroyImage(kmsvnc->va->dpy, kmsvnc->va->image->image_id));
+            va->derive_enabled = 0;
+        }
+    }
+    if (!va->derive_enabled) {
+        char success = 0;
+        for (int i = 0; i < KMSVNC_ARRAY_ELEMENTS(format_to_try); i++) {
+            if (is_alpha != format_to_try[i].is_alpha) continue;
+            VAImageFormat *fmt = format_to_try[i].fmt;
+            va->is_bgr = format_to_try[i].is_bgr;
+            va->is_xrgb = format_to_try[i].is_xrgb;
+            fmt->fourcc = format_to_try[i].fourcc;
+            if ((s = vaCreateImage(va->dpy, fmt, kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, va->image)) != VA_STATUS_SUCCESS) {
+                VA_MAY(s);
+                continue;
+            }
+            if ((s = vaMapBuffer(va->dpy, va->image->buf, (void**)&va->imgbuf)) != VA_STATUS_SUCCESS) {
+                VA_MAY(s);
+                VA_MAY(vaDestroyImage(kmsvnc->va->dpy, kmsvnc->va->image->image_id));
+                continue;
+            }
+            if ((s = vaGetImage(kmsvnc->va->dpy, kmsvnc->va->surface_id, 0, 0,
+                    kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height,
+                    kmsvnc->va->image->image_id)) != VA_STATUS_SUCCESS)
+            {
+                VA_MAY(s);
+                VA_MAY(vaUnmapBuffer(kmsvnc->va->dpy, kmsvnc->va->image->buf));
+                VA_MAY(vaDestroyImage(kmsvnc->va->dpy, kmsvnc->va->image->image_id));
+                continue;
+            }
+            else {
+                success = 1;
+                break;
+            }
+        }
+        if (!success) {
+            va->imgbuf = NULL;
+            KMSVNC_FATAL("failed to get vaapi image\n");
+        }
+    }
+    printf("vaapi %simage fourcc isbgr %hd isxrgb %hd\n", va->derive_enabled ? "derive " : "", va->is_bgr, va->is_xrgb);
+    print_va_image_fmt(&va->image->format);
+    return 0;
 }
 
 int va_hwframe_to_vaapi(char *out) {
-    VA_MUST(vaGetImage(kmsvnc->va->dpy, kmsvnc->va->surface_id, 0, 0,
+    if (!kmsvnc->va->derive_enabled) {
-                        kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, kmsvnc->va->image->image_id));
+        VA_MUST(vaGetImage(kmsvnc->va->dpy, kmsvnc->va->surface_id, 0, 0,
+                kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, kmsvnc->va->image->image_id));
+    }
     memcpy(out, kmsvnc->va->imgbuf, kmsvnc->drm->mfb->width * kmsvnc->drm->mfb->height * BYTES_PER_PIXEL);
 }

va.h

@@ -1,7 +1,7 @@
 #pragma once
 
-#define VA_MUST(x) do{VAStatus s; if ((s = (x)) != VA_STATUS_SUCCESS) KMSVNC_FATAL("va operation error %#x %s on line %d\n", s, vaErrorStr(s), __LINE__); } while (0)
+#define VA_MUST(x) do{VAStatus _s; if ((_s = (x)) != VA_STATUS_SUCCESS) KMSVNC_FATAL("va operation error %#x %s on line %d\n", _s, vaErrorStr(_s), __LINE__); } while (0)
-#define VA_MAY(x) do{VAStatus s; if ((s = (x)) != VA_STATUS_SUCCESS) fprintf(stderr, "va operation error %#x %s on line %d\n", s, vaErrorStr(s), __LINE__); } while (0)
+#define VA_MAY(x) do{VAStatus _s; if ((_s = (x)) != VA_STATUS_SUCCESS) fprintf(stderr, "va operation error %#x %s on line %d\n", _s, vaErrorStr(_s), __LINE__); } while (0)
 
 void va_cleanup();
 int va_init();