va done

CMakeLists.txt

@@ -10,6 +10,8 @@ find_package(PkgConfig REQUIRED)
 pkg_search_module(LIBDRM REQUIRED libdrm)
 pkg_search_module(LIBVNCSERVER REQUIRED libvncserver)
 pkg_search_module(XKBCOMMON REQUIRED xkbcommon)
+pkg_search_module(LIBVA REQUIRED libva)
+pkg_search_module(LIBVA_DRM REQUIRED libva-drm)
 
 include(CheckIncludeFiles)
 CHECK_INCLUDE_FILES("linux/uinput.h;linux/dma-buf.h" HAVE_LINUX_API_HEADERS)
@@ -17,17 +19,21 @@ IF(NOT HAVE_LINUX_API_HEADERS)
   message(FATAL_ERROR "linux-api-headers not found")
 ENDIF()
 
-add_executable(kmsvnc kmsvnc.c drm.c input.c keymap.c)
+add_executable(kmsvnc kmsvnc.c drm.c input.c keymap.c va.c)
 target_include_directories(kmsvnc PUBLIC
   ${LIBDRM_INCLUDEDIR}
   ${LIBDRM_INCLUDEDIR}/libdrm
   ${LIBVNCSERVER_INCLUDEDIR}
   ${XKBCOMMON_INCLUDEDIR}
+  ${LIBVA_INCLUDEDIR}
+  ${LIBVA_DRM_INCLUDEDIR}
 )
 target_link_libraries(kmsvnc PUBLIC
   m
   ${LIBDRM_LIBRARIES}
   ${LIBVNCSERVER_LIBRARIES}
   ${XKBCOMMON_LIBRARIES}
+  ${LIBVA_LIBRARIES}
+  ${LIBVA_DRM_LIBRARIES}
 )
 install(TARGETS kmsvnc RUNTIME DESTINATION bin)

README.md

@@ -10,6 +10,7 @@ Currently in very early stage.
  * libvncserver
  * libxkbcommon
  * libdrm
+ * libva
 
 ## Building
 ```

drm.c

@@ -5,8 +5,10 @@
 #include <string.h>
 #include <stdlib.h>
 #include <sys/ioctl.h>
+#include <libdrm/drm_fourcc.h>
 
 #include "drm.h"
+#include "va.h"
 
 extern struct kmsvnc_data *kmsvnc;
 
@@ -14,6 +16,10 @@ static void convert_copy(const char *in, int width, int height, char *buff) {
     memcpy(buff, in, width * height * 4);
 }
 
+static void convert_vaapi(const char *in, int width, int height, char *buff) {
+    va_hwframe_to_vaapi(buff);
+}
+
 static void convert_bgrx_to_rgb(const char *in, int width, int height, char *buff)
 {
     for (int y = 0; y < height; y++)
@@ -47,6 +53,7 @@ static inline void convert_x_tiled(const int tilex, const int tiley, const char
             if (kms_cpy_tmp_buf)
                 free(kms_convert_buf);
             kms_cpy_tmp_buf = malloc(max_offset * 4 + 4);
+            if (!kms_cpy_tmp_buf) return;
             kms_cpy_tmp_buf_len = max_offset * 4 + 4;
         }
         memcpy(kms_cpy_tmp_buf, in, max_offset * 4 + 4);
@@ -57,6 +64,7 @@ static inline void convert_x_tiled(const int tilex, const int tiley, const char
         if (kms_convert_buf)
             free(kms_convert_buf);
         kms_convert_buf = malloc(width * height * 4);
+        if (!kms_convert_buf) return;
         kms_convert_buf_len = width * height * 4;
     }
     for (int y = 0; y < height; y++)
@@ -83,13 +91,10 @@ void convert_intel_x_tiled_kmsbuf(const char *in, int width, int height, char *b
 
 static inline void drm_sync(int drmfd, uint64_t flags)
 {
-    int ioctl_err;
     struct dma_buf_sync sync = {
         .flags = flags,
     };
-    if (ioctl_err = ioctl(drmfd, DMA_BUF_IOCTL_SYNC, &sync)) {
-        fprintf(stderr, "DRM ioctl error %d on line %d\n", ioctl_err, __LINE__);
-    }
+    DRM_R_IOCTL_MAY(drmfd, DMA_BUF_IOCTL_SYNC, &sync);
 }
 
 void drm_sync_start(int drmfd)
@@ -110,6 +115,18 @@ void drm_cleanup() {
             drmFreeVersion(kmsvnc->drm->drm_ver);
             kmsvnc->drm->drm_ver = NULL;
         }
+        if (kmsvnc->drm->pixfmt_name) {
+            free(kmsvnc->drm->pixfmt_name);
+            kmsvnc->drm->pixfmt_name = NULL;
+        }
+        if (kmsvnc->drm->mod_vendor) {
+            free(kmsvnc->drm->mod_vendor);
+            kmsvnc->drm->mod_vendor = NULL;
+        }
+        if (kmsvnc->drm->mod_name) {
+            free(kmsvnc->drm->mod_name);
+            kmsvnc->drm->mod_name = NULL;
+        }
         if (kmsvnc->drm->plane) {
             drmModeFreePlane(kmsvnc->drm->plane);
             kmsvnc->drm->plane = NULL;
@@ -141,13 +158,14 @@ void drm_cleanup() {
 
 int drm_open() {
     struct kmsvnc_drm_data *drm = malloc(sizeof(struct kmsvnc_drm_data));
+    if (!drm) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
     memset(drm, 0, sizeof(struct kmsvnc_drm_data));
     kmsvnc->drm = drm;
 
     drm->drm_fd = open(kmsvnc->card, O_RDONLY);
     if (drm->drm_fd < 0)
     {
-        DRM_FATAL("card %s open failed: %s\n", kmsvnc->card, strerror(errno));
+        KMSVNC_FATAL("card %s open failed: %s\n", kmsvnc->card, strerror(errno));
     }
     drm->drm_ver = drmGetVersion(drm->drm_fd);
     printf("drm driver is %s\n", drm->drm_ver->name);
@@ -161,7 +179,7 @@ int drm_open() {
     {
         drm->plane = drmModeGetPlane(drm->drm_fd, kmsvnc->source_plane);
         if (!drm->plane)
-            DRM_FATAL("Failed to get plane %d: %s\n", kmsvnc->source_plane, strerror(errno));
+            KMSVNC_FATAL("Failed to get plane %d: %s\n", kmsvnc->source_plane, strerror(errno));
         if (drm->plane->fb_id == 0)
             fprintf(stderr, "Place %d does not have an attached framebuffer\n", kmsvnc->source_plane);
     }
@@ -169,7 +187,7 @@ int drm_open() {
     {
         drm->plane_res = drmModeGetPlaneResources(drm->drm_fd);
         if (!drm->plane_res)
-            DRM_FATAL("Failed to get plane resources: %s\n", strerror(errno));
+            KMSVNC_FATAL("Failed to get plane resources: %s\n", strerror(errno));
         int i;
         for (i = 0; i < drm->plane_res->count_planes; i++)
         {
@@ -194,11 +212,11 @@ int drm_open() {
         {
             if (kmsvnc->source_crtc != 0)
             {
-                DRM_FATAL("No usable planes found on CRTC %d\n", kmsvnc->source_crtc);
+                KMSVNC_FATAL("No usable planes found on CRTC %d\n", kmsvnc->source_crtc);
             }
             else
             {
-                DRM_FATAL("No usable planes found\n");
+                KMSVNC_FATAL("No usable planes found\n");
             }
         }
         printf("Using plane %u to locate framebuffers\n", drm->plane->plane_id);
@@ -208,32 +226,34 @@ int drm_open() {
 
     drm->mfb = drmModeGetFB2(drm->drm_fd, drm->plane->fb_id);
     if (!drm->mfb) {
-        DRM_FATAL("Failed to get framebuffer %u: %s\n", drm->plane->fb_id, strerror(errno));
+        KMSVNC_FATAL("Failed to get framebuffer %u: %s\n", drm->plane->fb_id, strerror(errno));
     }
+    drm->pixfmt_name = drmGetFormatName(drm->mfb->pixel_format);
+    drm->mod_vendor = drmGetFormatModifierVendor(drm->mfb->modifier);
+    drm->mod_name = drmGetFormatModifierName(drm->mfb->modifier);
     printf("Template framebuffer is %u: %ux%u fourcc:%u mod:%u flags:%u\n", drm->mfb->fb_id, drm->mfb->width, drm->mfb->height, drm->mfb->pixel_format, drm->mfb->modifier, drm->mfb->flags);
     printf("handles %u %u %u %u\n", drm->mfb->handles[0], drm->mfb->handles[1], drm->mfb->handles[2], drm->mfb->handles[3]);
     printf("offsets %u %u %u %u\n", drm->mfb->offsets[0], drm->mfb->offsets[1], drm->mfb->offsets[2], drm->mfb->offsets[3]);
     printf("pitches %u %u %u %u\n", drm->mfb->pitches[0], drm->mfb->pitches[1], drm->mfb->pitches[2], drm->mfb->pitches[3]);
-    printf("format %s, modifier %s:%s\n", drmGetFormatName(drm->mfb->pixel_format), drmGetFormatModifierVendor(drm->mfb->modifier), drmGetFormatModifierName(drm->mfb->modifier));
+    printf("format %s, modifier %s:%s\n", drm->pixfmt_name, drm->mod_vendor, drm->mod_name);
 
     if (
         drm->mfb->pixel_format != KMSVNC_FOURCC_TO_INT('X', 'R', '2', '4') &&
         drm->mfb->pixel_format != KMSVNC_FOURCC_TO_INT('A', 'R', '2', '4')
     )
     {
-        DRM_FATAL("Unsupported pixfmt\n");
+        KMSVNC_FATAL("Unsupported pixfmt %s, please create an issue with your pixfmt.\n", drm->pixfmt_name);
     }
 
     if (!drm->mfb->handles[0])
     {
-        DRM_FATAL("No handle set on framebuffer: maybe you need some additional capabilities?\n");
+        KMSVNC_FATAL("No handle set on framebuffer: maybe you need some additional capabilities?\n");
     }
 
-    int ioctl_err = 0;
-
     drm->mmap_fd = drm->drm_fd;
     drm->mmap_size = drm->mfb->width * drm->mfb->height * BYTES_PER_PIXEL;
     drm->funcs = malloc(sizeof(struct kmsvnc_drm_funcs));
+    if (!drm->funcs) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
     drm->funcs->convert = convert_bgrx_to_rgb;
     drm->funcs->sync_start = drm_sync_noop;
     drm->funcs->sync_end = drm_sync_noop;
@@ -250,7 +270,7 @@ static int drm_kmsbuf_prime() {
     int err = drmPrimeHandleToFD(drm->drm_fd, drm->mfb->handles[0], O_RDWR, &drm->prime_fd);
     if (err < 0 || drm->prime_fd < 0)
     {
-        DRM_FATAL("Failed to get PRIME fd from framebuffer handle");
+        KMSVNC_FATAL("Failed to get PRIME fd from framebuffer handle");
     }
     drm->funcs->sync_start = &drm_sync_start;
     drm->funcs->sync_end = &drm_sync_end;
@@ -258,6 +278,22 @@ static int drm_kmsbuf_prime() {
     return 0;
 }
 
+static int drm_kmsbuf_prime_vaapi() {
+    struct kmsvnc_drm_data *drm = kmsvnc->drm;
+
+    int err = drmPrimeHandleToFD(drm->drm_fd, drm->mfb->handles[0], O_RDWR, &drm->prime_fd);
+    if (err < 0 || drm->prime_fd < 0)
+    {
+        KMSVNC_FATAL("Failed to get PRIME fd from framebuffer handle");
+    }
+
+    if (va_init()) return 1;
+
+    drm->mmap_fd = drm->prime_fd;
+    drm->mapped = kmsvnc->va->imgbuf;
+    return 0;
+}
+
 static int drm_kmsbuf_dumb() {
     struct kmsvnc_drm_data *drm = kmsvnc->drm;
 
@@ -291,32 +327,19 @@ int drm_vendors() {
         driver_name = drm->drm_ver->name;
     }
 
-    if (strcmp(driver_name, "i915") == 0)
+    if (strcmp(driver_name, "i915") == 0 || strcmp(driver_name, "amdgpu") == 0)
     {
-        drm->funcs->convert = &convert_intel_x_tiled_kmsbuf;
-        if (drm_kmsbuf_prime()) return 1;
-    }
-    else if (strcmp(driver_name, "amdgpu") == 0)
-    {
-        struct drm_gem_flink flink;
-        flink.handle = drm->mfb->handles[0];
-        DRM_IOCTL_MUST(drm->drm_fd, DRM_IOCTL_GEM_FLINK, &flink);
-
-        struct drm_gem_open open_arg;
-        open_arg.name = flink.name;
-        DRM_IOCTL_MUST(drm->drm_fd, DRM_IOCTL_GEM_OPEN, &open_arg);
-
-        union drm_amdgpu_gem_mmap mmap_arg;
-        memset(&mmap_arg, 0, sizeof(mmap_arg));
-        mmap_arg.in.handle = open_arg.handle;
-        DRM_IOCTL_MUST(drm->drm_fd, DRM_IOCTL_AMDGPU_GEM_MMAP, &mmap_arg);
-
-        drm->mmap_size = open_arg.size;
-        drm->mmap_offset = mmap_arg.out.addr_ptr;
+        if (fourcc_mod_is_vendor(drm->mfb->modifier, INTEL)) {
+            if (strstr(drm->mod_name, "CCS")) {
+                printf("warn: intel with CCS modifier detected, please set INTEL_DEBUG=noccs\n");
+            }
+        };
+        drm->funcs->convert = &convert_vaapi;
+        if (drm_kmsbuf_prime_vaapi()) return 1;
     }
     else if (strcmp(driver_name, "nvidia-drm") == 0)
     {
-        // quirky and slow
+        printf("warn: nvidia card detected. Currently only x-tiled framebuffer is supported. Performance may suffer.\n");
         drm->funcs->convert = &convert_nvidia_x_tiled_kmsbuf;
         if (drm_kmsbuf_dumb()) return 1;
     }
@@ -325,6 +348,9 @@ int drm_vendors() {
              strcmp(driver_name, "virtio_gpu") == 0
     )
     {
+        if (drm->mfb->modifier != DRM_FORMAT_MOD_NONE && drm->mfb->modifier != DRM_FORMAT_MOD_LINEAR) {
+            printf("warn: modifier is not LINEAR, please create an issue with your modifier.\n");
+        }
         // virgl does not work
         if (drm_kmsbuf_dumb()) return 1;
     }
@@ -352,9 +378,17 @@ int drm_vendors() {
         drm->mmap_size = open_arg.size;
         drm->mmap_offset = mmap_arg.offset;
     }
+    else if (strcmp(driver_name, "test-i915-prime-xtiled") == 0)
+    {
+        drm->funcs->convert = &convert_intel_x_tiled_kmsbuf;
+        if (drm_kmsbuf_prime()) return 1;
+    }
     else
     {
         fprintf(stderr, "Untested drm driver, use at your own risk!\n");
+        if (drm->mfb->modifier != DRM_FORMAT_MOD_NONE && drm->mfb->modifier != DRM_FORMAT_MOD_LINEAR) {
+            printf("warn: modifier is not LINEAR, please create an issue with your driver and modifier.\n");
+        }
         if (drm_kmsbuf_dumb()) return 1;
     }
 
@@ -364,7 +398,7 @@ int drm_vendors() {
         drm->mapped = mmap(NULL, drm->mmap_size, PROT_READ, MAP_SHARED, drm->mmap_fd, drm->mmap_offset);
         if (drm->mapped == MAP_FAILED)
         {
-            DRM_FATAL("Failed to mmap: %s\n", strerror(errno));
+            KMSVNC_FATAL("Failed to mmap: %s\n", strerror(errno));
         }
     }
 

drm.h

@@ -2,11 +2,10 @@
 
 #include "kmsvnc.h"
 
-#define KMSVNC_FOURCC_TO_INT(a,b,c,d) (((a) << 0) + ((b) << 8) + ((c) << 16) + ((d) << 24))
+#define DRM_IOCTL_MUST(...) do{ int e; if (e = drmIoctl(__VA_ARGS__)) KMSVNC_FATAL("DRM ioctl error %d on line %d\n", e, __LINE__); } while(0)
+#define DRM_IOCTL_MAY(...) do{ int e; if (e = drmIoctl(__VA_ARGS__)) fprintf(stderr, "DRM ioctl error %d on line %d\n", e, __LINE__); } while(0)
+#define DRM_R_IOCTL_MAY(...) do{ int e; if (e = ioctl(__VA_ARGS__)) fprintf(stderr, "DRM ioctl error %d on line %d\n", e, __LINE__); } while(0)
 
-#define DRM_FATAL(...) { fprintf(stderr, __VA_ARGS__); return 1; }
-#define DRM_IOCTL_MUST(...) { int e; if (e = drmIoctl(__VA_ARGS__)) DRM_FATAL("DRM ioctl error %d on line %d\n", e, __LINE__) }
-#define DRM_IOCTL_MAY(...) { int e; if (e = drmIoctl(__VA_ARGS__)) fprintf(stderr, "DRM ioctl error %d on line %d\n", e, __LINE__); }
 
 void drm_cleanup();
 int drm_open();

input.c

@@ -29,13 +29,14 @@ void uinput_cleanup()
 int uinput_init()
 {
     struct kmsvnc_input_data *inp = malloc(sizeof(struct kmsvnc_input_data));
+    if (!inp) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
     memset(inp, 0, sizeof(struct kmsvnc_input_data));
     kmsvnc->input = inp;
 
     inp->uinput_fd = open("/dev/uinput", O_WRONLY | O_NONBLOCK);
     if (inp->uinput_fd <= 0)
     {
-        INP_FATAL("Failed to open uinput\n");
+        KMSVNC_FATAL("Failed to open uinput\n");
     }
     INP_IOCTL_MUST(inp->uinput_fd, UI_SET_EVBIT, EV_KEY);
     INP_IOCTL_MUST(inp->uinput_fd, UI_SET_EVBIT, EV_SYN);
@@ -74,6 +75,7 @@ int uinput_init()
     INP_IOCTL_MUST(inp->uinput_fd, UI_DEV_CREATE);
 
     inp->keystate = malloc(UINPUT_MAX_KEY);
+    if (!inp->keystate) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
     memset(inp->keystate, 0, UINPUT_MAX_KEY);
 
     return 0;
@@ -114,7 +116,7 @@ void rfb_key_hook(rfbBool down, rfbKeySym keysym, rfbClientPtr cl)
                 .value = 0,
             },
         };
-        for (int i = 0; i < ARRAY_SIZE(ies); i++)
+        for (int i = 0; i < KMSVNC_ARRAY_ELEMENTS(ies); i++)
         {
             write(kmsvnc->input->uinput_fd, &ies[i], sizeof(ies[0]));
         }
@@ -159,7 +161,7 @@ void rfb_ptr_hook(int mask, int screen_x, int screen_y, rfbClientPtr cl)
             .value = 0,
         },
     };
-    for (int i = 0; i < ARRAY_SIZE(ies1); i++)
+    for (int i = 0; i < KMSVNC_ARRAY_ELEMENTS(ies1); i++)
     {
         write(kmsvnc->input->uinput_fd, &ies1[i], sizeof(ies1[0]));
     }
@@ -177,7 +179,7 @@ void rfb_ptr_hook(int mask, int screen_x, int screen_y, rfbClientPtr cl)
                 .value = 0,
             },
         };
-        for (int i = 0; i < ARRAY_SIZE(ies2); i++)
+        for (int i = 0; i < KMSVNC_ARRAY_ELEMENTS(ies2); i++)
         {
             write(kmsvnc->input->uinput_fd, &ies2[i], sizeof(ies2[0]));
         }

input.h

@@ -4,14 +4,11 @@
 
 #include "kmsvnc.h"
 
-#define ARRAY_SIZE(x) (sizeof(x) / sizeof(x[0]))
-
 #define UINPUT_ABS_MAX INT16_MAX
 #define UINPUT_MAX_KEY 256
 
-#define INP_FATAL(...) { fprintf(stderr, __VA_ARGS__); return 1; }
-#define INP_IOCTL_MUST(...) { int e; if (e = ioctl(__VA_ARGS__)) INP_FATAL("uinput ioctl error %d on line %d\n", e, __LINE__) }
-#define INP_IOCTL_MAY(...) { int e; if (e = ioctl(__VA_ARGS__)) fprintf(stderr, "uinput ioctl error %d on line %d\n", e, __LINE__); }
+#define INP_IOCTL_MUST(...) do{ int e; if (e = ioctl(__VA_ARGS__)) KMSVNC_FATAL("uinput ioctl error %d on line %d\n", e, __LINE__); } while(0)
+#define INP_IOCTL_MAY(...) do{ int e; if (e = ioctl(__VA_ARGS__)) fprintf(stderr, "uinput ioctl error %d on line %d\n", e, __LINE__); } while(0)
 
 void uinput_cleanup();
 int uinput_init();

keymap.c

@@ -25,13 +25,14 @@ void xkb_cleanup() {
 int xkb_init()
 {
     struct kmsvnc_keymap_data *xkb = malloc(sizeof(struct kmsvnc_keymap_data));
+    if (!xkb) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
     memset(xkb, 0, sizeof(struct kmsvnc_keymap_data));
     kmsvnc->keymap = xkb;
 
     xkb->ctx = xkb_context_new(XKB_CONTEXT_NO_FLAGS);
     if (xkb->ctx == NULL)
     {
-        XKB_FATAL("Failed to create XKB context\n");
+        KMSVNC_FATAL("Failed to create XKB context\n");
     }
     struct xkb_rule_names names = {
         .rules = NULL,
@@ -43,7 +44,7 @@ int xkb_init()
     xkb->map = xkb_keymap_new_from_names(xkb->ctx, &names, 0);
     if (xkb->map == NULL)
     {
-        XKB_FATAL("Failed to create XKB keymap\n");
+        KMSVNC_FATAL("Failed to create XKB keymap\n");
     }
     // printf("xkb: keymap string\n%s\n", xkb_keymap_get_as_string(xkb->map, XKB_KEYMAP_USE_ORIGINAL_FORMAT));
     return 0;

keymap.h

@@ -2,8 +2,6 @@
 
 #include "kmsvnc.h"
 
-#define XKB_FATAL(...) { fprintf(stderr, __VA_ARGS__); return 1; }
-
 void xkb_cleanup();
 int xkb_init();
 void key_iter(struct xkb_keymap *xkb, xkb_keycode_t key, void *data);

kmsvnc.c

@@ -11,6 +11,7 @@
 #include "keymap.h"
 #include "input.h"
 #include "drm.h"
+#include "va.h"
 
 struct kmsvnc_data *kmsvnc = NULL;
 
@@ -98,6 +99,9 @@ static void cleanup() {
     if (kmsvnc->drm) {
         drm_cleanup();
     }
+    if (kmsvnc->va) {
+        va_cleanup();
+    }
     if (kmsvnc) {
         if (kmsvnc->vnc_opt) {
             free(kmsvnc->vnc_opt);
@@ -215,12 +219,17 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
 
 int main(int argc, char **argv)
 {
-    struct vnc_opt *vncopt = malloc(sizeof(struct vnc_opt));
-    memset(vncopt, 0, sizeof(struct vnc_opt));
-
     kmsvnc = malloc(sizeof(struct kmsvnc_data));
+    if (!kmsvnc) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
     memset(kmsvnc, 0, sizeof(struct kmsvnc_data));
 
+    struct vnc_opt *vncopt = malloc(sizeof(struct vnc_opt));
+    if (!vncopt) {
+        free(kmsvnc);
+        KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
+    }
+    memset(vncopt, 0, sizeof(struct vnc_opt));
+
     kmsvnc->vnc_opt = vncopt;
 
     kmsvnc->card = "/dev/dri/card0";
@@ -260,8 +269,14 @@ int main(int argc, char **argv)
 
     if (kmsvnc->debug_capture_fb) {
         int wfd = open(kmsvnc->debug_capture_fb, O_WRONLY | O_CREAT, 00644);
+        int max_size = 0;
+        for (int i = 0; i < 4; i++) {
+            int size = kmsvnc->drm->mfb->offsets[i] + kmsvnc->drm->mfb->height * kmsvnc->drm->mfb->pitches[i];
+            if (size > max_size) max_size = size;
+        }
         if (wfd > 0) {
-            write(wfd, kmsvnc->drm->mapped, kmsvnc->drm->mfb->width * kmsvnc->drm->mfb->height * BYTES_PER_PIXEL);
+            if (kmsvnc->va) va_hwframe_to_vaapi(kmsvnc->drm->mapped);
+            write(wfd, kmsvnc->drm->mapped, max_size);
             fsync(wfd);
             printf("wrote raw frame buffer to %s\n", kmsvnc->debug_capture_fb);
         }
@@ -274,8 +289,16 @@ int main(int argc, char **argv)
 
     size_t buflen = kmsvnc->drm->mfb->width * kmsvnc->drm->mfb->height * BYTES_PER_PIXEL;
     kmsvnc->buf = malloc(buflen);
+    if (!kmsvnc->buf) {
+        cleanup();
+        KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
+    }
     memset(kmsvnc->buf, 0, buflen);
     kmsvnc->buf1 = malloc(buflen);
+    if (!kmsvnc->buf1) {
+        cleanup();
+        KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
+    }
     memset(kmsvnc->buf1, 0, buflen);
 
     signal(SIGHUP, &signal_handler);

kmsvnc.h

@@ -8,6 +8,7 @@
 #include <amdgpu_drm.h>
 #include <xf86drmMode.h>
 #include <linux/dma-buf.h>
+#include <va/va.h>
 
 
 #define BYTES_PER_PIXEL 4
@@ -36,6 +37,7 @@ struct kmsvnc_data
     struct kmsvnc_drm_data *drm;
     struct kmsvnc_input_data *input;
     struct kmsvnc_keymap_data *keymap;
+    struct kmsvnc_va_data *va;
     rfbScreenInfoPtr server;
     char shutdown;
     char *buf;
@@ -86,5 +88,20 @@ struct kmsvnc_drm_data
     off_t mmap_offset;
     char *mapped;
     struct kmsvnc_drm_funcs *funcs;
+    char *pixfmt_name;
+    char *mod_vendor;
+    char *mod_name;
 };
 
+struct kmsvnc_va_data
+{
+    VADisplay dpy;
+    int render_node_fd;
+    VASurfaceID surface_id;
+    VAImage *image;
+    char *imgbuf;
+};
+
+#define KMSVNC_FATAL(...) do{ fprintf(stderr, __VA_ARGS__); return 1; } while(0)
+#define KMSVNC_ARRAY_ELEMENTS(x) (sizeof(x) / sizeof(x[0]))
+#define KMSVNC_FOURCC_TO_INT(a,b,c,d) (((a) << 0) + ((b) << 8) + ((c) << 16) + ((d) << 24))

va.c

@@ -0,0 +1,238 @@
+#include <va/va_drm.h>
+#include <va/va_drmcommon.h>
+#include <fcntl.h>
+
+#include "va.h"
+#include "kmsvnc.h"
+
+extern struct kmsvnc_data *kmsvnc;
+
+void va_cleanup() {
+    VAStatus s;
+    if (kmsvnc->va) {
+        if (kmsvnc->va->imgbuf) {
+            VA_MAY(vaUnmapBuffer(kmsvnc->va->dpy, kmsvnc->va->image->buf));
+            kmsvnc->va->imgbuf = NULL;
+        }
+        if (kmsvnc->va->image) {
+            if ((s = vaDestroyImage(kmsvnc->va->dpy, kmsvnc->va->image->image_id)) == VA_STATUS_SUCCESS) {
+                free(kmsvnc->va->image);
+            }
+            VA_MAY(s);
+            kmsvnc->va->image = NULL;
+        }
+        if (kmsvnc->va->surface_id > 0) {
+            VA_MAY(vaDestroySurfaces(kmsvnc->va->dpy, &kmsvnc->va->surface_id, 1));
+            kmsvnc->va->surface_id = 0;
+        }
+        if (kmsvnc->va->dpy) {
+            VA_MAY(vaTerminate(kmsvnc->va->dpy));
+            kmsvnc->va->dpy = NULL;
+        }
+        free(kmsvnc->va);
+        kmsvnc->va = NULL;
+    }
+}
+
+static void va_msg_callback(void *user_context, const char *message) {
+    #ifdef KMSVNC_VA_DEBUG
+    printf("va msg: %s");
+    #endif
+}
+
+static void va_error_callback(void *user_context, const char *message) {
+    printf("va error: %s");
+}
+
+int va_init() {
+    if (!kmsvnc->drm || !kmsvnc->drm->drm_fd || !kmsvnc->drm->prime_fd) {
+        KMSVNC_FATAL("drm is not initialized\n");
+    }
+
+    setenv("DISPLAY", "", 1);
+
+    struct kmsvnc_va_data *va = malloc(sizeof(struct kmsvnc_va_data));
+    if (!va) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
+    memset(va, 0, sizeof(struct kmsvnc_va_data));
+    kmsvnc->va = va;
+
+    char* render_node;
+    int effective_fd = 0;
+    if (render_node = drmGetRenderDeviceNameFromFd(kmsvnc->drm->drm_fd)) {
+        va->render_node_fd = open(render_node, O_RDWR);
+        free(render_node);
+    }
+    else {
+        printf("Using non-render node because the device does not have an associated render node.\n");
+    }
+    if (va->render_node_fd > 0) {
+        effective_fd = va->render_node_fd;
+    }
+    else {
+        printf("Using non-render node because render node fails to open.\n");
+        effective_fd = kmsvnc->drm->drm_fd;
+    }
+
+    va->dpy = vaGetDisplayDRM(effective_fd);
+    if (!va->dpy) {
+        KMSVNC_FATAL("vaGetDisplayDRM failed\n");
+    }
+
+    vaSetErrorCallback(va->dpy, &va_error_callback, NULL);
+    vaSetInfoCallback(va->dpy, &va_msg_callback, NULL);
+
+    int major, minor;
+    VAStatus status;
+    VA_MUST(vaInitialize(va->dpy, &major, &minor));
+
+    const char *vendor_string = vaQueryVendorString(va->dpy);
+    printf("vaapi vendor %s\n", vendor_string);
+
+    VADRMPRIMESurfaceDescriptor prime_desc;
+    VASurfaceAttrib prime_attrs[2] = {
+        {
+            .type  = VASurfaceAttribMemoryType,
+            .flags = VA_SURFACE_ATTRIB_SETTABLE,
+            .value.type    = VAGenericValueTypeInteger,
+            .value.value.i = VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME_2,
+        },
+        {
+            .type  = VASurfaceAttribExternalBufferDescriptor,
+            .flags = VA_SURFACE_ATTRIB_SETTABLE,
+            .value.type    = VAGenericValueTypePointer,
+            .value.value.p = &prime_desc,
+        }
+    };
+
+    prime_desc.fourcc = kmsvnc->drm->mfb->pixel_format == KMSVNC_FOURCC_TO_INT('X', 'R', '2', '4') ?
+        KMSVNC_FOURCC_TO_INT('B', 'G', 'R', 'X') :
+        KMSVNC_FOURCC_TO_INT('B', 'G', 'R', 'A') ;
+    prime_desc.width = kmsvnc->drm->mfb->width;
+    prime_desc.height = kmsvnc->drm->mfb->height;
+
+    int i;
+    int max_size = 0;
+    for (i = 0; i < 4; i++) {
+        int size = kmsvnc->drm->mfb->offsets[i] + kmsvnc->drm->mfb->height * kmsvnc->drm->mfb->pitches[i];
+        if (size > max_size) max_size = size;
+    }
+    for (i = 0; i < 4; i++) {
+        prime_desc.objects[i].fd = kmsvnc->drm->prime_fd;
+        prime_desc.objects[i].size = max_size;
+        prime_desc.objects[i].drm_format_modifier = kmsvnc->drm->mfb->modifier;
+    }
+
+    prime_desc.num_layers = 1;
+    prime_desc.layers[0].drm_format = kmsvnc->drm->mfb->pixel_format;
+    for (i = 0; i < 4; i++) {
+        prime_desc.layers[0].object_index[i] = 0;
+        prime_desc.layers[0].offset[i] = kmsvnc->drm->mfb->offsets[i];
+        prime_desc.layers[0].pitch[i] = kmsvnc->drm->mfb->pitches[i];
+    }
+    for (i = 0; i < 4; i++) {
+        if (!kmsvnc->drm->mfb->handles[i]) {
+            break;
+        }
+    }
+    prime_desc.layers[0].num_planes = i;
+    prime_desc.num_objects = 1;
+
+    VAStatus s;
+    if ((s = vaCreateSurfaces(va->dpy, VA_RT_FORMAT_RGB32,
+                            kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, &va->surface_id, 1,
+                            prime_attrs, KMSVNC_ARRAY_ELEMENTS(prime_attrs))) != VA_STATUS_SUCCESS)
+    {
+        printf("vaCreateSurfaces prime2 error %#x %s, trying prime\n", s, vaErrorStr(s));
+
+        VASurfaceAttribExternalBuffers buffer_desc;
+        VASurfaceAttrib buffer_attrs[2] = {
+            {
+                .type  = VASurfaceAttribMemoryType,
+                .flags = VA_SURFACE_ATTRIB_SETTABLE,
+                .value.type    = VAGenericValueTypeInteger,
+                .value.value.i = VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME,
+            },
+            {
+                .type  = VASurfaceAttribExternalBufferDescriptor,
+                .flags = VA_SURFACE_ATTRIB_SETTABLE,
+                .value.type    = VAGenericValueTypePointer,
+                .value.value.p = &buffer_desc,
+            }
+        };
+
+        unsigned long fd = kmsvnc->drm->prime_fd;
+
+        buffer_desc.pixel_format = prime_desc.fourcc;
+        buffer_desc.width        = kmsvnc->drm->mfb->width;
+        buffer_desc.height       = kmsvnc->drm->mfb->height;
+        buffer_desc.data_size    = max_size;
+        buffer_desc.buffers      = &fd;
+        buffer_desc.num_buffers  = 1;
+        buffer_desc.flags        = 0;
+
+        for (i = 0; i < 4; i++) {
+            buffer_desc.pitches[i] = kmsvnc->drm->mfb->pitches[i];
+            buffer_desc.offsets[i] = kmsvnc->drm->mfb->offsets[i];
+        }
+        buffer_desc.num_planes = prime_desc.layers[0].num_planes;
+
+
+        VA_MUST(vaCreateSurfaces(va->dpy, VA_RT_FORMAT_RGB32,
+                kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, &va->surface_id, 1,
+                buffer_attrs, KMSVNC_ARRAY_ELEMENTS(buffer_attrs)));
+    }
+
+    #ifdef KMSVNC_VA_DEBUG
+    int img_fmt_count = vaMaxNumImageFormats(va->dpy);
+    VAImageFormat *img_fmts = malloc(sizeof(VAImageFormat) * img_fmt_count);
+    if (!img_fmts) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
+    {
+        int got;
+        vaQueryImageFormats(va->dpy, img_fmts, &got);
+        if (got != img_fmt_count) {
+            KMSVNC_FATAL("got less VAImageFormats, %d instead of %d\n", got, img_fmt_count);
+        }
+    }
+
+    for (int i = 0; i < img_fmt_count; i++) {
+        printf("fmt %d: fourcc %d, %c%c%c%c, byte_order %s, bpp %d, depth %d, blue_mask %#x, green_mask %#x, red_mask %#x, reserved %#x\n", i, img_fmts[i].fourcc,
+            img_fmts[i].fourcc & 0xff,
+            img_fmts[i].fourcc >> 8 & 0xff,
+            img_fmts[i].fourcc >> 16 & 0xff,
+            img_fmts[i].fourcc >> 24 & 0xff,
+            img_fmts[i].byte_order - 1 ? "VA_LSB_FIRST" : "VA_MSB_FIRST",
+            img_fmts[i].bits_per_pixel,
+            img_fmts[i].depth,
+            img_fmts[i].blue_mask,
+            img_fmts[i].green_mask,
+            img_fmts[i].red_mask,
+            img_fmts[i].va_reserved
+        );
+    }
+    #endif
+
+    VAImageFormat format = {
+        .fourcc = KMSVNC_FOURCC_TO_INT('R','G','B','X'),
+        .byte_order = VA_LSB_FIRST,
+        .bits_per_pixel = 32,
+        .depth = 24,
+        .blue_mask = 0x000000ff,
+        .green_mask = 0x0000ff00,
+        .red_mask = 0x00ff0000,
+        .va_reserved = 0x00000000,
+    };
+    va->image = malloc(sizeof(VAImage));
+    if (!va->image) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
+    if ((s = vaCreateImage(va->dpy, &format, kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, va->image)) != VA_STATUS_SUCCESS) {
+        free(va->image);
+        va->image = NULL;
+        VA_MUST(s);
+    }
+    VA_MUST(vaMapBuffer(va->dpy, va->image->buf, (void**)&va->imgbuf));
+}
+
+int va_hwframe_to_vaapi(char *out) {
+    VA_MUST(vaGetImage(kmsvnc->va->dpy, kmsvnc->va->surface_id, 0, 0,
+                        kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, kmsvnc->va->image->image_id));
+    memcpy(out, kmsvnc->va->imgbuf, kmsvnc->drm->mfb->width * kmsvnc->drm->mfb->height * BYTES_PER_PIXEL);
+}

va.h

@@ -0,0 +1,8 @@
+#pragma once
+
+#define VA_MUST(x) do{VAStatus s; if ((s = (x)) != VA_STATUS_SUCCESS) KMSVNC_FATAL("va operation error %#x %s on line %d\n", s, vaErrorStr(s), __LINE__); } while (0)
+#define VA_MAY(x) do{VAStatus s; if ((s = (x)) != VA_STATUS_SUCCESS) fprintf(stderr, "va operation error %#x %s on line %d\n", s, vaErrorStr(s), __LINE__); } while (0)
+
+void va_cleanup();
+int va_init();
+int va_hwframe_to_vaapi(char *out);