resolve github #8

CMakeLists.txt

@@ -13,21 +13,41 @@ pkg_search_module(XKBCOMMON REQUIRED xkbcommon)
 pkg_search_module(LIBVA REQUIRED libva)
 pkg_search_module(LIBVA_DRM REQUIRED libva-drm)
 
+add_executable(kmsvnc)
+set(kmsvnc_SOURCES kmsvnc.c drm.c input.c keymap.c va.c drm_master.c)
+
 include(CheckIncludeFiles)
 CHECK_INCLUDE_FILES("linux/uinput.h;linux/dma-buf.h" HAVE_LINUX_API_HEADERS)
 IF(NOT HAVE_LINUX_API_HEADERS)
   message(FATAL_ERROR "linux-api-headers not found")
 ENDIF()
 
-add_executable(kmsvnc kmsvnc.c drm.c input.c keymap.c va.c drm_master.c)
+include(CheckSymbolExists)
+check_symbol_exists(SYS_pidfd_getfd "sys/syscall.h" HAVE_PIDFD_GETFD_SYSCALL)
+IF(NOT HAVE_PIDFD_GETFD_SYSCALL)
+  message(WARNING "pidfd_getfd syscall not found, the --screen-blank options will be disabled")
+  target_compile_options(kmsvnc PUBLIC -DDISABLE_KMSVNC_SCREEN_BLANK)
+  list(REMOVE_ITEM kmsvnc_SOURCES drm_master.c)
+ENDIF()
+
+
+target_sources(kmsvnc PUBLIC
+    ${kmsvnc_SOURCES}
+)
 target_include_directories(kmsvnc PUBLIC
   ${LIBDRM_INCLUDEDIR}
-  ${LIBDRM_INCLUDEDIR}/libdrm
   ${LIBVNCSERVER_INCLUDEDIR}
   ${XKBCOMMON_INCLUDEDIR}
   ${LIBVA_INCLUDEDIR}
   ${LIBVA_DRM_INCLUDEDIR}
 )
+target_compile_options(kmsvnc PUBLIC
+  ${LIBDRM_CFLAGS}
+  ${LIBVNCSERVER_CFLAGS}
+  ${XKBCOMMON_CFLAGS}
+  ${LIBVA_CFLAGS}
+  ${LIBVA_DRM_CFLAGS}
+)
 target_link_libraries(kmsvnc PUBLIC
   m
   ${LIBDRM_LIBRARIES}

drm.c

@@ -10,7 +10,15 @@
 
 #include "drm.h"
 #include "va.h"
-#include "drm_master.h"
+
+#ifndef DISABLE_KMSVNC_SCREEN_BLANK
+    #include "drm_master.h"
+#endif
+
+#ifndef fourcc_mod_is_vendor
+    #define fourcc_mod_is_vendor(modifier, vendor) \
+            (fourcc_mod_get_vendor(modifier) == DRM_FORMAT_MOD_VENDOR_## vendor)
+#endif
 
 extern struct kmsvnc_data *kmsvnc;
 
@@ -103,18 +111,8 @@ void convert_intel_x_tiled_kmsbuf(const char *in, int width, int height, char *b
 static void convert_vaapi(const char *in, int width, int height, char *buff) {
     va_hwframe_to_vaapi(buff);
     if (
-        !kmsvnc->va->selected_fmt->byte_order &&
         (KMSVNC_FOURCC_TO_INT('R','G','B',0) & kmsvnc->va->selected_fmt->fourcc) == KMSVNC_FOURCC_TO_INT('R','G','B',0)
     ) {}
-    else if (
-        kmsvnc->va->selected_fmt->byte_order &&
-        (KMSVNC_FOURCC_TO_INT(0,'B','G','R') & kmsvnc->va->selected_fmt->fourcc) == KMSVNC_FOURCC_TO_INT(0,'B','G','R')
-    ) {
-        for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) {
-            uint32_t *pixdata = (uint32_t*)(buff + i);
-            *pixdata = __builtin_bswap32(*pixdata);
-        }
-    }
     else {
         // is 30 depth?
         if (kmsvnc->va->selected_fmt->depth == 30) {
@@ -127,8 +125,8 @@ static void convert_vaapi(const char *in, int width, int height, char *buff) {
             }
         }
         else {
-            // handle ihd and mesa byte order quirk
-            if (kmsvnc->va->selected_fmt->byte_order) {
+            // actually, does anyone use this?
+            if (!kmsvnc->va->selected_fmt->byte_order) {
                 for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) {
                     uint32_t *pixdata = (uint32_t*)(buff + i);
                     *pixdata = __builtin_bswap32(*pixdata);
@@ -175,6 +173,7 @@ void drm_sync_noop(int drmfd)
 
 void drm_cleanup() {
     if (kmsvnc->drm) {
+#ifndef DISABLE_KMSVNC_SCREEN_BLANK
         if (kmsvnc->drm->gamma && kmsvnc->drm->gamma->size && kmsvnc->drm->gamma->red && kmsvnc->drm->gamma->green && kmsvnc->drm->gamma->blue) {
             if (drmModeCrtcSetGamma(kmsvnc->drm->drm_master_fd ?: kmsvnc->drm->drm_fd, kmsvnc->drm->plane->crtc_id, kmsvnc->drm->gamma->size, kmsvnc->drm->gamma->red, kmsvnc->drm->gamma->green, kmsvnc->drm->gamma->blue)) perror("Failed to restore gamma");
         }
@@ -186,6 +185,7 @@ void drm_cleanup() {
             free(kmsvnc->drm->gamma);
             kmsvnc->drm->gamma = NULL;
         }
+#endif
         if (kmsvnc->drm->drm_ver) {
             drmFreeVersion(kmsvnc->drm->drm_ver);
             kmsvnc->drm->drm_ver = NULL;
@@ -497,6 +497,7 @@ int drm_open() {
     if (!kmsvnc->screen_blank && drmIsMaster(drm->drm_fd)) {
         if (drmDropMaster(drm->drm_fd)) fprintf(stderr, "Failed to drop master");
     }
+#ifndef DISABLE_KMSVNC_SCREEN_BLANK
     if (kmsvnc->screen_blank && !drmIsMaster(drm->drm_fd)) {
         drm->drm_master_fd = drm_get_master_fd();
         drm->drm_master_fd = drm->drm_master_fd > 0 ? drm->drm_master_fd : 0;
@@ -504,6 +505,7 @@ int drm_open() {
             fprintf(stderr, "not master client, master fd %d\n", drm->drm_master_fd);
         }
     }
+#endif
 
     drm->drm_ver = drmGetVersion(drm->drm_fd);
     printf("drm driver is %s\n", drm->drm_ver->name);
@@ -516,6 +518,7 @@ int drm_open() {
 
     if (drm_refresh_planes(1)) return 1;
 
+#ifndef DISABLE_KMSVNC_SCREEN_BLANK
     if (kmsvnc->screen_blank) {
         drm->gamma = malloc(sizeof(struct kmsvnc_drm_gamma_data));
         if (!drm->gamma) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
@@ -575,6 +578,7 @@ int drm_open() {
             target_crtc = NULL;
         }
     }
+#endif
 
     drm->mfb = drmModeGetFB2(drm->drm_fd, drm->plane->fb_id);
     if (!drm->mfb) {

kmsvnc.c

@@ -241,8 +241,10 @@ static struct argp_option kmsvnc_main_options[] = {
     {"input-height", 0xff07, "0", 0, "Explicitly set input height"},
     {"input-offx", 0xff08, "0", 0, "Set input offset of x axis on a multi display system"},
     {"input-offy", 0xff09, "0", 0, "Set input offset of y axis on a multi display system"},
+#ifndef DISABLE_KMSVNC_SCREEN_BLANK
     {"screen-blank", 0xff0a, 0, OPTION_ARG_OPTIONAL, "Blank screen with gamma set on crtc"},
     {"screen-blank-restore-linear", 0xff0b, 0, OPTION_ARG_OPTIONAL, "Restore linear values on exit in case of messed up gamma"},
+#endif
     {"wakeup", 'w', 0, OPTION_ARG_OPTIONAL, "Move mouse to wake the system up before start"},
     {"disable-input", 'i', 0, OPTION_ARG_OPTIONAL, "Disable uinput"},
     {"desktop-name", 'n', "kmsvnc", 0, "Specify vnc desktop name"},

kmsvnc.h

@@ -139,6 +139,7 @@ struct kmsvnc_va_data
     VAImageFormat* img_fmts;
     int img_fmt_count;
     VAImageFormat* selected_fmt;
+    const char *vendor_string;
 };
 
 #define KMSVNC_FATAL(...) do{ fprintf(stderr, __VA_ARGS__); return 1; } while(0)

va.c

@@ -6,6 +6,7 @@
 #include <va/va_drm.h>
 #include <va/va_drmcommon.h>
 #include <fcntl.h>
+#include <string.h>
 
 #include "va.h"
 #include "kmsvnc.h"
@@ -38,6 +39,9 @@ void va_cleanup() {
             VA_MAY(vaTerminate(kmsvnc->va->dpy));
             kmsvnc->va->dpy = NULL;
         }
+        if (kmsvnc->va->vendor_string) {
+            kmsvnc->va->vendor_string = NULL;
+        }
         free(kmsvnc->va);
         kmsvnc->va = NULL;
     }
@@ -80,26 +84,17 @@ struct va_fmt_data {
     char is_alpha;
     uint32_t va_rt_format;
     uint32_t depth;
-    uint32_t blue_mask;
-    uint32_t green_mask;
-    uint32_t red_mask;
-    uint32_t byte_order;
 };
 
-static VAImageFormat* vaImgFmt_from_vaFmtData(struct va_fmt_data* data) {
+static VAImageFormat* vaImgFmt_apply_quirks(struct va_fmt_data* data) {
     static VAImageFormat ret = {0};
-    VAImageFormat fmt = {
-        .fourcc = data->va_fourcc,
-        .byte_order = data->byte_order,
-        .bits_per_pixel = 32,
-        .depth = data->depth,
-        .blue_mask = data->blue_mask,
-        .green_mask = data->green_mask,
-        .red_mask = data->red_mask,
-        .alpha_mask = 0,
-        .va_reserved = {0}
-    };
-    memcpy(&ret, &fmt, sizeof(VAImageFormat));
+    memcpy(&ret, data->fmt, sizeof(VAImageFormat));
+    if (!strncmp(kmsvnc->va->vendor_string, "Mesa", 4) && data->depth != 30) {
+        printf("applying mesa quirk\n");
+        ret.blue_mask = __builtin_bswap32(data->fmt->blue_mask);
+        ret.green_mask = __builtin_bswap32(data->fmt->green_mask);
+        ret.red_mask = __builtin_bswap32(data->fmt->red_mask);
+    }
     return &ret;
 }
 
@@ -126,6 +121,7 @@ int va_init() {
     }
 
     setenv("DISPLAY", "", 1);
+    setenv("WAYLAND_DISPLAY", "", 1);
 
     struct kmsvnc_va_data *va = malloc(sizeof(struct kmsvnc_va_data));
     if (!va) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
@@ -161,8 +157,8 @@ int va_init() {
     VAStatus status;
     VA_MUST(vaInitialize(va->dpy, &major, &minor));
 
-    const char *vendor_string = vaQueryVendorString(va->dpy);
-    printf("vaapi vendor %s\n", vendor_string);
+    va->vendor_string = vaQueryVendorString(va->dpy);
+    printf("vaapi vendor %s\n", va->vendor_string);
 
     VADRMPRIMESurfaceDescriptor prime_desc;
     VASurfaceAttrib prime_attrs[2] = {
@@ -291,45 +287,29 @@ int va_init() {
     }
 
     struct va_fmt_data format_to_try[] = {
-        {KMSVNC_FOURCC_TO_INT('R','G','B','X'), NULL, 0, VA_RT_FORMAT_RGB32, 24, 0xff00, 0xff0000, 0xff000000, 0},
-        {KMSVNC_FOURCC_TO_INT('R','G','B','A'), NULL, 1, VA_RT_FORMAT_RGB32, 32, 0xff00, 0xff0000, 0xff000000, 0},
+        {KMSVNC_FOURCC_TO_INT('R','G','B','X'), NULL, 0, VA_RT_FORMAT_RGB32, 24},
+        {KMSVNC_FOURCC_TO_INT('R','G','B','A'), NULL, 1, VA_RT_FORMAT_RGB32, 32},
 
-        {KMSVNC_FOURCC_TO_INT('X','B','G','R'), NULL, 0, VA_RT_FORMAT_RGB32, 24, 0xff0000, 0xff00, 0xff, 0},
-        {KMSVNC_FOURCC_TO_INT('A','B','G','R'), NULL, 1, VA_RT_FORMAT_RGB32, 32, 0xff0000, 0xff00, 0xff, 0},
+        {KMSVNC_FOURCC_TO_INT('X','B','G','R'), NULL, 0, VA_RT_FORMAT_RGB32, 24},
+        {KMSVNC_FOURCC_TO_INT('A','B','G','R'), NULL, 1, VA_RT_FORMAT_RGB32, 32},
 
-        {KMSVNC_FOURCC_TO_INT('X','R','G','B'), NULL, 0, VA_RT_FORMAT_RGB32, 24, 0xff, 0xff00, 0xff0000, 0},
-        {KMSVNC_FOURCC_TO_INT('A','R','G','B'), NULL, 1, VA_RT_FORMAT_RGB32, 32, 0xff, 0xff00, 0xff0000, 0},
+        {KMSVNC_FOURCC_TO_INT('X','R','G','B'), NULL, 0, VA_RT_FORMAT_RGB32, 24},
+        {KMSVNC_FOURCC_TO_INT('A','R','G','B'), NULL, 1, VA_RT_FORMAT_RGB32, 32},
 
-        {KMSVNC_FOURCC_TO_INT('B','G','R','X'), NULL, 0, VA_RT_FORMAT_RGB32, 24, 0xff000000, 0xff0000, 0xff00, 0},
-        {KMSVNC_FOURCC_TO_INT('B','G','R','A'), NULL, 1, VA_RT_FORMAT_RGB32, 32, 0xff000000, 0xff0000, 0xff00, 0},
+        {KMSVNC_FOURCC_TO_INT('B','G','R','X'), NULL, 0, VA_RT_FORMAT_RGB32, 24},
+        {KMSVNC_FOURCC_TO_INT('B','G','R','A'), NULL, 1, VA_RT_FORMAT_RGB32, 32},
 
 
-        {KMSVNC_FOURCC_TO_INT('X','R','3','0'), NULL, 0, VA_RT_FORMAT_RGB32_10, 30, 0x3ff, 0xffc00, 0x3ff00000, 0},
-        {KMSVNC_FOURCC_TO_INT('A','R','3','0'), NULL, 1, VA_RT_FORMAT_RGB32_10, 30, 0x3ff, 0xffc00, 0x3ff00000, 0},
-        {KMSVNC_FOURCC_TO_INT('X','B','3','0'), NULL, 0, VA_RT_FORMAT_RGB32_10, 30, 0x3ff00000, 0xffc00, 0x3ff, 0},
-        {KMSVNC_FOURCC_TO_INT('A','B','3','0'), NULL, 1, VA_RT_FORMAT_RGB32_10, 30, 0x3ff00000, 0xffc00, 0x3ff, 0},
+        {KMSVNC_FOURCC_TO_INT('X','R','3','0'), NULL, 0, VA_RT_FORMAT_RGB32_10, 30},
+        {KMSVNC_FOURCC_TO_INT('A','R','3','0'), NULL, 1, VA_RT_FORMAT_RGB32_10, 30},
+        {KMSVNC_FOURCC_TO_INT('X','B','3','0'), NULL, 0, VA_RT_FORMAT_RGB32_10, 30},
+        {KMSVNC_FOURCC_TO_INT('A','B','3','0'), NULL, 1, VA_RT_FORMAT_RGB32_10, 30},
     };
 
     for (int i = 0; i < va->img_fmt_count; i++) {
         for (int j = 0; j < KMSVNC_ARRAY_ELEMENTS(format_to_try); j++) {
             if (va->img_fmts[i].fourcc == format_to_try[j].va_fourcc) {
-                if (
-                    va->img_fmts[i].blue_mask == format_to_try[j].blue_mask &&
-                    va->img_fmts[i].green_mask == format_to_try[j].green_mask &&
-                    va->img_fmts[i].red_mask == format_to_try[j].red_mask
-                ) {
-                    format_to_try[j].fmt = va->img_fmts + i;
-                }
-                else if (
-                    format_to_try[j].depth != 30 &&
-                    va->img_fmts[i].blue_mask == __builtin_bswap32(format_to_try[j].blue_mask) &&
-                    va->img_fmts[i].green_mask == __builtin_bswap32(format_to_try[j].green_mask) &&
-                    va->img_fmts[i].red_mask == __builtin_bswap32(format_to_try[j].red_mask)
-                ) {
-                    // mesa quirk: mesa fourcc and pixel data is lsb_first for RGB32 formats, msb_first for RGB32_10 formats
-                    format_to_try[j].byte_order = 1u;
-                    format_to_try[j].fmt = va->img_fmts + i;
-                }
+                format_to_try[j].fmt = va->img_fmts + i;
             }
         }
     }
@@ -344,7 +324,7 @@ int va_init() {
             for (int i = 0; i < KMSVNC_ARRAY_ELEMENTS(format_to_try); i++) {
                 if (format_to_try[i].fmt == NULL) continue;
                 if (va->image->format.fourcc == format_to_try[i].fmt->fourcc) {
-                    va->selected_fmt = vaImgFmt_from_vaFmtData(format_to_try + i);
+                    va->selected_fmt = vaImgFmt_apply_quirks(format_to_try + i);
                     break;
                 }
             }
@@ -389,7 +369,7 @@ int va_init() {
                 continue;
             }
             else {
-                va->selected_fmt = vaImgFmt_from_vaFmtData(format_to_try + i);
+                va->selected_fmt = vaImgFmt_apply_quirks(format_to_try + i);
                 break;
             }
         }