switch to gitea actions

.drone.yml

@@ -1,21 +0,0 @@
-kind: pipeline
-type: docker
-name: default
-
-steps:
-- name: build
-  image: archlinux:latest
-  commands:
-  - pacman -Syu --noconfirm --needed base-devel libvncserver libxkbcommon libdrm libva git cmake
-  - mkdir build
-  - cd build
-  - cmake ..
-  - make
-
-trigger:
-  branch:
-  - master
-  - dev
-  event:
-    exclude:
-    - pull_request

.forgejo/workflows/default.yml

@@ -0,0 +1,27 @@
+name: Build with gcc + clang
+on:
+  push:
+    branches: [dev]
+jobs:
+  build:
+    if: "github.event_name != 'push' || !contains(github.event.head_commit.message, '[skip ci]')"
+    runs-on: docker
+    container:
+      image: archlinux:latest
+    env:
+      CFLAGS: "-pipe -fno-plt -fexceptions -fstack-clash-protection -fcf-protection -Wp,-D_FORTIFY_SOURCE=2 -Wformat -Werror=format-security"
+    steps:
+      - name: Prepare dependencies
+        run: |
+          pacman -Syu --noconfirm --needed nodejs git \
+            base-devel libvncserver libxkbcommon libdrm libva cmake clang
+      - name: Check out repository code
+        uses: actions/checkout@v4
+      - name: Build with gcc
+        run: |
+          CC=gcc cmake -B gcc-out
+          cmake --build gcc-out
+      - name: Build with clang
+        run: |
+          CC=clang cmake -B clang-out
+          cmake --build clang-out

CMakeLists.txt

@@ -13,13 +13,37 @@ pkg_search_module(XKBCOMMON REQUIRED xkbcommon)
 pkg_search_module(LIBVA REQUIRED libva)
 pkg_search_module(LIBVA_DRM REQUIRED libva-drm)
 
+add_executable(kmsvnc)
+set(kmsvnc_SOURCES kmsvnc.c drm.c input.c keymap.c va.c drm_master.c)
+
 include(CheckIncludeFiles)
 CHECK_INCLUDE_FILES("linux/uinput.h;linux/dma-buf.h" HAVE_LINUX_API_HEADERS)
 IF(NOT HAVE_LINUX_API_HEADERS)
   message(FATAL_ERROR "linux-api-headers not found")
 ENDIF()
 
-add_executable(kmsvnc kmsvnc.c drm.c input.c keymap.c va.c drm_master.c)
+include(CheckSymbolExists)
+check_symbol_exists(SYS_pidfd_getfd "sys/syscall.h" HAVE_LIBC_SYS_pidfd_getfd)
+IF(NOT HAVE_LIBC_SYS_pidfd_getfd)
+  message(WARNING "pidfd_getfd syscall not found, the --screen-blank options will be disabled")
+  target_compile_options(kmsvnc PUBLIC -DDISABLE_KMSVNC_SCREEN_BLANK)
+  list(REMOVE_ITEM kmsvnc_SOURCES drm_master.c)
+ENDIF()
+include(CMakePushCheckState)
+cmake_push_check_state()
+set(CMAKE_REQUIRED_INCLUDES ${LIBDRM_INCLUDEDIR}/libdrm) # can't do anything about that
+set(CMAKE_REQUIRED_LIBRARIES ${LIBDRM_LIBRARIES})
+check_symbol_exists(drmGetFormatName "xf86drm.h" HAVE_LIBDRM_drmGetFormatName)
+cmake_pop_check_state()
+IF(NOT HAVE_LIBDRM_drmGetFormatName)
+  message(WARNING "drmGetFormatName not found, format name printing will be disabled")
+  target_compile_options(kmsvnc PUBLIC -DDISABLE_KMSVNC_drmGetFormatName)
+ENDIF()
+
+
+target_sources(kmsvnc PUBLIC
+    ${kmsvnc_SOURCES}
+)
 target_include_directories(kmsvnc PUBLIC
   ${LIBDRM_INCLUDEDIR}
   ${LIBDRM_INCLUDEDIR}/libdrm

drm.c

@@ -10,7 +10,25 @@
 
 #include "drm.h"
 #include "va.h"
+
+#ifndef DISABLE_KMSVNC_SCREEN_BLANK
     #include "drm_master.h"
+#endif
+
+#ifndef fourcc_mod_is_vendor
+    #define fourcc_mod_is_vendor(modifier, vendor) \
+            (fourcc_mod_get_vendor(modifier) == DRM_FORMAT_MOD_VENDOR_## vendor)
+#endif
+#ifdef DISABLE_KMSVNC_drmGetFormatName
+    static char* drmGetFormatName(uint32_t data) {
+        char *name = "missing drmGetFormatName";
+        char *out = malloc(strlen(name)+1);
+        if (out) {
+            memcpy(out, name, strlen(name)+1);
+        }
+        return out;
+    }
+#endif
 
 extern struct kmsvnc_data *kmsvnc;
 
@@ -27,14 +45,18 @@ static int check_pixfmt_non_vaapi() {
 
 static void convert_copy(const char *in, int width, int height, char *buff)
 {
+    if (likely(in != buff)) {
         memcpy(buff, in, width * height * BYTES_PER_PIXEL);
     }
+}
 
 static void convert_bgra_to_rgba(const char *in, int width, int height, char *buff)
 {
+    if (likely(in != buff)) {
         memcpy(buff, in, width * height * BYTES_PER_PIXEL);
+    }
     for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) {
-        uint32_t pixdata = htonl(*((uint32_t*)(kmsvnc->drm->kms_convert_buf + i)));
+        uint32_t pixdata = htonl(*((uint32_t*)(buff + i)));
         buff[i+0] = (pixdata & 0x0000ff00) >> 8;
         buff[i+2] = (pixdata & 0xff000000) >> 24;
     }
@@ -97,39 +119,45 @@ void convert_intel_x_tiled_kmsbuf(const char *in, int width, int height, char *b
 }
 
 static void convert_vaapi(const char *in, int width, int height, char *buff) {
-    if (KMSVNC_FOURCC_TO_INT('R','G','B', 0) & kmsvnc->va->image->format.fourcc == KMSVNC_FOURCC_TO_INT('R','G','B', 0)) {
     va_hwframe_to_vaapi(buff);
-    }
+    if (
+        (KMSVNC_FOURCC_TO_INT('R','G','B',0) & kmsvnc->va->selected_fmt->fourcc) == KMSVNC_FOURCC_TO_INT('R','G','B',0)
+    ) {}
     else {
-        if (convert_buf_allocate(width * height * BYTES_PER_PIXEL)) return;
-        va_hwframe_to_vaapi(kmsvnc->drm->kms_convert_buf);
         // is 30 depth?
-        if (kmsvnc->va->image->format.depth == 30) {
+        if (kmsvnc->va->selected_fmt->depth == 30) {
             for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) {
                 // ensure little endianess
-                uint32_t pixdata = __builtin_bswap32(htonl(*((uint32_t*)(kmsvnc->drm->kms_convert_buf + i))));
+                uint32_t pixdata = __builtin_bswap32(htonl(*((uint32_t*)(buff + i))));
-                kmsvnc->drm->kms_convert_buf[i] = (pixdata & 0x3ff00000) >> 20 >> 2;
+                buff[i] = (pixdata & 0x3ff00000) >> 20 >> 2;
-                kmsvnc->drm->kms_convert_buf[i+1] = (pixdata & 0xffc00) >> 10 >> 2;
+                buff[i+1] = (pixdata & 0xffc00) >> 10 >> 2;
-                kmsvnc->drm->kms_convert_buf[i+2] = (pixdata & 0x3ff) >> 2;
+                buff[i+2] = (pixdata & 0x3ff) >> 2;
+            }
+        }
+        else {
+            // actually, does anyone use this?
+            if (!kmsvnc->va->selected_fmt->byte_order) {
+                for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) {
+                    uint32_t *pixdata = (uint32_t*)(buff + i);
+                    *pixdata = __builtin_bswap32(*pixdata);
+                }
             }
         }
         // is xrgb?
-        if ((kmsvnc->va->image->format.blue_mask | kmsvnc->va->image->format.red_mask) < 0x1000000) {
+        if ((kmsvnc->va->selected_fmt->blue_mask | kmsvnc->va->selected_fmt->red_mask) < 0x1000000) {
             for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) {
-                uint32_t *pixdata = (uint32_t*)(kmsvnc->drm->kms_convert_buf + i);
+                uint32_t *pixdata = (uint32_t*)(buff + i);
                 *pixdata = ntohl(htonl(*pixdata) << 8);
             }
         }
         // is bgrx?
-        if (kmsvnc->va->image->format.blue_mask > kmsvnc->va->image->format.red_mask) {
+        if (kmsvnc->va->selected_fmt->blue_mask > kmsvnc->va->selected_fmt->red_mask) {
             for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) {
-                uint32_t pixdata = htonl(*((uint32_t*)(kmsvnc->drm->kms_convert_buf + i)));
+                uint32_t pixdata = htonl(*((uint32_t*)(buff + i)));
                 buff[i+0] = (pixdata & 0x0000ff00) >> 8;
                 buff[i+2] = (pixdata & 0xff000000) >> 24;
             }
         }
-        // rgbx now
-        memcpy(buff, kmsvnc->drm->kms_convert_buf, width * height * BYTES_PER_PIXEL);
     }
 }
 
@@ -155,6 +183,7 @@ void drm_sync_noop(int drmfd)
 
 void drm_cleanup() {
     if (kmsvnc->drm) {
+#ifndef DISABLE_KMSVNC_SCREEN_BLANK
         if (kmsvnc->drm->gamma && kmsvnc->drm->gamma->size && kmsvnc->drm->gamma->red && kmsvnc->drm->gamma->green && kmsvnc->drm->gamma->blue) {
             if (drmModeCrtcSetGamma(kmsvnc->drm->drm_master_fd ?: kmsvnc->drm->drm_fd, kmsvnc->drm->plane->crtc_id, kmsvnc->drm->gamma->size, kmsvnc->drm->gamma->red, kmsvnc->drm->gamma->green, kmsvnc->drm->gamma->blue)) perror("Failed to restore gamma");
         }
@@ -166,6 +195,7 @@ void drm_cleanup() {
             free(kmsvnc->drm->gamma);
             kmsvnc->drm->gamma = NULL;
         }
+#endif
         if (kmsvnc->drm->drm_ver) {
             drmFreeVersion(kmsvnc->drm->drm_ver);
             kmsvnc->drm->drm_ver = NULL;
@@ -474,6 +504,10 @@ int drm_open() {
     {
         KMSVNC_FATAL("card %s open failed: %s\n", kmsvnc->card, strerror(errno));
     }
+    if (!kmsvnc->screen_blank && drmIsMaster(drm->drm_fd)) {
+        if (drmDropMaster(drm->drm_fd)) fprintf(stderr, "Failed to drop master");
+    }
+#ifndef DISABLE_KMSVNC_SCREEN_BLANK
     if (kmsvnc->screen_blank && !drmIsMaster(drm->drm_fd)) {
         drm->drm_master_fd = drm_get_master_fd();
         drm->drm_master_fd = drm->drm_master_fd > 0 ? drm->drm_master_fd : 0;
@@ -481,6 +515,7 @@ int drm_open() {
             fprintf(stderr, "not master client, master fd %d\n", drm->drm_master_fd);
         }
     }
+#endif
 
     drm->drm_ver = drmGetVersion(drm->drm_fd);
     printf("drm driver is %s\n", drm->drm_ver->name);
@@ -493,6 +528,7 @@ int drm_open() {
 
     if (drm_refresh_planes(1)) return 1;
 
+#ifndef DISABLE_KMSVNC_SCREEN_BLANK
     if (kmsvnc->screen_blank) {
         drm->gamma = malloc(sizeof(struct kmsvnc_drm_gamma_data));
         if (!drm->gamma) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
@@ -552,6 +588,7 @@ int drm_open() {
             target_crtc = NULL;
         }
     }
+#endif
 
     drm->mfb = drmModeGetFB2(drm->drm_fd, drm->plane->fb_id);
     if (!drm->mfb) {

drm.h

@@ -2,9 +2,9 @@
 
 #include "kmsvnc.h"
 
-#define DRM_IOCTL_MUST(...) do{ int e; if (e = drmIoctl(__VA_ARGS__)) KMSVNC_FATAL("DRM ioctl error %d on line %d\n", e, __LINE__); } while(0)
+#define DRM_IOCTL_MUST(...) do{ int e; if ((e = drmIoctl(__VA_ARGS__))) KMSVNC_FATAL("DRM ioctl error %d on line %d\n", e, __LINE__); } while(0)
-#define DRM_IOCTL_MAY(...) do{ int e; if (e = drmIoctl(__VA_ARGS__)) fprintf(stderr, "DRM ioctl error %d on line %d\n", e, __LINE__); } while(0)
+#define DRM_IOCTL_MAY(...) do{ int e; if ((e = drmIoctl(__VA_ARGS__))) fprintf(stderr, "DRM ioctl error %d on line %d\n", e, __LINE__); } while(0)
-#define DRM_R_IOCTL_MAY(...) do{ int e; if (e = ioctl(__VA_ARGS__)) fprintf(stderr, "DRM ioctl error %d on line %d\n", e, __LINE__); } while(0)
+#define DRM_R_IOCTL_MAY(...) do{ int e; if ((e = ioctl(__VA_ARGS__))) fprintf(stderr, "DRM ioctl error %d on line %d\n", e, __LINE__); } while(0)
 
 
 void drm_cleanup();

drm_master.c

@@ -1,4 +1,5 @@
 #define _GNU_SOURCE
+
 #include <stdio.h>
 #include <errno.h>
 #include <sys/types.h>
@@ -27,7 +28,6 @@ static inline int clone_fd(pid_t pid, int target_fd) {
 }
 
 static inline int cmp_fds(pid_t pid, const char *drm_pth) {
-    if (pid == 1) return -1;
     char path[PATH_MAX+1];
     snprintf(path, PATH_MAX+1, "/proc/%d/fd", pid);
 
@@ -39,18 +39,26 @@ static inline int cmp_fds(pid_t pid, const char *drm_pth) {
             if (ret == -1 && fdlist[n]->d_type == DT_LNK) {
                 char link_pth[PATH_MAX+1];
                 char real_pth[PATH_MAX+1];
+                #pragma GCC diagnostic push
+                #pragma GCC diagnostic ignored "-Wpragmas"
+                #pragma GCC diagnostic ignored "-Wunknown-warning-option"
+                #pragma GCC diagnostic ignored "-Wformat-truncation"
                 snprintf(link_pth, PATH_MAX+1, "%s/%s", path, fdlist[n]->d_name);
+                #pragma GCC diagnostic pop
                 memset(real_pth, 0, PATH_MAX+1);
+                #pragma GCC diagnostic push
+                #pragma GCC diagnostic ignored "-Wunused-result"
                 realpath(link_pth, real_pth);
+                #pragma GCC diagnostic pop
                 if (!strncmp(real_pth, drm_pth, PATH_MAX)) {
                     int fd = atoi(fdlist[n]->d_name);
                     if (fd > 0) {
                         int cloned = clone_fd(pid, fd);
                         if (cloned > 0 && drmIsMaster(cloned)) {
                             ret = cloned;
-                            //if (kmsvnc->debug_enabled) {
+                            if (kmsvnc->debug_enabled) {
                                 fprintf(stderr, "found drm master pid=%d, fd=%d, cloned=%d\n", pid, fd, cloned);
-                            //}
+                            }
                         }
                         else {
                             if (cloned > 0) close(cloned);
@@ -70,13 +78,16 @@ static inline int cmp_fds(pid_t pid, const char *drm_pth) {
 int drm_get_master_fd() {
     char drm_pth[PATH_MAX+1];
     memset(drm_pth, 0, PATH_MAX+1);
+    #pragma GCC diagnostic push
+    #pragma GCC diagnostic ignored "-Wunused-result"
     realpath(kmsvnc->card, drm_pth);
+    #pragma GCC diagnostic pop
 
     struct dirent **proclist;
     int count = scandir("/proc", &proclist, NULL, versionsort);
     int ret = -1;
     if (count >= 0) {
-        for (int n = count - 1; n > 0; n--) {
+        for (int n = 0; n < count; n++) {
             if (ret == -1 && proclist[n]->d_type == DT_DIR) {
                 pid_t pid = (pid_t)atoi(proclist[n]->d_name);
                 if (pid > 0) {

input.h

@@ -7,8 +7,8 @@
 #define UINPUT_ABS_MAX INT16_MAX
 #define UINPUT_MAX_KEY 256
 
-#define INP_IOCTL_MUST(...) do{ int e; if (e = ioctl(__VA_ARGS__)) KMSVNC_FATAL("uinput ioctl error %d on line %d\n", e, __LINE__); } while(0)
+#define INP_IOCTL_MUST(...) do{ int e; if ((e = ioctl(__VA_ARGS__))) KMSVNC_FATAL("uinput ioctl error %d on line %d\n", e, __LINE__); } while(0)
-#define INP_IOCTL_MAY(...) do{ int e; if (e = ioctl(__VA_ARGS__)) fprintf(stderr, "uinput ioctl error %d on line %d\n", e, __LINE__); } while(0)
+#define INP_IOCTL_MAY(...) do{ int e; if ((e = ioctl(__VA_ARGS__))) fprintf(stderr, "uinput ioctl error %d on line %d\n", e, __LINE__); } while(0)
 
 void uinput_cleanup();
 int uinput_init();

kmsvnc.c

@@ -1,9 +1,12 @@
+#define _GNU_SOURCE
+
 #include <fcntl.h>
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
 #include <signal.h>
 #include <time.h>
+#include <unistd.h>
 #include <argp.h>
 #include <arpa/inet.h>
 
@@ -127,7 +130,7 @@ static inline void update_vnc_cursor(char *data, int width, int height) {
         if (!kmsvnc->cursor_bitmap) return;
         kmsvnc->cursor_bitmap_len = rwidth * rheight * BYTES_PER_PIXEL;
     }
-    char *rich_source = malloc(rwidth * rheight * BYTES_PER_PIXEL);
+    unsigned char *rich_source = malloc(rwidth * rheight * BYTES_PER_PIXEL);
     if (!rich_source) return;
     char *maskString = malloc(rwidth * rheight);
     if (!maskString) {
@@ -208,6 +211,7 @@ static void cleanup() {
     }
 }
 
+void signal_handler_noop(int signum){}
 void signal_handler(int signum){
     if (kmsvnc->shutdown) {
         return;
@@ -219,7 +223,7 @@ void signal_handler(int signum){
 }
 
 static struct argp_option kmsvnc_main_options[] = {
-    {"device", 'd', "/dev/dri/card0", 0, "DRM device"},
+    {"device", 'd', "/dev/dri/cardX", 0, "DRM device"},
     {"source-plane", 0xfefc, "0", 0, "Use specific plane"},
     {"source-crtc", 0xfefd, "0", 0, "Use specific crtc (to list all crtcs and planes, set this to -1)"},
     {"force-driver", 0xfefe, "i915", 0, "force a certain driver (for debugging)"},
@@ -238,8 +242,11 @@ static struct argp_option kmsvnc_main_options[] = {
     {"input-height", 0xff07, "0", 0, "Explicitly set input height"},
     {"input-offx", 0xff08, "0", 0, "Set input offset of x axis on a multi display system"},
     {"input-offy", 0xff09, "0", 0, "Set input offset of y axis on a multi display system"},
+#ifndef DISABLE_KMSVNC_SCREEN_BLANK
     {"screen-blank", 0xff0a, 0, OPTION_ARG_OPTIONAL, "Blank screen with gamma set on crtc"},
     {"screen-blank-restore-linear", 0xff0b, 0, OPTION_ARG_OPTIONAL, "Restore linear values on exit in case of messed up gamma"},
+#endif
+    {"va-byteorder-swap", 0xff0c, 0, OPTION_ARG_OPTIONAL, "Force swap vaapi image rgb byteorder"},
     {"wakeup", 'w', 0, OPTION_ARG_OPTIONAL, "Move mouse to wake the system up before start"},
     {"disable-input", 'i', 0, OPTION_ARG_OPTIONAL, "Disable uinput"},
     {"desktop-name", 'n', "kmsvnc", 0, "Specify vnc desktop name"},
@@ -271,6 +278,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
             kmsvnc->vnc_opt->bind6 = arg;
             break;
         case 'p':
+            {
                 int port = atoi(arg);
                 if (port > 0 && port < 65536) {
                     kmsvnc->vnc_opt->port = port;
@@ -278,11 +286,13 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
                 else {
                     argp_error(state, "invalid port %s", arg);
                 }
+            }
             break;
         case '4':
             kmsvnc->vnc_opt->disable_ipv6 = 1;
             break;
         case 0xff00:
+            {
                 int fps = atoi(arg);
                 if (fps > 0 && fps < 1000) {
                     kmsvnc->vnc_opt->sleep_ns = NS_IN_S / fps;
@@ -290,6 +300,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
                 else {
                     argp_error(state, "invalid fps %s", arg);
                 }
+            }
             break;
         case 0xff01:
             kmsvnc->vnc_opt->always_shared = 0;
@@ -316,28 +327,36 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
             kmsvnc->debug_enabled = 1;
             break;
         case 0xff06:
+            {
                 int width = atoi(arg);
                 if (width > 0) {
                     kmsvnc->input_width = width;
                 }
+            }
             break;
         case 0xff07:
+            {
                 int height = atoi(arg);
                 if (height > 0) {
                     kmsvnc->input_height = height;
                 }
+            }
             break;
         case 0xff08:
+            {
                 int offset_x = atoi(arg);
                 if (offset_x > 0) {
                     kmsvnc->input_offx = offset_x;
                 }
+            }
             break;
         case 0xff09:
+            {
                 int offset_y = atoi(arg);
                 if (offset_y > 0) {
                     kmsvnc->input_offy = offset_y;
                 }
+            }
             break;
         case 0xff0a:
             kmsvnc->screen_blank = 1;
@@ -345,6 +364,9 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
         case 0xff0b:
             kmsvnc->screen_blank_restore = 1;
             break;
+        case 0xff0c:
+            kmsvnc->va_byteorder_swap = 1;
+            break;
         case 'w':
             kmsvnc->input_wakeup = 1;
             break;
@@ -377,7 +399,10 @@ int main(int argc, char **argv)
 
     kmsvnc->vnc_opt = vncopt;
 
-    kmsvnc->card = "/dev/dri/card0";
+    #define DEVICE_EXAMPLE_MAX_SIZE 15
+    #define DEVICE_EXAMPLE_FALLBACK "/dev/dri/card0"
+    static char device_example[DEVICE_EXAMPLE_MAX_SIZE] = DEVICE_EXAMPLE_FALLBACK;
+    kmsvnc->card = device_example;
     kmsvnc->va_derive_enabled = -1;
     kmsvnc->vnc_opt->bind = &(struct in_addr){0};
     kmsvnc->vnc_opt->always_shared = 1;
@@ -391,6 +416,18 @@ int main(int argc, char **argv)
     struct argp argp = {kmsvnc_main_options, parse_opt, args_doc, doc};
     argp_parse(&argp, argc, argv, 0, 0, NULL);
 
+    if (kmsvnc->card == device_example) {
+        for (int i = 0; i < 10; i++) {
+            snprintf(kmsvnc->card, DEVICE_EXAMPLE_MAX_SIZE, "/dev/dri/card%d", i);
+            if (!access(kmsvnc->card, F_OK)) {
+                break;
+            }
+            else {
+                snprintf(kmsvnc->card, DEVICE_EXAMPLE_MAX_SIZE, DEVICE_EXAMPLE_FALLBACK);
+            }
+        }
+    }
+
     if (!kmsvnc->disable_input) {
         const char* XKB_DEFAULT_LAYOUT = getenv("XKB_DEFAULT_LAYOUT");
         if (!XKB_DEFAULT_LAYOUT || strcmp(XKB_DEFAULT_LAYOUT, "") == 0) {
@@ -437,6 +474,20 @@ int main(int argc, char **argv)
         else {
             fprintf(stderr, "open file %s failed, %s\n", kmsvnc->debug_capture_fb, strerror(errno));
         }
+        if (kmsvnc->screen_blank) {
+            sigset_t signal_set;
+            int sig;
+            sigemptyset(&signal_set);
+            signal(SIGHUP, &signal_handler_noop);
+            signal(SIGINT, &signal_handler_noop);
+            signal(SIGTERM, &signal_handler_noop);
+            sigaddset(&signal_set, SIGHUP);
+            sigaddset(&signal_set, SIGINT);
+            sigaddset(&signal_set, SIGTERM);
+            fprintf(stderr, "blanking screen...\n");
+            sigwait(&signal_set, &sig);
+            fprintf(stderr, "got sig %d\n", sig);
+        }
         cleanup();
         return 0;
     }

kmsvnc.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <rfb/rfb.h>
+#include <stdint.h>
 #include <xkbcommon/xkbcommon.h>
 
 #include <xf86drm.h>
@@ -44,6 +45,7 @@ struct kmsvnc_data
     int input_offy;
     char screen_blank;
     char screen_blank_restore;
+    char va_byteorder_swap;
     struct kmsvnc_drm_data *drm;
     struct kmsvnc_input_data *input;
     struct kmsvnc_keymap_data *keymap;
@@ -137,6 +139,8 @@ struct kmsvnc_va_data
     char derive_enabled;
     VAImageFormat* img_fmts;
     int img_fmt_count;
+    VAImageFormat* selected_fmt;
+    const char *vendor_string;
 };
 
 #define KMSVNC_FATAL(...) do{ fprintf(stderr, __VA_ARGS__); return 1; } while(0)
@@ -145,3 +149,6 @@ struct kmsvnc_va_data
 #define KMSVNC_WRITE_MAY(fd,buf,count) do { ssize_t e = write((fd), (buf), (count)); if (e != (count)) fprintf(stderr, "should write %ld bytes, actually wrote %ld, on line %d\n", (count), e, __LINE__); } while (0)
 
 #define KMSVNC_DEBUG(...) do{ if (kmsvnc->debug_enabled) fprintf(stdout, __VA_ARGS__); } while(0)
+
+#define likely(x)   __builtin_expect(!!(x), 1)
+#define unlikely(x) __builtin_expect(!!(x), 0)

va.c

@@ -1,6 +1,12 @@
+#define _GNU_SOURCE
+
+#include <stdint.h>
+#include <stdio.h>
+#include <va/va.h>
 #include <va/va_drm.h>
 #include <va/va_drmcommon.h>
 #include <fcntl.h>
+#include <string.h>
 
 #include "va.h"
 #include "kmsvnc.h"
@@ -33,6 +39,9 @@ void va_cleanup() {
             VA_MAY(vaTerminate(kmsvnc->va->dpy));
             kmsvnc->va->dpy = NULL;
         }
+        if (kmsvnc->va->vendor_string) {
+            kmsvnc->va->vendor_string = NULL;
+        }
         free(kmsvnc->va);
         kmsvnc->va = NULL;
     }
@@ -69,8 +78,28 @@ static const struct {
     {KMSVNC_FOURCC_TO_INT('A', 'R', '3', '0'), KMSVNC_FOURCC_TO_INT('A', 'R', '3', '0'), VA_RT_FORMAT_RGB32_10, 1},
 };
 
+struct va_fmt_data {
+    uint32_t va_fourcc;
+    VAImageFormat *fmt;
+    char is_alpha;
+    uint32_t va_rt_format;
+    uint32_t depth;
+};
+
+static VAImageFormat* vaImgFmt_apply_quirks(struct va_fmt_data* data) {
+    static VAImageFormat ret = {0};
+    memcpy(&ret, data->fmt, sizeof(VAImageFormat));
+    if ((kmsvnc->va_byteorder_swap ^ !strncmp(kmsvnc->va->vendor_string, "Mesa", 4)) && data->depth != 30) {
+        printf("applying rgb mask byte order swap\n");
+        ret.blue_mask = __builtin_bswap32(data->fmt->blue_mask);
+        ret.green_mask = __builtin_bswap32(data->fmt->green_mask);
+        ret.red_mask = __builtin_bswap32(data->fmt->red_mask);
+    }
+    return &ret;
+}
+
 static void print_va_image_fmt(VAImageFormat *fmt) {
-        printf("image fmt: fourcc %d, %s, byte_order %s, bpp %d, depth %d, blue_mask %#x, green_mask %#x, red_mask %#x, reserved %#x %#x %#x %#x\n", fmt->fourcc,
+    printf("image fmt: fourcc %d, %s, byte_order %s, bpp %d, depth %d, blue_mask %#x, green_mask %#x, red_mask %#x, alpha_mask %#x, reserved %#x %#x %#x %#x\n", fmt->fourcc,
         fourcc_to_str(fmt->fourcc),
         fmt->byte_order == 1 ? "VA_LSB_FIRST" : "VA_MSB_FIRST",
         fmt->bits_per_pixel,
@@ -78,6 +107,7 @@ static void print_va_image_fmt(VAImageFormat *fmt) {
         fmt->blue_mask,
         fmt->green_mask,
         fmt->red_mask,
+        fmt->alpha_mask,
         fmt->va_reserved[0],
         fmt->va_reserved[1],
         fmt->va_reserved[2],
@@ -91,6 +121,7 @@ int va_init() {
     }
 
     setenv("DISPLAY", "", 1);
+    setenv("WAYLAND_DISPLAY", "", 1);
 
     struct kmsvnc_va_data *va = malloc(sizeof(struct kmsvnc_va_data));
     if (!va) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
@@ -99,7 +130,7 @@ int va_init() {
 
     char* render_node;
     int effective_fd = 0;
-    if (render_node = drmGetRenderDeviceNameFromFd(kmsvnc->drm->drm_fd)) {
+    if ((render_node = drmGetRenderDeviceNameFromFd(kmsvnc->drm->drm_fd))) {
         va->render_node_fd = open(render_node, O_RDWR);
         free(render_node);
     }
@@ -126,8 +157,8 @@ int va_init() {
     VAStatus status;
     VA_MUST(vaInitialize(va->dpy, &major, &minor));
 
-    const char *vendor_string = vaQueryVendorString(va->dpy);
+    va->vendor_string = vaQueryVendorString(va->dpy);
-    printf("vaapi vendor %s\n", vendor_string);
+    printf("vaapi vendor %s\n", va->vendor_string);
 
     VADRMPRIMESurfaceDescriptor prime_desc;
     VASurfaceAttrib prime_attrs[2] = {
@@ -255,28 +286,26 @@ int va_init() {
         }
     }
 
-    struct fourcc_data {
+    struct va_fmt_data format_to_try[] = {
-        uint32_t va_fourcc;
+        {KMSVNC_FOURCC_TO_INT('R','G','B','X'), NULL, 0, VA_RT_FORMAT_RGB32, 24},
-        VAImageFormat *fmt;
+        {KMSVNC_FOURCC_TO_INT('R','G','B','A'), NULL, 1, VA_RT_FORMAT_RGB32, 32},
-        char is_alpha;
-        uint32_t va_rt_format;
-    };
-    struct fourcc_data format_to_try[] = {
-        {KMSVNC_FOURCC_TO_INT('R','G','B','X'), NULL, 0, VA_RT_FORMAT_RGB32},
-        {KMSVNC_FOURCC_TO_INT('R','G','B','A'), NULL, 1, VA_RT_FORMAT_RGB32},
-        {KMSVNC_FOURCC_TO_INT('X','R','G','B'), NULL, 0, VA_RT_FORMAT_RGB32},
-        {KMSVNC_FOURCC_TO_INT('A','R','G','B'), NULL, 1, VA_RT_FORMAT_RGB32},
 
-        {KMSVNC_FOURCC_TO_INT('B','G','R','X'), NULL, 0, VA_RT_FORMAT_RGB32},
+        {KMSVNC_FOURCC_TO_INT('X','B','G','R'), NULL, 0, VA_RT_FORMAT_RGB32, 24},
-        {KMSVNC_FOURCC_TO_INT('B','G','R','A'), NULL, 1, VA_RT_FORMAT_RGB32},
+        {KMSVNC_FOURCC_TO_INT('A','B','G','R'), NULL, 1, VA_RT_FORMAT_RGB32, 32},
-        {KMSVNC_FOURCC_TO_INT('X','B','G','R'), NULL, 0, VA_RT_FORMAT_RGB32},
-        {KMSVNC_FOURCC_TO_INT('A','B','G','R'), NULL, 1, VA_RT_FORMAT_RGB32},
 
-        {KMSVNC_FOURCC_TO_INT('X','R','3','0'), NULL, 0, VA_RT_FORMAT_RGB32_10},
+        {KMSVNC_FOURCC_TO_INT('X','R','G','B'), NULL, 0, VA_RT_FORMAT_RGB32, 24},
-        {KMSVNC_FOURCC_TO_INT('A','R','3','0'), NULL, 1, VA_RT_FORMAT_RGB32_10},
+        {KMSVNC_FOURCC_TO_INT('A','R','G','B'), NULL, 1, VA_RT_FORMAT_RGB32, 32},
-        {KMSVNC_FOURCC_TO_INT('X','B','3','0'), NULL, 0, VA_RT_FORMAT_RGB32_10},
+
-        {KMSVNC_FOURCC_TO_INT('A','B','3','0'), NULL, 1, VA_RT_FORMAT_RGB32_10},
+        {KMSVNC_FOURCC_TO_INT('B','G','R','X'), NULL, 0, VA_RT_FORMAT_RGB32, 24},
+        {KMSVNC_FOURCC_TO_INT('B','G','R','A'), NULL, 1, VA_RT_FORMAT_RGB32, 32},
+
+
+        {KMSVNC_FOURCC_TO_INT('X','R','3','0'), NULL, 0, VA_RT_FORMAT_RGB32_10, 30},
+        {KMSVNC_FOURCC_TO_INT('A','R','3','0'), NULL, 1, VA_RT_FORMAT_RGB32_10, 30},
+        {KMSVNC_FOURCC_TO_INT('X','B','3','0'), NULL, 0, VA_RT_FORMAT_RGB32_10, 30},
+        {KMSVNC_FOURCC_TO_INT('A','B','3','0'), NULL, 1, VA_RT_FORMAT_RGB32_10, 30},
     };
+
     for (int i = 0; i < va->img_fmt_count; i++) {
         for (int j = 0; j < KMSVNC_ARRAY_ELEMENTS(format_to_try); j++) {
             if (va->img_fmts[i].fourcc == format_to_try[j].va_fourcc) {
@@ -292,14 +321,14 @@ int va_init() {
     va->derive_enabled = kmsvnc->va_derive_enabled < 0 ? va->derive_enabled : kmsvnc->va_derive_enabled != 0;
     if (va->derive_enabled) {
         if ((s = vaDeriveImage(va->dpy, va->surface_id, va->image)) == VA_STATUS_SUCCESS) {
-            char found = 0;
             for (int i = 0; i < KMSVNC_ARRAY_ELEMENTS(format_to_try); i++) {
+                if (format_to_try[i].fmt == NULL) continue;
                 if (va->image->format.fourcc == format_to_try[i].fmt->fourcc) {
-                    found = 1;
+                    va->selected_fmt = vaImgFmt_apply_quirks(format_to_try + i);
                     break;
                 }
             }
-            if (!found) {
+            if (!va->selected_fmt) {
                 va->derive_enabled = 0;
                 printf("vaDeriveImage returned unknown fourcc %d %s\n", va->image->format.fourcc, fourcc_to_str(va->image->format.fourcc));
                 VA_MAY(vaDestroyImage(kmsvnc->va->dpy, kmsvnc->va->image->image_id));
@@ -315,7 +344,6 @@ int va_init() {
         }
     }
     if (!va->derive_enabled) {
-        char success = 0;
         for (int i = 0; i < KMSVNC_ARRAY_ELEMENTS(format_to_try); i++) {
             if (format_to_try[i].fmt == NULL) continue;
             if (!kmsvnc->debug_enabled && rt_format != format_to_try[i].va_rt_format) continue;
@@ -341,17 +369,21 @@ int va_init() {
                 continue;
             }
             else {
-                success = 1;
+                va->selected_fmt = vaImgFmt_apply_quirks(format_to_try + i);
                 break;
             }
         }
-        if (!success) {
+        if (!va->selected_fmt) {
             va->imgbuf = NULL;
             KMSVNC_FATAL("failed to get vaapi image\n");
         }
     }
     printf("got vaapi %simage:\n", va->derive_enabled ? "derive " : "");
     print_va_image_fmt(&va->image->format);
+    if (kmsvnc->debug_enabled) {
+        fprintf(stderr, "selected image format:\n");
+        print_va_image_fmt(va->selected_fmt);
+    }
     return 0;
 }