test2

.drone.yml

@@ -0,0 +1,21 @@
+kind: pipeline
+type: docker
+name: default
+
+steps:
+- name: build
+  image: archlinux:latest
+  commands:
+  - pacman -Syu --noconfirm --needed base-devel libvncserver libxkbcommon libdrm libva git cmake
+  - mkdir build
+  - cd build
+  - cmake ..
+  - make
+
+trigger:
+  branch:
+  - master
+  - dev
+  event:
+    exclude:
+    - pull_request

.forgejo/workflows/default.yml

@@ -1,27 +0,0 @@
-name: Build with gcc + clang
-on:
-  push:
-    branches: [dev]
-jobs:
-  build:
-    if: "github.event_name != 'push' || !contains(github.event.head_commit.message, '[skip ci]')"
-    runs-on: docker
-    container:
-      image: archlinux:latest
-    env:
-      CFLAGS: "-pipe -fno-plt -fexceptions -fstack-clash-protection -fcf-protection -Wp,-D_FORTIFY_SOURCE=2 -Wformat -Werror=format-security"
-    steps:
-      - name: Prepare dependencies
-        run: |
-          pacman -Syu --noconfirm --needed nodejs git \
-            base-devel libvncserver libxkbcommon libdrm libva cmake clang
-      - name: Check out repository code
-        uses: actions/checkout@v4
-      - name: Build with gcc
-        run: |
-          CC=gcc cmake -B gcc-out
-          cmake --build gcc-out
-      - name: Build with clang
-        run: |
-          CC=clang cmake -B clang-out
-          cmake --build clang-out

CMakeLists.txt

@@ -13,37 +13,13 @@ pkg_search_module(XKBCOMMON REQUIRED xkbcommon)
 pkg_search_module(LIBVA REQUIRED libva)
 pkg_search_module(LIBVA_DRM REQUIRED libva-drm)
 
-add_executable(kmsvnc)
-set(kmsvnc_SOURCES kmsvnc.c drm.c input.c keymap.c va.c drm_master.c)
-
 include(CheckIncludeFiles)
 CHECK_INCLUDE_FILES("linux/uinput.h;linux/dma-buf.h" HAVE_LINUX_API_HEADERS)
 IF(NOT HAVE_LINUX_API_HEADERS)
   message(FATAL_ERROR "linux-api-headers not found")
 ENDIF()
 
-include(CheckSymbolExists)
-check_symbol_exists(SYS_pidfd_getfd "sys/syscall.h" HAVE_LIBC_SYS_pidfd_getfd)
-IF(NOT HAVE_LIBC_SYS_pidfd_getfd)
-  message(WARNING "pidfd_getfd syscall not found, the --screen-blank options will be disabled")
-  target_compile_options(kmsvnc PUBLIC -DDISABLE_KMSVNC_SCREEN_BLANK)
-  list(REMOVE_ITEM kmsvnc_SOURCES drm_master.c)
-ENDIF()
-include(CMakePushCheckState)
-cmake_push_check_state()
-set(CMAKE_REQUIRED_INCLUDES ${LIBDRM_INCLUDEDIR}/libdrm) # can't do anything about that
-set(CMAKE_REQUIRED_LIBRARIES ${LIBDRM_LIBRARIES})
-check_symbol_exists(drmGetFormatName "xf86drm.h" HAVE_LIBDRM_drmGetFormatName)
-cmake_pop_check_state()
-IF(NOT HAVE_LIBDRM_drmGetFormatName)
-  message(WARNING "drmGetFormatName not found, format name printing will be disabled")
-  target_compile_options(kmsvnc PUBLIC -DDISABLE_KMSVNC_drmGetFormatName)
-ENDIF()
-
-
-target_sources(kmsvnc PUBLIC
-    ${kmsvnc_SOURCES}
-)
+add_executable(kmsvnc kmsvnc.c drm.c input.c keymap.c va.c drm_master.c)
 target_include_directories(kmsvnc PUBLIC
   ${LIBDRM_INCLUDEDIR}
   ${LIBDRM_INCLUDEDIR}/libdrm

drm.c

@@ -10,25 +10,7 @@
 
 #include "drm.h"
 #include "va.h"
-
-#ifndef DISABLE_KMSVNC_SCREEN_BLANK
 #include "drm_master.h"
-#endif
-
-#ifndef fourcc_mod_is_vendor
-    #define fourcc_mod_is_vendor(modifier, vendor) \
-            (fourcc_mod_get_vendor(modifier) == DRM_FORMAT_MOD_VENDOR_## vendor)
-#endif
-#ifdef DISABLE_KMSVNC_drmGetFormatName
-    static char* drmGetFormatName(uint32_t data) {
-        char *name = "missing drmGetFormatName";
-        char *out = malloc(strlen(name)+1);
-        if (out) {
-            memcpy(out, name, strlen(name)+1);
-        }
-        return out;
-    }
-#endif
 
 extern struct kmsvnc_data *kmsvnc;
 
@@ -45,18 +27,14 @@ static int check_pixfmt_non_vaapi() {
 
 static void convert_copy(const char *in, int width, int height, char *buff)
 {
-    if (likely(in != buff)) {
     memcpy(buff, in, width * height * BYTES_PER_PIXEL);
 }
-}
 
 static void convert_bgra_to_rgba(const char *in, int width, int height, char *buff)
 {
-    if (likely(in != buff)) {
     memcpy(buff, in, width * height * BYTES_PER_PIXEL);
-    }
     for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) {
-        uint32_t pixdata = htonl(*((uint32_t*)(buff + i)));
+        uint32_t pixdata = htonl(*((uint32_t*)(kmsvnc->drm->kms_convert_buf + i)));
         buff[i+0] = (pixdata & 0x0000ff00) >> 8;
         buff[i+2] = (pixdata & 0xff000000) >> 24;
     }
@@ -119,45 +97,39 @@ void convert_intel_x_tiled_kmsbuf(const char *in, int width, int height, char *b
 }
 
 static void convert_vaapi(const char *in, int width, int height, char *buff) {
+    if (KMSVNC_FOURCC_TO_INT('R','G','B', 0) & kmsvnc->va->image->format.fourcc == KMSVNC_FOURCC_TO_INT('R','G','B', 0)) {
         va_hwframe_to_vaapi(buff);
-    if (
-        (KMSVNC_FOURCC_TO_INT('R','G','B',0) & kmsvnc->va->selected_fmt->fourcc) == KMSVNC_FOURCC_TO_INT('R','G','B',0)
-    ) {}
+    }
     else {
+        if (convert_buf_allocate(width * height * BYTES_PER_PIXEL)) return;
+        va_hwframe_to_vaapi(kmsvnc->drm->kms_convert_buf);
         // is 30 depth?
-        if (kmsvnc->va->selected_fmt->depth == 30) {
+        if (kmsvnc->va->image->format.depth == 30) {
             for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) {
                 // ensure little endianess
-                uint32_t pixdata = __builtin_bswap32(htonl(*((uint32_t*)(buff + i))));
-                buff[i] = (pixdata & 0x3ff00000) >> 20 >> 2;
-                buff[i+1] = (pixdata & 0xffc00) >> 10 >> 2;
-                buff[i+2] = (pixdata & 0x3ff) >> 2;
-            }
-        }
-        else {
-            // actually, does anyone use this?
-            if (!kmsvnc->va->selected_fmt->byte_order) {
-                for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) {
-                    uint32_t *pixdata = (uint32_t*)(buff + i);
-                    *pixdata = __builtin_bswap32(*pixdata);
-                }
+                uint32_t pixdata = __builtin_bswap32(htonl(*((uint32_t*)(kmsvnc->drm->kms_convert_buf + i))));
+                kmsvnc->drm->kms_convert_buf[i] = (pixdata & 0x3ff00000) >> 20 >> 2;
+                kmsvnc->drm->kms_convert_buf[i+1] = (pixdata & 0xffc00) >> 10 >> 2;
+                kmsvnc->drm->kms_convert_buf[i+2] = (pixdata & 0x3ff) >> 2;
             }
         }
         // is xrgb?
-        if ((kmsvnc->va->selected_fmt->blue_mask | kmsvnc->va->selected_fmt->red_mask) < 0x1000000) {
+        if ((kmsvnc->va->image->format.blue_mask | kmsvnc->va->image->format.red_mask) < 0x1000000) {
             for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) {
-                uint32_t *pixdata = (uint32_t*)(buff + i);
+                uint32_t *pixdata = (uint32_t*)(kmsvnc->drm->kms_convert_buf + i);
                 *pixdata = ntohl(htonl(*pixdata) << 8);
             }
         }
         // is bgrx?
-        if (kmsvnc->va->selected_fmt->blue_mask > kmsvnc->va->selected_fmt->red_mask) {
+        if (kmsvnc->va->image->format.blue_mask > kmsvnc->va->image->format.red_mask) {
             for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) {
-                uint32_t pixdata = htonl(*((uint32_t*)(buff + i)));
+                uint32_t pixdata = htonl(*((uint32_t*)(kmsvnc->drm->kms_convert_buf + i)));
                 buff[i+0] = (pixdata & 0x0000ff00) >> 8;
                 buff[i+2] = (pixdata & 0xff000000) >> 24;
             }
         }
+        // rgbx now
+        memcpy(buff, kmsvnc->drm->kms_convert_buf, width * height * BYTES_PER_PIXEL);
     }
 }
 
@@ -183,7 +155,6 @@ void drm_sync_noop(int drmfd)
 
 void drm_cleanup() {
     if (kmsvnc->drm) {
-#ifndef DISABLE_KMSVNC_SCREEN_BLANK
         if (kmsvnc->drm->gamma && kmsvnc->drm->gamma->size && kmsvnc->drm->gamma->red && kmsvnc->drm->gamma->green && kmsvnc->drm->gamma->blue) {
             if (drmModeCrtcSetGamma(kmsvnc->drm->drm_master_fd ?: kmsvnc->drm->drm_fd, kmsvnc->drm->plane->crtc_id, kmsvnc->drm->gamma->size, kmsvnc->drm->gamma->red, kmsvnc->drm->gamma->green, kmsvnc->drm->gamma->blue)) perror("Failed to restore gamma");
         }
@@ -195,7 +166,6 @@ void drm_cleanup() {
             free(kmsvnc->drm->gamma);
             kmsvnc->drm->gamma = NULL;
         }
-#endif
         if (kmsvnc->drm->drm_ver) {
             drmFreeVersion(kmsvnc->drm->drm_ver);
             kmsvnc->drm->drm_ver = NULL;
@@ -504,10 +474,6 @@ int drm_open() {
     {
         KMSVNC_FATAL("card %s open failed: %s\n", kmsvnc->card, strerror(errno));
     }
-    if (!kmsvnc->screen_blank && drmIsMaster(drm->drm_fd)) {
-        if (drmDropMaster(drm->drm_fd)) fprintf(stderr, "Failed to drop master");
-    }
-#ifndef DISABLE_KMSVNC_SCREEN_BLANK
     if (kmsvnc->screen_blank && !drmIsMaster(drm->drm_fd)) {
         drm->drm_master_fd = drm_get_master_fd();
         drm->drm_master_fd = drm->drm_master_fd > 0 ? drm->drm_master_fd : 0;
@@ -515,7 +481,6 @@ int drm_open() {
             fprintf(stderr, "not master client, master fd %d\n", drm->drm_master_fd);
         }
     }
-#endif
 
     drm->drm_ver = drmGetVersion(drm->drm_fd);
     printf("drm driver is %s\n", drm->drm_ver->name);
@@ -528,7 +493,6 @@ int drm_open() {
 
     if (drm_refresh_planes(1)) return 1;
 
-#ifndef DISABLE_KMSVNC_SCREEN_BLANK
     if (kmsvnc->screen_blank) {
         drm->gamma = malloc(sizeof(struct kmsvnc_drm_gamma_data));
         if (!drm->gamma) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
@@ -588,7 +552,6 @@ int drm_open() {
             target_crtc = NULL;
         }
     }
-#endif
 
     drm->mfb = drmModeGetFB2(drm->drm_fd, drm->plane->fb_id);
     if (!drm->mfb) {

drm.h

@@ -2,9 +2,9 @@
 
 #include "kmsvnc.h"
 
-#define DRM_IOCTL_MUST(...) do{ int e; if ((e = drmIoctl(__VA_ARGS__))) KMSVNC_FATAL("DRM ioctl error %d on line %d\n", e, __LINE__); } while(0)
-#define DRM_IOCTL_MAY(...) do{ int e; if ((e = drmIoctl(__VA_ARGS__))) fprintf(stderr, "DRM ioctl error %d on line %d\n", e, __LINE__); } while(0)
-#define DRM_R_IOCTL_MAY(...) do{ int e; if ((e = ioctl(__VA_ARGS__))) fprintf(stderr, "DRM ioctl error %d on line %d\n", e, __LINE__); } while(0)
+#define DRM_IOCTL_MUST(...) do{ int e; if (e = drmIoctl(__VA_ARGS__)) KMSVNC_FATAL("DRM ioctl error %d on line %d\n", e, __LINE__); } while(0)
+#define DRM_IOCTL_MAY(...) do{ int e; if (e = drmIoctl(__VA_ARGS__)) fprintf(stderr, "DRM ioctl error %d on line %d\n", e, __LINE__); } while(0)
+#define DRM_R_IOCTL_MAY(...) do{ int e; if (e = ioctl(__VA_ARGS__)) fprintf(stderr, "DRM ioctl error %d on line %d\n", e, __LINE__); } while(0)
 
 
 void drm_cleanup();

drm_master.c

@@ -1,5 +1,4 @@
 #define _GNU_SOURCE
-
 #include <stdio.h>
 #include <errno.h>
 #include <sys/types.h>
@@ -28,6 +27,7 @@ static inline int clone_fd(pid_t pid, int target_fd) {
 }
 
 static inline int cmp_fds(pid_t pid, const char *drm_pth) {
+    if (pid == 1) return -1;
     char path[PATH_MAX+1];
     snprintf(path, PATH_MAX+1, "/proc/%d/fd", pid);
 
@@ -39,26 +39,18 @@ static inline int cmp_fds(pid_t pid, const char *drm_pth) {
             if (ret == -1 && fdlist[n]->d_type == DT_LNK) {
                 char link_pth[PATH_MAX+1];
                 char real_pth[PATH_MAX+1];
-                #pragma GCC diagnostic push
-                #pragma GCC diagnostic ignored "-Wpragmas"
-                #pragma GCC diagnostic ignored "-Wunknown-warning-option"
-                #pragma GCC diagnostic ignored "-Wformat-truncation"
                 snprintf(link_pth, PATH_MAX+1, "%s/%s", path, fdlist[n]->d_name);
-                #pragma GCC diagnostic pop
                 memset(real_pth, 0, PATH_MAX+1);
-                #pragma GCC diagnostic push
-                #pragma GCC diagnostic ignored "-Wunused-result"
                 realpath(link_pth, real_pth);
-                #pragma GCC diagnostic pop
                 if (!strncmp(real_pth, drm_pth, PATH_MAX)) {
                     int fd = atoi(fdlist[n]->d_name);
                     if (fd > 0) {
                         int cloned = clone_fd(pid, fd);
                         if (cloned > 0 && drmIsMaster(cloned)) {
                             ret = cloned;
-                            if (kmsvnc->debug_enabled) {
+                            //if (kmsvnc->debug_enabled) {
                                 fprintf(stderr, "found drm master pid=%d, fd=%d, cloned=%d\n", pid, fd, cloned);
-                            }
+                            //}
                         }
                         else {
                             if (cloned > 0) close(cloned);
@@ -78,16 +70,13 @@ static inline int cmp_fds(pid_t pid, const char *drm_pth) {
 int drm_get_master_fd() {
     char drm_pth[PATH_MAX+1];
     memset(drm_pth, 0, PATH_MAX+1);
-    #pragma GCC diagnostic push
-    #pragma GCC diagnostic ignored "-Wunused-result"
     realpath(kmsvnc->card, drm_pth);
-    #pragma GCC diagnostic pop
 
     struct dirent **proclist;
     int count = scandir("/proc", &proclist, NULL, versionsort);
     int ret = -1;
     if (count >= 0) {
-        for (int n = 0; n < count; n++) {
+        for (int n = count - 1; n > 0; n--) {
             if (ret == -1 && proclist[n]->d_type == DT_DIR) {
                 pid_t pid = (pid_t)atoi(proclist[n]->d_name);
                 if (pid > 0) {

input.h

@@ -7,8 +7,8 @@
 #define UINPUT_ABS_MAX INT16_MAX
 #define UINPUT_MAX_KEY 256
 
-#define INP_IOCTL_MUST(...) do{ int e; if ((e = ioctl(__VA_ARGS__))) KMSVNC_FATAL("uinput ioctl error %d on line %d\n", e, __LINE__); } while(0)
-#define INP_IOCTL_MAY(...) do{ int e; if ((e = ioctl(__VA_ARGS__))) fprintf(stderr, "uinput ioctl error %d on line %d\n", e, __LINE__); } while(0)
+#define INP_IOCTL_MUST(...) do{ int e; if (e = ioctl(__VA_ARGS__)) KMSVNC_FATAL("uinput ioctl error %d on line %d\n", e, __LINE__); } while(0)
+#define INP_IOCTL_MAY(...) do{ int e; if (e = ioctl(__VA_ARGS__)) fprintf(stderr, "uinput ioctl error %d on line %d\n", e, __LINE__); } while(0)
 
 void uinput_cleanup();
 int uinput_init();

kmsvnc.c

@@ -1,12 +1,9 @@
-#define _GNU_SOURCE
-
 #include <fcntl.h>
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
 #include <signal.h>
 #include <time.h>
-#include <unistd.h>
 #include <argp.h>
 #include <arpa/inet.h>
 
@@ -130,7 +127,7 @@ static inline void update_vnc_cursor(char *data, int width, int height) {
         if (!kmsvnc->cursor_bitmap) return;
         kmsvnc->cursor_bitmap_len = rwidth * rheight * BYTES_PER_PIXEL;
     }
-    unsigned char *rich_source = malloc(rwidth * rheight * BYTES_PER_PIXEL);
+    char *rich_source = malloc(rwidth * rheight * BYTES_PER_PIXEL);
     if (!rich_source) return;
     char *maskString = malloc(rwidth * rheight);
     if (!maskString) {
@@ -211,7 +208,6 @@ static void cleanup() {
     }
 }
 
-void signal_handler_noop(int signum){}
 void signal_handler(int signum){
     if (kmsvnc->shutdown) {
         return;
@@ -223,7 +219,7 @@ void signal_handler(int signum){
 }
 
 static struct argp_option kmsvnc_main_options[] = {
-    {"device", 'd', "/dev/dri/cardX", 0, "DRM device"},
+    {"device", 'd', "/dev/dri/card0", 0, "DRM device"},
     {"source-plane", 0xfefc, "0", 0, "Use specific plane"},
     {"source-crtc", 0xfefd, "0", 0, "Use specific crtc (to list all crtcs and planes, set this to -1)"},
     {"force-driver", 0xfefe, "i915", 0, "force a certain driver (for debugging)"},
@@ -242,11 +238,8 @@ static struct argp_option kmsvnc_main_options[] = {
     {"input-height", 0xff07, "0", 0, "Explicitly set input height"},
     {"input-offx", 0xff08, "0", 0, "Set input offset of x axis on a multi display system"},
     {"input-offy", 0xff09, "0", 0, "Set input offset of y axis on a multi display system"},
-#ifndef DISABLE_KMSVNC_SCREEN_BLANK
     {"screen-blank", 0xff0a, 0, OPTION_ARG_OPTIONAL, "Blank screen with gamma set on crtc"},
     {"screen-blank-restore-linear", 0xff0b, 0, OPTION_ARG_OPTIONAL, "Restore linear values on exit in case of messed up gamma"},
-#endif
-    {"va-byteorder-swap", 0xff0c, 0, OPTION_ARG_OPTIONAL, "Force swap vaapi image rgb byteorder"},
     {"wakeup", 'w', 0, OPTION_ARG_OPTIONAL, "Move mouse to wake the system up before start"},
     {"disable-input", 'i', 0, OPTION_ARG_OPTIONAL, "Disable uinput"},
     {"desktop-name", 'n', "kmsvnc", 0, "Specify vnc desktop name"},
@@ -278,7 +271,6 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
             kmsvnc->vnc_opt->bind6 = arg;
             break;
         case 'p':
-            {
             int port = atoi(arg);
             if (port > 0 && port < 65536) {
                 kmsvnc->vnc_opt->port = port;
@@ -286,13 +278,11 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
             else {
                 argp_error(state, "invalid port %s", arg);
             }
-            }
             break;
         case '4':
             kmsvnc->vnc_opt->disable_ipv6 = 1;
             break;
         case 0xff00:
-            {
             int fps = atoi(arg);
             if (fps > 0 && fps < 1000) {
                 kmsvnc->vnc_opt->sleep_ns = NS_IN_S / fps;
@@ -300,7 +290,6 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
             else {
                 argp_error(state, "invalid fps %s", arg);
             }
-            }
             break;
         case 0xff01:
             kmsvnc->vnc_opt->always_shared = 0;
@@ -327,36 +316,28 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
             kmsvnc->debug_enabled = 1;
             break;
         case 0xff06:
-            {
             int width = atoi(arg);
             if (width > 0) {
                 kmsvnc->input_width = width;
             }
-            }
             break;
         case 0xff07:
-            {
             int height = atoi(arg);
             if (height > 0) {
                 kmsvnc->input_height = height;
             }
-            }
             break;
         case 0xff08:
-            {
             int offset_x = atoi(arg);
             if (offset_x > 0) {
                 kmsvnc->input_offx = offset_x;
             }
-            }
             break;
         case 0xff09:
-            {
             int offset_y = atoi(arg);
             if (offset_y > 0) {
                 kmsvnc->input_offy = offset_y;
             }
-            }
             break;
         case 0xff0a:
             kmsvnc->screen_blank = 1;
@@ -364,9 +345,6 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
         case 0xff0b:
             kmsvnc->screen_blank_restore = 1;
             break;
-        case 0xff0c:
-            kmsvnc->va_byteorder_swap = 1;
-            break;
         case 'w':
             kmsvnc->input_wakeup = 1;
             break;
@@ -399,10 +377,7 @@ int main(int argc, char **argv)
 
     kmsvnc->vnc_opt = vncopt;
 
-    #define DEVICE_EXAMPLE_MAX_SIZE 15
-    #define DEVICE_EXAMPLE_FALLBACK "/dev/dri/card0"
-    static char device_example[DEVICE_EXAMPLE_MAX_SIZE] = DEVICE_EXAMPLE_FALLBACK;
-    kmsvnc->card = device_example;
+    kmsvnc->card = "/dev/dri/card0";
     kmsvnc->va_derive_enabled = -1;
     kmsvnc->vnc_opt->bind = &(struct in_addr){0};
     kmsvnc->vnc_opt->always_shared = 1;
@@ -416,18 +391,6 @@ int main(int argc, char **argv)
     struct argp argp = {kmsvnc_main_options, parse_opt, args_doc, doc};
     argp_parse(&argp, argc, argv, 0, 0, NULL);
 
-    if (kmsvnc->card == device_example) {
-        for (int i = 0; i < 10; i++) {
-            snprintf(kmsvnc->card, DEVICE_EXAMPLE_MAX_SIZE, "/dev/dri/card%d", i);
-            if (!access(kmsvnc->card, F_OK)) {
-                break;
-            }
-            else {
-                snprintf(kmsvnc->card, DEVICE_EXAMPLE_MAX_SIZE, DEVICE_EXAMPLE_FALLBACK);
-            }
-        }
-    }
-
     if (!kmsvnc->disable_input) {
         const char* XKB_DEFAULT_LAYOUT = getenv("XKB_DEFAULT_LAYOUT");
         if (!XKB_DEFAULT_LAYOUT || strcmp(XKB_DEFAULT_LAYOUT, "") == 0) {
@@ -474,20 +437,6 @@ int main(int argc, char **argv)
         else {
             fprintf(stderr, "open file %s failed, %s\n", kmsvnc->debug_capture_fb, strerror(errno));
         }
-        if (kmsvnc->screen_blank) {
-            sigset_t signal_set;
-            int sig;
-            sigemptyset(&signal_set);
-            signal(SIGHUP, &signal_handler_noop);
-            signal(SIGINT, &signal_handler_noop);
-            signal(SIGTERM, &signal_handler_noop);
-            sigaddset(&signal_set, SIGHUP);
-            sigaddset(&signal_set, SIGINT);
-            sigaddset(&signal_set, SIGTERM);
-            fprintf(stderr, "blanking screen...\n");
-            sigwait(&signal_set, &sig);
-            fprintf(stderr, "got sig %d\n", sig);
-        }
         cleanup();
         return 0;
     }

kmsvnc.h

@@ -1,7 +1,6 @@
 #pragma once
 
 #include <rfb/rfb.h>
-#include <stdint.h>
 #include <xkbcommon/xkbcommon.h>
 
 #include <xf86drm.h>
@@ -45,7 +44,6 @@ struct kmsvnc_data
     int input_offy;
     char screen_blank;
     char screen_blank_restore;
-    char va_byteorder_swap;
     struct kmsvnc_drm_data *drm;
     struct kmsvnc_input_data *input;
     struct kmsvnc_keymap_data *keymap;
@@ -139,8 +137,6 @@ struct kmsvnc_va_data
     char derive_enabled;
     VAImageFormat* img_fmts;
     int img_fmt_count;
-    VAImageFormat* selected_fmt;
-    const char *vendor_string;
 };
 
 #define KMSVNC_FATAL(...) do{ fprintf(stderr, __VA_ARGS__); return 1; } while(0)
@@ -149,6 +145,3 @@ struct kmsvnc_va_data
 #define KMSVNC_WRITE_MAY(fd,buf,count) do { ssize_t e = write((fd), (buf), (count)); if (e != (count)) fprintf(stderr, "should write %ld bytes, actually wrote %ld, on line %d\n", (count), e, __LINE__); } while (0)
 
 #define KMSVNC_DEBUG(...) do{ if (kmsvnc->debug_enabled) fprintf(stdout, __VA_ARGS__); } while(0)
-
-#define likely(x)   __builtin_expect(!!(x), 1)
-#define unlikely(x) __builtin_expect(!!(x), 0)

va.c

@@ -1,12 +1,6 @@
-#define _GNU_SOURCE
-
-#include <stdint.h>
-#include <stdio.h>
-#include <va/va.h>
 #include <va/va_drm.h>
 #include <va/va_drmcommon.h>
 #include <fcntl.h>
-#include <string.h>
 
 #include "va.h"
 #include "kmsvnc.h"
@@ -39,9 +33,6 @@ void va_cleanup() {
             VA_MAY(vaTerminate(kmsvnc->va->dpy));
             kmsvnc->va->dpy = NULL;
         }
-        if (kmsvnc->va->vendor_string) {
-            kmsvnc->va->vendor_string = NULL;
-        }
         free(kmsvnc->va);
         kmsvnc->va = NULL;
     }
@@ -78,28 +69,8 @@ static const struct {
     {KMSVNC_FOURCC_TO_INT('A', 'R', '3', '0'), KMSVNC_FOURCC_TO_INT('A', 'R', '3', '0'), VA_RT_FORMAT_RGB32_10, 1},
 };
 
-struct va_fmt_data {
-    uint32_t va_fourcc;
-    VAImageFormat *fmt;
-    char is_alpha;
-    uint32_t va_rt_format;
-    uint32_t depth;
-};
-
-static VAImageFormat* vaImgFmt_apply_quirks(struct va_fmt_data* data) {
-    static VAImageFormat ret = {0};
-    memcpy(&ret, data->fmt, sizeof(VAImageFormat));
-    if ((kmsvnc->va_byteorder_swap ^ !strncmp(kmsvnc->va->vendor_string, "Mesa", 4)) && data->depth != 30) {
-        printf("applying rgb mask byte order swap\n");
-        ret.blue_mask = __builtin_bswap32(data->fmt->blue_mask);
-        ret.green_mask = __builtin_bswap32(data->fmt->green_mask);
-        ret.red_mask = __builtin_bswap32(data->fmt->red_mask);
-    }
-    return &ret;
-}
-
 static void print_va_image_fmt(VAImageFormat *fmt) {
-    printf("image fmt: fourcc %d, %s, byte_order %s, bpp %d, depth %d, blue_mask %#x, green_mask %#x, red_mask %#x, alpha_mask %#x, reserved %#x %#x %#x %#x\n", fmt->fourcc,
+        printf("image fmt: fourcc %d, %s, byte_order %s, bpp %d, depth %d, blue_mask %#x, green_mask %#x, red_mask %#x, reserved %#x %#x %#x %#x\n", fmt->fourcc,
             fourcc_to_str(fmt->fourcc),
             fmt->byte_order == 1 ? "VA_LSB_FIRST" : "VA_MSB_FIRST",
             fmt->bits_per_pixel,
@@ -107,7 +78,6 @@ static void print_va_image_fmt(VAImageFormat *fmt) {
             fmt->blue_mask,
             fmt->green_mask,
             fmt->red_mask,
-        fmt->alpha_mask,
             fmt->va_reserved[0],
             fmt->va_reserved[1],
             fmt->va_reserved[2],
@@ -121,7 +91,6 @@ int va_init() {
     }
 
     setenv("DISPLAY", "", 1);
-    setenv("WAYLAND_DISPLAY", "", 1);
 
     struct kmsvnc_va_data *va = malloc(sizeof(struct kmsvnc_va_data));
     if (!va) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
@@ -130,7 +99,7 @@ int va_init() {
 
     char* render_node;
     int effective_fd = 0;
-    if ((render_node = drmGetRenderDeviceNameFromFd(kmsvnc->drm->drm_fd))) {
+    if (render_node = drmGetRenderDeviceNameFromFd(kmsvnc->drm->drm_fd)) {
         va->render_node_fd = open(render_node, O_RDWR);
         free(render_node);
     }
@@ -157,8 +126,8 @@ int va_init() {
     VAStatus status;
     VA_MUST(vaInitialize(va->dpy, &major, &minor));
 
-    va->vendor_string = vaQueryVendorString(va->dpy);
-    printf("vaapi vendor %s\n", va->vendor_string);
+    const char *vendor_string = vaQueryVendorString(va->dpy);
+    printf("vaapi vendor %s\n", vendor_string);
 
     VADRMPRIMESurfaceDescriptor prime_desc;
     VASurfaceAttrib prime_attrs[2] = {
@@ -286,26 +255,28 @@ int va_init() {
         }
     }
 
-    struct va_fmt_data format_to_try[] = {
-        {KMSVNC_FOURCC_TO_INT('R','G','B','X'), NULL, 0, VA_RT_FORMAT_RGB32, 24},
-        {KMSVNC_FOURCC_TO_INT('R','G','B','A'), NULL, 1, VA_RT_FORMAT_RGB32, 32},
-
-        {KMSVNC_FOURCC_TO_INT('X','B','G','R'), NULL, 0, VA_RT_FORMAT_RGB32, 24},
-        {KMSVNC_FOURCC_TO_INT('A','B','G','R'), NULL, 1, VA_RT_FORMAT_RGB32, 32},
-
-        {KMSVNC_FOURCC_TO_INT('X','R','G','B'), NULL, 0, VA_RT_FORMAT_RGB32, 24},
-        {KMSVNC_FOURCC_TO_INT('A','R','G','B'), NULL, 1, VA_RT_FORMAT_RGB32, 32},
-
-        {KMSVNC_FOURCC_TO_INT('B','G','R','X'), NULL, 0, VA_RT_FORMAT_RGB32, 24},
-        {KMSVNC_FOURCC_TO_INT('B','G','R','A'), NULL, 1, VA_RT_FORMAT_RGB32, 32},
-
-
-        {KMSVNC_FOURCC_TO_INT('X','R','3','0'), NULL, 0, VA_RT_FORMAT_RGB32_10, 30},
-        {KMSVNC_FOURCC_TO_INT('A','R','3','0'), NULL, 1, VA_RT_FORMAT_RGB32_10, 30},
-        {KMSVNC_FOURCC_TO_INT('X','B','3','0'), NULL, 0, VA_RT_FORMAT_RGB32_10, 30},
-        {KMSVNC_FOURCC_TO_INT('A','B','3','0'), NULL, 1, VA_RT_FORMAT_RGB32_10, 30},
+    struct fourcc_data {
+        uint32_t va_fourcc;
+        VAImageFormat *fmt;
+        char is_alpha;
+        uint32_t va_rt_format;
     };
+    struct fourcc_data format_to_try[] = {
+        {KMSVNC_FOURCC_TO_INT('R','G','B','X'), NULL, 0, VA_RT_FORMAT_RGB32},
+        {KMSVNC_FOURCC_TO_INT('R','G','B','A'), NULL, 1, VA_RT_FORMAT_RGB32},
+        {KMSVNC_FOURCC_TO_INT('X','R','G','B'), NULL, 0, VA_RT_FORMAT_RGB32},
+        {KMSVNC_FOURCC_TO_INT('A','R','G','B'), NULL, 1, VA_RT_FORMAT_RGB32},
 
+        {KMSVNC_FOURCC_TO_INT('B','G','R','X'), NULL, 0, VA_RT_FORMAT_RGB32},
+        {KMSVNC_FOURCC_TO_INT('B','G','R','A'), NULL, 1, VA_RT_FORMAT_RGB32},
+        {KMSVNC_FOURCC_TO_INT('X','B','G','R'), NULL, 0, VA_RT_FORMAT_RGB32},
+        {KMSVNC_FOURCC_TO_INT('A','B','G','R'), NULL, 1, VA_RT_FORMAT_RGB32},
+
+        {KMSVNC_FOURCC_TO_INT('X','R','3','0'), NULL, 0, VA_RT_FORMAT_RGB32_10},
+        {KMSVNC_FOURCC_TO_INT('A','R','3','0'), NULL, 1, VA_RT_FORMAT_RGB32_10},
+        {KMSVNC_FOURCC_TO_INT('X','B','3','0'), NULL, 0, VA_RT_FORMAT_RGB32_10},
+        {KMSVNC_FOURCC_TO_INT('A','B','3','0'), NULL, 1, VA_RT_FORMAT_RGB32_10},
+    };
     for (int i = 0; i < va->img_fmt_count; i++) {
         for (int j = 0; j < KMSVNC_ARRAY_ELEMENTS(format_to_try); j++) {
             if (va->img_fmts[i].fourcc == format_to_try[j].va_fourcc) {
@@ -321,14 +292,14 @@ int va_init() {
     va->derive_enabled = kmsvnc->va_derive_enabled < 0 ? va->derive_enabled : kmsvnc->va_derive_enabled != 0;
     if (va->derive_enabled) {
         if ((s = vaDeriveImage(va->dpy, va->surface_id, va->image)) == VA_STATUS_SUCCESS) {
+            char found = 0;
             for (int i = 0; i < KMSVNC_ARRAY_ELEMENTS(format_to_try); i++) {
-                if (format_to_try[i].fmt == NULL) continue;
                 if (va->image->format.fourcc == format_to_try[i].fmt->fourcc) {
-                    va->selected_fmt = vaImgFmt_apply_quirks(format_to_try + i);
+                    found = 1;
                     break;
                 }
             }
-            if (!va->selected_fmt) {
+            if (!found) {
                 va->derive_enabled = 0;
                 printf("vaDeriveImage returned unknown fourcc %d %s\n", va->image->format.fourcc, fourcc_to_str(va->image->format.fourcc));
                 VA_MAY(vaDestroyImage(kmsvnc->va->dpy, kmsvnc->va->image->image_id));
@@ -344,6 +315,7 @@ int va_init() {
         }
     }
     if (!va->derive_enabled) {
+        char success = 0;
         for (int i = 0; i < KMSVNC_ARRAY_ELEMENTS(format_to_try); i++) {
             if (format_to_try[i].fmt == NULL) continue;
             if (!kmsvnc->debug_enabled && rt_format != format_to_try[i].va_rt_format) continue;
@@ -369,21 +341,17 @@ int va_init() {
                 continue;
             }
             else {
-                va->selected_fmt = vaImgFmt_apply_quirks(format_to_try + i);
+                success = 1;
                 break;
             }
         }
-        if (!va->selected_fmt) {
+        if (!success) {
             va->imgbuf = NULL;
             KMSVNC_FATAL("failed to get vaapi image\n");
         }
     }
     printf("got vaapi %simage:\n", va->derive_enabled ? "derive " : "");
     print_va_image_fmt(&va->image->format);
-    if (kmsvnc->debug_enabled) {
-        fprintf(stderr, "selected image format:\n");
-        print_va_image_fmt(va->selected_fmt);
-    }
     return 0;
 }