From 1666afec4e9933dff95452a37f889618e2d6c572 Mon Sep 17 00:00:00 2001 From: Jerry Date: Sat, 29 Apr 2023 02:05:13 +0800 Subject: [PATCH] vaapi: wip --- CMakeLists.txt | 8 +- README.md | 1 + drm.c | 112 +++++++++++++++-------- drm.h | 7 +- input.c | 10 ++- input.h | 7 +- keymap.c | 5 +- keymap.h | 2 - kmsvnc.c | 31 ++++++- kmsvnc.h | 17 ++++ va.c | 238 +++++++++++++++++++++++++++++++++++++++++++++++++ va.h | 8 ++ 12 files changed, 385 insertions(+), 61 deletions(-) create mode 100644 va.c create mode 100644 va.h diff --git a/CMakeLists.txt b/CMakeLists.txt index bf6b0c6..d4d1fdb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,6 +10,8 @@ find_package(PkgConfig REQUIRED) pkg_search_module(LIBDRM REQUIRED libdrm) pkg_search_module(LIBVNCSERVER REQUIRED libvncserver) pkg_search_module(XKBCOMMON REQUIRED xkbcommon) +pkg_search_module(LIBVA REQUIRED libva) +pkg_search_module(LIBVA_DRM REQUIRED libva-drm) include(CheckIncludeFiles) CHECK_INCLUDE_FILES("linux/uinput.h;linux/dma-buf.h" HAVE_LINUX_API_HEADERS) @@ -17,17 +19,21 @@ IF(NOT HAVE_LINUX_API_HEADERS) message(FATAL_ERROR "linux-api-headers not found") ENDIF() -add_executable(kmsvnc kmsvnc.c drm.c input.c keymap.c) +add_executable(kmsvnc kmsvnc.c drm.c input.c keymap.c va.c) target_include_directories(kmsvnc PUBLIC ${LIBDRM_INCLUDEDIR} ${LIBDRM_INCLUDEDIR}/libdrm ${LIBVNCSERVER_INCLUDEDIR} ${XKBCOMMON_INCLUDEDIR} + ${LIBVA_INCLUDEDIR} + ${LIBVA_DRM_INCLUDEDIR} ) target_link_libraries(kmsvnc PUBLIC m ${LIBDRM_LIBRARIES} ${LIBVNCSERVER_LIBRARIES} ${XKBCOMMON_LIBRARIES} + ${LIBVA_LIBRARIES} + ${LIBVA_DRM_LIBRARIES} ) install(TARGETS kmsvnc RUNTIME DESTINATION bin) diff --git a/README.md b/README.md index f723931..8b24047 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ Currently in very early stage. * libvncserver * libxkbcommon * libdrm + * libva ## Building ``` diff --git a/drm.c b/drm.c index d1b6a79..71ce110 100644 --- a/drm.c +++ b/drm.c @@ -5,8 +5,10 @@ #include #include #include +#include #include "drm.h" +#include "va.h" extern struct kmsvnc_data *kmsvnc; @@ -14,6 +16,10 @@ static void convert_copy(const char *in, int width, int height, char *buff) { memcpy(buff, in, width * height * 4); } +static void convert_vaapi(const char *in, int width, int height, char *buff) { + va_hwframe_to_vaapi(buff); +} + static void convert_bgrx_to_rgb(const char *in, int width, int height, char *buff) { for (int y = 0; y < height; y++) @@ -47,6 +53,7 @@ static inline void convert_x_tiled(const int tilex, const int tiley, const char if (kms_cpy_tmp_buf) free(kms_convert_buf); kms_cpy_tmp_buf = malloc(max_offset * 4 + 4); + if (!kms_cpy_tmp_buf) return; kms_cpy_tmp_buf_len = max_offset * 4 + 4; } memcpy(kms_cpy_tmp_buf, in, max_offset * 4 + 4); @@ -57,6 +64,7 @@ static inline void convert_x_tiled(const int tilex, const int tiley, const char if (kms_convert_buf) free(kms_convert_buf); kms_convert_buf = malloc(width * height * 4); + if (!kms_convert_buf) return; kms_convert_buf_len = width * height * 4; } for (int y = 0; y < height; y++) @@ -83,13 +91,10 @@ void convert_intel_x_tiled_kmsbuf(const char *in, int width, int height, char *b static inline void drm_sync(int drmfd, uint64_t flags) { - int ioctl_err; struct dma_buf_sync sync = { .flags = flags, }; - if (ioctl_err = ioctl(drmfd, DMA_BUF_IOCTL_SYNC, &sync)) { - fprintf(stderr, "DRM ioctl error %d on line %d\n", ioctl_err, __LINE__); - } + DRM_R_IOCTL_MAY(drmfd, DMA_BUF_IOCTL_SYNC, &sync); } void drm_sync_start(int drmfd) @@ -110,6 +115,18 @@ void drm_cleanup() { drmFreeVersion(kmsvnc->drm->drm_ver); kmsvnc->drm->drm_ver = NULL; } + if (kmsvnc->drm->pixfmt_name) { + free(kmsvnc->drm->pixfmt_name); + kmsvnc->drm->pixfmt_name = NULL; + } + if (kmsvnc->drm->mod_vendor) { + free(kmsvnc->drm->mod_vendor); + kmsvnc->drm->mod_vendor = NULL; + } + if (kmsvnc->drm->mod_name) { + free(kmsvnc->drm->mod_name); + kmsvnc->drm->mod_name = NULL; + } if (kmsvnc->drm->plane) { drmModeFreePlane(kmsvnc->drm->plane); kmsvnc->drm->plane = NULL; @@ -141,13 +158,14 @@ void drm_cleanup() { int drm_open() { struct kmsvnc_drm_data *drm = malloc(sizeof(struct kmsvnc_drm_data)); + if (!drm) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__); memset(drm, 0, sizeof(struct kmsvnc_drm_data)); kmsvnc->drm = drm; drm->drm_fd = open(kmsvnc->card, O_RDONLY); if (drm->drm_fd < 0) { - DRM_FATAL("card %s open failed: %s\n", kmsvnc->card, strerror(errno)); + KMSVNC_FATAL("card %s open failed: %s\n", kmsvnc->card, strerror(errno)); } drm->drm_ver = drmGetVersion(drm->drm_fd); printf("drm driver is %s\n", drm->drm_ver->name); @@ -161,7 +179,7 @@ int drm_open() { { drm->plane = drmModeGetPlane(drm->drm_fd, kmsvnc->source_plane); if (!drm->plane) - DRM_FATAL("Failed to get plane %d: %s\n", kmsvnc->source_plane, strerror(errno)); + KMSVNC_FATAL("Failed to get plane %d: %s\n", kmsvnc->source_plane, strerror(errno)); if (drm->plane->fb_id == 0) fprintf(stderr, "Place %d does not have an attached framebuffer\n", kmsvnc->source_plane); } @@ -169,7 +187,7 @@ int drm_open() { { drm->plane_res = drmModeGetPlaneResources(drm->drm_fd); if (!drm->plane_res) - DRM_FATAL("Failed to get plane resources: %s\n", strerror(errno)); + KMSVNC_FATAL("Failed to get plane resources: %s\n", strerror(errno)); int i; for (i = 0; i < drm->plane_res->count_planes; i++) { @@ -194,11 +212,11 @@ int drm_open() { { if (kmsvnc->source_crtc != 0) { - DRM_FATAL("No usable planes found on CRTC %d\n", kmsvnc->source_crtc); + KMSVNC_FATAL("No usable planes found on CRTC %d\n", kmsvnc->source_crtc); } else { - DRM_FATAL("No usable planes found\n"); + KMSVNC_FATAL("No usable planes found\n"); } } printf("Using plane %u to locate framebuffers\n", drm->plane->plane_id); @@ -208,32 +226,34 @@ int drm_open() { drm->mfb = drmModeGetFB2(drm->drm_fd, drm->plane->fb_id); if (!drm->mfb) { - DRM_FATAL("Failed to get framebuffer %u: %s\n", drm->plane->fb_id, strerror(errno)); + KMSVNC_FATAL("Failed to get framebuffer %u: %s\n", drm->plane->fb_id, strerror(errno)); } + drm->pixfmt_name = drmGetFormatName(drm->mfb->pixel_format); + drm->mod_vendor = drmGetFormatModifierVendor(drm->mfb->modifier); + drm->mod_name = drmGetFormatModifierName(drm->mfb->modifier); printf("Template framebuffer is %u: %ux%u fourcc:%u mod:%u flags:%u\n", drm->mfb->fb_id, drm->mfb->width, drm->mfb->height, drm->mfb->pixel_format, drm->mfb->modifier, drm->mfb->flags); printf("handles %u %u %u %u\n", drm->mfb->handles[0], drm->mfb->handles[1], drm->mfb->handles[2], drm->mfb->handles[3]); printf("offsets %u %u %u %u\n", drm->mfb->offsets[0], drm->mfb->offsets[1], drm->mfb->offsets[2], drm->mfb->offsets[3]); printf("pitches %u %u %u %u\n", drm->mfb->pitches[0], drm->mfb->pitches[1], drm->mfb->pitches[2], drm->mfb->pitches[3]); - printf("format %s, modifier %s:%s\n", drmGetFormatName(drm->mfb->pixel_format), drmGetFormatModifierVendor(drm->mfb->modifier), drmGetFormatModifierName(drm->mfb->modifier)); + printf("format %s, modifier %s:%s\n", drm->pixfmt_name, drm->mod_vendor, drm->mod_name); if ( drm->mfb->pixel_format != KMSVNC_FOURCC_TO_INT('X', 'R', '2', '4') && drm->mfb->pixel_format != KMSVNC_FOURCC_TO_INT('A', 'R', '2', '4') ) { - DRM_FATAL("Unsupported pixfmt\n"); + KMSVNC_FATAL("Unsupported pixfmt %s, please create an issue with your pixfmt.\n", drm->pixfmt_name); } if (!drm->mfb->handles[0]) { - DRM_FATAL("No handle set on framebuffer: maybe you need some additional capabilities?\n"); + KMSVNC_FATAL("No handle set on framebuffer: maybe you need some additional capabilities?\n"); } - int ioctl_err = 0; - drm->mmap_fd = drm->drm_fd; drm->mmap_size = drm->mfb->width * drm->mfb->height * BYTES_PER_PIXEL; drm->funcs = malloc(sizeof(struct kmsvnc_drm_funcs)); + if (!drm->funcs) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__); drm->funcs->convert = convert_bgrx_to_rgb; drm->funcs->sync_start = drm_sync_noop; drm->funcs->sync_end = drm_sync_noop; @@ -250,7 +270,7 @@ static int drm_kmsbuf_prime() { int err = drmPrimeHandleToFD(drm->drm_fd, drm->mfb->handles[0], O_RDWR, &drm->prime_fd); if (err < 0 || drm->prime_fd < 0) { - DRM_FATAL("Failed to get PRIME fd from framebuffer handle"); + KMSVNC_FATAL("Failed to get PRIME fd from framebuffer handle"); } drm->funcs->sync_start = &drm_sync_start; drm->funcs->sync_end = &drm_sync_end; @@ -258,6 +278,22 @@ static int drm_kmsbuf_prime() { return 0; } +static int drm_kmsbuf_prime_vaapi() { + struct kmsvnc_drm_data *drm = kmsvnc->drm; + + int err = drmPrimeHandleToFD(drm->drm_fd, drm->mfb->handles[0], O_RDWR, &drm->prime_fd); + if (err < 0 || drm->prime_fd < 0) + { + KMSVNC_FATAL("Failed to get PRIME fd from framebuffer handle"); + } + + if (va_init()) return 1; + + drm->mmap_fd = drm->prime_fd; + drm->mapped = kmsvnc->va->imgbuf; + return 0; +} + static int drm_kmsbuf_dumb() { struct kmsvnc_drm_data *drm = kmsvnc->drm; @@ -291,32 +327,19 @@ int drm_vendors() { driver_name = drm->drm_ver->name; } - if (strcmp(driver_name, "i915") == 0) + if (strcmp(driver_name, "i915") == 0 || strcmp(driver_name, "amdgpu") == 0) { - drm->funcs->convert = &convert_intel_x_tiled_kmsbuf; - if (drm_kmsbuf_prime()) return 1; - } - else if (strcmp(driver_name, "amdgpu") == 0) - { - struct drm_gem_flink flink; - flink.handle = drm->mfb->handles[0]; - DRM_IOCTL_MUST(drm->drm_fd, DRM_IOCTL_GEM_FLINK, &flink); - - struct drm_gem_open open_arg; - open_arg.name = flink.name; - DRM_IOCTL_MUST(drm->drm_fd, DRM_IOCTL_GEM_OPEN, &open_arg); - - union drm_amdgpu_gem_mmap mmap_arg; - memset(&mmap_arg, 0, sizeof(mmap_arg)); - mmap_arg.in.handle = open_arg.handle; - DRM_IOCTL_MUST(drm->drm_fd, DRM_IOCTL_AMDGPU_GEM_MMAP, &mmap_arg); - - drm->mmap_size = open_arg.size; - drm->mmap_offset = mmap_arg.out.addr_ptr; + if (fourcc_mod_is_vendor(drm->mfb->modifier, INTEL)) { + if (strstr(drm->mod_name, "CCS")) { + printf("warn: intel with CCS modifier detected, please set INTEL_DEBUG=noccs\n"); + } + }; + drm->funcs->convert = &convert_vaapi; + if (drm_kmsbuf_prime_vaapi()) return 1; } else if (strcmp(driver_name, "nvidia-drm") == 0) { - // quirky and slow + printf("warn: nvidia card detected. Currently only x-tiled framebuffer is supported. Performance may suffer.\n"); drm->funcs->convert = &convert_nvidia_x_tiled_kmsbuf; if (drm_kmsbuf_dumb()) return 1; } @@ -325,6 +348,9 @@ int drm_vendors() { strcmp(driver_name, "virtio_gpu") == 0 ) { + if (drm->mfb->modifier != DRM_FORMAT_MOD_NONE && drm->mfb->modifier != DRM_FORMAT_MOD_LINEAR) { + printf("warn: modifier is not LINEAR, please create an issue with your modifier.\n"); + } // virgl does not work if (drm_kmsbuf_dumb()) return 1; } @@ -352,9 +378,17 @@ int drm_vendors() { drm->mmap_size = open_arg.size; drm->mmap_offset = mmap_arg.offset; } + else if (strcmp(driver_name, "test-i915-prime-xtiled") == 0) + { + drm->funcs->convert = &convert_intel_x_tiled_kmsbuf; + if (drm_kmsbuf_prime()) return 1; + } else { fprintf(stderr, "Untested drm driver, use at your own risk!\n"); + if (drm->mfb->modifier != DRM_FORMAT_MOD_NONE && drm->mfb->modifier != DRM_FORMAT_MOD_LINEAR) { + printf("warn: modifier is not LINEAR, please create an issue with your driver and modifier.\n"); + } if (drm_kmsbuf_dumb()) return 1; } @@ -364,7 +398,7 @@ int drm_vendors() { drm->mapped = mmap(NULL, drm->mmap_size, PROT_READ, MAP_SHARED, drm->mmap_fd, drm->mmap_offset); if (drm->mapped == MAP_FAILED) { - DRM_FATAL("Failed to mmap: %s\n", strerror(errno)); + KMSVNC_FATAL("Failed to mmap: %s\n", strerror(errno)); } } diff --git a/drm.h b/drm.h index 12257a4..91e7c06 100644 --- a/drm.h +++ b/drm.h @@ -2,11 +2,10 @@ #include "kmsvnc.h" -#define KMSVNC_FOURCC_TO_INT(a,b,c,d) (((a) << 0) + ((b) << 8) + ((c) << 16) + ((d) << 24)) +#define DRM_IOCTL_MUST(...) do{ int e; if (e = drmIoctl(__VA_ARGS__)) KMSVNC_FATAL("DRM ioctl error %d on line %d\n", e, __LINE__); } while(0) +#define DRM_IOCTL_MAY(...) do{ int e; if (e = drmIoctl(__VA_ARGS__)) fprintf(stderr, "DRM ioctl error %d on line %d\n", e, __LINE__); } while(0) +#define DRM_R_IOCTL_MAY(...) do{ int e; if (e = ioctl(__VA_ARGS__)) fprintf(stderr, "DRM ioctl error %d on line %d\n", e, __LINE__); } while(0) -#define DRM_FATAL(...) { fprintf(stderr, __VA_ARGS__); return 1; } -#define DRM_IOCTL_MUST(...) { int e; if (e = drmIoctl(__VA_ARGS__)) DRM_FATAL("DRM ioctl error %d on line %d\n", e, __LINE__) } -#define DRM_IOCTL_MAY(...) { int e; if (e = drmIoctl(__VA_ARGS__)) fprintf(stderr, "DRM ioctl error %d on line %d\n", e, __LINE__); } void drm_cleanup(); int drm_open(); diff --git a/input.c b/input.c index 60eea70..95f3107 100644 --- a/input.c +++ b/input.c @@ -29,13 +29,14 @@ void uinput_cleanup() int uinput_init() { struct kmsvnc_input_data *inp = malloc(sizeof(struct kmsvnc_input_data)); + if (!inp) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__); memset(inp, 0, sizeof(struct kmsvnc_input_data)); kmsvnc->input = inp; inp->uinput_fd = open("/dev/uinput", O_WRONLY | O_NONBLOCK); if (inp->uinput_fd <= 0) { - INP_FATAL("Failed to open uinput\n"); + KMSVNC_FATAL("Failed to open uinput\n"); } INP_IOCTL_MUST(inp->uinput_fd, UI_SET_EVBIT, EV_KEY); INP_IOCTL_MUST(inp->uinput_fd, UI_SET_EVBIT, EV_SYN); @@ -74,6 +75,7 @@ int uinput_init() INP_IOCTL_MUST(inp->uinput_fd, UI_DEV_CREATE); inp->keystate = malloc(UINPUT_MAX_KEY); + if (!inp->keystate) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__); memset(inp->keystate, 0, UINPUT_MAX_KEY); return 0; @@ -114,7 +116,7 @@ void rfb_key_hook(rfbBool down, rfbKeySym keysym, rfbClientPtr cl) .value = 0, }, }; - for (int i = 0; i < ARRAY_SIZE(ies); i++) + for (int i = 0; i < KMSVNC_ARRAY_ELEMENTS(ies); i++) { write(kmsvnc->input->uinput_fd, &ies[i], sizeof(ies[0])); } @@ -159,7 +161,7 @@ void rfb_ptr_hook(int mask, int screen_x, int screen_y, rfbClientPtr cl) .value = 0, }, }; - for (int i = 0; i < ARRAY_SIZE(ies1); i++) + for (int i = 0; i < KMSVNC_ARRAY_ELEMENTS(ies1); i++) { write(kmsvnc->input->uinput_fd, &ies1[i], sizeof(ies1[0])); } @@ -177,7 +179,7 @@ void rfb_ptr_hook(int mask, int screen_x, int screen_y, rfbClientPtr cl) .value = 0, }, }; - for (int i = 0; i < ARRAY_SIZE(ies2); i++) + for (int i = 0; i < KMSVNC_ARRAY_ELEMENTS(ies2); i++) { write(kmsvnc->input->uinput_fd, &ies2[i], sizeof(ies2[0])); } diff --git a/input.h b/input.h index efa7d8b..5877bcf 100644 --- a/input.h +++ b/input.h @@ -4,14 +4,11 @@ #include "kmsvnc.h" -#define ARRAY_SIZE(x) (sizeof(x) / sizeof(x[0])) - #define UINPUT_ABS_MAX INT16_MAX #define UINPUT_MAX_KEY 256 -#define INP_FATAL(...) { fprintf(stderr, __VA_ARGS__); return 1; } -#define INP_IOCTL_MUST(...) { int e; if (e = ioctl(__VA_ARGS__)) INP_FATAL("uinput ioctl error %d on line %d\n", e, __LINE__) } -#define INP_IOCTL_MAY(...) { int e; if (e = ioctl(__VA_ARGS__)) fprintf(stderr, "uinput ioctl error %d on line %d\n", e, __LINE__); } +#define INP_IOCTL_MUST(...) do{ int e; if (e = ioctl(__VA_ARGS__)) KMSVNC_FATAL("uinput ioctl error %d on line %d\n", e, __LINE__); } while(0) +#define INP_IOCTL_MAY(...) do{ int e; if (e = ioctl(__VA_ARGS__)) fprintf(stderr, "uinput ioctl error %d on line %d\n", e, __LINE__); } while(0) void uinput_cleanup(); int uinput_init(); diff --git a/keymap.c b/keymap.c index 6acd19f..893b443 100644 --- a/keymap.c +++ b/keymap.c @@ -25,13 +25,14 @@ void xkb_cleanup() { int xkb_init() { struct kmsvnc_keymap_data *xkb = malloc(sizeof(struct kmsvnc_keymap_data)); + if (!xkb) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__); memset(xkb, 0, sizeof(struct kmsvnc_keymap_data)); kmsvnc->keymap = xkb; xkb->ctx = xkb_context_new(XKB_CONTEXT_NO_FLAGS); if (xkb->ctx == NULL) { - XKB_FATAL("Failed to create XKB context\n"); + KMSVNC_FATAL("Failed to create XKB context\n"); } struct xkb_rule_names names = { .rules = NULL, @@ -43,7 +44,7 @@ int xkb_init() xkb->map = xkb_keymap_new_from_names(xkb->ctx, &names, 0); if (xkb->map == NULL) { - XKB_FATAL("Failed to create XKB keymap\n"); + KMSVNC_FATAL("Failed to create XKB keymap\n"); } // printf("xkb: keymap string\n%s\n", xkb_keymap_get_as_string(xkb->map, XKB_KEYMAP_USE_ORIGINAL_FORMAT)); return 0; diff --git a/keymap.h b/keymap.h index bd1ccda..964fbbc 100644 --- a/keymap.h +++ b/keymap.h @@ -2,8 +2,6 @@ #include "kmsvnc.h" -#define XKB_FATAL(...) { fprintf(stderr, __VA_ARGS__); return 1; } - void xkb_cleanup(); int xkb_init(); void key_iter(struct xkb_keymap *xkb, xkb_keycode_t key, void *data); diff --git a/kmsvnc.c b/kmsvnc.c index ea47470..452e576 100644 --- a/kmsvnc.c +++ b/kmsvnc.c @@ -11,6 +11,7 @@ #include "keymap.h" #include "input.h" #include "drm.h" +#include "va.h" struct kmsvnc_data *kmsvnc = NULL; @@ -98,6 +99,9 @@ static void cleanup() { if (kmsvnc->drm) { drm_cleanup(); } + if (kmsvnc->va) { + va_cleanup(); + } if (kmsvnc) { if (kmsvnc->vnc_opt) { free(kmsvnc->vnc_opt); @@ -215,12 +219,17 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { int main(int argc, char **argv) { - struct vnc_opt *vncopt = malloc(sizeof(struct vnc_opt)); - memset(vncopt, 0, sizeof(struct vnc_opt)); - kmsvnc = malloc(sizeof(struct kmsvnc_data)); + if (!kmsvnc) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__); memset(kmsvnc, 0, sizeof(struct kmsvnc_data)); + struct vnc_opt *vncopt = malloc(sizeof(struct vnc_opt)); + if (!vncopt) { + free(kmsvnc); + KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__); + } + memset(vncopt, 0, sizeof(struct vnc_opt)); + kmsvnc->vnc_opt = vncopt; kmsvnc->card = "/dev/dri/card0"; @@ -260,8 +269,14 @@ int main(int argc, char **argv) if (kmsvnc->debug_capture_fb) { int wfd = open(kmsvnc->debug_capture_fb, O_WRONLY | O_CREAT, 00644); + int max_size = 0; + for (int i = 0; i < 4; i++) { + int size = kmsvnc->drm->mfb->offsets[i] + kmsvnc->drm->mfb->height * kmsvnc->drm->mfb->pitches[i]; + if (size > max_size) max_size = size; + } if (wfd > 0) { - write(wfd, kmsvnc->drm->mapped, kmsvnc->drm->mfb->width * kmsvnc->drm->mfb->height * BYTES_PER_PIXEL); + if (kmsvnc->va) va_hwframe_to_vaapi(kmsvnc->drm->mapped); + write(wfd, kmsvnc->drm->mapped, max_size); fsync(wfd); printf("wrote raw frame buffer to %s\n", kmsvnc->debug_capture_fb); } @@ -274,8 +289,16 @@ int main(int argc, char **argv) size_t buflen = kmsvnc->drm->mfb->width * kmsvnc->drm->mfb->height * BYTES_PER_PIXEL; kmsvnc->buf = malloc(buflen); + if (!kmsvnc->buf) { + cleanup(); + KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__); + } memset(kmsvnc->buf, 0, buflen); kmsvnc->buf1 = malloc(buflen); + if (!kmsvnc->buf1) { + cleanup(); + KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__); + } memset(kmsvnc->buf1, 0, buflen); signal(SIGHUP, &signal_handler); diff --git a/kmsvnc.h b/kmsvnc.h index a9c1048..b8b5204 100644 --- a/kmsvnc.h +++ b/kmsvnc.h @@ -8,6 +8,7 @@ #include #include #include +#include #define BYTES_PER_PIXEL 4 @@ -36,6 +37,7 @@ struct kmsvnc_data struct kmsvnc_drm_data *drm; struct kmsvnc_input_data *input; struct kmsvnc_keymap_data *keymap; + struct kmsvnc_va_data *va; rfbScreenInfoPtr server; char shutdown; char *buf; @@ -86,5 +88,20 @@ struct kmsvnc_drm_data off_t mmap_offset; char *mapped; struct kmsvnc_drm_funcs *funcs; + char *pixfmt_name; + char *mod_vendor; + char *mod_name; }; +struct kmsvnc_va_data +{ + VADisplay dpy; + int render_node_fd; + VASurfaceID surface_id; + VAImage *image; + char *imgbuf; +}; + +#define KMSVNC_FATAL(...) do{ fprintf(stderr, __VA_ARGS__); return 1; } while(0) +#define KMSVNC_ARRAY_ELEMENTS(x) (sizeof(x) / sizeof(x[0])) +#define KMSVNC_FOURCC_TO_INT(a,b,c,d) (((a) << 0) + ((b) << 8) + ((c) << 16) + ((d) << 24)) diff --git a/va.c b/va.c new file mode 100644 index 0000000..be54c8a --- /dev/null +++ b/va.c @@ -0,0 +1,238 @@ +#include +#include +#include + +#include "va.h" +#include "kmsvnc.h" + +extern struct kmsvnc_data *kmsvnc; + +void va_cleanup() { + VAStatus s; + if (kmsvnc->va) { + if (kmsvnc->va->imgbuf) { + VA_MAY(vaUnmapBuffer(kmsvnc->va->dpy, kmsvnc->va->image->buf)); + kmsvnc->va->imgbuf = NULL; + } + if (kmsvnc->va->image) { + if ((s = vaDestroyImage(kmsvnc->va->dpy, kmsvnc->va->image->image_id)) == VA_STATUS_SUCCESS) { + free(kmsvnc->va->image); + } + VA_MAY(s); + kmsvnc->va->image = NULL; + } + if (kmsvnc->va->surface_id > 0) { + VA_MAY(vaDestroySurfaces(kmsvnc->va->dpy, &kmsvnc->va->surface_id, 1)); + kmsvnc->va->surface_id = 0; + } + if (kmsvnc->va->dpy) { + VA_MAY(vaTerminate(kmsvnc->va->dpy)); + kmsvnc->va->dpy = NULL; + } + free(kmsvnc->va); + kmsvnc->va = NULL; + } +} + +static void va_msg_callback(void *user_context, const char *message) { + #ifdef KMSVNC_VA_DEBUG + printf("va msg: %s"); + #endif +} + +static void va_error_callback(void *user_context, const char *message) { + printf("va error: %s"); +} + +int va_init() { + if (!kmsvnc->drm || !kmsvnc->drm->drm_fd || !kmsvnc->drm->prime_fd) { + KMSVNC_FATAL("drm is not initialized\n"); + } + + setenv("DISPLAY", "", 1); + + struct kmsvnc_va_data *va = malloc(sizeof(struct kmsvnc_va_data)); + if (!va) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__); + memset(va, 0, sizeof(struct kmsvnc_va_data)); + kmsvnc->va = va; + + char* render_node; + int effective_fd = 0; + if (render_node = drmGetRenderDeviceNameFromFd(kmsvnc->drm->drm_fd)) { + va->render_node_fd = open(render_node, O_RDWR); + free(render_node); + } + else { + printf("Using non-render node because the device does not have an associated render node.\n"); + } + if (va->render_node_fd > 0) { + effective_fd = va->render_node_fd; + } + else { + printf("Using non-render node because render node fails to open.\n"); + effective_fd = kmsvnc->drm->drm_fd; + } + + va->dpy = vaGetDisplayDRM(effective_fd); + if (!va->dpy) { + KMSVNC_FATAL("vaGetDisplayDRM failed\n"); + } + + vaSetErrorCallback(va->dpy, &va_error_callback, NULL); + vaSetInfoCallback(va->dpy, &va_msg_callback, NULL); + + int major, minor; + VAStatus status; + VA_MUST(vaInitialize(va->dpy, &major, &minor)); + + const char *vendor_string = vaQueryVendorString(va->dpy); + printf("vaapi vendor %s\n", vendor_string); + + VADRMPRIMESurfaceDescriptor prime_desc; + VASurfaceAttrib prime_attrs[2] = { + { + .type = VASurfaceAttribMemoryType, + .flags = VA_SURFACE_ATTRIB_SETTABLE, + .value.type = VAGenericValueTypeInteger, + .value.value.i = VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME_2, + }, + { + .type = VASurfaceAttribExternalBufferDescriptor, + .flags = VA_SURFACE_ATTRIB_SETTABLE, + .value.type = VAGenericValueTypePointer, + .value.value.p = &prime_desc, + } + }; + + prime_desc.fourcc = kmsvnc->drm->mfb->pixel_format == KMSVNC_FOURCC_TO_INT('X', 'R', '2', '4') ? + KMSVNC_FOURCC_TO_INT('B', 'G', 'R', 'X') : + KMSVNC_FOURCC_TO_INT('B', 'G', 'R', 'A') ; + prime_desc.width = kmsvnc->drm->mfb->width; + prime_desc.height = kmsvnc->drm->mfb->height; + + int i; + int max_size = 0; + for (i = 0; i < 4; i++) { + int size = kmsvnc->drm->mfb->offsets[i] + kmsvnc->drm->mfb->height * kmsvnc->drm->mfb->pitches[i]; + if (size > max_size) max_size = size; + } + for (i = 0; i < 4; i++) { + prime_desc.objects[i].fd = kmsvnc->drm->prime_fd; + prime_desc.objects[i].size = max_size; + prime_desc.objects[i].drm_format_modifier = kmsvnc->drm->mfb->modifier; + } + + prime_desc.num_layers = 1; + prime_desc.layers[0].drm_format = kmsvnc->drm->mfb->pixel_format; + for (i = 0; i < 4; i++) { + prime_desc.layers[0].object_index[i] = 0; + prime_desc.layers[0].offset[i] = kmsvnc->drm->mfb->offsets[i]; + prime_desc.layers[0].pitch[i] = kmsvnc->drm->mfb->pitches[i]; + } + for (i = 0; i < 4; i++) { + if (!kmsvnc->drm->mfb->handles[i]) { + break; + } + } + prime_desc.layers[0].num_planes = i; + prime_desc.num_objects = 1; + + VAStatus s; + if ((s = vaCreateSurfaces(va->dpy, VA_RT_FORMAT_RGB32, + kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, &va->surface_id, 1, + prime_attrs, KMSVNC_ARRAY_ELEMENTS(prime_attrs))) != VA_STATUS_SUCCESS) + { + printf("vaCreateSurfaces prime2 error %#x %s, trying prime\n", s, vaErrorStr(s)); + + VASurfaceAttribExternalBuffers buffer_desc; + VASurfaceAttrib buffer_attrs[2] = { + { + .type = VASurfaceAttribMemoryType, + .flags = VA_SURFACE_ATTRIB_SETTABLE, + .value.type = VAGenericValueTypeInteger, + .value.value.i = VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME, + }, + { + .type = VASurfaceAttribExternalBufferDescriptor, + .flags = VA_SURFACE_ATTRIB_SETTABLE, + .value.type = VAGenericValueTypePointer, + .value.value.p = &buffer_desc, + } + }; + + unsigned long fd = kmsvnc->drm->prime_fd; + + buffer_desc.pixel_format = prime_desc.fourcc; + buffer_desc.width = kmsvnc->drm->mfb->width; + buffer_desc.height = kmsvnc->drm->mfb->height; + buffer_desc.data_size = max_size; + buffer_desc.buffers = &fd; + buffer_desc.num_buffers = 1; + buffer_desc.flags = 0; + + for (i = 0; i < 4; i++) { + buffer_desc.pitches[i] = kmsvnc->drm->mfb->pitches[i]; + buffer_desc.offsets[i] = kmsvnc->drm->mfb->offsets[i]; + } + buffer_desc.num_planes = prime_desc.layers[0].num_planes; + + + VA_MUST(vaCreateSurfaces(va->dpy, VA_RT_FORMAT_RGB32, + kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, &va->surface_id, 1, + buffer_attrs, KMSVNC_ARRAY_ELEMENTS(buffer_attrs))); + } + + #ifdef KMSVNC_VA_DEBUG + int img_fmt_count = vaMaxNumImageFormats(va->dpy); + VAImageFormat *img_fmts = malloc(sizeof(VAImageFormat) * img_fmt_count); + if (!img_fmts) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__); + { + int got; + vaQueryImageFormats(va->dpy, img_fmts, &got); + if (got != img_fmt_count) { + KMSVNC_FATAL("got less VAImageFormats, %d instead of %d\n", got, img_fmt_count); + } + } + + for (int i = 0; i < img_fmt_count; i++) { + printf("fmt %d: fourcc %d, %c%c%c%c, byte_order %s, bpp %d, depth %d, blue_mask %#x, green_mask %#x, red_mask %#x, reserved %#x\n", i, img_fmts[i].fourcc, + img_fmts[i].fourcc & 0xff, + img_fmts[i].fourcc >> 8 & 0xff, + img_fmts[i].fourcc >> 16 & 0xff, + img_fmts[i].fourcc >> 24 & 0xff, + img_fmts[i].byte_order - 1 ? "VA_LSB_FIRST" : "VA_MSB_FIRST", + img_fmts[i].bits_per_pixel, + img_fmts[i].depth, + img_fmts[i].blue_mask, + img_fmts[i].green_mask, + img_fmts[i].red_mask, + img_fmts[i].va_reserved + ); + } + #endif + + VAImageFormat format = { + .fourcc = KMSVNC_FOURCC_TO_INT('R','G','B','X'), + .byte_order = VA_LSB_FIRST, + .bits_per_pixel = 32, + .depth = 24, + .blue_mask = 0x000000ff, + .green_mask = 0x0000ff00, + .red_mask = 0x00ff0000, + .va_reserved = 0x00000000, + }; + va->image = malloc(sizeof(VAImage)); + if (!va->image) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__); + if ((s = vaCreateImage(va->dpy, &format, kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, va->image)) != VA_STATUS_SUCCESS) { + free(va->image); + va->image = NULL; + VA_MUST(s); + } + VA_MUST(vaMapBuffer(va->dpy, va->image->buf, (void**)&va->imgbuf)); +} + +int va_hwframe_to_vaapi(char *out) { + VA_MUST(vaGetImage(kmsvnc->va->dpy, kmsvnc->va->surface_id, 0, 0, + kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, kmsvnc->va->image->image_id)); + memcpy(out, kmsvnc->va->imgbuf, kmsvnc->drm->mfb->width * kmsvnc->drm->mfb->height * BYTES_PER_PIXEL); +} diff --git a/va.h b/va.h new file mode 100644 index 0000000..0e1e67d --- /dev/null +++ b/va.h @@ -0,0 +1,8 @@ +#pragma once + +#define VA_MUST(x) do{VAStatus s; if ((s = (x)) != VA_STATUS_SUCCESS) KMSVNC_FATAL("va operation error %#x %s on line %d\n", s, vaErrorStr(s), __LINE__); } while (0) +#define VA_MAY(x) do{VAStatus s; if ((s = (x)) != VA_STATUS_SUCCESS) fprintf(stderr, "va operation error %#x %s on line %d\n", s, vaErrorStr(s), __LINE__); } while (0) + +void va_cleanup(); +int va_init(); +int va_hwframe_to_vaapi(char *out);