Jerry/kmsvnc
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

vaapi: wip

Browse source
This commit is contained in:
JerryXiao 2023-04-29 02:05:13 +08:00
parent f3465fe871
commit 1666afec4e
Signed by: Jerry
GPG key ID: 22618F758B5BE2E5
12 changed files with 385 additions and 61 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
CMakeLists.txt
View file

@ -10,6 +10,8 @@ find_package(PkgConfig REQUIRED)
pkg_search_module(LIBDRM REQUIRED libdrm)
pkg_search_module(LIBVNCSERVER REQUIRED libvncserver)
pkg_search_module(XKBCOMMON REQUIRED xkbcommon)
pkg_search_module(LIBVA REQUIRED libva)
pkg_search_module(LIBVA_DRM REQUIRED libva-drm)
include(CheckIncludeFiles)
CHECK_INCLUDE_FILES("linux/uinput.h;linux/dma-buf.h" HAVE_LINUX_API_HEADERS)
@ -17,17 +19,21 @@ IF(NOT HAVE_LINUX_API_HEADERS)
message(FATAL_ERROR "linux-api-headers not found")
ENDIF()
add_executable(kmsvnc kmsvnc.c drm.c input.c keymap.c)
add_executable(kmsvnc kmsvnc.c drm.c input.c keymap.c va.c)
target_include_directories(kmsvnc PUBLIC
${LIBDRM_INCLUDEDIR}
${LIBDRM_INCLUDEDIR}/libdrm
${LIBVNCSERVER_INCLUDEDIR}
${XKBCOMMON_INCLUDEDIR}
${LIBVA_INCLUDEDIR}
${LIBVA_DRM_INCLUDEDIR}
)
target_link_libraries(kmsvnc PUBLIC
m
${LIBDRM_LIBRARIES}
${LIBVNCSERVER_LIBRARIES}
${XKBCOMMON_LIBRARIES}
${LIBVA_LIBRARIES}
${LIBVA_DRM_LIBRARIES}
)
install(TARGETS kmsvnc RUNTIME DESTINATION bin)

1
README.md
View file

@ -10,6 +10,7 @@ Currently in very early stage.
* libvncserver
* libxkbcommon
* libdrm
* libva
## Building
```

112
drm.c
View file

@ -5,8 +5,10 @@
#include <string.h>
#include <stdlib.h>
#include <sys/ioctl.h>
#include <libdrm/drm_fourcc.h>
#include "drm.h"
#include "va.h"
extern struct kmsvnc_data *kmsvnc;
@ -14,6 +16,10 @@ static void convert_copy(const char *in, int width, int height, char *buff) {
memcpy(buff, in, width * height * 4);
}
static void convert_vaapi(const char *in, int width, int height, char *buff) {
va_hwframe_to_vaapi(buff);
}
static void convert_bgrx_to_rgb(const char *in, int width, int height, char *buff)
{
for (int y = 0; y < height; y++)
@ -47,6 +53,7 @@ static inline void convert_x_tiled(const int tilex, const int tiley, const char
if (kms_cpy_tmp_buf)
free(kms_convert_buf);
kms_cpy_tmp_buf = malloc(max_offset * 4 + 4);
if (!kms_cpy_tmp_buf) return;
kms_cpy_tmp_buf_len = max_offset * 4 + 4;
}
memcpy(kms_cpy_tmp_buf, in, max_offset * 4 + 4);
@ -57,6 +64,7 @@ static inline void convert_x_tiled(const int tilex, const int tiley, const char
if (kms_convert_buf)
free(kms_convert_buf);
kms_convert_buf = malloc(width * height * 4);
if (!kms_convert_buf) return;
kms_convert_buf_len = width * height * 4;
}
for (int y = 0; y < height; y++)
@ -83,13 +91,10 @@ void convert_intel_x_tiled_kmsbuf(const char *in, int width, int height, char *b
static inline void drm_sync(int drmfd, uint64_t flags)
{
int ioctl_err;
struct dma_buf_sync sync = {
.flags = flags,
};
if (ioctl_err = ioctl(drmfd, DMA_BUF_IOCTL_SYNC, &sync)) {
fprintf(stderr, "DRM ioctl error %d on line %d\n", ioctl_err, __LINE__);
}
DRM_R_IOCTL_MAY(drmfd, DMA_BUF_IOCTL_SYNC, &sync);
}
void drm_sync_start(int drmfd)
@ -110,6 +115,18 @@ void drm_cleanup() {
drmFreeVersion(kmsvnc->drm->drm_ver);
kmsvnc->drm->drm_ver = NULL;
}
if (kmsvnc->drm->pixfmt_name) {
free(kmsvnc->drm->pixfmt_name);
kmsvnc->drm->pixfmt_name = NULL;
}
if (kmsvnc->drm->mod_vendor) {
free(kmsvnc->drm->mod_vendor);
kmsvnc->drm->mod_vendor = NULL;
}
if (kmsvnc->drm->mod_name) {
free(kmsvnc->drm->mod_name);
kmsvnc->drm->mod_name = NULL;
}
if (kmsvnc->drm->plane) {
drmModeFreePlane(kmsvnc->drm->plane);
kmsvnc->drm->plane = NULL;
@ -141,13 +158,14 @@ void drm_cleanup() {
int drm_open() {
struct kmsvnc_drm_data *drm = malloc(sizeof(struct kmsvnc_drm_data));
if (!drm) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
memset(drm, 0, sizeof(struct kmsvnc_drm_data));
kmsvnc->drm = drm;
drm->drm_fd = open(kmsvnc->card, O_RDONLY);
if (drm->drm_fd < 0)
{
DRM_FATAL("card %s open failed: %s\n", kmsvnc->card, strerror(errno));
KMSVNC_FATAL("card %s open failed: %s\n", kmsvnc->card, strerror(errno));
}
drm->drm_ver = drmGetVersion(drm->drm_fd);
printf("drm driver is %s\n", drm->drm_ver->name);
@ -161,7 +179,7 @@ int drm_open() {
{
drm->plane = drmModeGetPlane(drm->drm_fd, kmsvnc->source_plane);
if (!drm->plane)
DRM_FATAL("Failed to get plane %d: %s\n", kmsvnc->source_plane, strerror(errno));
KMSVNC_FATAL("Failed to get plane %d: %s\n", kmsvnc->source_plane, strerror(errno));
if (drm->plane->fb_id == 0)
fprintf(stderr, "Place %d does not have an attached framebuffer\n", kmsvnc->source_plane);
}
@ -169,7 +187,7 @@ int drm_open() {
{
drm->plane_res = drmModeGetPlaneResources(drm->drm_fd);
if (!drm->plane_res)
DRM_FATAL("Failed to get plane resources: %s\n", strerror(errno));
KMSVNC_FATAL("Failed to get plane resources: %s\n", strerror(errno));
int i;
for (i = 0; i < drm->plane_res->count_planes; i++)
{
@ -194,11 +212,11 @@ int drm_open() {
{
if (kmsvnc->source_crtc != 0)
{
DRM_FATAL("No usable planes found on CRTC %d\n", kmsvnc->source_crtc);
KMSVNC_FATAL("No usable planes found on CRTC %d\n", kmsvnc->source_crtc);
}
else
{
DRM_FATAL("No usable planes found\n");
KMSVNC_FATAL("No usable planes found\n");
}
}
printf("Using plane %u to locate framebuffers\n", drm->plane->plane_id);
@ -208,32 +226,34 @@ int drm_open() {
drm->mfb = drmModeGetFB2(drm->drm_fd, drm->plane->fb_id);
if (!drm->mfb) {
DRM_FATAL("Failed to get framebuffer %u: %s\n", drm->plane->fb_id, strerror(errno));
KMSVNC_FATAL("Failed to get framebuffer %u: %s\n", drm->plane->fb_id, strerror(errno));
}
drm->pixfmt_name = drmGetFormatName(drm->mfb->pixel_format);
drm->mod_vendor = drmGetFormatModifierVendor(drm->mfb->modifier);
drm->mod_name = drmGetFormatModifierName(drm->mfb->modifier);
printf("Template framebuffer is %u: %ux%u fourcc:%u mod:%u flags:%u\n", drm->mfb->fb_id, drm->mfb->width, drm->mfb->height, drm->mfb->pixel_format, drm->mfb->modifier, drm->mfb->flags);
printf("handles %u %u %u %u\n", drm->mfb->handles[0], drm->mfb->handles[1], drm->mfb->handles[2], drm->mfb->handles[3]);
printf("offsets %u %u %u %u\n", drm->mfb->offsets[0], drm->mfb->offsets[1], drm->mfb->offsets[2], drm->mfb->offsets[3]);
printf("pitches %u %u %u %u\n", drm->mfb->pitches[0], drm->mfb->pitches[1], drm->mfb->pitches[2], drm->mfb->pitches[3]);
printf("format %s, modifier %s:%s\n", drmGetFormatName(drm->mfb->pixel_format), drmGetFormatModifierVendor(drm->mfb->modifier), drmGetFormatModifierName(drm->mfb->modifier));
printf("format %s, modifier %s:%s\n", drm->pixfmt_name, drm->mod_vendor, drm->mod_name);
if (
drm->mfb->pixel_format != KMSVNC_FOURCC_TO_INT('X', 'R', '2', '4') &&
drm->mfb->pixel_format != KMSVNC_FOURCC_TO_INT('A', 'R', '2', '4')
)
{
DRM_FATAL("Unsupported pixfmt\n");
KMSVNC_FATAL("Unsupported pixfmt %s, please create an issue with your pixfmt.\n", drm->pixfmt_name);
}
if (!drm->mfb->handles[0])
{
DRM_FATAL("No handle set on framebuffer: maybe you need some additional capabilities?\n");
KMSVNC_FATAL("No handle set on framebuffer: maybe you need some additional capabilities?\n");
}
int ioctl_err = 0;
drm->mmap_fd = drm->drm_fd;
drm->mmap_size = drm->mfb->width * drm->mfb->height * BYTES_PER_PIXEL;
drm->funcs = malloc(sizeof(struct kmsvnc_drm_funcs));
if (!drm->funcs) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
drm->funcs->convert = convert_bgrx_to_rgb;
drm->funcs->sync_start = drm_sync_noop;
drm->funcs->sync_end = drm_sync_noop;
@ -250,7 +270,7 @@ static int drm_kmsbuf_prime() {
int err = drmPrimeHandleToFD(drm->drm_fd, drm->mfb->handles[0], O_RDWR, &drm->prime_fd);
if (err < 0 || drm->prime_fd < 0)
{
DRM_FATAL("Failed to get PRIME fd from framebuffer handle");
KMSVNC_FATAL("Failed to get PRIME fd from framebuffer handle");
}
drm->funcs->sync_start = &drm_sync_start;
drm->funcs->sync_end = &drm_sync_end;
@ -258,6 +278,22 @@ static int drm_kmsbuf_prime() {
return 0;
}
static int drm_kmsbuf_prime_vaapi() {
struct kmsvnc_drm_data *drm = kmsvnc->drm;
int err = drmPrimeHandleToFD(drm->drm_fd, drm->mfb->handles[0], O_RDWR, &drm->prime_fd);
if (err < 0 || drm->prime_fd < 0)
{
KMSVNC_FATAL("Failed to get PRIME fd from framebuffer handle");
}
if (va_init()) return 1;
drm->mmap_fd = drm->prime_fd;
drm->mapped = kmsvnc->va->imgbuf;
return 0;
}
static int drm_kmsbuf_dumb() {
struct kmsvnc_drm_data *drm = kmsvnc->drm;
@ -291,32 +327,19 @@ int drm_vendors() {
driver_name = drm->drm_ver->name;
}
if (strcmp(driver_name, "i915") == 0)
if (strcmp(driver_name, "i915") == 0 || strcmp(driver_name, "amdgpu") == 0)
{
drm->funcs->convert = &convert_intel_x_tiled_kmsbuf;
if (drm_kmsbuf_prime()) return 1;
}
else if (strcmp(driver_name, "amdgpu") == 0)
{
struct drm_gem_flink flink;
flink.handle = drm->mfb->handles[0];
DRM_IOCTL_MUST(drm->drm_fd, DRM_IOCTL_GEM_FLINK, &flink);
struct drm_gem_open open_arg;
open_arg.name = flink.name;
DRM_IOCTL_MUST(drm->drm_fd, DRM_IOCTL_GEM_OPEN, &open_arg);
union drm_amdgpu_gem_mmap mmap_arg;
memset(&mmap_arg, 0, sizeof(mmap_arg));
mmap_arg.in.handle = open_arg.handle;
DRM_IOCTL_MUST(drm->drm_fd, DRM_IOCTL_AMDGPU_GEM_MMAP, &mmap_arg);
drm->mmap_size = open_arg.size;
drm->mmap_offset = mmap_arg.out.addr_ptr;
if (fourcc_mod_is_vendor(drm->mfb->modifier, INTEL)) {
if (strstr(drm->mod_name, "CCS")) {
printf("warn: intel with CCS modifier detected, please set INTEL_DEBUG=noccs\n");
}
};
drm->funcs->convert = &convert_vaapi;
if (drm_kmsbuf_prime_vaapi()) return 1;
}
else if (strcmp(driver_name, "nvidia-drm") == 0)
{
// quirky and slow
printf("warn: nvidia card detected. Currently only x-tiled framebuffer is supported. Performance may suffer.\n");
drm->funcs->convert = &convert_nvidia_x_tiled_kmsbuf;
if (drm_kmsbuf_dumb()) return 1;
}
@ -325,6 +348,9 @@ int drm_vendors() {
strcmp(driver_name, "virtio_gpu") == 0
)
{
if (drm->mfb->modifier != DRM_FORMAT_MOD_NONE && drm->mfb->modifier != DRM_FORMAT_MOD_LINEAR) {
printf("warn: modifier is not LINEAR, please create an issue with your modifier.\n");
}
// virgl does not work
if (drm_kmsbuf_dumb()) return 1;
}
@ -352,9 +378,17 @@ int drm_vendors() {
drm->mmap_size = open_arg.size;
drm->mmap_offset = mmap_arg.offset;
}
else if (strcmp(driver_name, "test-i915-prime-xtiled") == 0)
{
drm->funcs->convert = &convert_intel_x_tiled_kmsbuf;
if (drm_kmsbuf_prime()) return 1;
}
else
{
fprintf(stderr, "Untested drm driver, use at your own risk!\n");
if (drm->mfb->modifier != DRM_FORMAT_MOD_NONE && drm->mfb->modifier != DRM_FORMAT_MOD_LINEAR) {
printf("warn: modifier is not LINEAR, please create an issue with your driver and modifier.\n");
}
if (drm_kmsbuf_dumb()) return 1;
}
@ -364,7 +398,7 @@ int drm_vendors() {
drm->mapped = mmap(NULL, drm->mmap_size, PROT_READ, MAP_SHARED, drm->mmap_fd, drm->mmap_offset);
if (drm->mapped == MAP_FAILED)
{
DRM_FATAL("Failed to mmap: %s\n", strerror(errno));
KMSVNC_FATAL("Failed to mmap: %s\n", strerror(errno));
}
}

7
drm.h
View file

@ -2,11 +2,10 @@
#include "kmsvnc.h"
#define KMSVNC_FOURCC_TO_INT(a,b,c,d) (((a) << 0) + ((b) << 8) + ((c) << 16) + ((d) << 24))
#define DRM_IOCTL_MUST(...) do{ int e; if (e = drmIoctl(__VA_ARGS__)) KMSVNC_FATAL("DRM ioctl error %d on line %d\n", e, __LINE__); } while(0)
#define DRM_IOCTL_MAY(...) do{ int e; if (e = drmIoctl(__VA_ARGS__)) fprintf(stderr, "DRM ioctl error %d on line %d\n", e, __LINE__); } while(0)
#define DRM_R_IOCTL_MAY(...) do{ int e; if (e = ioctl(__VA_ARGS__)) fprintf(stderr, "DRM ioctl error %d on line %d\n", e, __LINE__); } while(0)
#define DRM_FATAL(...) { fprintf(stderr, __VA_ARGS__); return 1; }
#define DRM_IOCTL_MUST(...) { int e; if (e = drmIoctl(__VA_ARGS__)) DRM_FATAL("DRM ioctl error %d on line %d\n", e, __LINE__) }
#define DRM_IOCTL_MAY(...) { int e; if (e = drmIoctl(__VA_ARGS__)) fprintf(stderr, "DRM ioctl error %d on line %d\n", e, __LINE__); }
void drm_cleanup();
int drm_open();

10
input.c
View file

@ -29,13 +29,14 @@ void uinput_cleanup()
int uinput_init()
{
struct kmsvnc_input_data *inp = malloc(sizeof(struct kmsvnc_input_data));
if (!inp) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
memset(inp, 0, sizeof(struct kmsvnc_input_data));
kmsvnc->input = inp;
inp->uinput_fd = open("/dev/uinput", O_WRONLY | O_NONBLOCK);
if (inp->uinput_fd <= 0)
{
INP_FATAL("Failed to open uinput\n");
KMSVNC_FATAL("Failed to open uinput\n");
}
INP_IOCTL_MUST(inp->uinput_fd, UI_SET_EVBIT, EV_KEY);
INP_IOCTL_MUST(inp->uinput_fd, UI_SET_EVBIT, EV_SYN);
@ -74,6 +75,7 @@ int uinput_init()
INP_IOCTL_MUST(inp->uinput_fd, UI_DEV_CREATE);
inp->keystate = malloc(UINPUT_MAX_KEY);
if (!inp->keystate) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
memset(inp->keystate, 0, UINPUT_MAX_KEY);
return 0;
@ -114,7 +116,7 @@ void rfb_key_hook(rfbBool down, rfbKeySym keysym, rfbClientPtr cl)
.value = 0,
},
};
for (int i = 0; i < ARRAY_SIZE(ies); i++)
for (int i = 0; i < KMSVNC_ARRAY_ELEMENTS(ies); i++)
{
write(kmsvnc->input->uinput_fd, &ies[i], sizeof(ies[0]));
}
@ -159,7 +161,7 @@ void rfb_ptr_hook(int mask, int screen_x, int screen_y, rfbClientPtr cl)
.value = 0,
},
};
for (int i = 0; i < ARRAY_SIZE(ies1); i++)
for (int i = 0; i < KMSVNC_ARRAY_ELEMENTS(ies1); i++)
{
write(kmsvnc->input->uinput_fd, &ies1[i], sizeof(ies1[0]));
}
@ -177,7 +179,7 @@ void rfb_ptr_hook(int mask, int screen_x, int screen_y, rfbClientPtr cl)
.value = 0,
},
};
for (int i = 0; i < ARRAY_SIZE(ies2); i++)
for (int i = 0; i < KMSVNC_ARRAY_ELEMENTS(ies2); i++)
{
write(kmsvnc->input->uinput_fd, &ies2[i], sizeof(ies2[0]));
}

7
input.h
View file

@ -4,14 +4,11 @@
#include "kmsvnc.h"
#define ARRAY_SIZE(x) (sizeof(x) / sizeof(x[0]))
#define UINPUT_ABS_MAX INT16_MAX
#define UINPUT_MAX_KEY 256
#define INP_FATAL(...) { fprintf(stderr, __VA_ARGS__); return 1; }
#define INP_IOCTL_MUST(...) { int e; if (e = ioctl(__VA_ARGS__)) INP_FATAL("uinput ioctl error %d on line %d\n", e, __LINE__) }
#define INP_IOCTL_MAY(...) { int e; if (e = ioctl(__VA_ARGS__)) fprintf(stderr, "uinput ioctl error %d on line %d\n", e, __LINE__); }
#define INP_IOCTL_MUST(...) do{ int e; if (e = ioctl(__VA_ARGS__)) KMSVNC_FATAL("uinput ioctl error %d on line %d\n", e, __LINE__); } while(0)
#define INP_IOCTL_MAY(...) do{ int e; if (e = ioctl(__VA_ARGS__)) fprintf(stderr, "uinput ioctl error %d on line %d\n", e, __LINE__); } while(0)
void uinput_cleanup();
int uinput_init();

5
keymap.c
View file

@ -25,13 +25,14 @@ void xkb_cleanup() {
int xkb_init()
{
struct kmsvnc_keymap_data *xkb = malloc(sizeof(struct kmsvnc_keymap_data));
if (!xkb) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
memset(xkb, 0, sizeof(struct kmsvnc_keymap_data));
kmsvnc->keymap = xkb;
xkb->ctx = xkb_context_new(XKB_CONTEXT_NO_FLAGS);
if (xkb->ctx == NULL)
{
XKB_FATAL("Failed to create XKB context\n");
KMSVNC_FATAL("Failed to create XKB context\n");
}
struct xkb_rule_names names = {
.rules = NULL,
@ -43,7 +44,7 @@ int xkb_init()
xkb->map = xkb_keymap_new_from_names(xkb->ctx, &names, 0);
if (xkb->map == NULL)
{
XKB_FATAL("Failed to create XKB keymap\n");
KMSVNC_FATAL("Failed to create XKB keymap\n");
}
// printf("xkb: keymap string\n%s\n", xkb_keymap_get_as_string(xkb->map, XKB_KEYMAP_USE_ORIGINAL_FORMAT));
return 0;

2
keymap.h
View file

@ -2,8 +2,6 @@
#include "kmsvnc.h"
#define XKB_FATAL(...) { fprintf(stderr, __VA_ARGS__); return 1; }
void xkb_cleanup();
int xkb_init();
void key_iter(struct xkb_keymap *xkb, xkb_keycode_t key, void *data);

31
kmsvnc.c
View file

@ -11,6 +11,7 @@
#include "keymap.h"
#include "input.h"
#include "drm.h"
#include "va.h"
struct kmsvnc_data *kmsvnc = NULL;
@ -98,6 +99,9 @@ static void cleanup() {
if (kmsvnc->drm) {
drm_cleanup();
}
if (kmsvnc->va) {
va_cleanup();
}
if (kmsvnc) {
if (kmsvnc->vnc_opt) {
free(kmsvnc->vnc_opt);
@ -215,12 +219,17 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
int main(int argc, char **argv)
{
struct vnc_opt *vncopt = malloc(sizeof(struct vnc_opt));
memset(vncopt, 0, sizeof(struct vnc_opt));
kmsvnc = malloc(sizeof(struct kmsvnc_data));
if (!kmsvnc) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
memset(kmsvnc, 0, sizeof(struct kmsvnc_data));
struct vnc_opt *vncopt = malloc(sizeof(struct vnc_opt));
if (!vncopt) {
free(kmsvnc);
KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
}
memset(vncopt, 0, sizeof(struct vnc_opt));
kmsvnc->vnc_opt = vncopt;
kmsvnc->card = "/dev/dri/card0";
@ -260,8 +269,14 @@ int main(int argc, char **argv)
if (kmsvnc->debug_capture_fb) {
int wfd = open(kmsvnc->debug_capture_fb, O_WRONLY | O_CREAT, 00644);
int max_size = 0;
for (int i = 0; i < 4; i++) {
int size = kmsvnc->drm->mfb->offsets[i] + kmsvnc->drm->mfb->height * kmsvnc->drm->mfb->pitches[i];
if (size > max_size) max_size = size;
}
if (wfd > 0) {
write(wfd, kmsvnc->drm->mapped, kmsvnc->drm->mfb->width * kmsvnc->drm->mfb->height * BYTES_PER_PIXEL);
if (kmsvnc->va) va_hwframe_to_vaapi(kmsvnc->drm->mapped);
write(wfd, kmsvnc->drm->mapped, max_size);
fsync(wfd);
printf("wrote raw frame buffer to %s\n", kmsvnc->debug_capture_fb);
}
@ -274,8 +289,16 @@ int main(int argc, char **argv)
size_t buflen = kmsvnc->drm->mfb->width * kmsvnc->drm->mfb->height * BYTES_PER_PIXEL;
kmsvnc->buf = malloc(buflen);
if (!kmsvnc->buf) {
cleanup();
KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
}
memset(kmsvnc->buf, 0, buflen);
kmsvnc->buf1 = malloc(buflen);
if (!kmsvnc->buf1) {
cleanup();
KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
}
memset(kmsvnc->buf1, 0, buflen);
signal(SIGHUP, &signal_handler);

17
kmsvnc.h
View file

@ -8,6 +8,7 @@
#include <amdgpu_drm.h>
#include <xf86drmMode.h>
#include <linux/dma-buf.h>
#include <va/va.h>
#define BYTES_PER_PIXEL 4
@ -36,6 +37,7 @@ struct kmsvnc_data
struct kmsvnc_drm_data *drm;
struct kmsvnc_input_data *input;
struct kmsvnc_keymap_data *keymap;
struct kmsvnc_va_data *va;
rfbScreenInfoPtr server;
char shutdown;
char *buf;
@ -86,5 +88,20 @@ struct kmsvnc_drm_data
off_t mmap_offset;
char *mapped;
struct kmsvnc_drm_funcs *funcs;
char *pixfmt_name;
char *mod_vendor;
char *mod_name;
};
struct kmsvnc_va_data
{
VADisplay dpy;
int render_node_fd;
VASurfaceID surface_id;
VAImage *image;
char *imgbuf;
};
#define KMSVNC_FATAL(...) do{ fprintf(stderr, __VA_ARGS__); return 1; } while(0)
#define KMSVNC_ARRAY_ELEMENTS(x) (sizeof(x) / sizeof(x[0]))
#define KMSVNC_FOURCC_TO_INT(a,b,c,d) (((a) << 0) + ((b) << 8) + ((c) << 16) + ((d) << 24))

238
va.c Normal file
View file

@ -0,0 +1,238 @@
#include <va/va_drm.h>
#include <va/va_drmcommon.h>
#include <fcntl.h>
#include "va.h"
#include "kmsvnc.h"
extern struct kmsvnc_data *kmsvnc;
void va_cleanup() {
VAStatus s;
if (kmsvnc->va) {
if (kmsvnc->va->imgbuf) {
VA_MAY(vaUnmapBuffer(kmsvnc->va->dpy, kmsvnc->va->image->buf));
kmsvnc->va->imgbuf = NULL;
}
if (kmsvnc->va->image) {
if ((s = vaDestroyImage(kmsvnc->va->dpy, kmsvnc->va->image->image_id)) == VA_STATUS_SUCCESS) {
free(kmsvnc->va->image);
}
VA_MAY(s);
kmsvnc->va->image = NULL;
}
if (kmsvnc->va->surface_id > 0) {
VA_MAY(vaDestroySurfaces(kmsvnc->va->dpy, &kmsvnc->va->surface_id, 1));
kmsvnc->va->surface_id = 0;
}
if (kmsvnc->va->dpy) {
VA_MAY(vaTerminate(kmsvnc->va->dpy));
kmsvnc->va->dpy = NULL;
}
free(kmsvnc->va);
kmsvnc->va = NULL;
}
}
static void va_msg_callback(void *user_context, const char *message) {
#ifdef KMSVNC_VA_DEBUG
printf("va msg: %s");
#endif
}
static void va_error_callback(void *user_context, const char *message) {
printf("va error: %s");
}
int va_init() {
if (!kmsvnc->drm || !kmsvnc->drm->drm_fd || !kmsvnc->drm->prime_fd) {
KMSVNC_FATAL("drm is not initialized\n");
}
setenv("DISPLAY", "", 1);
struct kmsvnc_va_data *va = malloc(sizeof(struct kmsvnc_va_data));
if (!va) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
memset(va, 0, sizeof(struct kmsvnc_va_data));
kmsvnc->va = va;
char* render_node;
int effective_fd = 0;
if (render_node = drmGetRenderDeviceNameFromFd(kmsvnc->drm->drm_fd)) {
va->render_node_fd = open(render_node, O_RDWR);
free(render_node);
}
else {
printf("Using non-render node because the device does not have an associated render node.\n");
}
if (va->render_node_fd > 0) {
effective_fd = va->render_node_fd;
}
else {
printf("Using non-render node because render node fails to open.\n");
effective_fd = kmsvnc->drm->drm_fd;
}
va->dpy = vaGetDisplayDRM(effective_fd);
if (!va->dpy) {
KMSVNC_FATAL("vaGetDisplayDRM failed\n");
}
vaSetErrorCallback(va->dpy, &va_error_callback, NULL);
vaSetInfoCallback(va->dpy, &va_msg_callback, NULL);
int major, minor;
VAStatus status;
VA_MUST(vaInitialize(va->dpy, &major, &minor));
const char *vendor_string = vaQueryVendorString(va->dpy);
printf("vaapi vendor %s\n", vendor_string);
VADRMPRIMESurfaceDescriptor prime_desc;
VASurfaceAttrib prime_attrs[2] = {
{
.type = VASurfaceAttribMemoryType,
.flags = VA_SURFACE_ATTRIB_SETTABLE,
.value.type = VAGenericValueTypeInteger,
.value.value.i = VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME_2,
},
{
.type = VASurfaceAttribExternalBufferDescriptor,
.flags = VA_SURFACE_ATTRIB_SETTABLE,
.value.type = VAGenericValueTypePointer,
.value.value.p = &prime_desc,
}
};
prime_desc.fourcc = kmsvnc->drm->mfb->pixel_format == KMSVNC_FOURCC_TO_INT('X', 'R', '2', '4') ?
KMSVNC_FOURCC_TO_INT('B', 'G', 'R', 'X') :
KMSVNC_FOURCC_TO_INT('B', 'G', 'R', 'A') ;
prime_desc.width = kmsvnc->drm->mfb->width;
prime_desc.height = kmsvnc->drm->mfb->height;
int i;
int max_size = 0;
for (i = 0; i < 4; i++) {
int size = kmsvnc->drm->mfb->offsets[i] + kmsvnc->drm->mfb->height * kmsvnc->drm->mfb->pitches[i];
if (size > max_size) max_size = size;
}
for (i = 0; i < 4; i++) {
prime_desc.objects[i].fd = kmsvnc->drm->prime_fd;
prime_desc.objects[i].size = max_size;
prime_desc.objects[i].drm_format_modifier = kmsvnc->drm->mfb->modifier;
}
prime_desc.num_layers = 1;
prime_desc.layers[0].drm_format = kmsvnc->drm->mfb->pixel_format;
for (i = 0; i < 4; i++) {
prime_desc.layers[0].object_index[i] = 0;
prime_desc.layers[0].offset[i] = kmsvnc->drm->mfb->offsets[i];
prime_desc.layers[0].pitch[i] = kmsvnc->drm->mfb->pitches[i];
}
for (i = 0; i < 4; i++) {
if (!kmsvnc->drm->mfb->handles[i]) {
break;
}
}
prime_desc.layers[0].num_planes = i;
prime_desc.num_objects = 1;
VAStatus s;
if ((s = vaCreateSurfaces(va->dpy, VA_RT_FORMAT_RGB32,
kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, &va->surface_id, 1,
prime_attrs, KMSVNC_ARRAY_ELEMENTS(prime_attrs))) != VA_STATUS_SUCCESS)
{
printf("vaCreateSurfaces prime2 error %#x %s, trying prime\n", s, vaErrorStr(s));
VASurfaceAttribExternalBuffers buffer_desc;
VASurfaceAttrib buffer_attrs[2] = {
{
.type = VASurfaceAttribMemoryType,
.flags = VA_SURFACE_ATTRIB_SETTABLE,
.value.type = VAGenericValueTypeInteger,
.value.value.i = VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME,
},
{
.type = VASurfaceAttribExternalBufferDescriptor,
.flags = VA_SURFACE_ATTRIB_SETTABLE,
.value.type = VAGenericValueTypePointer,
.value.value.p = &buffer_desc,
}
};
unsigned long fd = kmsvnc->drm->prime_fd;
buffer_desc.pixel_format = prime_desc.fourcc;
buffer_desc.width = kmsvnc->drm->mfb->width;
buffer_desc.height = kmsvnc->drm->mfb->height;
buffer_desc.data_size = max_size;
buffer_desc.buffers = &fd;
buffer_desc.num_buffers = 1;
buffer_desc.flags = 0;
for (i = 0; i < 4; i++) {
buffer_desc.pitches[i] = kmsvnc->drm->mfb->pitches[i];
buffer_desc.offsets[i] = kmsvnc->drm->mfb->offsets[i];
}
buffer_desc.num_planes = prime_desc.layers[0].num_planes;
VA_MUST(vaCreateSurfaces(va->dpy, VA_RT_FORMAT_RGB32,
kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, &va->surface_id, 1,
buffer_attrs, KMSVNC_ARRAY_ELEMENTS(buffer_attrs)));
}
#ifdef KMSVNC_VA_DEBUG
int img_fmt_count = vaMaxNumImageFormats(va->dpy);
VAImageFormat *img_fmts = malloc(sizeof(VAImageFormat) * img_fmt_count);
if (!img_fmts) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
{
int got;
vaQueryImageFormats(va->dpy, img_fmts, &got);
if (got != img_fmt_count) {
KMSVNC_FATAL("got less VAImageFormats, %d instead of %d\n", got, img_fmt_count);
}
}
for (int i = 0; i < img_fmt_count; i++) {
printf("fmt %d: fourcc %d, %c%c%c%c, byte_order %s, bpp %d, depth %d, blue_mask %#x, green_mask %#x, red_mask %#x, reserved %#x\n", i, img_fmts[i].fourcc,
img_fmts[i].fourcc & 0xff,
img_fmts[i].fourcc >> 8 & 0xff,
img_fmts[i].fourcc >> 16 & 0xff,
img_fmts[i].fourcc >> 24 & 0xff,
img_fmts[i].byte_order - 1 ? "VA_LSB_FIRST" : "VA_MSB_FIRST",
img_fmts[i].bits_per_pixel,
img_fmts[i].depth,
img_fmts[i].blue_mask,
img_fmts[i].green_mask,
img_fmts[i].red_mask,
img_fmts[i].va_reserved
);
}
#endif
VAImageFormat format = {
.fourcc = KMSVNC_FOURCC_TO_INT('R','G','B','X'),
.byte_order = VA_LSB_FIRST,
.bits_per_pixel = 32,
.depth = 24,
.blue_mask = 0x000000ff,
.green_mask = 0x0000ff00,
.red_mask = 0x00ff0000,
.va_reserved = 0x00000000,
};
va->image = malloc(sizeof(VAImage));
if (!va->image) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
if ((s = vaCreateImage(va->dpy, &format, kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, va->image)) != VA_STATUS_SUCCESS) {
free(va->image);
va->image = NULL;
VA_MUST(s);
}
VA_MUST(vaMapBuffer(va->dpy, va->image->buf, (void**)&va->imgbuf));
}
int va_hwframe_to_vaapi(char *out) {
VA_MUST(vaGetImage(kmsvnc->va->dpy, kmsvnc->va->surface_id, 0, 0,
kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, kmsvnc->va->image->image_id));
memcpy(out, kmsvnc->va->imgbuf, kmsvnc->drm->mfb->width * kmsvnc->drm->mfb->height * BYTES_PER_PIXEL);
}

8
va.h Normal file
View file

@ -0,0 +1,8 @@
#pragma once
#define VA_MUST(x) do{VAStatus s; if ((s = (x)) != VA_STATUS_SUCCESS) KMSVNC_FATAL("va operation error %#x %s on line %d\n", s, vaErrorStr(s), __LINE__); } while (0)
#define VA_MAY(x) do{VAStatus s; if ((s = (x)) != VA_STATUS_SUCCESS) fprintf(stderr, "va operation error %#x %s on line %d\n", s, vaErrorStr(s), __LINE__); } while (0)
void va_cleanup();
int va_init();
int va_hwframe_to_vaapi(char *out);