Jerry/kmsvnc
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

vaapi: wip

Browse source
This commit is contained in:
JerryXiao 2023-04-29 02:05:13 +08:00
parent f3465fe871
commit 38a2d8e64d
Signed by: Jerry
GPG key ID: 22618F758B5BE2E5
11 changed files with 282 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
README.md
View file

@ -10,6 +10,7 @@ Currently in very early stage.
* libvncserver
* libxkbcommon
* libdrm
* libva
## Building
```

53
drm.c
View file

@ -7,6 +7,7 @@
#include <sys/ioctl.h>
#include "drm.h"
#include "va.h"
extern struct kmsvnc_data *kmsvnc;
@ -14,6 +15,10 @@ static void convert_copy(const char *in, int width, int height, char *buff) {
memcpy(buff, in, width * height * 4);
}
static void convert_vaapi(const char *in, int width, int height, char *buff) {
va_hwframe_to_vaapi();
}
static void convert_bgrx_to_rgb(const char *in, int width, int height, char *buff)
{
for (int y = 0; y < height; y++)
@ -83,13 +88,10 @@ void convert_intel_x_tiled_kmsbuf(const char *in, int width, int height, char *b
static inline void drm_sync(int drmfd, uint64_t flags)
{
int ioctl_err;
struct dma_buf_sync sync = {
.flags = flags,
};
if (ioctl_err = ioctl(drmfd, DMA_BUF_IOCTL_SYNC, &sync)) {
fprintf(stderr, "DRM ioctl error %d on line %d\n", ioctl_err, __LINE__);
}
DRM_R_IOCTL_MAY(drmfd, DMA_BUF_IOCTL_SYNC, &sync);
}
void drm_sync_start(int drmfd)
@ -147,7 +149,7 @@ int drm_open() {
drm->drm_fd = open(kmsvnc->card, O_RDONLY);
if (drm->drm_fd < 0)
{
DRM_FATAL("card %s open failed: %s\n", kmsvnc->card, strerror(errno));
KMSVNC_FATAL("card %s open failed: %s\n", kmsvnc->card, strerror(errno));
}
drm->drm_ver = drmGetVersion(drm->drm_fd);
printf("drm driver is %s\n", drm->drm_ver->name);
@ -161,7 +163,7 @@ int drm_open() {
{
drm->plane = drmModeGetPlane(drm->drm_fd, kmsvnc->source_plane);
if (!drm->plane)
DRM_FATAL("Failed to get plane %d: %s\n", kmsvnc->source_plane, strerror(errno));
KMSVNC_FATAL("Failed to get plane %d: %s\n", kmsvnc->source_plane, strerror(errno));
if (drm->plane->fb_id == 0)
fprintf(stderr, "Place %d does not have an attached framebuffer\n", kmsvnc->source_plane);
}
@ -169,7 +171,7 @@ int drm_open() {
{
drm->plane_res = drmModeGetPlaneResources(drm->drm_fd);
if (!drm->plane_res)
DRM_FATAL("Failed to get plane resources: %s\n", strerror(errno));
KMSVNC_FATAL("Failed to get plane resources: %s\n", strerror(errno));
int i;
for (i = 0; i < drm->plane_res->count_planes; i++)
{
@ -194,11 +196,11 @@ int drm_open() {
{
if (kmsvnc->source_crtc != 0)
{
DRM_FATAL("No usable planes found on CRTC %d\n", kmsvnc->source_crtc);
KMSVNC_FATAL("No usable planes found on CRTC %d\n", kmsvnc->source_crtc);
}
else
{
DRM_FATAL("No usable planes found\n");
KMSVNC_FATAL("No usable planes found\n");
}
}
printf("Using plane %u to locate framebuffers\n", drm->plane->plane_id);
@ -208,7 +210,7 @@ int drm_open() {
drm->mfb = drmModeGetFB2(drm->drm_fd, drm->plane->fb_id);
if (!drm->mfb) {
DRM_FATAL("Failed to get framebuffer %u: %s\n", drm->plane->fb_id, strerror(errno));
KMSVNC_FATAL("Failed to get framebuffer %u: %s\n", drm->plane->fb_id, strerror(errno));
}
printf("Template framebuffer is %u: %ux%u fourcc:%u mod:%u flags:%u\n", drm->mfb->fb_id, drm->mfb->width, drm->mfb->height, drm->mfb->pixel_format, drm->mfb->modifier, drm->mfb->flags);
printf("handles %u %u %u %u\n", drm->mfb->handles[0], drm->mfb->handles[1], drm->mfb->handles[2], drm->mfb->handles[3]);
@ -221,16 +223,14 @@ int drm_open() {
drm->mfb->pixel_format != KMSVNC_FOURCC_TO_INT('A', 'R', '2', '4')
)
{
DRM_FATAL("Unsupported pixfmt\n");
KMSVNC_FATAL("Unsupported pixfmt\n");
}
if (!drm->mfb->handles[0])
{
DRM_FATAL("No handle set on framebuffer: maybe you need some additional capabilities?\n");
KMSVNC_FATAL("No handle set on framebuffer: maybe you need some additional capabilities?\n");
}
int ioctl_err = 0;
drm->mmap_fd = drm->drm_fd;
drm->mmap_size = drm->mfb->width * drm->mfb->height * BYTES_PER_PIXEL;
drm->funcs = malloc(sizeof(struct kmsvnc_drm_funcs));
@ -250,7 +250,7 @@ static int drm_kmsbuf_prime() {
int err = drmPrimeHandleToFD(drm->drm_fd, drm->mfb->handles[0], O_RDWR, &drm->prime_fd);
if (err < 0 || drm->prime_fd < 0)
{
DRM_FATAL("Failed to get PRIME fd from framebuffer handle");
KMSVNC_FATAL("Failed to get PRIME fd from framebuffer handle");
}
drm->funcs->sync_start = &drm_sync_start;
drm->funcs->sync_end = &drm_sync_end;
@ -258,6 +258,24 @@ static int drm_kmsbuf_prime() {
return 0;
}
static int drm_kmsbuf_prime_vaapi() {
struct kmsvnc_drm_data *drm = kmsvnc->drm;
int err = drmPrimeHandleToFD(drm->drm_fd, drm->mfb->handles[0], O_RDWR, &drm->prime_fd);
if (err < 0 || drm->prime_fd < 0)
{
KMSVNC_FATAL("Failed to get PRIME fd from framebuffer handle");
}
if (va_init()) return 1;
drm->funcs->sync_start = &drm_sync_start;
drm->funcs->sync_end = &drm_sync_end;
drm->mmap_fd = drm->prime_fd;
drm->mapped = kmsvnc->buf;
return 0;
}
static int drm_kmsbuf_dumb() {
struct kmsvnc_drm_data *drm = kmsvnc->drm;
@ -294,7 +312,8 @@ int drm_vendors() {
if (strcmp(driver_name, "i915") == 0)
{
drm->funcs->convert = &convert_intel_x_tiled_kmsbuf;
if (drm_kmsbuf_prime()) return 1;
drm->funcs->convert = &convert_vaapi;
if (drm_kmsbuf_prime_vaapi()) return 1;
}
else if (strcmp(driver_name, "amdgpu") == 0)
{
@ -364,7 +383,7 @@ int drm_vendors() {
drm->mapped = mmap(NULL, drm->mmap_size, PROT_READ, MAP_SHARED, drm->mmap_fd, drm->mmap_offset);
if (drm->mapped == MAP_FAILED)
{
DRM_FATAL("Failed to mmap: %s\n", strerror(errno));
KMSVNC_FATAL("Failed to mmap: %s\n", strerror(errno));
}
}

7
drm.h
View file

@ -2,11 +2,10 @@
#include "kmsvnc.h"
#define KMSVNC_FOURCC_TO_INT(a,b,c,d) (((a) << 0) + ((b) << 8) + ((c) << 16) + ((d) << 24))
#define DRM_IOCTL_MUST(...) do{ int e; if (e = drmIoctl(__VA_ARGS__)) KMSVNC_FATAL("DRM ioctl error %d on line %d\n", e, __LINE__); } while(0)
#define DRM_IOCTL_MAY(...) do{ int e; if (e = drmIoctl(__VA_ARGS__)) fprintf(stderr, "DRM ioctl error %d on line %d\n", e, __LINE__); } while(0)
#define DRM_R_IOCTL_MAY(...) do{ int e; if (e = ioctl(__VA_ARGS__)) fprintf(stderr, "DRM ioctl error %d on line %d\n", e, __LINE__); } while(0)
#define DRM_FATAL(...) { fprintf(stderr, __VA_ARGS__); return 1; }
#define DRM_IOCTL_MUST(...) { int e; if (e = drmIoctl(__VA_ARGS__)) DRM_FATAL("DRM ioctl error %d on line %d\n", e, __LINE__) }
#define DRM_IOCTL_MAY(...) { int e; if (e = drmIoctl(__VA_ARGS__)) fprintf(stderr, "DRM ioctl error %d on line %d\n", e, __LINE__); }
void drm_cleanup();
int drm_open();

8
input.c
View file

@ -35,7 +35,7 @@ int uinput_init()
inp->uinput_fd = open("/dev/uinput", O_WRONLY | O_NONBLOCK);
if (inp->uinput_fd <= 0)
{
INP_FATAL("Failed to open uinput\n");
KMSVNC_FATAL("Failed to open uinput\n");
}
INP_IOCTL_MUST(inp->uinput_fd, UI_SET_EVBIT, EV_KEY);
INP_IOCTL_MUST(inp->uinput_fd, UI_SET_EVBIT, EV_SYN);
@ -114,7 +114,7 @@ void rfb_key_hook(rfbBool down, rfbKeySym keysym, rfbClientPtr cl)
.value = 0,
},
};
for (int i = 0; i < ARRAY_SIZE(ies); i++)
for (int i = 0; i < KMSVNC_ARRAY_ELEMENTS(ies); i++)
{
write(kmsvnc->input->uinput_fd, &ies[i], sizeof(ies[0]));
}
@ -159,7 +159,7 @@ void rfb_ptr_hook(int mask, int screen_x, int screen_y, rfbClientPtr cl)
.value = 0,
},
};
for (int i = 0; i < ARRAY_SIZE(ies1); i++)
for (int i = 0; i < KMSVNC_ARRAY_ELEMENTS(ies1); i++)
{
write(kmsvnc->input->uinput_fd, &ies1[i], sizeof(ies1[0]));
}
@ -177,7 +177,7 @@ void rfb_ptr_hook(int mask, int screen_x, int screen_y, rfbClientPtr cl)
.value = 0,
},
};
for (int i = 0; i < ARRAY_SIZE(ies2); i++)
for (int i = 0; i < KMSVNC_ARRAY_ELEMENTS(ies2); i++)
{
write(kmsvnc->input->uinput_fd, &ies2[i], sizeof(ies2[0]));
}

7
input.h
View file

@ -4,14 +4,11 @@
#include "kmsvnc.h"
#define ARRAY_SIZE(x) (sizeof(x) / sizeof(x[0]))
#define UINPUT_ABS_MAX INT16_MAX
#define UINPUT_MAX_KEY 256
#define INP_FATAL(...) { fprintf(stderr, __VA_ARGS__); return 1; }
#define INP_IOCTL_MUST(...) { int e; if (e = ioctl(__VA_ARGS__)) INP_FATAL("uinput ioctl error %d on line %d\n", e, __LINE__) }
#define INP_IOCTL_MAY(...) { int e; if (e = ioctl(__VA_ARGS__)) fprintf(stderr, "uinput ioctl error %d on line %d\n", e, __LINE__); }
#define INP_IOCTL_MUST(...) do{ int e; if (e = ioctl(__VA_ARGS__)) KMSVNC_FATAL("uinput ioctl error %d on line %d\n", e, __LINE__); } while(0)
#define INP_IOCTL_MAY(...) do{ int e; if (e = ioctl(__VA_ARGS__)) fprintf(stderr, "uinput ioctl error %d on line %d\n", e, __LINE__); } while(0)
void uinput_cleanup();
int uinput_init();

4
keymap.c
View file

@ -31,7 +31,7 @@ int xkb_init()
xkb->ctx = xkb_context_new(XKB_CONTEXT_NO_FLAGS);
if (xkb->ctx == NULL)
{
XKB_FATAL("Failed to create XKB context\n");
KMSVNC_FATAL("Failed to create XKB context\n");
}
struct xkb_rule_names names = {
.rules = NULL,
@ -43,7 +43,7 @@ int xkb_init()
xkb->map = xkb_keymap_new_from_names(xkb->ctx, &names, 0);
if (xkb->map == NULL)
{
XKB_FATAL("Failed to create XKB keymap\n");
KMSVNC_FATAL("Failed to create XKB keymap\n");
}
// printf("xkb: keymap string\n%s\n", xkb_keymap_get_as_string(xkb->map, XKB_KEYMAP_USE_ORIGINAL_FORMAT));
return 0;

2
keymap.h
View file

@ -2,8 +2,6 @@
#include "kmsvnc.h"
#define XKB_FATAL(...) { fprintf(stderr, __VA_ARGS__); return 1; }
void xkb_cleanup();
int xkb_init();
void key_iter(struct xkb_keymap *xkb, xkb_keycode_t key, void *data);

7
kmsvnc.c
View file

@ -11,6 +11,7 @@
#include "keymap.h"
#include "input.h"
#include "drm.h"
#include "va.h"
struct kmsvnc_data *kmsvnc = NULL;
@ -260,8 +261,12 @@ int main(int argc, char **argv)
if (kmsvnc->debug_capture_fb) {
int wfd = open(kmsvnc->debug_capture_fb, O_WRONLY | O_CREAT, 00644);
size_t len = kmsvnc->drm->mfb->width * kmsvnc->drm->mfb->height * BYTES_PER_PIXEL;
if (kmsvnc->drm->mfb->offsets[1]) {
len = kmsvnc->drm->mfb->offsets[1] + 61440;
}
if (wfd > 0) {
write(wfd, kmsvnc->drm->mapped, kmsvnc->drm->mfb->width * kmsvnc->drm->mfb->height * BYTES_PER_PIXEL);
write(wfd, kmsvnc->drm->mapped, len);
fsync(wfd);
printf("wrote raw frame buffer to %s\n", kmsvnc->debug_capture_fb);
}

13
kmsvnc.h
View file

@ -8,6 +8,7 @@
#include <amdgpu_drm.h>
#include <xf86drmMode.h>
#include <linux/dma-buf.h>
#include <va/va.h>
#define BYTES_PER_PIXEL 4
@ -36,6 +37,7 @@ struct kmsvnc_data
struct kmsvnc_drm_data *drm;
struct kmsvnc_input_data *input;
struct kmsvnc_keymap_data *keymap;
struct kmsvnc_va_data *va;
rfbScreenInfoPtr server;
char shutdown;
char *buf;
@ -88,3 +90,14 @@ struct kmsvnc_drm_data
struct kmsvnc_drm_funcs *funcs;
};
struct kmsvnc_va_data
{
VADisplay dpy;
VASurfaceID surface_id;
VAImage *image;
char *imgbuf;
};
#define KMSVNC_FATAL(...) do{ fprintf(stderr, __VA_ARGS__); return 1; } while(0)
#define KMSVNC_ARRAY_ELEMENTS(x) (sizeof(x) / sizeof(x[0]))
#define KMSVNC_FOURCC_TO_INT(a,b,c,d) (((a) << 0) + ((b) << 8) + ((c) << 16) + ((d) << 24))

208
va.c Normal file
View file

@ -0,0 +1,208 @@
#include <va/va_drm.h>
#include <va/va_drmcommon.h>
#include <fcntl.h>
#include "va.h"
#include "kmsvnc.h"
extern struct kmsvnc_data *kmsvnc;
void va_cleanup () {
}
static void va_msg_callback(void *user_context, const char *message) {
#ifdef KMSVNC_VA_DEBUG
printf("va msg: %s");
#endif
}
static void va_error_callback(void *user_context, const char *message) {
printf("va error: %s");
}
int va_init() {
if (!kmsvnc->drm || !kmsvnc->drm->drm_fd || !kmsvnc->drm->prime_fd) {
KMSVNC_FATAL("drm is not initialized\n");
}
setenv("DISPLAY", "", 1);
struct kmsvnc_va_data *va = malloc(sizeof(struct kmsvnc_va_data));
memset(va, 0, sizeof(struct kmsvnc_va_data));
kmsvnc->va = va;
char* render_node;
int drm_fd = 0;
if (render_node = drmGetRenderDeviceNameFromFd(kmsvnc->drm->drm_fd)) {
drm_fd = open(render_node, O_RDWR);
}
else {
printf("Using non-render node because the device does not have an associated render node.\n");
}
if (drm_fd <= 0) {
printf("Using non-render node because render node fails to open.\n");
drm_fd = kmsvnc->drm->drm_fd;
}
va->dpy = vaGetDisplayDRM(drm_fd);
if (!va->dpy) {
KMSVNC_FATAL("vaGetDisplayDRM failed\n");
}
vaSetErrorCallback(va->dpy, &va_error_callback, NULL);
vaSetInfoCallback(va->dpy, &va_msg_callback, NULL);
int major, minor;
VAStatus status;
VA_CHECK(vaInitialize(va->dpy, &major, &minor));
const char *vendor_string = vaQueryVendorString(va->dpy);
printf("vaapi vendor %s\n", vendor_string);
VADRMPRIMESurfaceDescriptor prime_desc;
VASurfaceAttrib prime_attrs[2] = {
{
.type = VASurfaceAttribMemoryType,
.flags = VA_SURFACE_ATTRIB_SETTABLE,
.value.type = VAGenericValueTypeInteger,
.value.value.i = VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME_2,
},
{
.type = VASurfaceAttribExternalBufferDescriptor,
.flags = VA_SURFACE_ATTRIB_SETTABLE,
.value.type = VAGenericValueTypePointer,
.value.value.p = &prime_desc,
}
};
prime_desc.fourcc = kmsvnc->drm->mfb->pixel_format == KMSVNC_FOURCC_TO_INT('X', 'R', '2', '4') ?
KMSVNC_FOURCC_TO_INT('B', 'G', 'R', 'X') :
KMSVNC_FOURCC_TO_INT('B', 'G', 'R', 'A') ;
prime_desc.width = kmsvnc->drm->mfb->width;
prime_desc.height = kmsvnc->drm->mfb->height;
int i;
int max_size = 0;
for (i = 0; i < 4; i++) {
int size = kmsvnc->drm->mfb->offsets[i] + kmsvnc->drm->mfb->height * kmsvnc->drm->mfb->pitches[i];
if (size > max_size) max_size = size;
}
for (i = 0; i < 4; i++) {
prime_desc.objects[i].fd = kmsvnc->drm->prime_fd;
prime_desc.objects[i].size = max_size;
prime_desc.objects[i].drm_format_modifier = kmsvnc->drm->mfb->modifier;
}
prime_desc.num_layers = 1;
prime_desc.layers[0].drm_format = kmsvnc->drm->mfb->pixel_format;
for (i = 0; i < 4; i++) {
prime_desc.layers[0].object_index[i] = 0;
prime_desc.layers[0].offset[i] = kmsvnc->drm->mfb->offsets[i];
prime_desc.layers[0].pitch[i] = kmsvnc->drm->mfb->pitches[i];
}
for (i = 0; i < 4; i++) {
if (!kmsvnc->drm->mfb->handles[i]) {
break;
}
}
prime_desc.layers[0].num_planes = i;
prime_desc.num_objects = 1;
VAStatus s;
if (s = vaCreateSurfaces(va->dpy, VA_RT_FORMAT_RGB32,
kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, &va->surface_id, 1,
prime_attrs, KMSVNC_ARRAY_ELEMENTS(prime_attrs)))
{
printf("vaCreateSurfaces prime2 error %#x %s, trying prime\n", s, vaErrorStr(s));
VASurfaceAttribExternalBuffers buffer_desc;
VASurfaceAttrib buffer_attrs[2] = {
{
.type = VASurfaceAttribMemoryType,
.flags = VA_SURFACE_ATTRIB_SETTABLE,
.value.type = VAGenericValueTypeInteger,
.value.value.i = VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME,
},
{
.type = VASurfaceAttribExternalBufferDescriptor,
.flags = VA_SURFACE_ATTRIB_SETTABLE,
.value.type = VAGenericValueTypePointer,
.value.value.p = &buffer_desc,
}
};
unsigned long fd = kmsvnc->drm->prime_fd;
buffer_desc.pixel_format = prime_desc.fourcc;
buffer_desc.width = kmsvnc->drm->mfb->width;
buffer_desc.height = kmsvnc->drm->mfb->height;
buffer_desc.data_size = max_size;
buffer_desc.buffers = &fd;
buffer_desc.num_buffers = 1;
buffer_desc.flags = 0;
for (i = 0; i < 4; i++) {
buffer_desc.pitches[i] = kmsvnc->drm->mfb->pitches[i];
buffer_desc.offsets[i] = kmsvnc->drm->mfb->offsets[i];
}
buffer_desc.num_planes = prime_desc.layers[0].num_planes;
VA_CHECK(vaCreateSurfaces(va->dpy, VA_RT_FORMAT_RGB32,
kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, &va->surface_id, 1,
buffer_attrs, KMSVNC_ARRAY_ELEMENTS(buffer_attrs)));
}
#ifdef KMSVNC_VA_DEBUG
int img_fmt_count = vaMaxNumImageFormats(va->dpy);
VAImageFormat *img_fmts = malloc(sizeof(VAImageFormat) * img_fmt_count);
{
int got;
vaQueryImageFormats(va->dpy, img_fmts, &got);
if (got != img_fmt_count) {
KMSVNC_FATAL("got less VAImageFormats, %d instead of %d\n", got, img_fmt_count);
}
}
for (int i = 0; i < img_fmt_count; i++) {
printf("fmt %d: fourcc %d, %c%c%c%c, byte_order %s, bpp %d, depth %d, blue_mask %#x, green_mask %#x, red_mask %#x, reserved %#x\n", i, img_fmts[i].fourcc,
img_fmts[i].fourcc & 0xff,
img_fmts[i].fourcc >> 8 & 0xff,
img_fmts[i].fourcc >> 16 & 0xff,
img_fmts[i].fourcc >> 24 & 0xff,
img_fmts[i].byte_order - 1 ? "VA_LSB_FIRST" : "VA_MSB_FIRST",
img_fmts[i].bits_per_pixel,
img_fmts[i].depth,
img_fmts[i].blue_mask,
img_fmts[i].green_mask,
img_fmts[i].red_mask,
img_fmts[i].va_reserved
);
}
#endif
VAImageFormat format = {
.fourcc = KMSVNC_FOURCC_TO_INT('R','G','B','X'),
.byte_order = VA_LSB_FIRST,
.bits_per_pixel = 32,
.depth = 24,
.blue_mask = 0x000000ff,
.green_mask = 0x0000ff00,
.red_mask = 0x00ff0000,
.va_reserved = 0x00000000,
};
va->image = malloc(sizeof(VAImage));
VA_CHECK(vaCreateImage(va->dpy, &format, kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, va->image));
//VA_CHECK(vaDeriveImage(va->dpy, va->surface_id, va->image));
va->imgbuf = malloc(va->image->data_size);
VA_CHECK(vaMapBuffer(va->dpy, va->image->buf, (void**)&va->imgbuf));
}
int va_hwframe_to_vaapi() {
//VA_CHECK(vaSyncSurface(kmsvnc->va->dpy, kmsvnc->va->surface_id));
VA_CHECK(vaGetImage(kmsvnc->va->dpy, kmsvnc->va->surface_id, 0, 0,
kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, kmsvnc->va->image->image_id));
memcpy(kmsvnc->buf1, kmsvnc->va->imgbuf, kmsvnc->drm->mfb->width * kmsvnc->drm->mfb->height * BYTES_PER_PIXEL);
//printf("image data size: %d\n", kmsvnc->va->image->data_size);
}

7
va.h Normal file
View file

@ -0,0 +1,7 @@
#pragma once
#define VA_CHECK(x) do{VAStatus s; if ((s = (x)) != VA_STATUS_SUCCESS) KMSVNC_FATAL("va operation error %#x %s on line %d\n", s, vaErrorStr(s), __LINE__); } while (0)
void va_cleanup ();
int va_init();
int va_hwframe_to_vaapi();