Jerry/kmsvnc
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

WIP: allow capture cursor

Browse source
This commit is contained in:
JerryXiao 2023-07-16 01:34:58 +08:00
parent e7d59d0e5e
commit daf3db0b19
Signed by: Jerry
GPG key ID: 22618F758B5BE2E5
5 changed files with 413 additions and 100 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

375
drm.c
View file

@ -1,4 +1,5 @@
#include <stdio.h> #include <stdio.h>
#include <assert.h>
#include <errno.h> #include <errno.h>
#include <fcntl.h> #include <fcntl.h>
#include <sys/mman.h> #include <sys/mman.h>
@ -23,35 +24,29 @@ static int check_pixfmt_non_vaapi() {
return 0; return 0;
} }
static void convert_copy(const char *in, int width, int height, char *buff) { static void convert_copy(const char *in, int width, int height, char *buff)
memcpy(buff, in, width * height * 4); {
memcpy(buff, in, width * height * BYTES_PER_PIXEL);
} }
static void convert_bgrx_to_rgb(const char *in, int width, int height, char *buff) static void convert_bgra_to_rgba(const char *in, int width, int height, char *buff)
{ {
for (int y = 0; y < height; y++) memcpy(buff, in, width * height * BYTES_PER_PIXEL);
{ for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) {
for (int x = 0; x < width; x++) uint32_t pixdata = htonl(*((uint32_t*)(kmsvnc->drm->kms_convert_buf + i)));
{ buff[i+0] = (pixdata & 0x0000ff00) >> 8;
buff[(y * width + x) * 4] = in[(y * width + x) * 4 + 2]; buff[i+2] = (pixdata & 0xff000000) >> 24;
buff[(y * width + x) * 4 + 1] = in[(y * width + x) * 4 + 1];
buff[(y * width + x) * 4 + 2] = in[(y * width + x) * 4];
}
} }
} }
static char *kms_convert_buf = NULL;
static size_t kms_convert_buf_len = 0;
static char *kms_cpy_tmp_buf = NULL;
static size_t kms_cpy_tmp_buf_len = 0;
static inline char convert_buf_allocate(size_t len) { static inline char convert_buf_allocate(size_t len) {
if (kms_convert_buf_len < len) if (kmsvnc->drm->kms_convert_buf_len < len)
{ {
if (kms_convert_buf) if (kmsvnc->drm->kms_convert_buf)
free(kms_convert_buf); free(kmsvnc->drm->kms_convert_buf);
kms_convert_buf = malloc(len); kmsvnc->drm->kms_convert_buf = malloc(len);
if (!kms_convert_buf) return 1; if (!kmsvnc->drm->kms_convert_buf) return 1;
kms_convert_buf_len = len; kmsvnc->drm->kms_convert_buf_len = len;
} }
return 0; return 0;
} }
@ -66,16 +61,16 @@ static inline void convert_x_tiled(const int tilex, const int tiley, const char
int sno = (width / tilex) + (height / tiley) * (width / tilex); int sno = (width / tilex) + (height / tiley) * (width / tilex);
int ord = (width % tilex) + (height % tiley) * tilex; int ord = (width % tilex) + (height % tiley) * tilex;
int max_offset = sno * tilex * tiley + ord; int max_offset = sno * tilex * tiley + ord;
if (kms_cpy_tmp_buf_len < max_offset * 4 + 4) if (kmsvnc->drm->kms_cpy_tmp_buf_len < max_offset * 4 + 4)
{ {
if (kms_cpy_tmp_buf) if (kmsvnc->drm->kms_cpy_tmp_buf)
free(kms_convert_buf); free(kmsvnc->drm->kms_convert_buf);
kms_cpy_tmp_buf = malloc(max_offset * 4 + 4); kmsvnc->drm->kms_cpy_tmp_buf = malloc(max_offset * 4 + 4);
if (!kms_cpy_tmp_buf) return; if (!kmsvnc->drm->kms_cpy_tmp_buf) return;
kms_cpy_tmp_buf_len = max_offset * 4 + 4; kmsvnc->drm->kms_cpy_tmp_buf_len = max_offset * 4 + 4;
} }
memcpy(kms_cpy_tmp_buf, in, max_offset * 4 + 4); memcpy(kmsvnc->drm->kms_cpy_tmp_buf, in, max_offset * 4 + 4);
in = (const char *)kms_cpy_tmp_buf; in = (const char *)kmsvnc->drm->kms_cpy_tmp_buf;
} }
if (convert_buf_allocate(width * height * 4)) return; if (convert_buf_allocate(width * height * 4)) return;
for (int y = 0; y < height; y++) for (int y = 0; y < height; y++)
@ -85,10 +80,10 @@ static inline void convert_x_tiled(const int tilex, const int tiley, const char
int sno = (x / tilex) + (y / tiley) * (width / tilex); int sno = (x / tilex) + (y / tiley) * (width / tilex);
int ord = (x % tilex) + (y % tiley) * tilex; int ord = (x % tilex) + (y % tiley) * tilex;
int offset = sno * tilex * tiley + ord; int offset = sno * tilex * tiley + ord;
memcpy(kms_convert_buf + (x + y * width) * 4, in + offset * 4, 4); memcpy(kmsvnc->drm->kms_convert_buf + (x + y * width) * 4, in + offset * 4, 4);
} }
} }
convert_bgrx_to_rgb(kms_convert_buf, width, height, buff); convert_bgra_to_rgba(kmsvnc->drm->kms_convert_buf, width, height, buff);
} }
void convert_nvidia_x_tiled_kmsbuf(const char *in, int width, int height, char *buff) void convert_nvidia_x_tiled_kmsbuf(const char *in, int width, int height, char *buff)
@ -106,29 +101,34 @@ static void convert_vaapi(const char *in, int width, int height, char *buff) {
} }
else { else {
if (convert_buf_allocate(width * height * BYTES_PER_PIXEL)) return; if (convert_buf_allocate(width * height * BYTES_PER_PIXEL)) return;
va_hwframe_to_vaapi(kms_convert_buf); va_hwframe_to_vaapi(kmsvnc->drm->kms_convert_buf);
// is 30 depth? // is 30 depth?
if (kmsvnc->va->image->format.depth == 30) { if (kmsvnc->va->image->format.depth == 30) {
for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) { for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) {
uint32_t pixdata = *((uint32_t*)(kms_convert_buf + i)); // ensure little endianess
kms_convert_buf[i] = (pixdata & 0x3ff00000) >> 20 >> 2; uint32_t pixdata = __builtin_bswap32(htonl(*((uint32_t*)(kmsvnc->drm->kms_convert_buf + i))));
kms_convert_buf[i+1] = (pixdata & 0xffc00) >> 10 >> 2; kmsvnc->drm->kms_convert_buf[i] = (pixdata & 0x3ff00000) >> 20 >> 2;
kms_convert_buf[i+2] = (pixdata & 0x3ff) >> 2; kmsvnc->drm->kms_convert_buf[i+1] = (pixdata & 0xffc00) >> 10 >> 2;
kmsvnc->drm->kms_convert_buf[i+2] = (pixdata & 0x3ff) >> 2;
} }
} }
// is xrgb? // is xrgb?
if ((kmsvnc->va->image->format.blue_mask | kmsvnc->va->image->format.red_mask) < 0x1000000) { if ((kmsvnc->va->image->format.blue_mask | kmsvnc->va->image->format.red_mask) < 0x1000000) {
for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) { for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) {
*((uint32_t*)(kms_convert_buf + i)) <<= 8; uint32_t *pixdata = (uint32_t*)(kmsvnc->drm->kms_convert_buf + i);
*pixdata = ntohl(htonl(*pixdata) << 8);
} }
} }
// is bgr? // is bgrx?
if (kmsvnc->va->image->format.blue_mask > kmsvnc->va->image->format.red_mask) { if (kmsvnc->va->image->format.blue_mask > kmsvnc->va->image->format.red_mask) {
convert_bgrx_to_rgb(kms_convert_buf, width, height, buff); for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) {
uint32_t pixdata = htonl(*((uint32_t*)(kmsvnc->drm->kms_convert_buf + i)));
buff[i+0] = (pixdata & 0x0000ff00) >> 8;
buff[i+2] = (pixdata & 0xff000000) >> 24;
} }
else {
memcpy(buff, kms_convert_buf, width * height * BYTES_PER_PIXEL);
} }
// rgbx now
memcpy(buff, kmsvnc->drm->kms_convert_buf, width * height * BYTES_PER_PIXEL);
} }
} }
@ -174,14 +174,26 @@ void drm_cleanup() {
drmModeFreePlane(kmsvnc->drm->plane); drmModeFreePlane(kmsvnc->drm->plane);
kmsvnc->drm->plane = NULL; kmsvnc->drm->plane = NULL;
} }
if (kmsvnc->drm->cursor_plane) {
drmModeFreePlane(kmsvnc->drm->cursor_plane);
kmsvnc->drm->cursor_plane = NULL;
}
if (kmsvnc->drm->mfb) { if (kmsvnc->drm->mfb) {
drmModeFreeFB2(kmsvnc->drm->mfb); drmModeFreeFB2(kmsvnc->drm->mfb);
kmsvnc->drm->mfb = NULL; kmsvnc->drm->mfb = NULL;
} }
if (kmsvnc->drm->mapped) { if (kmsvnc->drm->cursor_mfb) {
drmModeFreeFB2(kmsvnc->drm->cursor_mfb);
kmsvnc->drm->cursor_mfb = NULL;
}
if (kmsvnc->drm->mapped && kmsvnc->drm->mapped != MAP_FAILED) {
munmap(kmsvnc->drm->mapped, kmsvnc->drm->mmap_size); munmap(kmsvnc->drm->mapped, kmsvnc->drm->mmap_size);
kmsvnc->drm->mapped = NULL; kmsvnc->drm->mapped = NULL;
} }
if (kmsvnc->drm->cursor_mapped && kmsvnc->drm->cursor_mapped != MAP_FAILED) {
munmap(kmsvnc->drm->cursor_mapped, kmsvnc->drm->cursor_mmap_size);
kmsvnc->drm->cursor_mapped = NULL;
}
if (kmsvnc->drm->prime_fd > 0) { if (kmsvnc->drm->prime_fd > 0) {
close(kmsvnc->drm->prime_fd); close(kmsvnc->drm->prime_fd);
kmsvnc->drm->prime_fd = 0; kmsvnc->drm->prime_fd = 0;
@ -194,11 +206,236 @@ void drm_cleanup() {
drmModeFreePlaneResources(kmsvnc->drm->plane_res); drmModeFreePlaneResources(kmsvnc->drm->plane_res);
kmsvnc->drm->plane_res = NULL; kmsvnc->drm->plane_res = NULL;
} }
if (kmsvnc->drm->kms_convert_buf) {
free(kmsvnc->drm->kms_convert_buf);
kmsvnc->drm->kms_convert_buf = NULL;
}
kmsvnc->drm->kms_convert_buf_len = 0;
if (kmsvnc->drm->kms_cpy_tmp_buf) {
free(kmsvnc->drm->kms_cpy_tmp_buf);
kmsvnc->drm->kms_cpy_tmp_buf = NULL;
}
kmsvnc->drm->kms_cpy_tmp_buf_len = 0;
if (kmsvnc->drm->kms_cursor_buf) {
free(kmsvnc->drm->kms_cursor_buf);
kmsvnc->drm->kms_cursor_buf = NULL;
}
kmsvnc->drm->kms_cursor_buf_len = 0;
free(kmsvnc->drm); free(kmsvnc->drm);
kmsvnc->drm = NULL; kmsvnc->drm = NULL;
} }
} }
static const char* drm_get_plane_type_name(uint64_t plane_type) {
switch (plane_type) {
case DRM_PLANE_TYPE_OVERLAY:
return "overlay";
case DRM_PLANE_TYPE_PRIMARY:
return "primary";
case DRM_PLANE_TYPE_CURSOR:
return "cursor";
default:
return "unknown";
}
};
static int drm_refresh_planes(char first_time) {
struct kmsvnc_drm_data *drm = kmsvnc->drm;
if (!drm->plane && kmsvnc->source_plane > 0)
{
drm->plane = drmModeGetPlane(drm->drm_fd, kmsvnc->source_plane);
if (!drm->plane)
KMSVNC_FATAL("Failed to get plane %d: %s\n", kmsvnc->source_plane, strerror(errno));
if (drm->plane->fb_id == 0)
fprintf(stderr, "Place %d does not have an attached framebuffer\n", kmsvnc->source_plane);
}
if (!drm->plane || (kmsvnc->capture_cursor && !drm->cursor_plane)) {
drmModePlane *current_plane = NULL;
drm->plane_res = drmModeGetPlaneResources(drm->drm_fd);
if (!drm->plane_res)
KMSVNC_FATAL("Failed to get plane resources: %s\n", strerror(errno));
int i;
for (i = 0; i < drm->plane_res->count_planes; i++)
{
current_plane = drmModeGetPlane(drm->drm_fd, drm->plane_res->planes[i]);
if (!current_plane)
{
fprintf(stderr, "Failed to get plane %u: %s\n", drm->plane_res->planes[i], strerror(errno));
continue;
}
// get plane type
uint64_t plane_type = 114514;
drmModeObjectPropertiesPtr plane_props = drmModeObjectGetProperties(drm->drm_fd, current_plane->plane_id, DRM_MODE_OBJECT_PLANE);
if (!plane_props) {
fprintf(stderr, "Failed to get plane prop %u: %s\n", drm->plane_res->planes[i], strerror(errno));
}
else {
for (int i = 0; i < plane_props->count_props; i++) {
drmModePropertyPtr plane_prop = drmModeGetProperty(drm->drm_fd, plane_props->props[i]);
if (strcmp(plane_prop->name, "type") == 0) {
plane_type = plane_props->prop_values[i];
}
drmModeFreeProperty(plane_prop);
}
drmModeFreeObjectProperties(plane_props);
}
assert(drm->plane_res->planes[i] == current_plane->plane_id);
if (first_time) {
printf("Plane %u CRTC %u FB %u Type %s\n", current_plane->plane_id, current_plane->crtc_id, current_plane->fb_id, drm_get_plane_type_name(plane_type));
}
// populate drm->plane and drm->cursor_plane
char nofree = 0;
if (current_plane->fb_id != 0) {
if (!drm->plane) {
if (kmsvnc->source_crtc == 0 || current_plane->crtc_id == kmsvnc->source_crtc) {
nofree = 1;
drm->plane = current_plane;
}
}
// assume cursor plane is always after primary plane
if (!drm->cursor_plane) {
if (drm->plane && drm->plane->crtc_id == current_plane->crtc_id && plane_type == DRM_PLANE_TYPE_CURSOR) {
nofree = 1;
drm->cursor_plane = current_plane;
}
}
}
if ((!kmsvnc->capture_cursor || drm->cursor_plane) && drm->plane) {
break;
}
if (!nofree) {
drmModeFreePlane(current_plane);
}
current_plane = NULL;
}
if (!first_time) return 0;
if (i == drm->plane_res->count_planes)
{
if (!drm->plane) {
if (kmsvnc->source_crtc != 0)
{
KMSVNC_FATAL("No usable planes found on CRTC %d\n", kmsvnc->source_crtc);
}
else
{
KMSVNC_FATAL("No usable planes found\n");
}
}
else if (!drm->cursor_plane) {
fprintf(stderr, "No usable cursor plane found, cursor capture currently unavailable\n");
}
}
printf("Using plane %u to locate framebuffers\n", drm->plane->plane_id);
if (drm->cursor_plane) {
printf("Using cursor plane %u\n", drm->cursor_plane->plane_id);
}
}
return 0;
}
int drm_dump_cursor_plane(char **data, int *width, int *height) {
struct kmsvnc_drm_data *drm = kmsvnc->drm;
if (drm->cursor_plane) {
drmModeFreePlane(drm->cursor_plane);
drm->cursor_plane = NULL;
}
drm_refresh_planes(0); // ignore error
if (!drm->cursor_plane) {
data = NULL;
return 1;
}
if (drm->cursor_mfb) drmModeFreeFB2(drm->cursor_mfb);
drm->cursor_mfb = drmModeGetFB2(drm->drm_fd, drm->cursor_plane->fb_id);
if (!drm->cursor_mfb) {
KMSVNC_DEBUG("Cursor framebuffer missing\n");
return 1;
}
if (drm->cursor_mfb->modifier != DRM_FORMAT_MOD_NONE && drm->cursor_mfb->modifier != DRM_FORMAT_MOD_LINEAR) {
//kmsvnc->capture_cursor = 0;
KMSVNC_DEBUG("Cursor plane modifier is not linear: %lu\n", drm->cursor_mfb->modifier);
return 1;
}
if (
drm->cursor_mfb->pixel_format != KMSVNC_FOURCC_TO_INT('A', 'R', '2', '4') &&
drm->cursor_mfb->pixel_format != KMSVNC_FOURCC_TO_INT('A', 'R', '3', '0')
)
{
//kmsvnc->capture_cursor = 0;
char *fmtname = drmGetFormatName(drm->cursor_mfb->pixel_format);
KMSVNC_DEBUG("Cursor plane pixel format unsupported (%u, %s)\n", drm->cursor_mfb->pixel_format, fmtname);
free(fmtname);
return 1;
}
struct drm_gem_flink flink;
flink.handle = drm->cursor_mfb->handles[0];
DRM_IOCTL_MUST(drm->drm_fd, DRM_IOCTL_GEM_FLINK, &flink);
struct drm_gem_open open_arg;
open_arg.name = flink.name;
DRM_IOCTL_MUST(drm->drm_fd, DRM_IOCTL_GEM_OPEN, &open_arg);
struct drm_mode_map_dumb mreq;
memset(&mreq, 0, sizeof(mreq));
mreq.handle = open_arg.handle;
DRM_IOCTL_MUST(drm->drm_fd, DRM_IOCTL_MODE_MAP_DUMB, &mreq);
size_t mmap_size = open_arg.size;
if (mmap_size != drm->cursor_mfb->width * drm->cursor_mfb->height * BYTES_PER_PIXEL) {
KMSVNC_DEBUG("Cursor plane mmap_size != calculated size (%d, %d)\n", mmap_size, drm->cursor_mfb->width * drm->cursor_mfb->height * BYTES_PER_PIXEL);
return 1;
}
off_t mmap_offset = mreq.offset;
if (drm->cursor_mapped && drm->cursor_mapped != MAP_FAILED) munmap(drm->cursor_mapped, drm->cursor_mmap_size);
drm->cursor_mapped = mmap(NULL, mmap_size, PROT_READ, MAP_SHARED, drm->drm_fd, mmap_offset);
if (drm->cursor_mapped == MAP_FAILED)
{
KMSVNC_DEBUG("Failed to mmap cursor: %s\n", strerror(errno));
return 1;
}
else
{
if (kmsvnc->drm->kms_cursor_buf_len < mmap_size)
{
if (kmsvnc->drm->kms_cursor_buf)
free(kmsvnc->drm->kms_cursor_buf);
kmsvnc->drm->kms_cursor_buf = malloc(mmap_size);
if (!kmsvnc->drm->kms_cursor_buf) return 1;
kmsvnc->drm->kms_cursor_buf_len = mmap_size;
}
memcpy(drm->kms_cursor_buf, drm->cursor_mapped, mmap_size);
if (drm->cursor_mfb->pixel_format == KMSVNC_FOURCC_TO_INT('X', 'R', '3', '0') ||
drm->cursor_mfb->pixel_format == KMSVNC_FOURCC_TO_INT('A', 'R', '3', '0'))
{
for (int i = 0; i < drm->cursor_mfb->width * drm->cursor_mfb->height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) {
uint32_t pixdata = __builtin_bswap32(htonl(*((uint32_t*)(kmsvnc->drm->kms_cursor_buf + i))));
kmsvnc->drm->kms_cursor_buf[i] = (pixdata & 0x3ff00000) >> 20 >> 2;
kmsvnc->drm->kms_cursor_buf[i+1] = (pixdata & 0xffc00) >> 10 >> 2;
kmsvnc->drm->kms_cursor_buf[i+2] = (pixdata & 0x3ff) >> 2;
kmsvnc->drm->kms_cursor_buf[i+3] = (pixdata & 0xc0000000) >> 30 << 6;
}
}
if (drm->cursor_mfb->pixel_format == KMSVNC_FOURCC_TO_INT('X', 'R', '2', '4') ||
drm->cursor_mfb->pixel_format == KMSVNC_FOURCC_TO_INT('A', 'R', '2', '4'))
{
// bgra to rgba
for (int i = 0; i < drm->cursor_mfb->width * drm->cursor_mfb->height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) {
uint32_t pixdata = htonl(*((uint32_t*)(kmsvnc->drm->kms_cursor_buf + i)));
kmsvnc->drm->kms_cursor_buf[i+0] = (pixdata & 0x0000ff00) >> 8;
kmsvnc->drm->kms_cursor_buf[i+2] = (pixdata & 0xff000000) >> 24;
}
}
*width = drm->cursor_mfb->width;
*height = drm->cursor_mfb->height;
*data = drm->kms_cursor_buf;
}
return 0;
}
int drm_open() { int drm_open() {
struct kmsvnc_drm_data *drm = malloc(sizeof(struct kmsvnc_drm_data)); struct kmsvnc_drm_data *drm = malloc(sizeof(struct kmsvnc_drm_data));
if (!drm) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__); if (!drm) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
@ -218,54 +455,8 @@ int drm_open() {
{ {
perror("Failed to set universal planes capability: primary planes will not be usable"); perror("Failed to set universal planes capability: primary planes will not be usable");
} }
if (kmsvnc->source_plane > 0)
{
drm->plane = drmModeGetPlane(drm->drm_fd, kmsvnc->source_plane);
if (!drm->plane)
KMSVNC_FATAL("Failed to get plane %d: %s\n", kmsvnc->source_plane, strerror(errno));
if (drm->plane->fb_id == 0)
fprintf(stderr, "Place %d does not have an attached framebuffer\n", kmsvnc->source_plane);
}
else
{
drm->plane_res = drmModeGetPlaneResources(drm->drm_fd);
if (!drm->plane_res)
KMSVNC_FATAL("Failed to get plane resources: %s\n", strerror(errno));
int i;
for (i = 0; i < drm->plane_res->count_planes; i++)
{
drm->plane = drmModeGetPlane(drm->drm_fd, drm->plane_res->planes[i]);
if (!drm->plane)
{
fprintf(stderr, "Failed to get plane %u: %s\n", drm->plane_res->planes[i], strerror(errno));
continue;
}
printf("Plane %u CRTC %u FB %u\n", drm->plane->plane_id, drm->plane->crtc_id, drm->plane->fb_id);
if ((kmsvnc->source_crtc != 0 && drm->plane->crtc_id != kmsvnc->source_crtc) || drm->plane->fb_id == 0)
{
// Either not connected to the target source CRTC
// or not active.
drmModeFreePlane(drm->plane);
drm->plane = NULL;
continue;
}
break;
}
if (i == drm->plane_res->count_planes)
{
if (kmsvnc->source_crtc != 0)
{
KMSVNC_FATAL("No usable planes found on CRTC %d\n", kmsvnc->source_crtc);
}
else
{
KMSVNC_FATAL("No usable planes found\n");
}
}
printf("Using plane %u to locate framebuffers\n", drm->plane->plane_id);
}
uint32_t plane_id = drm->plane->plane_id;
if (drm_refresh_planes(1)) return 1;
drm->mfb = drmModeGetFB2(drm->drm_fd, drm->plane->fb_id); drm->mfb = drmModeGetFB2(drm->drm_fd, drm->plane->fb_id);
if (!drm->mfb) { if (!drm->mfb) {
@ -289,7 +480,7 @@ int drm_open() {
drm->mmap_size = drm->mfb->width * drm->mfb->height * BYTES_PER_PIXEL; drm->mmap_size = drm->mfb->width * drm->mfb->height * BYTES_PER_PIXEL;
drm->funcs = malloc(sizeof(struct kmsvnc_drm_funcs)); drm->funcs = malloc(sizeof(struct kmsvnc_drm_funcs));
if (!drm->funcs) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__); if (!drm->funcs) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
drm->funcs->convert = convert_bgrx_to_rgb; drm->funcs->convert = convert_bgra_to_rgba;
drm->funcs->sync_start = drm_sync_noop; drm->funcs->sync_start = drm_sync_noop;
drm->funcs->sync_end = drm_sync_noop; drm->funcs->sync_end = drm_sync_noop;
@ -305,7 +496,7 @@ static int drm_kmsbuf_prime() {
int err = drmPrimeHandleToFD(drm->drm_fd, drm->mfb->handles[0], O_RDWR, &drm->prime_fd); int err = drmPrimeHandleToFD(drm->drm_fd, drm->mfb->handles[0], O_RDWR, &drm->prime_fd);
if (err < 0 || drm->prime_fd < 0) if (err < 0 || drm->prime_fd < 0)
{ {
KMSVNC_FATAL("Failed to get PRIME fd from framebuffer handle"); KMSVNC_FATAL("Failed to get PRIME fd from framebuffer handle\n");
} }
drm->funcs->sync_start = &drm_sync_start; drm->funcs->sync_start = &drm_sync_start;
drm->funcs->sync_end = &drm_sync_end; drm->funcs->sync_end = &drm_sync_end;
@ -319,7 +510,7 @@ static int drm_kmsbuf_prime_vaapi() {
int err = drmPrimeHandleToFD(drm->drm_fd, drm->mfb->handles[0], O_RDWR, &drm->prime_fd); int err = drmPrimeHandleToFD(drm->drm_fd, drm->mfb->handles[0], O_RDWR, &drm->prime_fd);
if (err < 0 || drm->prime_fd < 0) if (err < 0 || drm->prime_fd < 0)
{ {
KMSVNC_FATAL("Failed to get PRIME fd from framebuffer handle"); KMSVNC_FATAL("Failed to get PRIME fd from framebuffer handle\n");
} }
if (va_init()) return 1; if (va_init()) return 1;

1
drm.h
View file

@ -10,3 +10,4 @@
void drm_cleanup(); void drm_cleanup();
int drm_open(); int drm_open();
int drm_vendors(); int drm_vendors();
int drm_dump_cursor_plane(char **data, int *width, int *height);

109
kmsvnc.c
View file

@ -89,6 +89,89 @@ static void update_screen_buf(char* to, char *from, int width, int height) {
} }
} }
static inline void update_vnc_cursor(char *data, int width, int height) {
uint8_t r, g, b, a;
#define CURSOR_MIN_A 160 // ~63%
int min_x = width;
int max_x = -1;
int min_y = height;
int max_y = -1;
int x, y;
for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) {
uint32_t pixdata = htonl(*((uint32_t*)(data + i)));
//r = (pixdata & 0xff000000u) >> 24;
//g = (pixdata & 0x00ff0000u) >> 16;
//b = (pixdata & 0x0000ff00u) >> 8;
a = pixdata & 0xff;
if (a > CURSOR_MIN_A) {
x = (i / BYTES_PER_PIXEL) % width;
y = (i / BYTES_PER_PIXEL) / width;
if (x < min_x) min_x = x;
if (y < min_y) min_y = y;
if (x > max_x) max_x = x;
if (y > max_y) max_y = y;
}
}
if (min_x > max_x || min_y > max_y) {
// no cursor detected
return;
}
int rwidth = max_x - min_x + 1;
int rheight = max_y - min_y + 1;
if (kmsvnc->cursor_bitmap_len < rwidth * rheight * BYTES_PER_PIXEL)
{
if (kmsvnc->cursor_bitmap)
free(kmsvnc->cursor_bitmap);
kmsvnc->cursor_bitmap = malloc(rwidth * rheight * BYTES_PER_PIXEL);
if (!kmsvnc->cursor_bitmap) return;
kmsvnc->cursor_bitmap_len = rwidth * rheight * BYTES_PER_PIXEL;
}
char *rich_source = malloc(rwidth * rheight * BYTES_PER_PIXEL);
if (!rich_source) return;
char *maskString = malloc(rwidth * rheight);
if (!maskString) {
free(rich_source);
return;
}
memset(maskString, ' ', rwidth * rheight);
for (int i = 0; i < rwidth; i++) {
for (int j = 0; j < rheight; j++) {
int t = (i + j * rwidth) * BYTES_PER_PIXEL;
int s = ((i+min_x) + (j+min_y) * width) * BYTES_PER_PIXEL;
*((uint32_t*)(rich_source + t)) = *((uint32_t*)(data + s));
if ((uint8_t)*(rich_source + t + 3) > CURSOR_MIN_A) {
maskString[i + j * rwidth] = 'x';
}
}
}
if ((kmsvnc->server->cursor->width != rwidth || kmsvnc->server->cursor->height != rheight) || memcmp(kmsvnc->cursor_bitmap, rich_source, rwidth * rheight * BYTES_PER_PIXEL)) {
KMSVNC_DEBUG("cursor update %dx%d\n", rwidth, rheight);
memcpy(kmsvnc->cursor_bitmap, rich_source, kmsvnc->cursor_bitmap_len);
char *cursorString = malloc(rwidth * rheight);
if (!cursorString) {
free(rich_source);
free(maskString);
return;
}
memset(cursorString, 'x', rwidth * rheight);
rfbCursorPtr cursor = rfbMakeXCursor(rwidth, rheight, cursorString, maskString);
free(cursorString);
cursor->richSource = rich_source;
cursor->cleanupRichSource = TRUE;
cursor->xhot = 0;
cursor->yhot = 0;
rfbSetCursor(kmsvnc->server, cursor);
}
else {
free(rich_source);
free(maskString);
}
}
static void cleanup() { static void cleanup() {
if (kmsvnc->keymap) { if (kmsvnc->keymap) {
xkb_cleanup(); xkb_cleanup();
@ -115,6 +198,11 @@ static void cleanup() {
free(kmsvnc->buf); free(kmsvnc->buf);
kmsvnc->buf = NULL; kmsvnc->buf = NULL;
} }
if (kmsvnc->cursor_bitmap) {
free(kmsvnc->cursor_bitmap);
kmsvnc->cursor_bitmap = NULL;
}
kmsvnc->cursor_bitmap_len = 0;
free(kmsvnc); free(kmsvnc);
kmsvnc = NULL; kmsvnc = NULL;
} }
@ -142,9 +230,10 @@ static struct argp_option kmsvnc_main_options[] = {
{"fps", 0xff00, "30", 0, "Target frames per second"}, {"fps", 0xff00, "30", 0, "Target frames per second"},
{"disable-always-shared", 0xff01, 0, OPTION_ARG_OPTIONAL, "Do not always treat incoming connections as shared"}, {"disable-always-shared", 0xff01, 0, OPTION_ARG_OPTIONAL, "Do not always treat incoming connections as shared"},
{"disable-compare-fb", 0xff02, 0, OPTION_ARG_OPTIONAL, "Do not compare pixels"}, {"disable-compare-fb", 0xff02, 0, OPTION_ARG_OPTIONAL, "Do not compare pixels"},
{"capture-cursor", 'c', 0, OPTION_ARG_OPTIONAL, "Do not capture cursor fb"},
{"capture-raw-fb", 0xff03, "/tmp/rawfb.bin", 0, "Capture RAW framebuffer instead of starting the vnc server (for debugging)"}, {"capture-raw-fb", 0xff03, "/tmp/rawfb.bin", 0, "Capture RAW framebuffer instead of starting the vnc server (for debugging)"},
{"va-derive", 0xff04, "off", 0, "Enable derive with vaapi"}, {"va-derive", 0xff04, "off", 0, "Enable derive with vaapi"},
{"va-debug", 0xff05, 0, OPTION_ARG_OPTIONAL, "Print va debug message"}, {"debug", 0xff05, 0, OPTION_ARG_OPTIONAL, "Print debug message"},
{"input-width", 0xff06, "0", 0, "Explicitly set input width, normally this is inferred from screen width on a single display system"}, {"input-width", 0xff06, "0", 0, "Explicitly set input width, normally this is inferred from screen width on a single display system"},
{"input-height", 0xff07, "0", 0, "Explicitly set input height"}, {"input-height", 0xff07, "0", 0, "Explicitly set input height"},
{"input-offx", 0xff08, "0", 0, "Set input offset of x axis on a multi display system"}, {"input-offx", 0xff08, "0", 0, "Set input offset of x axis on a multi display system"},
@ -206,6 +295,9 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
case 0xff02: case 0xff02:
kmsvnc->vnc_opt->disable_cmpfb = 1; kmsvnc->vnc_opt->disable_cmpfb = 1;
break; break;
case 'c':
kmsvnc->capture_cursor = 1;
break;
case 0xff03: case 0xff03:
kmsvnc->debug_capture_fb = arg; kmsvnc->debug_capture_fb = arg;
kmsvnc->disable_input = 1; kmsvnc->disable_input = 1;
@ -219,7 +311,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
} }
break; break;
case 0xff05: case 0xff05:
kmsvnc->va_debug = 1; kmsvnc->debug_enabled = 1;
break; break;
case 0xff06: case 0xff06:
int width = atoi(arg); int width = atoi(arg);
@ -377,6 +469,7 @@ int main(int argc, char **argv)
} }
rfbInitServer(kmsvnc->server); rfbInitServer(kmsvnc->server);
rfbRunEventLoop(kmsvnc->server, -1, TRUE); rfbRunEventLoop(kmsvnc->server, -1, TRUE);
int cursor_frame = 0;
while (rfbIsActive(kmsvnc->server)) while (rfbIsActive(kmsvnc->server))
{ {
between_frames(); between_frames();
@ -386,6 +479,18 @@ int main(int argc, char **argv)
kmsvnc->drm->funcs->convert(kmsvnc->drm->mapped, kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, kmsvnc->buf1); kmsvnc->drm->funcs->convert(kmsvnc->drm->mapped, kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, kmsvnc->buf1);
kmsvnc->drm->funcs->sync_end(kmsvnc->drm->prime_fd); kmsvnc->drm->funcs->sync_end(kmsvnc->drm->prime_fd);
update_screen_buf(kmsvnc->buf, kmsvnc->buf1, kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height); update_screen_buf(kmsvnc->buf, kmsvnc->buf1, kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height);
if (kmsvnc->capture_cursor) {
cursor_frame++;
cursor_frame %= CURSOR_FRAMESKIP;
if (!cursor_frame) {
char *data = NULL;
int width, height;
int err = drm_dump_cursor_plane(&data, &width, &height);
if (!err && data) {
update_vnc_cursor(data, width, height);
}
}
}
} }
} }
cleanup(); cleanup();

18
kmsvnc.h
View file

@ -12,6 +12,7 @@
#define BYTES_PER_PIXEL 4 #define BYTES_PER_PIXEL 4
#define CURSOR_FRAMESKIP 15
struct vnc_opt struct vnc_opt
{ {
@ -34,7 +35,7 @@ struct kmsvnc_data
char input_wakeup; char input_wakeup;
char disable_input; char disable_input;
int va_derive_enabled; int va_derive_enabled;
int va_debug; char debug_enabled;
int source_plane; int source_plane;
int source_crtc; int source_crtc;
int input_width; int input_width;
@ -47,6 +48,9 @@ struct kmsvnc_data
struct kmsvnc_va_data *va; struct kmsvnc_va_data *va;
rfbScreenInfoPtr server; rfbScreenInfoPtr server;
char shutdown; char shutdown;
char capture_cursor;
char *cursor_bitmap;
int cursor_bitmap_len;
char *buf; char *buf;
char *buf1; char *buf1;
}; };
@ -87,18 +91,28 @@ struct kmsvnc_drm_data
drmVersionPtr drm_ver; drmVersionPtr drm_ver;
int prime_fd; int prime_fd;
drmModePlane *plane; drmModePlane *plane;
drmModePlane *cursor_plane;
drmModePlaneRes *plane_res; drmModePlaneRes *plane_res;
drmModeFB2 *mfb; drmModeFB2 *mfb;
drmModeFB2 *cursor_mfb;
u_int32_t plane_id; u_int32_t plane_id;
int mmap_fd; int mmap_fd;
size_t mmap_size; size_t mmap_size;
off_t mmap_offset; off_t mmap_offset;
char *mapped; char *mapped;
char *cursor_mapped;
size_t cursor_mmap_size;
char skip_map; char skip_map;
struct kmsvnc_drm_funcs *funcs; struct kmsvnc_drm_funcs *funcs;
char *pixfmt_name; char *pixfmt_name;
char *mod_vendor; char *mod_vendor;
char *mod_name; char *mod_name;
char *kms_convert_buf;
size_t kms_convert_buf_len;
char *kms_cpy_tmp_buf;
size_t kms_cpy_tmp_buf_len;
char *kms_cursor_buf;
size_t kms_cursor_buf_len;
}; };
struct kmsvnc_va_data struct kmsvnc_va_data
@ -117,3 +131,5 @@ struct kmsvnc_va_data
#define KMSVNC_ARRAY_ELEMENTS(x) (sizeof(x) / sizeof(x[0])) #define KMSVNC_ARRAY_ELEMENTS(x) (sizeof(x) / sizeof(x[0]))
#define KMSVNC_FOURCC_TO_INT(a,b,c,d) (((a) << 0) + ((b) << 8) + ((c) << 16) + ((d) << 24)) #define KMSVNC_FOURCC_TO_INT(a,b,c,d) (((a) << 0) + ((b) << 8) + ((c) << 16) + ((d) << 24))
#define KMSVNC_WRITE_MAY(fd,buf,count) do { ssize_t e = write((fd), (buf), (count)); if (e != (count)) fprintf(stderr, "should write %ld bytes, actually wrote %ld, on line %d\n", (count), e, __LINE__); } while (0) #define KMSVNC_WRITE_MAY(fd,buf,count) do { ssize_t e = write((fd), (buf), (count)); if (e != (count)) fprintf(stderr, "should write %ld bytes, actually wrote %ld, on line %d\n", (count), e, __LINE__); } while (0)
#define KMSVNC_DEBUG(...) do{ if (kmsvnc->debug_enabled) fprintf(stdout, __VA_ARGS__); } while(0)

8
va.c
View file

@ -39,7 +39,7 @@ void va_cleanup() {
} }
static void va_msg_callback(void *user_context, const char *message) { static void va_msg_callback(void *user_context, const char *message) {
if (kmsvnc->va_debug) { if (kmsvnc->debug_enabled) {
printf("va msg: %s", message); printf("va msg: %s", message);
} }
} }
@ -158,7 +158,7 @@ int va_init() {
if (!rt_format) { if (!rt_format) {
KMSVNC_FATAL("Unsupported pixfmt %s for vaapi, please create an issue with your pixfmt.", kmsvnc->drm->pixfmt_name); KMSVNC_FATAL("Unsupported pixfmt %s for vaapi, please create an issue with your pixfmt.", kmsvnc->drm->pixfmt_name);
} }
if (kmsvnc->va_debug) { if (kmsvnc->debug_enabled) {
printf("selected rt_format %u, alpha %d\n", rt_format, is_alpha); printf("selected rt_format %u, alpha %d\n", rt_format, is_alpha);
} }
prime_desc.width = kmsvnc->drm->mfb->width; prime_desc.width = kmsvnc->drm->mfb->width;
@ -249,7 +249,7 @@ int va_init() {
} }
} }
if (kmsvnc->va_debug) { if (kmsvnc->debug_enabled) {
for (int i = 0; i < va->img_fmt_count; i++) { for (int i = 0; i < va->img_fmt_count; i++) {
print_va_image_fmt(va->img_fmts + i); print_va_image_fmt(va->img_fmts + i);
} }
@ -318,7 +318,7 @@ int va_init() {
char success = 0; char success = 0;
for (int i = 0; i < KMSVNC_ARRAY_ELEMENTS(format_to_try); i++) { for (int i = 0; i < KMSVNC_ARRAY_ELEMENTS(format_to_try); i++) {
if (format_to_try[i].fmt == NULL) continue; if (format_to_try[i].fmt == NULL) continue;
if (!kmsvnc->va_debug && rt_format != format_to_try[i].va_rt_format) continue; if (!kmsvnc->debug_enabled && rt_format != format_to_try[i].va_rt_format) continue;
if (is_alpha != format_to_try[i].is_alpha) continue; if (is_alpha != format_to_try[i].is_alpha) continue;
VAImageFormat *fmt = format_to_try[i].fmt; VAImageFormat *fmt = format_to_try[i].fmt;