From daf3db0b1922b828a19c699cbfe4f5be2042deca Mon Sep 17 00:00:00 2001 From: Jerry Date: Sun, 16 Jul 2023 01:34:58 +0800 Subject: [PATCH] WIP: allow capture cursor --- drm.c | 377 +++++++++++++++++++++++++++++++++++++++++-------------- drm.h | 1 + kmsvnc.c | 109 +++++++++++++++- kmsvnc.h | 18 ++- va.c | 8 +- 5 files changed, 413 insertions(+), 100 deletions(-) diff --git a/drm.c b/drm.c index 69c2553..3ed39b9 100644 --- a/drm.c +++ b/drm.c @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -23,35 +24,29 @@ static int check_pixfmt_non_vaapi() { return 0; } -static void convert_copy(const char *in, int width, int height, char *buff) { - memcpy(buff, in, width * height * 4); +static void convert_copy(const char *in, int width, int height, char *buff) +{ + memcpy(buff, in, width * height * BYTES_PER_PIXEL); } -static void convert_bgrx_to_rgb(const char *in, int width, int height, char *buff) +static void convert_bgra_to_rgba(const char *in, int width, int height, char *buff) { - for (int y = 0; y < height; y++) - { - for (int x = 0; x < width; x++) - { - buff[(y * width + x) * 4] = in[(y * width + x) * 4 + 2]; - buff[(y * width + x) * 4 + 1] = in[(y * width + x) * 4 + 1]; - buff[(y * width + x) * 4 + 2] = in[(y * width + x) * 4]; - } + memcpy(buff, in, width * height * BYTES_PER_PIXEL); + for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) { + uint32_t pixdata = htonl(*((uint32_t*)(kmsvnc->drm->kms_convert_buf + i))); + buff[i+0] = (pixdata & 0x0000ff00) >> 8; + buff[i+2] = (pixdata & 0xff000000) >> 24; } } -static char *kms_convert_buf = NULL; -static size_t kms_convert_buf_len = 0; -static char *kms_cpy_tmp_buf = NULL; -static size_t kms_cpy_tmp_buf_len = 0; static inline char convert_buf_allocate(size_t len) { - if (kms_convert_buf_len < len) + if (kmsvnc->drm->kms_convert_buf_len < len) { - if (kms_convert_buf) - free(kms_convert_buf); - kms_convert_buf = malloc(len); - if (!kms_convert_buf) return 1; - kms_convert_buf_len = len; + if (kmsvnc->drm->kms_convert_buf) + free(kmsvnc->drm->kms_convert_buf); + kmsvnc->drm->kms_convert_buf = malloc(len); + if (!kmsvnc->drm->kms_convert_buf) return 1; + kmsvnc->drm->kms_convert_buf_len = len; } return 0; } @@ -66,16 +61,16 @@ static inline void convert_x_tiled(const int tilex, const int tiley, const char int sno = (width / tilex) + (height / tiley) * (width / tilex); int ord = (width % tilex) + (height % tiley) * tilex; int max_offset = sno * tilex * tiley + ord; - if (kms_cpy_tmp_buf_len < max_offset * 4 + 4) + if (kmsvnc->drm->kms_cpy_tmp_buf_len < max_offset * 4 + 4) { - if (kms_cpy_tmp_buf) - free(kms_convert_buf); - kms_cpy_tmp_buf = malloc(max_offset * 4 + 4); - if (!kms_cpy_tmp_buf) return; - kms_cpy_tmp_buf_len = max_offset * 4 + 4; + if (kmsvnc->drm->kms_cpy_tmp_buf) + free(kmsvnc->drm->kms_convert_buf); + kmsvnc->drm->kms_cpy_tmp_buf = malloc(max_offset * 4 + 4); + if (!kmsvnc->drm->kms_cpy_tmp_buf) return; + kmsvnc->drm->kms_cpy_tmp_buf_len = max_offset * 4 + 4; } - memcpy(kms_cpy_tmp_buf, in, max_offset * 4 + 4); - in = (const char *)kms_cpy_tmp_buf; + memcpy(kmsvnc->drm->kms_cpy_tmp_buf, in, max_offset * 4 + 4); + in = (const char *)kmsvnc->drm->kms_cpy_tmp_buf; } if (convert_buf_allocate(width * height * 4)) return; for (int y = 0; y < height; y++) @@ -85,10 +80,10 @@ static inline void convert_x_tiled(const int tilex, const int tiley, const char int sno = (x / tilex) + (y / tiley) * (width / tilex); int ord = (x % tilex) + (y % tiley) * tilex; int offset = sno * tilex * tiley + ord; - memcpy(kms_convert_buf + (x + y * width) * 4, in + offset * 4, 4); + memcpy(kmsvnc->drm->kms_convert_buf + (x + y * width) * 4, in + offset * 4, 4); } } - convert_bgrx_to_rgb(kms_convert_buf, width, height, buff); + convert_bgra_to_rgba(kmsvnc->drm->kms_convert_buf, width, height, buff); } void convert_nvidia_x_tiled_kmsbuf(const char *in, int width, int height, char *buff) @@ -106,29 +101,34 @@ static void convert_vaapi(const char *in, int width, int height, char *buff) { } else { if (convert_buf_allocate(width * height * BYTES_PER_PIXEL)) return; - va_hwframe_to_vaapi(kms_convert_buf); + va_hwframe_to_vaapi(kmsvnc->drm->kms_convert_buf); // is 30 depth? if (kmsvnc->va->image->format.depth == 30) { for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) { - uint32_t pixdata = *((uint32_t*)(kms_convert_buf + i)); - kms_convert_buf[i] = (pixdata & 0x3ff00000) >> 20 >> 2; - kms_convert_buf[i+1] = (pixdata & 0xffc00) >> 10 >> 2; - kms_convert_buf[i+2] = (pixdata & 0x3ff) >> 2; + // ensure little endianess + uint32_t pixdata = __builtin_bswap32(htonl(*((uint32_t*)(kmsvnc->drm->kms_convert_buf + i)))); + kmsvnc->drm->kms_convert_buf[i] = (pixdata & 0x3ff00000) >> 20 >> 2; + kmsvnc->drm->kms_convert_buf[i+1] = (pixdata & 0xffc00) >> 10 >> 2; + kmsvnc->drm->kms_convert_buf[i+2] = (pixdata & 0x3ff) >> 2; } } // is xrgb? if ((kmsvnc->va->image->format.blue_mask | kmsvnc->va->image->format.red_mask) < 0x1000000) { for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) { - *((uint32_t*)(kms_convert_buf + i)) <<= 8; + uint32_t *pixdata = (uint32_t*)(kmsvnc->drm->kms_convert_buf + i); + *pixdata = ntohl(htonl(*pixdata) << 8); } } - // is bgr? + // is bgrx? if (kmsvnc->va->image->format.blue_mask > kmsvnc->va->image->format.red_mask) { - convert_bgrx_to_rgb(kms_convert_buf, width, height, buff); - } - else { - memcpy(buff, kms_convert_buf, width * height * BYTES_PER_PIXEL); + for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) { + uint32_t pixdata = htonl(*((uint32_t*)(kmsvnc->drm->kms_convert_buf + i))); + buff[i+0] = (pixdata & 0x0000ff00) >> 8; + buff[i+2] = (pixdata & 0xff000000) >> 24; + } } + // rgbx now + memcpy(buff, kmsvnc->drm->kms_convert_buf, width * height * BYTES_PER_PIXEL); } } @@ -174,14 +174,26 @@ void drm_cleanup() { drmModeFreePlane(kmsvnc->drm->plane); kmsvnc->drm->plane = NULL; } + if (kmsvnc->drm->cursor_plane) { + drmModeFreePlane(kmsvnc->drm->cursor_plane); + kmsvnc->drm->cursor_plane = NULL; + } if (kmsvnc->drm->mfb) { drmModeFreeFB2(kmsvnc->drm->mfb); kmsvnc->drm->mfb = NULL; } - if (kmsvnc->drm->mapped) { + if (kmsvnc->drm->cursor_mfb) { + drmModeFreeFB2(kmsvnc->drm->cursor_mfb); + kmsvnc->drm->cursor_mfb = NULL; + } + if (kmsvnc->drm->mapped && kmsvnc->drm->mapped != MAP_FAILED) { munmap(kmsvnc->drm->mapped, kmsvnc->drm->mmap_size); kmsvnc->drm->mapped = NULL; } + if (kmsvnc->drm->cursor_mapped && kmsvnc->drm->cursor_mapped != MAP_FAILED) { + munmap(kmsvnc->drm->cursor_mapped, kmsvnc->drm->cursor_mmap_size); + kmsvnc->drm->cursor_mapped = NULL; + } if (kmsvnc->drm->prime_fd > 0) { close(kmsvnc->drm->prime_fd); kmsvnc->drm->prime_fd = 0; @@ -194,11 +206,236 @@ void drm_cleanup() { drmModeFreePlaneResources(kmsvnc->drm->plane_res); kmsvnc->drm->plane_res = NULL; } + if (kmsvnc->drm->kms_convert_buf) { + free(kmsvnc->drm->kms_convert_buf); + kmsvnc->drm->kms_convert_buf = NULL; + } + kmsvnc->drm->kms_convert_buf_len = 0; + if (kmsvnc->drm->kms_cpy_tmp_buf) { + free(kmsvnc->drm->kms_cpy_tmp_buf); + kmsvnc->drm->kms_cpy_tmp_buf = NULL; + } + kmsvnc->drm->kms_cpy_tmp_buf_len = 0; + if (kmsvnc->drm->kms_cursor_buf) { + free(kmsvnc->drm->kms_cursor_buf); + kmsvnc->drm->kms_cursor_buf = NULL; + } + kmsvnc->drm->kms_cursor_buf_len = 0; free(kmsvnc->drm); kmsvnc->drm = NULL; } } +static const char* drm_get_plane_type_name(uint64_t plane_type) { + switch (plane_type) { + case DRM_PLANE_TYPE_OVERLAY: + return "overlay"; + case DRM_PLANE_TYPE_PRIMARY: + return "primary"; + case DRM_PLANE_TYPE_CURSOR: + return "cursor"; + default: + return "unknown"; + } +}; + +static int drm_refresh_planes(char first_time) { + struct kmsvnc_drm_data *drm = kmsvnc->drm; + if (!drm->plane && kmsvnc->source_plane > 0) + { + drm->plane = drmModeGetPlane(drm->drm_fd, kmsvnc->source_plane); + if (!drm->plane) + KMSVNC_FATAL("Failed to get plane %d: %s\n", kmsvnc->source_plane, strerror(errno)); + if (drm->plane->fb_id == 0) + fprintf(stderr, "Place %d does not have an attached framebuffer\n", kmsvnc->source_plane); + } + if (!drm->plane || (kmsvnc->capture_cursor && !drm->cursor_plane)) { + drmModePlane *current_plane = NULL; + drm->plane_res = drmModeGetPlaneResources(drm->drm_fd); + if (!drm->plane_res) + KMSVNC_FATAL("Failed to get plane resources: %s\n", strerror(errno)); + int i; + for (i = 0; i < drm->plane_res->count_planes; i++) + { + current_plane = drmModeGetPlane(drm->drm_fd, drm->plane_res->planes[i]); + if (!current_plane) + { + fprintf(stderr, "Failed to get plane %u: %s\n", drm->plane_res->planes[i], strerror(errno)); + continue; + } + // get plane type + uint64_t plane_type = 114514; + drmModeObjectPropertiesPtr plane_props = drmModeObjectGetProperties(drm->drm_fd, current_plane->plane_id, DRM_MODE_OBJECT_PLANE); + if (!plane_props) { + fprintf(stderr, "Failed to get plane prop %u: %s\n", drm->plane_res->planes[i], strerror(errno)); + } + else { + for (int i = 0; i < plane_props->count_props; i++) { + drmModePropertyPtr plane_prop = drmModeGetProperty(drm->drm_fd, plane_props->props[i]); + if (strcmp(plane_prop->name, "type") == 0) { + plane_type = plane_props->prop_values[i]; + } + drmModeFreeProperty(plane_prop); + } + drmModeFreeObjectProperties(plane_props); + } + assert(drm->plane_res->planes[i] == current_plane->plane_id); + if (first_time) { + printf("Plane %u CRTC %u FB %u Type %s\n", current_plane->plane_id, current_plane->crtc_id, current_plane->fb_id, drm_get_plane_type_name(plane_type)); + } + // populate drm->plane and drm->cursor_plane + char nofree = 0; + if (current_plane->fb_id != 0) { + if (!drm->plane) { + if (kmsvnc->source_crtc == 0 || current_plane->crtc_id == kmsvnc->source_crtc) { + nofree = 1; + drm->plane = current_plane; + } + } + // assume cursor plane is always after primary plane + if (!drm->cursor_plane) { + if (drm->plane && drm->plane->crtc_id == current_plane->crtc_id && plane_type == DRM_PLANE_TYPE_CURSOR) { + nofree = 1; + drm->cursor_plane = current_plane; + } + } + } + if ((!kmsvnc->capture_cursor || drm->cursor_plane) && drm->plane) { + break; + } + if (!nofree) { + drmModeFreePlane(current_plane); + } + current_plane = NULL; + } + if (!first_time) return 0; + if (i == drm->plane_res->count_planes) + { + if (!drm->plane) { + if (kmsvnc->source_crtc != 0) + { + KMSVNC_FATAL("No usable planes found on CRTC %d\n", kmsvnc->source_crtc); + } + else + { + KMSVNC_FATAL("No usable planes found\n"); + } + } + else if (!drm->cursor_plane) { + fprintf(stderr, "No usable cursor plane found, cursor capture currently unavailable\n"); + } + } + printf("Using plane %u to locate framebuffers\n", drm->plane->plane_id); + if (drm->cursor_plane) { + printf("Using cursor plane %u\n", drm->cursor_plane->plane_id); + } + } + return 0; +} + +int drm_dump_cursor_plane(char **data, int *width, int *height) { + struct kmsvnc_drm_data *drm = kmsvnc->drm; + + if (drm->cursor_plane) { + drmModeFreePlane(drm->cursor_plane); + drm->cursor_plane = NULL; + } + drm_refresh_planes(0); // ignore error + if (!drm->cursor_plane) { + data = NULL; + return 1; + } + if (drm->cursor_mfb) drmModeFreeFB2(drm->cursor_mfb); + drm->cursor_mfb = drmModeGetFB2(drm->drm_fd, drm->cursor_plane->fb_id); + if (!drm->cursor_mfb) { + KMSVNC_DEBUG("Cursor framebuffer missing\n"); + return 1; + } + + if (drm->cursor_mfb->modifier != DRM_FORMAT_MOD_NONE && drm->cursor_mfb->modifier != DRM_FORMAT_MOD_LINEAR) { + //kmsvnc->capture_cursor = 0; + KMSVNC_DEBUG("Cursor plane modifier is not linear: %lu\n", drm->cursor_mfb->modifier); + return 1; + } + + if ( + drm->cursor_mfb->pixel_format != KMSVNC_FOURCC_TO_INT('A', 'R', '2', '4') && + drm->cursor_mfb->pixel_format != KMSVNC_FOURCC_TO_INT('A', 'R', '3', '0') + ) + { + //kmsvnc->capture_cursor = 0; + char *fmtname = drmGetFormatName(drm->cursor_mfb->pixel_format); + KMSVNC_DEBUG("Cursor plane pixel format unsupported (%u, %s)\n", drm->cursor_mfb->pixel_format, fmtname); + free(fmtname); + return 1; + } + + struct drm_gem_flink flink; + flink.handle = drm->cursor_mfb->handles[0]; + DRM_IOCTL_MUST(drm->drm_fd, DRM_IOCTL_GEM_FLINK, &flink); + + struct drm_gem_open open_arg; + open_arg.name = flink.name; + DRM_IOCTL_MUST(drm->drm_fd, DRM_IOCTL_GEM_OPEN, &open_arg); + + struct drm_mode_map_dumb mreq; + memset(&mreq, 0, sizeof(mreq)); + mreq.handle = open_arg.handle; + DRM_IOCTL_MUST(drm->drm_fd, DRM_IOCTL_MODE_MAP_DUMB, &mreq); + + size_t mmap_size = open_arg.size; + if (mmap_size != drm->cursor_mfb->width * drm->cursor_mfb->height * BYTES_PER_PIXEL) { + KMSVNC_DEBUG("Cursor plane mmap_size != calculated size (%d, %d)\n", mmap_size, drm->cursor_mfb->width * drm->cursor_mfb->height * BYTES_PER_PIXEL); + return 1; + } + + off_t mmap_offset = mreq.offset; + if (drm->cursor_mapped && drm->cursor_mapped != MAP_FAILED) munmap(drm->cursor_mapped, drm->cursor_mmap_size); + drm->cursor_mapped = mmap(NULL, mmap_size, PROT_READ, MAP_SHARED, drm->drm_fd, mmap_offset); + if (drm->cursor_mapped == MAP_FAILED) + { + KMSVNC_DEBUG("Failed to mmap cursor: %s\n", strerror(errno)); + return 1; + } + else + { + if (kmsvnc->drm->kms_cursor_buf_len < mmap_size) + { + if (kmsvnc->drm->kms_cursor_buf) + free(kmsvnc->drm->kms_cursor_buf); + kmsvnc->drm->kms_cursor_buf = malloc(mmap_size); + if (!kmsvnc->drm->kms_cursor_buf) return 1; + kmsvnc->drm->kms_cursor_buf_len = mmap_size; + } + memcpy(drm->kms_cursor_buf, drm->cursor_mapped, mmap_size); + if (drm->cursor_mfb->pixel_format == KMSVNC_FOURCC_TO_INT('X', 'R', '3', '0') || + drm->cursor_mfb->pixel_format == KMSVNC_FOURCC_TO_INT('A', 'R', '3', '0')) + { + for (int i = 0; i < drm->cursor_mfb->width * drm->cursor_mfb->height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) { + uint32_t pixdata = __builtin_bswap32(htonl(*((uint32_t*)(kmsvnc->drm->kms_cursor_buf + i)))); + kmsvnc->drm->kms_cursor_buf[i] = (pixdata & 0x3ff00000) >> 20 >> 2; + kmsvnc->drm->kms_cursor_buf[i+1] = (pixdata & 0xffc00) >> 10 >> 2; + kmsvnc->drm->kms_cursor_buf[i+2] = (pixdata & 0x3ff) >> 2; + kmsvnc->drm->kms_cursor_buf[i+3] = (pixdata & 0xc0000000) >> 30 << 6; + } + } + if (drm->cursor_mfb->pixel_format == KMSVNC_FOURCC_TO_INT('X', 'R', '2', '4') || + drm->cursor_mfb->pixel_format == KMSVNC_FOURCC_TO_INT('A', 'R', '2', '4')) + { + // bgra to rgba + for (int i = 0; i < drm->cursor_mfb->width * drm->cursor_mfb->height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) { + uint32_t pixdata = htonl(*((uint32_t*)(kmsvnc->drm->kms_cursor_buf + i))); + kmsvnc->drm->kms_cursor_buf[i+0] = (pixdata & 0x0000ff00) >> 8; + kmsvnc->drm->kms_cursor_buf[i+2] = (pixdata & 0xff000000) >> 24; + } + } + *width = drm->cursor_mfb->width; + *height = drm->cursor_mfb->height; + *data = drm->kms_cursor_buf; + } + return 0; +} + int drm_open() { struct kmsvnc_drm_data *drm = malloc(sizeof(struct kmsvnc_drm_data)); if (!drm) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__); @@ -218,54 +455,8 @@ int drm_open() { { perror("Failed to set universal planes capability: primary planes will not be usable"); } - if (kmsvnc->source_plane > 0) - { - drm->plane = drmModeGetPlane(drm->drm_fd, kmsvnc->source_plane); - if (!drm->plane) - KMSVNC_FATAL("Failed to get plane %d: %s\n", kmsvnc->source_plane, strerror(errno)); - if (drm->plane->fb_id == 0) - fprintf(stderr, "Place %d does not have an attached framebuffer\n", kmsvnc->source_plane); - } - else - { - drm->plane_res = drmModeGetPlaneResources(drm->drm_fd); - if (!drm->plane_res) - KMSVNC_FATAL("Failed to get plane resources: %s\n", strerror(errno)); - int i; - for (i = 0; i < drm->plane_res->count_planes; i++) - { - drm->plane = drmModeGetPlane(drm->drm_fd, drm->plane_res->planes[i]); - if (!drm->plane) - { - fprintf(stderr, "Failed to get plane %u: %s\n", drm->plane_res->planes[i], strerror(errno)); - continue; - } - printf("Plane %u CRTC %u FB %u\n", drm->plane->plane_id, drm->plane->crtc_id, drm->plane->fb_id); - if ((kmsvnc->source_crtc != 0 && drm->plane->crtc_id != kmsvnc->source_crtc) || drm->plane->fb_id == 0) - { - // Either not connected to the target source CRTC - // or not active. - drmModeFreePlane(drm->plane); - drm->plane = NULL; - continue; - } - break; - } - if (i == drm->plane_res->count_planes) - { - if (kmsvnc->source_crtc != 0) - { - KMSVNC_FATAL("No usable planes found on CRTC %d\n", kmsvnc->source_crtc); - } - else - { - KMSVNC_FATAL("No usable planes found\n"); - } - } - printf("Using plane %u to locate framebuffers\n", drm->plane->plane_id); - } - uint32_t plane_id = drm->plane->plane_id; + if (drm_refresh_planes(1)) return 1; drm->mfb = drmModeGetFB2(drm->drm_fd, drm->plane->fb_id); if (!drm->mfb) { @@ -289,7 +480,7 @@ int drm_open() { drm->mmap_size = drm->mfb->width * drm->mfb->height * BYTES_PER_PIXEL; drm->funcs = malloc(sizeof(struct kmsvnc_drm_funcs)); if (!drm->funcs) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__); - drm->funcs->convert = convert_bgrx_to_rgb; + drm->funcs->convert = convert_bgra_to_rgba; drm->funcs->sync_start = drm_sync_noop; drm->funcs->sync_end = drm_sync_noop; @@ -305,7 +496,7 @@ static int drm_kmsbuf_prime() { int err = drmPrimeHandleToFD(drm->drm_fd, drm->mfb->handles[0], O_RDWR, &drm->prime_fd); if (err < 0 || drm->prime_fd < 0) { - KMSVNC_FATAL("Failed to get PRIME fd from framebuffer handle"); + KMSVNC_FATAL("Failed to get PRIME fd from framebuffer handle\n"); } drm->funcs->sync_start = &drm_sync_start; drm->funcs->sync_end = &drm_sync_end; @@ -319,7 +510,7 @@ static int drm_kmsbuf_prime_vaapi() { int err = drmPrimeHandleToFD(drm->drm_fd, drm->mfb->handles[0], O_RDWR, &drm->prime_fd); if (err < 0 || drm->prime_fd < 0) { - KMSVNC_FATAL("Failed to get PRIME fd from framebuffer handle"); + KMSVNC_FATAL("Failed to get PRIME fd from framebuffer handle\n"); } if (va_init()) return 1; diff --git a/drm.h b/drm.h index 91e7c06..a2ba9b2 100644 --- a/drm.h +++ b/drm.h @@ -10,3 +10,4 @@ void drm_cleanup(); int drm_open(); int drm_vendors(); +int drm_dump_cursor_plane(char **data, int *width, int *height); diff --git a/kmsvnc.c b/kmsvnc.c index 73bd12b..c1af49b 100644 --- a/kmsvnc.c +++ b/kmsvnc.c @@ -89,6 +89,89 @@ static void update_screen_buf(char* to, char *from, int width, int height) { } } +static inline void update_vnc_cursor(char *data, int width, int height) { + uint8_t r, g, b, a; + #define CURSOR_MIN_A 160 // ~63% + int min_x = width; + int max_x = -1; + int min_y = height; + int max_y = -1; + int x, y; + + for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) { + uint32_t pixdata = htonl(*((uint32_t*)(data + i))); + //r = (pixdata & 0xff000000u) >> 24; + //g = (pixdata & 0x00ff0000u) >> 16; + //b = (pixdata & 0x0000ff00u) >> 8; + a = pixdata & 0xff; + if (a > CURSOR_MIN_A) { + x = (i / BYTES_PER_PIXEL) % width; + y = (i / BYTES_PER_PIXEL) / width; + if (x < min_x) min_x = x; + if (y < min_y) min_y = y; + if (x > max_x) max_x = x; + if (y > max_y) max_y = y; + } + } + if (min_x > max_x || min_y > max_y) { + // no cursor detected + return; + } + int rwidth = max_x - min_x + 1; + int rheight = max_y - min_y + 1; + if (kmsvnc->cursor_bitmap_len < rwidth * rheight * BYTES_PER_PIXEL) + { + if (kmsvnc->cursor_bitmap) + free(kmsvnc->cursor_bitmap); + kmsvnc->cursor_bitmap = malloc(rwidth * rheight * BYTES_PER_PIXEL); + if (!kmsvnc->cursor_bitmap) return; + kmsvnc->cursor_bitmap_len = rwidth * rheight * BYTES_PER_PIXEL; + } + char *rich_source = malloc(rwidth * rheight * BYTES_PER_PIXEL); + if (!rich_source) return; + char *maskString = malloc(rwidth * rheight); + if (!maskString) { + free(rich_source); + return; + } + memset(maskString, ' ', rwidth * rheight); + for (int i = 0; i < rwidth; i++) { + for (int j = 0; j < rheight; j++) { + int t = (i + j * rwidth) * BYTES_PER_PIXEL; + int s = ((i+min_x) + (j+min_y) * width) * BYTES_PER_PIXEL; + *((uint32_t*)(rich_source + t)) = *((uint32_t*)(data + s)); + if ((uint8_t)*(rich_source + t + 3) > CURSOR_MIN_A) { + maskString[i + j * rwidth] = 'x'; + } + } + } + + if ((kmsvnc->server->cursor->width != rwidth || kmsvnc->server->cursor->height != rheight) || memcmp(kmsvnc->cursor_bitmap, rich_source, rwidth * rheight * BYTES_PER_PIXEL)) { + KMSVNC_DEBUG("cursor update %dx%d\n", rwidth, rheight); + memcpy(kmsvnc->cursor_bitmap, rich_source, kmsvnc->cursor_bitmap_len); + char *cursorString = malloc(rwidth * rheight); + if (!cursorString) { + free(rich_source); + free(maskString); + return; + } + + memset(cursorString, 'x', rwidth * rheight); + + rfbCursorPtr cursor = rfbMakeXCursor(rwidth, rheight, cursorString, maskString); + free(cursorString); + cursor->richSource = rich_source; + cursor->cleanupRichSource = TRUE; + cursor->xhot = 0; + cursor->yhot = 0; + rfbSetCursor(kmsvnc->server, cursor); + } + else { + free(rich_source); + free(maskString); + } +} + static void cleanup() { if (kmsvnc->keymap) { xkb_cleanup(); @@ -115,6 +198,11 @@ static void cleanup() { free(kmsvnc->buf); kmsvnc->buf = NULL; } + if (kmsvnc->cursor_bitmap) { + free(kmsvnc->cursor_bitmap); + kmsvnc->cursor_bitmap = NULL; + } + kmsvnc->cursor_bitmap_len = 0; free(kmsvnc); kmsvnc = NULL; } @@ -142,9 +230,10 @@ static struct argp_option kmsvnc_main_options[] = { {"fps", 0xff00, "30", 0, "Target frames per second"}, {"disable-always-shared", 0xff01, 0, OPTION_ARG_OPTIONAL, "Do not always treat incoming connections as shared"}, {"disable-compare-fb", 0xff02, 0, OPTION_ARG_OPTIONAL, "Do not compare pixels"}, + {"capture-cursor", 'c', 0, OPTION_ARG_OPTIONAL, "Do not capture cursor fb"}, {"capture-raw-fb", 0xff03, "/tmp/rawfb.bin", 0, "Capture RAW framebuffer instead of starting the vnc server (for debugging)"}, {"va-derive", 0xff04, "off", 0, "Enable derive with vaapi"}, - {"va-debug", 0xff05, 0, OPTION_ARG_OPTIONAL, "Print va debug message"}, + {"debug", 0xff05, 0, OPTION_ARG_OPTIONAL, "Print debug message"}, {"input-width", 0xff06, "0", 0, "Explicitly set input width, normally this is inferred from screen width on a single display system"}, {"input-height", 0xff07, "0", 0, "Explicitly set input height"}, {"input-offx", 0xff08, "0", 0, "Set input offset of x axis on a multi display system"}, @@ -206,6 +295,9 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { case 0xff02: kmsvnc->vnc_opt->disable_cmpfb = 1; break; + case 'c': + kmsvnc->capture_cursor = 1; + break; case 0xff03: kmsvnc->debug_capture_fb = arg; kmsvnc->disable_input = 1; @@ -219,7 +311,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { } break; case 0xff05: - kmsvnc->va_debug = 1; + kmsvnc->debug_enabled = 1; break; case 0xff06: int width = atoi(arg); @@ -377,6 +469,7 @@ int main(int argc, char **argv) } rfbInitServer(kmsvnc->server); rfbRunEventLoop(kmsvnc->server, -1, TRUE); + int cursor_frame = 0; while (rfbIsActive(kmsvnc->server)) { between_frames(); @@ -386,6 +479,18 @@ int main(int argc, char **argv) kmsvnc->drm->funcs->convert(kmsvnc->drm->mapped, kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, kmsvnc->buf1); kmsvnc->drm->funcs->sync_end(kmsvnc->drm->prime_fd); update_screen_buf(kmsvnc->buf, kmsvnc->buf1, kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height); + if (kmsvnc->capture_cursor) { + cursor_frame++; + cursor_frame %= CURSOR_FRAMESKIP; + if (!cursor_frame) { + char *data = NULL; + int width, height; + int err = drm_dump_cursor_plane(&data, &width, &height); + if (!err && data) { + update_vnc_cursor(data, width, height); + } + } + } } } cleanup(); diff --git a/kmsvnc.h b/kmsvnc.h index 27970dc..9593ed8 100644 --- a/kmsvnc.h +++ b/kmsvnc.h @@ -12,6 +12,7 @@ #define BYTES_PER_PIXEL 4 +#define CURSOR_FRAMESKIP 15 struct vnc_opt { @@ -34,7 +35,7 @@ struct kmsvnc_data char input_wakeup; char disable_input; int va_derive_enabled; - int va_debug; + char debug_enabled; int source_plane; int source_crtc; int input_width; @@ -47,6 +48,9 @@ struct kmsvnc_data struct kmsvnc_va_data *va; rfbScreenInfoPtr server; char shutdown; + char capture_cursor; + char *cursor_bitmap; + int cursor_bitmap_len; char *buf; char *buf1; }; @@ -87,18 +91,28 @@ struct kmsvnc_drm_data drmVersionPtr drm_ver; int prime_fd; drmModePlane *plane; + drmModePlane *cursor_plane; drmModePlaneRes *plane_res; drmModeFB2 *mfb; + drmModeFB2 *cursor_mfb; u_int32_t plane_id; int mmap_fd; size_t mmap_size; off_t mmap_offset; char *mapped; + char *cursor_mapped; + size_t cursor_mmap_size; char skip_map; struct kmsvnc_drm_funcs *funcs; char *pixfmt_name; char *mod_vendor; char *mod_name; + char *kms_convert_buf; + size_t kms_convert_buf_len; + char *kms_cpy_tmp_buf; + size_t kms_cpy_tmp_buf_len; + char *kms_cursor_buf; + size_t kms_cursor_buf_len; }; struct kmsvnc_va_data @@ -117,3 +131,5 @@ struct kmsvnc_va_data #define KMSVNC_ARRAY_ELEMENTS(x) (sizeof(x) / sizeof(x[0])) #define KMSVNC_FOURCC_TO_INT(a,b,c,d) (((a) << 0) + ((b) << 8) + ((c) << 16) + ((d) << 24)) #define KMSVNC_WRITE_MAY(fd,buf,count) do { ssize_t e = write((fd), (buf), (count)); if (e != (count)) fprintf(stderr, "should write %ld bytes, actually wrote %ld, on line %d\n", (count), e, __LINE__); } while (0) + +#define KMSVNC_DEBUG(...) do{ if (kmsvnc->debug_enabled) fprintf(stdout, __VA_ARGS__); } while(0) diff --git a/va.c b/va.c index 20990fc..49d13a8 100644 --- a/va.c +++ b/va.c @@ -39,7 +39,7 @@ void va_cleanup() { } static void va_msg_callback(void *user_context, const char *message) { - if (kmsvnc->va_debug) { + if (kmsvnc->debug_enabled) { printf("va msg: %s", message); } } @@ -158,7 +158,7 @@ int va_init() { if (!rt_format) { KMSVNC_FATAL("Unsupported pixfmt %s for vaapi, please create an issue with your pixfmt.", kmsvnc->drm->pixfmt_name); } - if (kmsvnc->va_debug) { + if (kmsvnc->debug_enabled) { printf("selected rt_format %u, alpha %d\n", rt_format, is_alpha); } prime_desc.width = kmsvnc->drm->mfb->width; @@ -249,7 +249,7 @@ int va_init() { } } - if (kmsvnc->va_debug) { + if (kmsvnc->debug_enabled) { for (int i = 0; i < va->img_fmt_count; i++) { print_va_image_fmt(va->img_fmts + i); } @@ -318,7 +318,7 @@ int va_init() { char success = 0; for (int i = 0; i < KMSVNC_ARRAY_ELEMENTS(format_to_try); i++) { if (format_to_try[i].fmt == NULL) continue; - if (!kmsvnc->va_debug && rt_format != format_to_try[i].va_rt_format) continue; + if (!kmsvnc->debug_enabled && rt_format != format_to_try[i].va_rt_format) continue; if (is_alpha != format_to_try[i].is_alpha) continue; VAImageFormat *fmt = format_to_try[i].fmt;