diff --git a/drm.c b/drm.c index ac79b06..b4facfc 100644 --- a/drm.c +++ b/drm.c @@ -16,7 +16,7 @@ static void convert_copy(const char *in, int width, int height, char *buff) { } static void convert_vaapi(const char *in, int width, int height, char *buff) { - va_hwframe_to_vaapi(); + va_hwframe_to_vaapi(buff); } static void convert_bgrx_to_rgb(const char *in, int width, int height, char *buff) @@ -52,6 +52,7 @@ static inline void convert_x_tiled(const int tilex, const int tiley, const char if (kms_cpy_tmp_buf) free(kms_convert_buf); kms_cpy_tmp_buf = malloc(max_offset * 4 + 4); + if (!kms_cpy_tmp_buf) return; kms_cpy_tmp_buf_len = max_offset * 4 + 4; } memcpy(kms_cpy_tmp_buf, in, max_offset * 4 + 4); @@ -62,6 +63,7 @@ static inline void convert_x_tiled(const int tilex, const int tiley, const char if (kms_convert_buf) free(kms_convert_buf); kms_convert_buf = malloc(width * height * 4); + if (!kms_convert_buf) return; kms_convert_buf_len = width * height * 4; } for (int y = 0; y < height; y++) @@ -143,6 +145,7 @@ void drm_cleanup() { int drm_open() { struct kmsvnc_drm_data *drm = malloc(sizeof(struct kmsvnc_drm_data)); + if (!drm) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__); memset(drm, 0, sizeof(struct kmsvnc_drm_data)); kmsvnc->drm = drm; @@ -234,6 +237,7 @@ int drm_open() { drm->mmap_fd = drm->drm_fd; drm->mmap_size = drm->mfb->width * drm->mfb->height * BYTES_PER_PIXEL; drm->funcs = malloc(sizeof(struct kmsvnc_drm_funcs)); + if (!drm->funcs) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__); drm->funcs->convert = convert_bgrx_to_rgb; drm->funcs->sync_start = drm_sync_noop; drm->funcs->sync_end = drm_sync_noop; @@ -269,8 +273,6 @@ static int drm_kmsbuf_prime_vaapi() { if (va_init()) return 1; - drm->funcs->sync_start = &drm_sync_start; - drm->funcs->sync_end = &drm_sync_end; drm->mmap_fd = drm->prime_fd; drm->mapped = kmsvnc->va->imgbuf; return 0; @@ -309,33 +311,14 @@ int drm_vendors() { driver_name = drm->drm_ver->name; } - if (strcmp(driver_name, "i915") == 0) + if (strcmp(driver_name, "i915") == 0 || strcmp(driver_name, "amdgpu") == 0) { - drm->funcs->convert = &convert_intel_x_tiled_kmsbuf; drm->funcs->convert = &convert_vaapi; if (drm_kmsbuf_prime_vaapi()) return 1; } - else if (strcmp(driver_name, "amdgpu") == 0) - { - struct drm_gem_flink flink; - flink.handle = drm->mfb->handles[0]; - DRM_IOCTL_MUST(drm->drm_fd, DRM_IOCTL_GEM_FLINK, &flink); - - struct drm_gem_open open_arg; - open_arg.name = flink.name; - DRM_IOCTL_MUST(drm->drm_fd, DRM_IOCTL_GEM_OPEN, &open_arg); - - union drm_amdgpu_gem_mmap mmap_arg; - memset(&mmap_arg, 0, sizeof(mmap_arg)); - mmap_arg.in.handle = open_arg.handle; - DRM_IOCTL_MUST(drm->drm_fd, DRM_IOCTL_AMDGPU_GEM_MMAP, &mmap_arg); - - drm->mmap_size = open_arg.size; - drm->mmap_offset = mmap_arg.out.addr_ptr; - } else if (strcmp(driver_name, "nvidia-drm") == 0) { - // quirky and slow + printf("warn: nvidia card detected. Currently only x-tiled framebuffer is supported. Performance may suffer.\n"); drm->funcs->convert = &convert_nvidia_x_tiled_kmsbuf; if (drm_kmsbuf_dumb()) return 1; } @@ -371,6 +354,11 @@ int drm_vendors() { drm->mmap_size = open_arg.size; drm->mmap_offset = mmap_arg.offset; } + else if (strcmp(driver_name, "test-i915-prime-xtiled") == 0) + { + drm->funcs->convert = &convert_intel_x_tiled_kmsbuf; + if (drm_kmsbuf_prime()) return 1; + } else { fprintf(stderr, "Untested drm driver, use at your own risk!\n"); diff --git a/input.c b/input.c index dbc0864..95f3107 100644 --- a/input.c +++ b/input.c @@ -29,6 +29,7 @@ void uinput_cleanup() int uinput_init() { struct kmsvnc_input_data *inp = malloc(sizeof(struct kmsvnc_input_data)); + if (!inp) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__); memset(inp, 0, sizeof(struct kmsvnc_input_data)); kmsvnc->input = inp; @@ -74,6 +75,7 @@ int uinput_init() INP_IOCTL_MUST(inp->uinput_fd, UI_DEV_CREATE); inp->keystate = malloc(UINPUT_MAX_KEY); + if (!inp->keystate) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__); memset(inp->keystate, 0, UINPUT_MAX_KEY); return 0; diff --git a/keymap.c b/keymap.c index bd49040..893b443 100644 --- a/keymap.c +++ b/keymap.c @@ -25,6 +25,7 @@ void xkb_cleanup() { int xkb_init() { struct kmsvnc_keymap_data *xkb = malloc(sizeof(struct kmsvnc_keymap_data)); + if (!xkb) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__); memset(xkb, 0, sizeof(struct kmsvnc_keymap_data)); kmsvnc->keymap = xkb; diff --git a/kmsvnc.c b/kmsvnc.c index dd8837b..452e576 100644 --- a/kmsvnc.c +++ b/kmsvnc.c @@ -99,6 +99,9 @@ static void cleanup() { if (kmsvnc->drm) { drm_cleanup(); } + if (kmsvnc->va) { + va_cleanup(); + } if (kmsvnc) { if (kmsvnc->vnc_opt) { free(kmsvnc->vnc_opt); @@ -216,12 +219,17 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { int main(int argc, char **argv) { - struct vnc_opt *vncopt = malloc(sizeof(struct vnc_opt)); - memset(vncopt, 0, sizeof(struct vnc_opt)); - kmsvnc = malloc(sizeof(struct kmsvnc_data)); + if (!kmsvnc) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__); memset(kmsvnc, 0, sizeof(struct kmsvnc_data)); + struct vnc_opt *vncopt = malloc(sizeof(struct vnc_opt)); + if (!vncopt) { + free(kmsvnc); + KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__); + } + memset(vncopt, 0, sizeof(struct vnc_opt)); + kmsvnc->vnc_opt = vncopt; kmsvnc->card = "/dev/dri/card0"; @@ -261,12 +269,14 @@ int main(int argc, char **argv) if (kmsvnc->debug_capture_fb) { int wfd = open(kmsvnc->debug_capture_fb, O_WRONLY | O_CREAT, 00644); - size_t len = kmsvnc->drm->mfb->width * kmsvnc->drm->mfb->height * BYTES_PER_PIXEL; - if (kmsvnc->drm->mfb->offsets[1]) { - len = kmsvnc->drm->mfb->offsets[1] + 61440; + int max_size = 0; + for (int i = 0; i < 4; i++) { + int size = kmsvnc->drm->mfb->offsets[i] + kmsvnc->drm->mfb->height * kmsvnc->drm->mfb->pitches[i]; + if (size > max_size) max_size = size; } if (wfd > 0) { - write(wfd, kmsvnc->drm->mapped, len); + if (kmsvnc->va) va_hwframe_to_vaapi(kmsvnc->drm->mapped); + write(wfd, kmsvnc->drm->mapped, max_size); fsync(wfd); printf("wrote raw frame buffer to %s\n", kmsvnc->debug_capture_fb); } @@ -279,8 +289,16 @@ int main(int argc, char **argv) size_t buflen = kmsvnc->drm->mfb->width * kmsvnc->drm->mfb->height * BYTES_PER_PIXEL; kmsvnc->buf = malloc(buflen); + if (!kmsvnc->buf) { + cleanup(); + KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__); + } memset(kmsvnc->buf, 0, buflen); kmsvnc->buf1 = malloc(buflen); + if (!kmsvnc->buf1) { + cleanup(); + KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__); + } memset(kmsvnc->buf1, 0, buflen); signal(SIGHUP, &signal_handler); diff --git a/kmsvnc.h b/kmsvnc.h index 98795c2..6c99368 100644 --- a/kmsvnc.h +++ b/kmsvnc.h @@ -93,6 +93,7 @@ struct kmsvnc_drm_data struct kmsvnc_va_data { VADisplay dpy; + int render_node_fd; VASurfaceID surface_id; VAImage *image; char *imgbuf; diff --git a/va.c b/va.c index fc9e9cb..be54c8a 100644 --- a/va.c +++ b/va.c @@ -7,8 +7,31 @@ extern struct kmsvnc_data *kmsvnc; -void va_cleanup () { - +void va_cleanup() { + VAStatus s; + if (kmsvnc->va) { + if (kmsvnc->va->imgbuf) { + VA_MAY(vaUnmapBuffer(kmsvnc->va->dpy, kmsvnc->va->image->buf)); + kmsvnc->va->imgbuf = NULL; + } + if (kmsvnc->va->image) { + if ((s = vaDestroyImage(kmsvnc->va->dpy, kmsvnc->va->image->image_id)) == VA_STATUS_SUCCESS) { + free(kmsvnc->va->image); + } + VA_MAY(s); + kmsvnc->va->image = NULL; + } + if (kmsvnc->va->surface_id > 0) { + VA_MAY(vaDestroySurfaces(kmsvnc->va->dpy, &kmsvnc->va->surface_id, 1)); + kmsvnc->va->surface_id = 0; + } + if (kmsvnc->va->dpy) { + VA_MAY(vaTerminate(kmsvnc->va->dpy)); + kmsvnc->va->dpy = NULL; + } + free(kmsvnc->va); + kmsvnc->va = NULL; + } } static void va_msg_callback(void *user_context, const char *message) { @@ -29,23 +52,28 @@ int va_init() { setenv("DISPLAY", "", 1); struct kmsvnc_va_data *va = malloc(sizeof(struct kmsvnc_va_data)); + if (!va) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__); memset(va, 0, sizeof(struct kmsvnc_va_data)); kmsvnc->va = va; char* render_node; - int drm_fd = 0; + int effective_fd = 0; if (render_node = drmGetRenderDeviceNameFromFd(kmsvnc->drm->drm_fd)) { - drm_fd = open(render_node, O_RDWR); + va->render_node_fd = open(render_node, O_RDWR); + free(render_node); } else { printf("Using non-render node because the device does not have an associated render node.\n"); } - if (drm_fd <= 0) { + if (va->render_node_fd > 0) { + effective_fd = va->render_node_fd; + } + else { printf("Using non-render node because render node fails to open.\n"); - drm_fd = kmsvnc->drm->drm_fd; + effective_fd = kmsvnc->drm->drm_fd; } - va->dpy = vaGetDisplayDRM(drm_fd); + va->dpy = vaGetDisplayDRM(effective_fd); if (!va->dpy) { KMSVNC_FATAL("vaGetDisplayDRM failed\n"); } @@ -55,7 +83,7 @@ int va_init() { int major, minor; VAStatus status; - VA_CHECK(vaInitialize(va->dpy, &major, &minor)); + VA_MUST(vaInitialize(va->dpy, &major, &minor)); const char *vendor_string = vaQueryVendorString(va->dpy); printf("vaapi vendor %s\n", vendor_string); @@ -110,9 +138,9 @@ int va_init() { prime_desc.num_objects = 1; VAStatus s; - if (s = vaCreateSurfaces(va->dpy, VA_RT_FORMAT_RGB32, + if ((s = vaCreateSurfaces(va->dpy, VA_RT_FORMAT_RGB32, kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, &va->surface_id, 1, - prime_attrs, KMSVNC_ARRAY_ELEMENTS(prime_attrs))) + prime_attrs, KMSVNC_ARRAY_ELEMENTS(prime_attrs))) != VA_STATUS_SUCCESS) { printf("vaCreateSurfaces prime2 error %#x %s, trying prime\n", s, vaErrorStr(s)); @@ -149,7 +177,7 @@ int va_init() { buffer_desc.num_planes = prime_desc.layers[0].num_planes; - VA_CHECK(vaCreateSurfaces(va->dpy, VA_RT_FORMAT_RGB32, + VA_MUST(vaCreateSurfaces(va->dpy, VA_RT_FORMAT_RGB32, kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, &va->surface_id, 1, buffer_attrs, KMSVNC_ARRAY_ELEMENTS(buffer_attrs))); } @@ -157,6 +185,7 @@ int va_init() { #ifdef KMSVNC_VA_DEBUG int img_fmt_count = vaMaxNumImageFormats(va->dpy); VAImageFormat *img_fmts = malloc(sizeof(VAImageFormat) * img_fmt_count); + if (!img_fmts) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__); { int got; vaQueryImageFormats(va->dpy, img_fmts, &got); @@ -193,16 +222,17 @@ int va_init() { .va_reserved = 0x00000000, }; va->image = malloc(sizeof(VAImage)); - VA_CHECK(vaCreateImage(va->dpy, &format, kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, va->image)); - //VA_CHECK(vaDeriveImage(va->dpy, va->surface_id, va->image)); - va->imgbuf = malloc(va->image->data_size); - VA_CHECK(vaMapBuffer(va->dpy, va->image->buf, (void**)&va->imgbuf)); + if (!va->image) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__); + if ((s = vaCreateImage(va->dpy, &format, kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, va->image)) != VA_STATUS_SUCCESS) { + free(va->image); + va->image = NULL; + VA_MUST(s); + } + VA_MUST(vaMapBuffer(va->dpy, va->image->buf, (void**)&va->imgbuf)); } -int va_hwframe_to_vaapi() { - //VA_CHECK(vaSyncSurface(kmsvnc->va->dpy, kmsvnc->va->surface_id)); - VA_CHECK(vaGetImage(kmsvnc->va->dpy, kmsvnc->va->surface_id, 0, 0, +int va_hwframe_to_vaapi(char *out) { + VA_MUST(vaGetImage(kmsvnc->va->dpy, kmsvnc->va->surface_id, 0, 0, kmsvnc->drm->mfb->width, kmsvnc->drm->mfb->height, kmsvnc->va->image->image_id)); - memcpy(kmsvnc->buf1, kmsvnc->va->imgbuf, kmsvnc->drm->mfb->width * kmsvnc->drm->mfb->height * BYTES_PER_PIXEL); - //printf("image data size: %d\n", kmsvnc->va->image->data_size); + memcpy(out, kmsvnc->va->imgbuf, kmsvnc->drm->mfb->width * kmsvnc->drm->mfb->height * BYTES_PER_PIXEL); } diff --git a/va.h b/va.h index 7ab4dc5..0e1e67d 100644 --- a/va.h +++ b/va.h @@ -1,7 +1,8 @@ #pragma once -#define VA_CHECK(x) do{VAStatus s; if ((s = (x)) != VA_STATUS_SUCCESS) KMSVNC_FATAL("va operation error %#x %s on line %d\n", s, vaErrorStr(s), __LINE__); } while (0) +#define VA_MUST(x) do{VAStatus s; if ((s = (x)) != VA_STATUS_SUCCESS) KMSVNC_FATAL("va operation error %#x %s on line %d\n", s, vaErrorStr(s), __LINE__); } while (0) +#define VA_MAY(x) do{VAStatus s; if ((s = (x)) != VA_STATUS_SUCCESS) fprintf(stderr, "va operation error %#x %s on line %d\n", s, vaErrorStr(s), __LINE__); } while (0) -void va_cleanup (); +void va_cleanup(); int va_init(); -int va_hwframe_to_vaapi(); +int va_hwframe_to_vaapi(char *out);