Jerry/kmsvnc
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

mesa quirk finally fixed

Browse source
This commit is contained in:
JerryXiao 2024-01-23 10:40:35 +08:00
parent 0d40834b17
commit a5a2ba0d82
Signed by: Jerry
GPG key ID: 22618F758B5BE2E5
3 changed files with 33 additions and 62 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
drm.c
View file

@ -103,18 +103,8 @@ void convert_intel_x_tiled_kmsbuf(const char *in, int width, int height, char *b
static void convert_vaapi(const char *in, int width, int height, char *buff) { static void convert_vaapi(const char *in, int width, int height, char *buff) {
va_hwframe_to_vaapi(buff); va_hwframe_to_vaapi(buff);
if ( if (
!kmsvnc->va->selected_fmt->byte_order &&
(KMSVNC_FOURCC_TO_INT('R','G','B',0) & kmsvnc->va->selected_fmt->fourcc) == KMSVNC_FOURCC_TO_INT('R','G','B',0) (KMSVNC_FOURCC_TO_INT('R','G','B',0) & kmsvnc->va->selected_fmt->fourcc) == KMSVNC_FOURCC_TO_INT('R','G','B',0)
) {} ) {}
else if (
kmsvnc->va->selected_fmt->byte_order &&
(KMSVNC_FOURCC_TO_INT(0,'B','G','R') & kmsvnc->va->selected_fmt->fourcc) == KMSVNC_FOURCC_TO_INT(0,'B','G','R')
) {
for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) {
uint32_t *pixdata = (uint32_t*)(buff + i);
*pixdata = __builtin_bswap32(*pixdata);
}
}
else { else {
// is 30 depth? // is 30 depth?
if (kmsvnc->va->selected_fmt->depth == 30) { if (kmsvnc->va->selected_fmt->depth == 30) {
@ -127,8 +117,8 @@ static void convert_vaapi(const char *in, int width, int height, char *buff) {
} }
} }
else { else {
// handle ihd and mesa byte order quirk // actually, does anyone use this?
if (kmsvnc->va->selected_fmt->byte_order) { if (!kmsvnc->va->selected_fmt->byte_order) {
for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) { for (int i = 0; i < width * height * BYTES_PER_PIXEL; i += BYTES_PER_PIXEL) {
uint32_t *pixdata = (uint32_t*)(buff + i); uint32_t *pixdata = (uint32_t*)(buff + i);
*pixdata = __builtin_bswap32(*pixdata); *pixdata = __builtin_bswap32(*pixdata);

1
kmsvnc.h
View file

@ -139,6 +139,7 @@ struct kmsvnc_va_data
VAImageFormat* img_fmts; VAImageFormat* img_fmts;
int img_fmt_count; int img_fmt_count;
VAImageFormat* selected_fmt; VAImageFormat* selected_fmt;
const char *vendor_string;
}; };
#define KMSVNC_FATAL(...) do{ fprintf(stderr, __VA_ARGS__); return 1; } while(0) #define KMSVNC_FATAL(...) do{ fprintf(stderr, __VA_ARGS__); return 1; } while(0)

80
va.c
View file

@ -6,6 +6,7 @@
#include <va/va_drm.h> #include <va/va_drm.h>
#include <va/va_drmcommon.h> #include <va/va_drmcommon.h>
#include <fcntl.h> #include <fcntl.h>
#include <string.h>
#include "va.h" #include "va.h"
#include "kmsvnc.h" #include "kmsvnc.h"
@ -38,6 +39,9 @@ void va_cleanup() {
VA_MAY(vaTerminate(kmsvnc->va->dpy)); VA_MAY(vaTerminate(kmsvnc->va->dpy));
kmsvnc->va->dpy = NULL; kmsvnc->va->dpy = NULL;
} }
if (kmsvnc->va->vendor_string) {
kmsvnc->va->vendor_string = NULL;
}
free(kmsvnc->va); free(kmsvnc->va);
kmsvnc->va = NULL; kmsvnc->va = NULL;
} }
@ -80,26 +84,17 @@ struct va_fmt_data {
char is_alpha; char is_alpha;
uint32_t va_rt_format; uint32_t va_rt_format;
uint32_t depth; uint32_t depth;
uint32_t blue_mask;
uint32_t green_mask;
uint32_t red_mask;
uint32_t byte_order;
}; };
static VAImageFormat* vaImgFmt_from_vaFmtData(struct va_fmt_data* data) { static VAImageFormat* vaImgFmt_apply_quirks(struct va_fmt_data* data) {
static VAImageFormat ret = {0}; static VAImageFormat ret = {0};
VAImageFormat fmt = { memcpy(&ret, data->fmt, sizeof(VAImageFormat));
.fourcc = data->va_fourcc, if (!strncmp(kmsvnc->va->vendor_string, "Mesa", 4) && data->depth != 30) {
.byte_order = data->byte_order, printf("applying mesa quirk\n");
.bits_per_pixel = 32, ret.blue_mask = __builtin_bswap32(data->fmt->blue_mask);
.depth = data->depth, ret.green_mask = __builtin_bswap32(data->fmt->green_mask);
.blue_mask = data->blue_mask, ret.red_mask = __builtin_bswap32(data->fmt->red_mask);
.green_mask = data->green_mask, }
.red_mask = data->red_mask,
.alpha_mask = 0,
.va_reserved = {0}
};
memcpy(&ret, &fmt, sizeof(VAImageFormat));
return &ret; return &ret;
} }
@ -126,6 +121,7 @@ int va_init() {
} }
setenv("DISPLAY", "", 1); setenv("DISPLAY", "", 1);
setenv("WAYLAND_DISPLAY", "", 1);
struct kmsvnc_va_data *va = malloc(sizeof(struct kmsvnc_va_data)); struct kmsvnc_va_data *va = malloc(sizeof(struct kmsvnc_va_data));
if (!va) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__); if (!va) KMSVNC_FATAL("memory allocation error at %s:%d\n", __FILE__, __LINE__);
@ -161,8 +157,8 @@ int va_init() {
VAStatus status; VAStatus status;
VA_MUST(vaInitialize(va->dpy, &major, &minor)); VA_MUST(vaInitialize(va->dpy, &major, &minor));
const char *vendor_string = vaQueryVendorString(va->dpy); va->vendor_string = vaQueryVendorString(va->dpy);
printf("vaapi vendor %s\n", vendor_string); printf("vaapi vendor %s\n", va->vendor_string);
VADRMPRIMESurfaceDescriptor prime_desc; VADRMPRIMESurfaceDescriptor prime_desc;
VASurfaceAttrib prime_attrs[2] = { VASurfaceAttrib prime_attrs[2] = {
@ -291,45 +287,29 @@ int va_init() {
} }
struct va_fmt_data format_to_try[] = { struct va_fmt_data format_to_try[] = {
{KMSVNC_FOURCC_TO_INT('R','G','B','X'), NULL, 0, VA_RT_FORMAT_RGB32, 24, 0xff00, 0xff0000, 0xff000000, 0}, {KMSVNC_FOURCC_TO_INT('R','G','B','X'), NULL, 0, VA_RT_FORMAT_RGB32, 24},
{KMSVNC_FOURCC_TO_INT('R','G','B','A'), NULL, 1, VA_RT_FORMAT_RGB32, 32, 0xff00, 0xff0000, 0xff000000, 0}, {KMSVNC_FOURCC_TO_INT('R','G','B','A'), NULL, 1, VA_RT_FORMAT_RGB32, 32},
{KMSVNC_FOURCC_TO_INT('X','B','G','R'), NULL, 0, VA_RT_FORMAT_RGB32, 24, 0xff0000, 0xff00, 0xff, 0}, {KMSVNC_FOURCC_TO_INT('X','B','G','R'), NULL, 0, VA_RT_FORMAT_RGB32, 24},
{KMSVNC_FOURCC_TO_INT('A','B','G','R'), NULL, 1, VA_RT_FORMAT_RGB32, 32, 0xff0000, 0xff00, 0xff, 0}, {KMSVNC_FOURCC_TO_INT('A','B','G','R'), NULL, 1, VA_RT_FORMAT_RGB32, 32},
{KMSVNC_FOURCC_TO_INT('X','R','G','B'), NULL, 0, VA_RT_FORMAT_RGB32, 24, 0xff, 0xff00, 0xff0000, 0}, {KMSVNC_FOURCC_TO_INT('X','R','G','B'), NULL, 0, VA_RT_FORMAT_RGB32, 24},
{KMSVNC_FOURCC_TO_INT('A','R','G','B'), NULL, 1, VA_RT_FORMAT_RGB32, 32, 0xff, 0xff00, 0xff0000, 0}, {KMSVNC_FOURCC_TO_INT('A','R','G','B'), NULL, 1, VA_RT_FORMAT_RGB32, 32},
{KMSVNC_FOURCC_TO_INT('B','G','R','X'), NULL, 0, VA_RT_FORMAT_RGB32, 24, 0xff000000, 0xff0000, 0xff00, 0}, {KMSVNC_FOURCC_TO_INT('B','G','R','X'), NULL, 0, VA_RT_FORMAT_RGB32, 24},
{KMSVNC_FOURCC_TO_INT('B','G','R','A'), NULL, 1, VA_RT_FORMAT_RGB32, 32, 0xff000000, 0xff0000, 0xff00, 0}, {KMSVNC_FOURCC_TO_INT('B','G','R','A'), NULL, 1, VA_RT_FORMAT_RGB32, 32},
{KMSVNC_FOURCC_TO_INT('X','R','3','0'), NULL, 0, VA_RT_FORMAT_RGB32_10, 30, 0x3ff, 0xffc00, 0x3ff00000, 0}, {KMSVNC_FOURCC_TO_INT('X','R','3','0'), NULL, 0, VA_RT_FORMAT_RGB32_10, 30},
{KMSVNC_FOURCC_TO_INT('A','R','3','0'), NULL, 1, VA_RT_FORMAT_RGB32_10, 30, 0x3ff, 0xffc00, 0x3ff00000, 0}, {KMSVNC_FOURCC_TO_INT('A','R','3','0'), NULL, 1, VA_RT_FORMAT_RGB32_10, 30},
{KMSVNC_FOURCC_TO_INT('X','B','3','0'), NULL, 0, VA_RT_FORMAT_RGB32_10, 30, 0x3ff00000, 0xffc00, 0x3ff, 0}, {KMSVNC_FOURCC_TO_INT('X','B','3','0'), NULL, 0, VA_RT_FORMAT_RGB32_10, 30},
{KMSVNC_FOURCC_TO_INT('A','B','3','0'), NULL, 1, VA_RT_FORMAT_RGB32_10, 30, 0x3ff00000, 0xffc00, 0x3ff, 0}, {KMSVNC_FOURCC_TO_INT('A','B','3','0'), NULL, 1, VA_RT_FORMAT_RGB32_10, 30},
}; };
for (int i = 0; i < va->img_fmt_count; i++) { for (int i = 0; i < va->img_fmt_count; i++) {
for (int j = 0; j < KMSVNC_ARRAY_ELEMENTS(format_to_try); j++) { for (int j = 0; j < KMSVNC_ARRAY_ELEMENTS(format_to_try); j++) {
if (va->img_fmts[i].fourcc == format_to_try[j].va_fourcc) { if (va->img_fmts[i].fourcc == format_to_try[j].va_fourcc) {
if ( format_to_try[j].fmt = va->img_fmts + i;
va->img_fmts[i].blue_mask == format_to_try[j].blue_mask &&
va->img_fmts[i].green_mask == format_to_try[j].green_mask &&
va->img_fmts[i].red_mask == format_to_try[j].red_mask
) {
format_to_try[j].fmt = va->img_fmts + i;
}
else if (
format_to_try[j].depth != 30 &&
va->img_fmts[i].blue_mask == __builtin_bswap32(format_to_try[j].blue_mask) &&
va->img_fmts[i].green_mask == __builtin_bswap32(format_to_try[j].green_mask) &&
va->img_fmts[i].red_mask == __builtin_bswap32(format_to_try[j].red_mask)
) {
// mesa quirk: mesa fourcc and pixel data is lsb_first for RGB32 formats, msb_first for RGB32_10 formats
format_to_try[j].byte_order = 1u;
format_to_try[j].fmt = va->img_fmts + i;
}
} }
} }
} }
@ -344,7 +324,7 @@ int va_init() {
for (int i = 0; i < KMSVNC_ARRAY_ELEMENTS(format_to_try); i++) { for (int i = 0; i < KMSVNC_ARRAY_ELEMENTS(format_to_try); i++) {
if (format_to_try[i].fmt == NULL) continue; if (format_to_try[i].fmt == NULL) continue;
if (va->image->format.fourcc == format_to_try[i].fmt->fourcc) { if (va->image->format.fourcc == format_to_try[i].fmt->fourcc) {
va->selected_fmt = vaImgFmt_from_vaFmtData(format_to_try + i); va->selected_fmt = vaImgFmt_apply_quirks(format_to_try + i);
break; break;
} }
} }
@ -389,7 +369,7 @@ int va_init() {
continue; continue;
} }
else { else {
va->selected_fmt = vaImgFmt_from_vaFmtData(format_to_try + i); va->selected_fmt = vaImgFmt_apply_quirks(format_to_try + i);
break; break;
} }
} }