kms_vaapi: fix regression, screen rotation not working when capturing screen

This commit is contained in:
dec05eba 2023-05-12 05:03:12 +02:00
parent 99dbbd07ab
commit 26571094aa
3 changed files with 68 additions and 23 deletions

3
TODO
View File

@ -64,4 +64,5 @@ Enable opus/flac again. It's broken right now when merging audio inputs. The aud
Support vp8/vp9/av1. This is especially important on amd which on some distros (such as Manjaro) where hardware accelerated h264/hevc is disabled in the mesa package. Support vp8/vp9/av1. This is especially important on amd which on some distros (such as Manjaro) where hardware accelerated h264/hevc is disabled in the mesa package.
Support screen (all monitors) capture on amd/intel when no combined plane is found. Support screen (all monitors) capture on amd/intel when no combined plane is found.
Use separate plane (which has offset and pitch) from combined plane instead of the combined plane.

View File

@ -14,6 +14,7 @@
#include <xf86drm.h> #include <xf86drm.h>
#include <xf86drmMode.h> #include <xf86drmMode.h>
#include <libdrm/drm_mode.h>
#define MAX_CONNECTORS 32 #define MAX_CONNECTORS 32
@ -85,6 +86,37 @@ static bool connector_get_property_by_name(int drmfd, drmModeConnectorPtr props,
return false; return false;
} }
static bool plane_is_cursor_plane(int drmfd, uint32_t plane_id) {
drmModeObjectPropertiesPtr props = drmModeObjectGetProperties(drmfd, plane_id, DRM_MODE_OBJECT_PLANE);
if(!props)
return false;
for(uint32_t i = 0; i < props->count_props; ++i) {
drmModePropertyPtr prop = drmModeGetProperty(drmfd, props->props[i]);
if(prop) {
if(strcmp(prop->name, "type") == 0) {
const uint64_t current_enum_value = props->prop_values[i];
bool is_cursor = false;
for(int j = 0; j < prop->count_enums; ++j) {
if(prop->enums[j].value == current_enum_value && strcmp(prop->enums[j].name, "Cursor") == 0) {
is_cursor = true;
break;
}
}
drmModeFreeProperty(prop);
return is_cursor;
}
drmModeFreeProperty(prop);
}
}
drmModeFreeObjectProperties(props);
return false;
}
/* Returns 0 if not found */ /* Returns 0 if not found */
static uint32_t get_connector_by_crtc_id(const connector_to_crtc_map *c2crtc_map, uint32_t crtc_id) { static uint32_t get_connector_by_crtc_id(const connector_to_crtc_map *c2crtc_map, uint32_t crtc_id) {
for(int i = 0; i < c2crtc_map->num_maps; ++i) { for(int i = 0; i < c2crtc_map->num_maps; ++i) {
@ -147,6 +179,9 @@ static int kms_get_plane_ids(gsr_drm *drm) {
continue; continue;
} }
if(plane_is_cursor_plane(drm->drmfd, plane->plane_id))
continue;
// TODO: Fallback to getfb(1)? // TODO: Fallback to getfb(1)?
drmModeFB2Ptr drmfb = drmModeGetFB2(drm->drmfd, plane->fb_id); drmModeFB2Ptr drmfb = drmModeGetFB2(drm->drmfd, plane->fb_id);
if(drmfb) { if(drmfb) {

View File

@ -45,6 +45,7 @@ typedef struct {
gsr_kms_client kms_client; gsr_kms_client kms_client;
gsr_kms_response kms_response; gsr_kms_response kms_response;
vec2i screen_size;
vec2i capture_pos; vec2i capture_pos;
vec2i capture_size; vec2i capture_size;
bool screen_capture; bool screen_capture;
@ -136,6 +137,10 @@ static bool properties_has_atom(Atom *props, int nprop, Atom atom) {
static void monitor_callback(const XRROutputInfo *output_info, const XRRCrtcInfo *crt_info, const XRRModeInfo *mode_info, void *userdata) { static void monitor_callback(const XRROutputInfo *output_info, const XRRCrtcInfo *crt_info, const XRRModeInfo *mode_info, void *userdata) {
MonitorCallbackUserdata *monitor_callback_userdata = userdata; MonitorCallbackUserdata *monitor_callback_userdata = userdata;
++monitor_callback_userdata->num_monitors; ++monitor_callback_userdata->num_monitors;
if(strcmp(monitor_callback_userdata->monitor_to_capture, "screen") == 0)
monitor_callback_userdata->rotation = crt_info->rotation;
if(monitor_callback_userdata->monitor_to_capture_len != output_info->nameLen || memcmp(monitor_callback_userdata->monitor_to_capture, output_info->name, output_info->nameLen) != 0) if(monitor_callback_userdata->monitor_to_capture_len != output_info->nameLen || memcmp(monitor_callback_userdata->monitor_to_capture, output_info->name, output_info->nameLen) != 0)
return; return;
@ -190,12 +195,14 @@ static int gsr_capture_kms_vaapi_start(gsr_capture *cap, AVCodecContext *video_c
}; };
for_each_active_monitor_output(cap_kms->dpy, monitor_callback, &monitor_callback_userdata); for_each_active_monitor_output(cap_kms->dpy, monitor_callback, &monitor_callback_userdata);
cap_kms->screen_size.x = WidthOfScreen(DefaultScreenOfDisplay(cap_kms->dpy));
cap_kms->screen_size.y = HeightOfScreen(DefaultScreenOfDisplay(cap_kms->dpy));
gsr_monitor monitor; gsr_monitor monitor;
if(strcmp(cap_kms->params.display_to_capture, "screen") == 0) { if(strcmp(cap_kms->params.display_to_capture, "screen") == 0) {
monitor.pos.x = 0; monitor.pos.x = 0;
monitor.pos.y = 0; monitor.pos.y = 0;
monitor.size.x = WidthOfScreen(DefaultScreenOfDisplay(cap_kms->dpy)); monitor.size = cap_kms->screen_size;
monitor.size.y = HeightOfScreen(DefaultScreenOfDisplay(cap_kms->dpy));
cap_kms->screen_capture = true; cap_kms->screen_capture = true;
} else if(!get_monitor_by_name(cap_kms->dpy, cap_kms->params.display_to_capture, &monitor)) { } else if(!get_monitor_by_name(cap_kms->dpy, cap_kms->params.display_to_capture, &monitor)) {
fprintf(stderr, "gsr error: gsr_capture_kms_vaapi_start: failed to find monitor by name \"%s\"\n", cap_kms->params.display_to_capture); fprintf(stderr, "gsr error: gsr_capture_kms_vaapi_start: failed to find monitor by name \"%s\"\n", cap_kms->params.display_to_capture);
@ -412,6 +419,22 @@ static gsr_kms_response_fd* find_first_combined_drm(gsr_kms_response *kms_respon
return NULL; return NULL;
} }
static gsr_kms_response_fd* find_largest_drm(gsr_kms_response *kms_response) {
if(kms_response->num_fds == 0)
return NULL;
int64_t largest_size = 0;
gsr_kms_response_fd *largest_drm = &kms_response->fds[0];
for(int i = 0; i < kms_response->num_fds; ++i) {
const int64_t size = (int64_t)kms_response->fds[i].width * (int64_t)kms_response->fds[i].height;
if(size > largest_size) {
largest_size = size;
largest_drm = &kms_response->fds[i];
}
}
return largest_drm;
}
static int gsr_capture_kms_vaapi_capture(gsr_capture *cap, AVFrame *frame) { static int gsr_capture_kms_vaapi_capture(gsr_capture *cap, AVFrame *frame) {
gsr_capture_kms_vaapi *cap_kms = cap->priv; gsr_capture_kms_vaapi *cap_kms = cap->priv;
@ -442,16 +465,9 @@ static int gsr_capture_kms_vaapi_capture(gsr_capture *cap, AVFrame *frame) {
gsr_kms_response_fd *drm_fd = NULL; gsr_kms_response_fd *drm_fd = NULL;
if(cap_kms->screen_capture) { if(cap_kms->screen_capture) {
drm_fd = find_first_combined_drm(&cap_kms->kms_response); drm_fd = find_first_combined_drm(&cap_kms->kms_response);
if(drm_fd) { if(!drm_fd)
capture_is_combined_plane = true; drm_fd = find_largest_drm(&cap_kms->kms_response);
} else { capture_is_combined_plane = drm_fd->is_combined_plane || ((int)drm_fd->width == cap_kms->screen_size.x && (int)drm_fd->height == cap_kms->screen_size.y);
static bool error_shown = false;
if(!error_shown) {
error_shown = true;
fprintf(stderr, "gsr warning: no combined drm found, screen capture will capture the first monitor found instead\n");
}
drm_fd = &cap_kms->kms_response.fds[0];
}
} else { } else {
for(int i = 0; i < cap_kms->monitor_id.num_connector_ids; ++i) { for(int i = 0; i < cap_kms->monitor_id.num_connector_ids; ++i) {
drm_fd = find_drm_by_connector_id(&cap_kms->kms_response, cap_kms->monitor_id.connector_ids[i]); drm_fd = find_drm_by_connector_id(&cap_kms->kms_response, cap_kms->monitor_id.connector_ids[i]);
@ -464,16 +480,9 @@ static int gsr_capture_kms_vaapi_capture(gsr_capture *cap, AVFrame *frame) {
if(!drm_fd) { if(!drm_fd) {
drm_fd = find_first_combined_drm(&cap_kms->kms_response); drm_fd = find_first_combined_drm(&cap_kms->kms_response);
if(drm_fd) { if(!drm_fd)
capture_is_combined_plane = true; drm_fd = find_largest_drm(&cap_kms->kms_response);
} else { capture_is_combined_plane = drm_fd->is_combined_plane || ((int)drm_fd->width == cap_kms->screen_size.x && (int)drm_fd->height == cap_kms->screen_size.y);
static bool error_shown = false;
if(!error_shown) {
error_shown = true;
fprintf(stderr, "gsr error: no drm found for monitor and no combined drm found, capture will fail\n");
}
return -1;
}
} }
} }