Fix wayland capture on amd/intel

TODO

@@ -79,5 +79,7 @@ Test kms_cuda on hyprland and other wlroots based compositor to see if it works.
 Support "screen" (all monitors) capture on wayland. This should be done by getting all drm fds and multiple EGL_DMA_BUF_PLANEX_FD_EXT to create one egl image with all fds combined.
 
 Support pipewire screen capture?
-Support screen rotation in nvidia wayland.
+Support screen rotation in amd/intel/nvidia wayland.
 Support wlroots dmabuf screen recording, because it doesn't require root access unlike kms grab.
+Capture cursor on amd/intel wayland without xwayland.
+When nvidia supports hardware cursor then capture the cursor. Right now the cursor is captured because it's a software cursor so it's composed on the dma buf.

build.sh

@@ -45,11 +45,11 @@ build_gsr() {
     $CC -c src/cursor.c $opts $includes
     $CC -c src/utils.c $opts $includes
     $CC -c src/library_loader.c $opts $includes
-    $CC -c external/wlr-export-dmabuf-unstable-v1-protocol.c $opts $includes
+    #$CC -c external/wlr-export-dmabuf-unstable-v1-protocol.c $opts $includes
     $CXX -c src/sound.cpp $opts $includes
     $CXX -c src/main.cpp $opts $includes
     $CXX -o gpu-screen-recorder -O2 capture.o nvfbc.o kms_client.o egl.o cuda.o xnvctrl.o overclock.o window_texture.o shader.o \
-        color_conversion.o cursor.o utils.o library_loader.o xcomposite_cuda.o xcomposite_vaapi.o kms_vaapi.o kms_cuda.o wlr-export-dmabuf-unstable-v1-protocol.o sound.o main.o $libs $opts
+        color_conversion.o cursor.o utils.o library_loader.o xcomposite_cuda.o xcomposite_vaapi.o kms_vaapi.o kms_cuda.o sound.o main.o $libs $opts
 }
 
 #build_wayland_protocol

kms/server/kms_server.c

@@ -92,8 +92,11 @@ static bool plane_is_cursor_plane(int drmfd, uint32_t plane_id) {
 
     for(uint32_t i = 0; i < props->count_props; ++i) {
         drmModePropertyPtr prop = drmModeGetProperty(drmfd, props->props[i]);
-        if(prop) {
+        if(!prop)
-            if(strcmp(prop->name, "type") == 0) {
+            continue;
+
+        const uint32_t type = prop->flags & (DRM_MODE_PROP_LEGACY_TYPE | DRM_MODE_PROP_EXTENDED_TYPE);
+        if((type & DRM_MODE_PROP_ENUM) && strcmp(prop->name, "type") == 0) {
             const uint64_t current_enum_value = props->prop_values[i];
             bool is_cursor = false;
 
@@ -102,7 +105,6 @@ static bool plane_is_cursor_plane(int drmfd, uint32_t plane_id) {
                     is_cursor = true;
                     break;
                 }
-
             }
 
             drmModeFreeProperty(prop);
@@ -110,7 +112,6 @@ static bool plane_is_cursor_plane(int drmfd, uint32_t plane_id) {
         }
         drmModeFreeProperty(prop);
     }
-    }
 
     drmModeFreeObjectProperties(props);
     return false;

src/capture/kms_cuda.c

@@ -119,13 +119,12 @@ static void monitor_callback(const gsr_monitor *monitor, void *userdata) {
     if(monitor_callback_userdata->monitor_to_capture_len != monitor->name_len || memcmp(monitor_callback_userdata->monitor_to_capture, monitor->name, monitor->name_len) != 0)
         return;
 
-    const int connector_index = monitor_callback_userdata->cap_kms->monitor_id.num_connector_ids;
+    if(monitor_callback_userdata->cap_kms->monitor_id.num_connector_ids < MAX_CONNECTOR_IDS) {
-    if(connector_index < MAX_CONNECTOR_IDS) {
+        monitor_callback_userdata->cap_kms->monitor_id.connector_ids[monitor_callback_userdata->cap_kms->monitor_id.num_connector_ids] = monitor->connector_id;
-        monitor_callback_userdata->cap_kms->monitor_id.connector_ids[connector_index] = monitor->connector_id;
         ++monitor_callback_userdata->cap_kms->monitor_id.num_connector_ids;
     }
 
-    if(connector_index == MAX_CONNECTOR_IDS)
+    if(monitor_callback_userdata->cap_kms->monitor_id.num_connector_ids == MAX_CONNECTOR_IDS)
         fprintf(stderr, "gsr warning: reached max connector ids\n");
 }
 

src/capture/kms_vaapi.c

@@ -119,11 +119,12 @@ static bool drm_create_codec_context(gsr_capture_kms_vaapi *cap_kms, AVCodecCont
 
 typedef struct {
     gsr_capture_kms_vaapi *cap_kms;
-    const Atom randr_connector_id_atom;
+    Atom randr_connector_id_atom;
     const char *monitor_to_capture;
     int monitor_to_capture_len;
     int num_monitors;
     int rotation;
+    bool wayland;
 } MonitorCallbackUserdata;
 
 static bool properties_has_atom(Atom *props, int nprop, Atom atom) {
@@ -135,15 +136,22 @@ static bool properties_has_atom(Atom *props, int nprop, Atom atom) {
 }
 
 static void monitor_callback(const gsr_monitor *monitor, void *userdata) {
+    (void)monitor;
     MonitorCallbackUserdata *monitor_callback_userdata = userdata;
     ++monitor_callback_userdata->num_monitors;
 
-    if(strcmp(monitor_callback_userdata->monitor_to_capture, "screen") == 0)
-        monitor_callback_userdata->rotation = monitor->crt_info->rotation;
-
     if(monitor_callback_userdata->monitor_to_capture_len != monitor->name_len || memcmp(monitor_callback_userdata->monitor_to_capture, monitor->name, monitor->name_len) != 0)
         return;
 
+    if(monitor_callback_userdata->wayland) {
+        if(monitor_callback_userdata->cap_kms->monitor_id.num_connector_ids < MAX_CONNECTOR_IDS) {
+            monitor_callback_userdata->cap_kms->monitor_id.connector_ids[monitor_callback_userdata->cap_kms->monitor_id.num_connector_ids] = monitor->connector_id;
+            ++monitor_callback_userdata->cap_kms->monitor_id.num_connector_ids;
+        }
+    } else {
+        if(strcmp(monitor_callback_userdata->monitor_to_capture, "screen") == 0)
+            monitor_callback_userdata->rotation = monitor->crt_info->rotation;
+
         monitor_callback_userdata->rotation = monitor->crt_info->rotation;
         for(int i = 0; i < monitor->crt_info->noutput && monitor_callback_userdata->cap_kms->monitor_id.num_connector_ids < MAX_CONNECTOR_IDS; ++i) {
             int nprop = 0;
@@ -173,6 +181,7 @@ static void monitor_callback(const gsr_monitor *monitor, void *userdata) {
 
             XFree(props);
         }
+    }
 
     if(monitor_callback_userdata->cap_kms->monitor_id.num_connector_ids == MAX_CONNECTOR_IDS)
         fprintf(stderr, "gsr warning: reached max connector ids\n");
@@ -188,18 +197,30 @@ static int gsr_capture_kms_vaapi_start(gsr_capture *cap, AVCodecContext *video_c
     void *connection = cap_kms->params.wayland ? (void*)cap_kms->params.card_path : (void*)cap_kms->dpy;
     const gsr_connection_type connection_type = cap_kms->params.wayland ? GSR_CONNECTION_DRM : GSR_CONNECTION_X11;
 
-    const Atom randr_connector_id_atom = XInternAtom(cap_kms->dpy, "CONNECTOR_ID", False);
-    cap_kms->monitor_id.num_connector_ids = 0;
     MonitorCallbackUserdata monitor_callback_userdata = {
-        cap_kms, randr_connector_id_atom,
+        cap_kms, None,
         cap_kms->params.display_to_capture, strlen(cap_kms->params.display_to_capture),
         0,
-        X11_ROT_0
+        X11_ROT_0,
+        true
     };
+
+    if(cap_kms->params.wayland) {
+        cap_kms->monitor_id.num_connector_ids = 0;
+        for_each_active_monitor_output(connection, connection_type, monitor_callback, &monitor_callback_userdata);
+
+        cap_kms->screen_size.x = 0;
+        cap_kms->screen_size.y = 0;
+    } else {
+        const Atom randr_connector_id_atom = XInternAtom(cap_kms->dpy, "CONNECTOR_ID", False);
+        cap_kms->monitor_id.num_connector_ids = 0;
+        monitor_callback_userdata.randr_connector_id_atom = randr_connector_id_atom;
+        monitor_callback_userdata.wayland = false;
         for_each_active_monitor_output(connection, connection_type, monitor_callback, &monitor_callback_userdata);
 
         cap_kms->screen_size.x = WidthOfScreen(DefaultScreenOfDisplay(cap_kms->dpy));
         cap_kms->screen_size.y = HeightOfScreen(DefaultScreenOfDisplay(cap_kms->dpy));
+    }
 
     gsr_monitor monitor;
     if(strcmp(cap_kms->params.display_to_capture, "screen") == 0) {
@@ -244,6 +265,7 @@ static int gsr_capture_kms_vaapi_start(gsr_capture *cap, AVCodecContext *video_c
         return -1;
     }
 
+    if(cap_kms->dpy) {
         if(gsr_cursor_init(&cap_kms->cursor, &cap_kms->egl, cap_kms->dpy) != 0) {
             gsr_capture_kms_vaapi_stop(cap, video_codec_context);
             return -1;
@@ -251,6 +273,7 @@ static int gsr_capture_kms_vaapi_start(gsr_capture *cap, AVCodecContext *video_c
 
         gsr_cursor_change_window_target(&cap_kms->cursor, DefaultRootWindow(cap_kms->dpy));
         gsr_cursor_update(&cap_kms->cursor, &cap_kms->xev);
+    }
 
     return 0;
 }
@@ -267,10 +290,12 @@ static void gsr_capture_kms_vaapi_tick(gsr_capture *cap, AVCodecContext *video_c
     // TODO:
     cap_kms->egl.glClear(GL_COLOR_BUFFER_BIT);
 
+    if(cap_kms->dpy) {
         while(XPending(cap_kms->dpy)) {
             XNextEvent(cap_kms->dpy, &cap_kms->xev);
             gsr_cursor_update(&cap_kms->cursor, &cap_kms->xev);
         }
+    }
 
     if(!cap_kms->created_hw_frame) {
         cap_kms->created_hw_frame = true;
@@ -596,6 +621,7 @@ static void gsr_capture_kms_vaapi_stop(gsr_capture *cap, AVCodecContext *video_c
         }
     }
 
+    if(cap_kms->egl.egl_context) {
         if(cap_kms->input_texture) {
             cap_kms->egl.glDeleteTextures(1, &cap_kms->input_texture);
             cap_kms->input_texture = 0;
@@ -604,6 +630,7 @@ static void gsr_capture_kms_vaapi_stop(gsr_capture *cap, AVCodecContext *video_c
         cap_kms->egl.glDeleteTextures(2, cap_kms->target_textures);
         cap_kms->target_textures[0] = 0;
         cap_kms->target_textures[1] = 0;
+    }
 
     for(int i = 0; i < cap_kms->kms_response.num_fds; ++i) {
         if(cap_kms->kms_response.fds[i].fd > 0)
@@ -656,7 +683,7 @@ gsr_capture* gsr_capture_kms_vaapi_create(const gsr_capture_kms_vaapi_params *pa
     }
 
     Display *display = XOpenDisplay(NULL);
-    if(!display) {
+    if(!params->wayland && !display) {
         fprintf(stderr, "gsr error: gsr_capture_kms_vaapi_create failed: XOpenDisplay failed\n");
         free(cap);
         free(cap_kms);

src/main.cpp

@@ -1449,7 +1449,7 @@ int main(int argc, char **argv) {
             kms_params.display_to_capture = capture_target;
             kms_params.gpu_inf = gpu_inf;
             kms_params.card_path = card_path;
-            kms_params.wayland = false;//wayland;
+            kms_params.wayland = wayland;
             capture = gsr_capture_kms_vaapi_create(&kms_params);
             if(!capture)
                 _exit(1);