macOS: Fix a regression causing *burn-in* of text when resizing semi-transparent OS windows

Fixes #6439

docs/changelog.rst

@@ -40,6 +40,8 @@ Detailed list of changes
 
 - macOS: Fix a regression that caused rendering to hang when transitioning to full screen with :opt:`macos_colorspace` set to ``default`` (:iss:`6435`)
 
+- macOS: Fix a regression causing *burn-in* of text when resizing semi-transparent OS windows (:iss:`6439`)
+
 - Remote control launch: Fix ``--env`` not implemented when using ``--cwd=current`` with the SSH kitten (:iss:`6438`)
 
 0.29.0 [2023-07-10]

kitty/cocoa_window.m

@@ -737,6 +737,21 @@ cocoa_focus_window(void *w) {
     [window makeKeyWindow];
 }
 
+bool
+cocoa_set_shadow(void *w, bool has_shadow) {
+    NSWindow *window = (NSWindow*)w;
+    bool current = window.hasShadow;
+    [window setHasShadow:has_shadow];
+    if (has_shadow) {  // invalidate the shadow to clear the ghosting
+        [window invalidateShadow];
+        // invalidateShadow does not work with layer backed views, see http://www.openradar.me/34184270
+        // so do the frame resize dance to force AppKit to invalidate the shadow even with the layer
+        [window setFrame:NSMakeRect(NSMinX(window.frame), NSMinY(window.frame), NSWidth(window.frame)+1, NSHeight(window.frame)) display:NO animate:NO];
+        [window setFrame:NSMakeRect(NSMinX(window.frame), NSMinY(window.frame), NSWidth(window.frame)-1, NSHeight(window.frame)) display:NO animate:NO];
+    }
+    return current;
+}
+
 long
 cocoa_window_number(void *w) {
     NSWindow *window = (NSWindow*)w;

kitty/glfw.c

@@ -305,6 +305,7 @@ cocoa_out_of_sequence_render(OSWindow *window) {
 static void
 cocoa_os_window_resized(GLFWwindow *w) {
     if (!set_callback_window(w)) return;
+    if (global_state.callback_os_window->ignore_resize_events) return;
     cocoa_out_of_sequence_render(global_state.callback_os_window);
     global_state.callback_os_window = NULL;
 }
@@ -319,6 +320,24 @@ change_live_resize_state(OSWindow *w, bool in_progress) {
         w->live_resize.num_of_resize_events = 0;
         apply_swap_interval(in_progress ? 0 : -1);
 #ifdef __APPLE__
+        extern bool cocoa_set_shadow(void*, bool);
+        if (w->is_semi_transparent && w->handle) {
+            // shadow with transparency during resize causes burn-in of window contents
+            // aka ghosting of the text after resize is completed. So avoid that by turning
+            // the shadow on and off.
+            if (in_progress) {
+                bool had_shadow = cocoa_set_shadow(glfwGetCocoaWindow(w->handle), false);
+                w->has_cocoa_shadow = had_shadow;
+            } else {
+                if (w->has_cocoa_shadow) {
+                    // turning on the shadow will cause a resize event which will be delivered before cocoa_set_shadow returns
+                    w->ignore_resize_events = true;
+                    cocoa_set_shadow(glfwGetCocoaWindow(w->handle), true);
+                    w->ignore_resize_events = false;
+                    w->has_cocoa_shadow = false;
+                }
+            }
+        }
         cocoa_out_of_sequence_render(w);
 #endif
     }
@@ -327,6 +346,7 @@ change_live_resize_state(OSWindow *w, bool in_progress) {
 static void
 live_resize_callback(GLFWwindow *w, bool started) {
     if (!set_callback_window(w)) return;
+    if (global_state.callback_os_window->ignore_resize_events) return;
     global_state.callback_os_window->live_resize.from_os_notification = true;
     change_live_resize_state(global_state.callback_os_window, true);
     global_state.has_pending_resizes = true;
@@ -340,6 +360,7 @@ live_resize_callback(GLFWwindow *w, bool started) {
 static void
 framebuffer_size_callback(GLFWwindow *w, int width, int height) {
     if (!set_callback_window(w)) return;
+    if (global_state.callback_os_window->ignore_resize_events) return;
     int min_width, min_height; min_size_for_os_window(global_state.callback_os_window, &min_width, &min_height);
     if (width >= min_width && height >= min_height) {
         OSWindow *window = global_state.callback_os_window;
@@ -358,6 +379,7 @@ framebuffer_size_callback(GLFWwindow *w, int width, int height) {
 static void
 dpi_change_callback(GLFWwindow *w, float x_scale UNUSED, float y_scale UNUSED) {
     if (!set_callback_window(w)) return;
+    if (global_state.callback_os_window->ignore_resize_events) return;
     // Ensure update_os_window_viewport() is called in the near future, it will
     // take care of DPI changes.
     OSWindow *window = global_state.callback_os_window;

kitty/state.h

@@ -229,7 +229,7 @@ typedef struct {
     bool viewport_size_dirty, viewport_updated_at_least_once;
     monotonic_t viewport_resized_at;
     LiveResizeInfo live_resize;
-    bool has_pending_resizes, is_semi_transparent, shown_once, is_damaged;
+    bool has_pending_resizes, is_semi_transparent, shown_once, is_damaged, has_cocoa_shadow, ignore_resize_events;
     unsigned int clear_count;
     WindowChromeState last_window_chrome;
     float background_opacity;