diff --git a/kitty/glfw.c b/kitty/glfw.c
index a65079b7d..9bc39b1f9 100644
--- a/kitty/glfw.c
+++ b/kitty/glfw.c
@@ -29,13 +29,11 @@ extern bool cocoa_make_window_resizable(void *w);
 
 static GLFWcursor *standard_cursor = NULL, *click_cursor = NULL, *arrow_cursor = NULL;
 
-#define GLFW_WINDOW(w) ((GLFWwindow*)((w)->handle))
-
 static void 
 update_viewport(OSWindow *window) {
     int w, h;
-    glfwGetFramebufferSize(GLFW_WINDOW(window), &window->viewport_width, &window->viewport_height);
-    glfwGetWindowSize(GLFW_WINDOW(window), &w, &h);
+    glfwGetFramebufferSize(window->handle, &window->viewport_width, &window->viewport_height);
+    glfwGetWindowSize(window->handle, &w, &h);
     window->viewport_x_ratio = (double)window->viewport_width / (double)w;
     window->viewport_y_ratio = (double)window->viewport_height / (double)h;
     window->viewport_size_dirty = true;
@@ -43,6 +41,17 @@ update_viewport(OSWindow *window) {
 
 // callbacks {{{
 
+void
+remove_os_window_reference(OSWindow *w) { if (w->handle) glfwSetWindowUserPointer(w->handle, NULL); }
+
+static inline void 
+update_os_window_references() {
+    for (size_t i = 0; i < global_state.num_os_windows; i++) {
+        OSWindow *w = global_state.os_windows + i;
+        if (w->handle) glfwSetWindowUserPointer(w->handle, w);
+    }
+}
+
 static inline bool
 set_callback_window(GLFWwindow *w) {
     global_state.callback_os_window = glfwGetWindowUserPointer(w);
@@ -229,8 +238,8 @@ create_os_window(PyObject UNUSED *self, PyObject *args) {
     OSWindow *w = add_os_window();
     w->id = ++global_state.os_window_id_counter;
     w->handle = glfw_window;
+    update_os_window_references();
     if (logo.pixels && logo.width && logo.height) glfwSetWindowIcon(glfw_window, 1, &logo);
-    glfwSetWindowUserPointer(glfw_window, w);
     glfwSetCursor(glfw_window, standard_cursor);
     update_viewport(w);
     glfwSetFramebufferSizeCallback(glfw_window, framebuffer_size_callback);
diff --git a/kitty/state.c b/kitty/state.c
index 48b8df1b9..00fab939c 100644
--- a/kitty/state.c
+++ b/kitty/state.c
@@ -36,6 +36,20 @@ GlobalState global_state = {{0}};
                     Tab *tab = osw->tabs + t;
 #define END_WITH_TAB break; }}}}
 
+#define WITH_OS_WINDOW_REFS \
+    id_type cb_window_id = 0, focused_window_id = 0; \
+    if (global_state.callback_os_window) cb_window_id = global_state.callback_os_window->id; \
+    if (global_state.focused_os_window) focused_window_id = global_state.focused_os_window->id;
+
+#define END_WITH_OS_WINDOW_REFS \
+    if (cb_window_id || focused_window_id) { \
+        for (size_t wn = 0; wn < global_state.num_os_windows; wn++) { \
+            OSWindow *wp = global_state.os_windows + wn; \
+            if (wp->id == cb_window_id && cb_window_id) global_state.callback_os_window = wp; \
+            if (wp->id == focused_window_id && focused_window_id) global_state.focused_os_window = wp; \
+    }}
+
+
 OSWindow* 
 current_os_window() {
     if (global_state.callback_os_window) return global_state.callback_os_window;
@@ -59,10 +73,12 @@ os_window_for_kitty_window(id_type kitty_window_id) {
 
 OSWindow*
 add_os_window() {
+    WITH_OS_WINDOW_REFS
     ensure_space_for(&global_state, os_windows, OSWindow, global_state.num_os_windows + 1, capacity, 1, true);
     OSWindow *ans = global_state.os_windows + global_state.num_os_windows++;
     memset(ans, 0, sizeof(OSWindow));
     ans->tab_bar_render_data.vao_idx = create_cell_vao();
+    END_WITH_OS_WINDOW_REFS
     return ans;
 }
 
@@ -159,11 +175,14 @@ bool
 remove_os_window(id_type os_window_id, int *viewport_width, int *viewport_height) {
     bool found = false;
     WITH_OS_WINDOW(os_window_id)
+        remove_os_window_reference(os_window);
         *viewport_width = os_window->viewport_width; *viewport_height = os_window->viewport_height;
         found = true;
     END_WITH_OS_WINDOW
     if (found) { 
-        REMOVER(global_state.os_windows, os_window_id, global_state.num_os_windows, OSWindow, destroy_os_window, global_state.capacity);
+        WITH_OS_WINDOW_REFS
+            REMOVER(global_state.os_windows, os_window_id, global_state.num_os_windows, OSWindow, destroy_os_window, global_state.capacity);
+        END_WITH_OS_WINDOW_REFS
     }
     return found;
 }
diff --git a/kitty/state.h b/kitty/state.h
index ef3f20149..8576f3176 100644
--- a/kitty/state.h
+++ b/kitty/state.h
@@ -129,6 +129,7 @@ extern GlobalState global_state;
 void gl_init();
 void remove_vao(ssize_t vao_idx);
 bool remove_os_window(id_type os_window_id, int*, int*);
+void remove_os_window_reference(OSWindow *w);
 void mark_os_window_for_close(OSWindow* w, bool yes);
 bool should_os_window_close(OSWindow* w);
 bool should_os_window_be_rendered(OSWindow* w);