diff --git a/kitty/glfw.c b/kitty/glfw.c
index 2eb1e659e..a21c9238e 100644
--- a/kitty/glfw.c
+++ b/kitty/glfw.c
@@ -227,10 +227,38 @@ refresh_callback(GLFWwindow *w) {
 
 static int mods_at_last_key_or_button_event = 0;
 
+static inline int
+key_to_modifier(int key) {
+    switch(key) {
+        case GLFW_KEY_LEFT_SHIFT:
+        case GLFW_KEY_RIGHT_SHIFT:
+            return GLFW_MOD_SHIFT;
+        case GLFW_KEY_LEFT_CONTROL:
+        case GLFW_KEY_RIGHT_CONTROL:
+            return GLFW_MOD_CONTROL;
+        case GLFW_KEY_LEFT_ALT:
+        case GLFW_KEY_RIGHT_ALT:
+            return GLFW_MOD_ALT;
+        case GLFW_KEY_LEFT_SUPER:
+        case GLFW_KEY_RIGHT_SUPER:
+            return GLFW_MOD_SUPER;
+        default:
+            return -1;
+    }
+}
+
 static void
 key_callback(GLFWwindow *w, GLFWkeyevent *ev) {
     if (!set_callback_window(w)) return;
     mods_at_last_key_or_button_event = ev->mods;
+    int key_modifier = key_to_modifier(ev->key);
+    if (key_modifier != -1) {
+        if (ev->action == GLFW_RELEASE) {
+            mods_at_last_key_or_button_event &= ~key_modifier;
+        } else {
+            mods_at_last_key_or_button_event |= key_modifier;
+        }
+    }
     global_state.callback_os_window->cursor_blink_zero_time = monotonic();
     if (ev->key >= 0 && ev->key <= GLFW_KEY_LAST) {
         global_state.callback_os_window->is_key_pressed[ev->key] = ev->action == GLFW_RELEASE ? false : true;