diff --git a/kitty/screen.c b/kitty/screen.c
index 9469e2d24..05a20b64f 100644
--- a/kitty/screen.c
+++ b/kitty/screen.c
@@ -270,10 +270,42 @@ is_selection_empty(const Selection *s) {
     return s->start.x == s->end.x && s->start.in_left_half_of_cell == s->end.in_left_half_of_cell && start_y == end_y;
 }
 
+static bool
+selection_intersects_screen_lines(const Selections *selections, int a, int b) {
+    if (a > b) SWAP(a, b);
+    for (size_t i = 0; i < selections->count; i++) {
+        const Selection *s = selections->items + i;
+        if (!is_selection_empty(s)) {
+            int start = (int)s->start.y - s->start_scrolled_by;
+            int end = (int)s->end.y - s->end_scrolled_by;
+            int top = MIN(start, end);
+            int bottom = MAX(start, end);
+            if ((top <= a && bottom >= a) || (top >= a && top <= b)) return true;
+        }
+    }
+    return false;
+}
+
+
 static void
-index_selection(const Screen *self, Selections *selections, bool up) {
+index_selection(const Screen *self, Selections *selections, bool up, index_type top, index_type bottom) {
+    const bool needs_special_handling = self->linebuf == self->alt_linebuf && (top > 0 || bottom < self->lines - 1);
     for (size_t i = 0; i < selections->count; i++) {
         Selection *s = selections->items + i;
+        if (is_selection_empty(s)) continue;
+        if (needs_special_handling) {
+            int start = (int)s->start.y - s->start_scrolled_by;
+            int end = (int)s->end.y - s->end_scrolled_by;
+            int stop = MIN(start, end);
+            int sbottom = MAX(start, end);
+            if (stop < (int)top) {
+                if (sbottom < (int)top) continue;
+                clear_selection(selections); return;
+            } else {
+                if (stop > (int)bottom) continue;
+                if (sbottom > (int)bottom) { clear_selection(selections); return; }
+            }
+        }
         if (up) {
             if (s->start.y == 0) s->start_scrolled_by += 1;
             else {
@@ -319,7 +351,7 @@ index_selection(const Screen *self, Selections *selections, bool up) {
     } \
     INDEX_GRAPHICS(1) \
     self->is_dirty = true; \
-    index_selection(self, &self->selections, false); \
+    index_selection(self, &self->selections, false, top, bottom); \
     clear_selection(&self->url_ranges);
 
 
@@ -674,23 +706,6 @@ selection_has_screen_line(const Selections *selections, const int y) {
     return false;
 }
 
-static bool
-selection_intersects_screen_lines(const Selections *selections, int a, int b) {
-    if (a > b) SWAP(a, b);
-    for (size_t i = 0; i < selections->count; i++) {
-        const Selection *s = selections->items + i;
-        if (!is_selection_empty(s)) {
-            int start = (int)s->start.y - s->start_scrolled_by;
-            int end = (int)s->end.y - s->end_scrolled_by;
-            int top = MIN(start, end);
-            int bottom = MAX(start, end);
-            if ((top <= a && bottom >= a) || (top >= a && top <= b)) return true;
-        }
-    }
-    return false;
-}
-
-
 static void
 clear_intersecting_selections(Screen *self, index_type y) {
     if (selection_has_screen_line(&self->selections, y)) clear_selection(&self->selections);
@@ -1998,7 +2013,7 @@ screen_cursor_to_column(Screen *self, unsigned int column) {
     } \
     linebuf_clear_line(self->linebuf, bottom, true); \
     self->is_dirty = true; \
-    index_selection(self, &self->selections, true); \
+    index_selection(self, &self->selections, true, top, bottom); \
     clear_selection(&self->url_ranges);
 
 void