diff --git a/kitty/graphics.c b/kitty/graphics.c
index e5d5b11f4..b99d7f1d0 100644
--- a/kitty/graphics.c
+++ b/kitty/graphics.c
@@ -1299,14 +1299,18 @@ filter_refs(GraphicsManager *self, const void* data, bool free_images, bool (*fi
 }
 
 
-static inline void
-modify_refs(GraphicsManager *self, const void* data, bool free_images, bool (*filter_func)(ImageRef*, Image*, const void*, CellPixelSize), CellPixelSize cell) {
+static void
+modify_refs(GraphicsManager *self, const void* data, bool (*filter_func)(ImageRef*, Image*, const void*, CellPixelSize), CellPixelSize cell) {
     for (size_t i = self->image_count; i-- > 0;) {
         Image *img = self->images + i;
         for (size_t j = img->refcnt; j-- > 0;) {
             if (filter_func(img->refs + j, img, data, cell)) remove_i_from_array(img->refs, j, img->refcnt);
         }
-        if (img->refcnt == 0 && (free_images || img->client_id == 0)) remove_image(self, i);
+        if (img->refcnt == 0 && img->client_id == 0 && img->client_number == 0) {
+            // references have all scrolled off the history buffer and the image has no way to reference it
+            // to create new references so remove it.
+            remove_image(self, i);
+        }
     }
 }
 
@@ -1363,7 +1367,7 @@ void
 grman_scroll_images(GraphicsManager *self, const ScrollData *data, CellPixelSize cell) {
     if (self->image_count) {
         self->layers_dirty = true;
-        modify_refs(self, data, true, data->has_margins ? scroll_filter_margins_func : scroll_filter_func, cell);
+        modify_refs(self, data, data->has_margins ? scroll_filter_margins_func : scroll_filter_func, cell);
     }
 }
 
diff --git a/kitty_tests/graphics.py b/kitty_tests/graphics.py
index e39c655a0..a13313e60 100644
--- a/kitty_tests/graphics.py
+++ b/kitty_tests/graphics.py
@@ -138,7 +138,11 @@ def put_helpers(self, cw, ch):
         nonlocal iid
         iid += 1
         imgid = kw.pop('id', None) or iid
-        cmd = 'a=T,f=24,i=%d,s=%d,v=%d,%s' % (imgid, w, h, put_cmd(**kw))
+        no_id = kw.pop('no_id', False)
+        if no_id:
+            cmd = 'a=T,f=24,s=%d,v=%d,%s' % (w, h, put_cmd(**kw))
+        else:
+            cmd = 'a=T,f=24,i=%d,s=%d,v=%d,%s' % (imgid, w, h, put_cmd(**kw))
         data = b'x' * w * h * 3
         res = send_command(screen, cmd, data)
         return imgid, parse_response(res)
@@ -524,7 +528,7 @@ class TestGraphics(BaseTest):
     def test_gr_scroll(self):
         cw, ch = 10, 20
         s, dx, dy, put_image, put_ref, layers, rect_eq = put_helpers(self, cw, ch)
-        put_image(s, 10, 20)  # a one cell image at (0, 0)
+        put_image(s, 10, 20, no_id=True)  # a one cell image at (0, 0)
         self.ae(len(layers(s)), 1)
         for i in range(s.lines):
             s.index()
@@ -550,7 +554,7 @@ class TestGraphics(BaseTest):
         for i in range(s.lines):  # ensure cursor is at top margin
             s.reverse_index()
         # Test clipped scrolling during index
-        put_image(s, cw, 2*ch, z=-1)  # 1x2 cell image
+        put_image(s, cw, 2*ch, z=-1, no_id=True)  # 1x2 cell image
         self.ae(s.grman.image_count, 3)
         self.ae(layers(s)[0]['src_rect'], {'left': 0.0, 'top': 0.0, 'right': 1.0, 'bottom': 1.0})
         s.index(), s.index()
@@ -562,7 +566,7 @@ class TestGraphics(BaseTest):
         # Test clipped scrolling during reverse_index
         for i in range(s.lines):
             s.reverse_index()
-        put_image(s, cw, 2*ch, z=-1)  # 1x2 cell image
+        put_image(s, cw, 2*ch, z=-1, no_id=True)  # 1x2 cell image
         self.ae(s.grman.image_count, 3)
         self.ae(layers(s)[0]['src_rect'], {'left': 0.0, 'top': 0.0, 'right': 1.0, 'bottom': 1.0})
         while s.cursor.y != 1: