diff --git a/kitty/boss.py b/kitty/boss.py
index 59357b201..d768166f8 100644
--- a/kitty/boss.py
+++ b/kitty/boss.py
@@ -218,6 +218,10 @@ class Boss:
             self.opts, override_font_size=self.current_font_size)
         self.sprites.do_layout(cell_size.width, cell_size.height)
         self.resize_windows_after_font_size_change()
+        for window in self.window_id_map.values():
+            if window is not None:
+                window.screen.refresh_sprite_positions()
+        self.tab_manager.refresh_sprite_positions()
 
     def resize_windows_after_font_size_change(self):
         self.tab_manager.resize()
diff --git a/kitty/data-types.h b/kitty/data-types.h
index d50169996..87a656578 100644
--- a/kitty/data-types.h
+++ b/kitty/data-types.h
@@ -353,10 +353,12 @@ void linebuf_delete_lines(LineBuf *self, index_type num, index_type y, index_typ
 void linebuf_set_attribute(LineBuf *, unsigned int , unsigned int );
 void linebuf_rewrap(LineBuf *self, LineBuf *other, int *cursor_y_out, HistoryBuf *);
 unsigned int linebuf_char_width_at(LineBuf *self, index_type x, index_type y);
+void linebuf_refresh_sprite_positions(LineBuf *self);
 bool historybuf_resize(HistoryBuf *self, index_type lines);
 void historybuf_add_line(HistoryBuf *self, const Line *line);
 void historybuf_rewrap(HistoryBuf *self, HistoryBuf *other);
 void historybuf_init_line(HistoryBuf *self, index_type num, Line *l);
+void historybuf_refresh_sprite_positions(HistoryBuf *self);
 
 double monotonic();
 double timers_timeout(Timers*);
diff --git a/kitty/history.c b/kitty/history.c
index e669f18ca..c5ab589b7 100644
--- a/kitty/history.c
+++ b/kitty/history.c
@@ -56,6 +56,13 @@ dealloc(HistoryBuf* self) {
     Py_TYPE(self)->tp_free((PyObject*)self);
 }
 
+void 
+historybuf_refresh_sprite_positions(HistoryBuf *self) {
+    for (index_type i = 0; i < self->ynum; i++) {
+        update_sprites_in_line(lineptr(self, i), self->xnum);
+    }
+}
+
 static inline index_type 
 index_of(HistoryBuf *self, index_type lnum) {
     // The index (buffer position) of the line with line number lnum
diff --git a/kitty/line-buf.c b/kitty/line-buf.c
index 98001bd97..46e45cf6f 100644
--- a/kitty/line-buf.c
+++ b/kitty/line-buf.c
@@ -359,6 +359,13 @@ as_ansi(LineBuf *self, PyObject *callback) {
     Py_RETURN_NONE;
 }
 
+void 
+linebuf_refresh_sprite_positions(LineBuf *self) {
+    for (index_type i = 0; i < self->ynum; i++) {
+        update_sprites_in_line(lineptr(self, i), self->xnum);
+    }
+}
+
 // Boilerplate {{{
 static PyObject*
 copy_old(LineBuf *self, PyObject *y);
diff --git a/kitty/lineops.h b/kitty/lineops.h
index b27f21ed3..db7a3e1d8 100644
--- a/kitty/lineops.h
+++ b/kitty/lineops.h
@@ -11,9 +11,11 @@
 static inline void
 update_sprites_in_line(Cell *cells, index_type xnum) {
     if (LIKELY(xnum > 0)) {
-        set_sprite_position(cells, NULL);
+        if (CHAR_IS_BLANK(cells->ch)) { clear_sprite_position(cells[0]); }
+        else set_sprite_position(cells, NULL);
         for (index_type i = 1; i < xnum; i++) {
-            set_sprite_position(cells + i, cells + i - 1);
+            if (CHAR_IS_BLANK(cells[i].ch)) { clear_sprite_position(cells[i]); }
+            else set_sprite_position(cells + i, cells + i - 1);
         }
     }
 }
diff --git a/kitty/screen.c b/kitty/screen.c
index f9f76fe93..4cffdda9f 100644
--- a/kitty/screen.c
+++ b/kitty/screen.c
@@ -168,6 +168,15 @@ screen_resize(Screen *self, unsigned int lines, unsigned int columns) {
     return true;
 }
 
+
+static void
+screen_refresh_sprite_positions(Screen *self) {
+    linebuf_refresh_sprite_positions(self->main_linebuf);
+    linebuf_refresh_sprite_positions(self->alt_linebuf);
+    historybuf_refresh_sprite_positions(self->historybuf);
+}
+
+
 static bool 
 screen_change_scrollback_size(Screen *self, unsigned int size) {
     if (size != self->historybuf->ynum) return historybuf_resize(self->historybuf, size);
@@ -1160,6 +1169,7 @@ cursor_up(Screen *self, PyObject *args) {
 
 WRAP0x(index)
 WRAP0(reverse_index)
+WRAP0(refresh_sprite_positions)
 WRAP0(reset)
 WRAP0(set_tab_stop)
 WRAP1(clear_tab_stop, 0)
@@ -1316,6 +1326,7 @@ static PyMethodDef methods[] = {
     MND(set_tab_stop, METH_NOARGS)
     MND(clear_tab_stop, METH_VARARGS)
     MND(reverse_index, METH_NOARGS)
+    MND(refresh_sprite_positions, METH_NOARGS)
     MND(is_dirty, METH_NOARGS)
     MND(mark_as_dirty, METH_NOARGS)
     MND(resize, METH_VARARGS)
diff --git a/kitty/sprites.c b/kitty/sprites.c
index ea1539f20..0e732468d 100644
--- a/kitty/sprites.c
+++ b/kitty/sprites.c
@@ -169,29 +169,22 @@ sprite_position_for(PyObject UNUSED *self, PyObject *args) {
 
 bool
 update_cell_range_data(ScreenModes *modes, Line *line, unsigned int xstart, unsigned int xmax, unsigned int *data) {
-    SpritePosition *sp;
-    char_type previous_ch=0, ch;
-    uint8_t previous_width = 0;
-    int err = 0;
+    char_type ch;
     const bool screen_reversed = modes->mDECSCNM;
 
     size_t base = line->ynum * line->xnum * DATA_CELL_SIZE;
     for (size_t i = xstart, offset = base + xstart * DATA_CELL_SIZE; i <= xmax; i++, offset += DATA_CELL_SIZE) {
         ch = line->cells[i].ch;
-        if (previous_width == 2) sp = sprite_map_position_for(previous_ch, 0, true, &err);
-        else sp = sprite_map_position_for(ch, line->cells[i].cc, false, &err);
-        if (sp == NULL) { sprite_map_set_error(err); return false; }
         char_type attrs = ch >> ATTRS_SHIFT;
         unsigned int decoration = (attrs >> DECORATION_SHIFT) & DECORATION_MASK;
         unsigned int strikethrough = ((attrs >> STRIKE_SHIFT) & 1) ? 3 : 0;
         bool reverse = ((attrs >> REVERSE_SHIFT) & 1) ^ screen_reversed;
-        data[offset] = sp->x;
-        data[offset+1] = sp->y;
-        data[offset+2] = sp->z | (decoration << 24) | (strikethrough << 26);
+        data[offset] = line->cells[i].sprite_x;
+        data[offset+1] = line->cells[i].sprite_y;
+        data[offset+2] = line->cells[i].sprite_z | (decoration << 24) | (strikethrough << 26);
         data[offset+(reverse ? 4 : 3)] = line->cells[i].fg;
         data[offset+(reverse ? 3 : 4)] = line->cells[i].bg;
         data[offset+5] = line->cells[i].fg;
-        previous_ch = ch; previous_width = (attrs) & WIDTH_MASK;
     }
     return true;
 }
diff --git a/kitty/tabs.py b/kitty/tabs.py
index 72e88eb81..193951d00 100644
--- a/kitty/tabs.py
+++ b/kitty/tabs.py
@@ -278,7 +278,6 @@ class TabBar:
 
     def render(self, cell_program, sprites):
         if self.render_buf is not None:
-            sprites.render_dirty_sprites()
             if self.vao_id is None:
                 self.vao_id = cell_program.create_sprite_map()
             if self.dirty:
@@ -302,6 +301,7 @@ class TabManager:
         self.opts, self.args = opts, args
         self.tabs = []
         self.tab_bar = TabBar(opts)
+        self.refresh_sprite_positions = self.tab_bar.screen.refresh_sprite_positions
         self.tab_bar.layout(*self.tab_bar_layout_data)
         self.active_tab_idx = 0