diff --git a/kitty/config.py b/kitty/config.py
index 96f1434a4..4eff5981e 100644
--- a/kitty/config.py
+++ b/kitty/config.py
@@ -124,4 +124,13 @@ def build_ansi_color_tables(opts: Options) -> Tuple[dict, dict]:
     bg = {40 + i: getattr(opts, 'color{}'.format(i)) for i in range(8)}
     bg[49] = opts.background
     bg.update({100 + i: getattr(opts, 'color{}'.format(i + 8)) for i in range(8)})
-    return fg, bg
+    build_ansi_color_tables.fg, build_ansi_color_tables.bg = fg, bg
+build_ansi_color_tables(defaults)
+
+
+def fg_color_table():
+    return build_ansi_color_tables.fg
+
+
+def bg_color_table():
+    return build_ansi_color_tables.bg
diff --git a/kitty/data_types.py b/kitty/data_types.py
index 0920e43c4..5f3126776 100644
--- a/kitty/data_types.py
+++ b/kitty/data_types.py
@@ -8,6 +8,8 @@ from itertools import repeat
 
 from PyQt5.QtGui import QColor
 
+from .config import fg_color_table, bg_color_table
+
 code = 'I' if array.array('I').itemsize >= 4 else 'L'
 lcode = 'L' if array.array('L').itemsize >= 8 else 'Q'
 
@@ -59,6 +61,9 @@ class Cursor:
         return self.__class__.__name__ + '({})'.format(', '.join(
             '{}={}'.format(x, getattr(self, x)) for x in self.__slots__))
 
+    def colors(self):
+        return as_color(self.fg, fg_color_table()), as_color(self.bg, bg_color_table()), as_color(self.decoration_fg, fg_color_table())
+
 CHAR_MASK = 0xFFFFFF
 ATTRS_SHIFT = 24
 ATTRS_MASK = 0xFF << ATTRS_SHIFT
diff --git a/kitty/term.py b/kitty/term.py
index c22b14452..442f9891c 100644
--- a/kitty/term.py
+++ b/kitty/term.py
@@ -9,7 +9,7 @@ from PyQt5.QtGui import QColor, QPainter, QFont, QFontMetrics, QRegion, QPen
 from PyQt5.QtWidgets import QWidget
 
 from .config import build_ansi_color_tables, Options
-from .data_types import Line, as_color
+from .data_types import Line
 from .utils import set_current_font_metrics
 
 
@@ -44,7 +44,7 @@ class TerminalWidget(QWidget):
         self.setPalette(pal)
         self.current_bg = pal.color(pal.Window)
         self.current_fg = pal.color(pal.WindowText)
-        self.ansi_fg, self.ansi_bg = build_ansi_color_tables(opts)
+        build_ansi_color_tables(opts)
         f = QFont(opts.font_family)
         f.setPointSizeF(opts.font_size)
         self.setFont(f)
@@ -104,10 +104,10 @@ class TerminalWidget(QWidget):
 
     def paint_cell(self, painter: QPainter, line: Line, col: int, y: int) -> None:
         x = self.cell_positions[col]
-        fg = as_color(line.fg[col], self.ansi_fg)
+        c = line.cursor_from(x)
+        fg, bg, decoration_fg = c.colors()
         if fg is not None:
             painter.setPen(QPen(fg))
-        bg = as_color(line.bg[col], self.ansi_bg)
         if bg is not None:
             r = QRect(x, y, self.cell_width, self.cell_height)
             painter.fillRect(r, bg)