Allow dynamically generating configuration by running an arbitrary program using the new geninclude directive

docs/changelog.rst

@@ -89,6 +89,8 @@ Detailed list of changes
 
 - diff kitten: Automatically use dark/light color scheme based on the color scheme of the parent terminal. Can be controlled via the new :opt:`kitten-diff.color_scheme` option. Note that this is a **default behavior change** (:iss:`8170`)
 
+- Allow dynamically generating configuration by running an arbitrary program using the new :code:`geninclude` directive in :file:`kitty.conf`
+
 - When a program running in kitty reports progress of a task display it as a percentage in the tab title. Controlled by the :opt:`tab_title_template` option
 
 - When mapping a custom kitten allow using shell escaping for the kitten path (:iss:`8178`)

docs/conf.rst

@@ -45,13 +45,21 @@ expanded, so :code:`${USER}.conf` becomes :file:`name.conf` if
 to detect the operating system. It is ``linux``, ``macos`` or ``bsd``.
 Also, you can use :code:`globinclude` to include files
 matching a shell glob pattern and :code:`envinclude` to include configuration
-from environment variables. For example::
+from environment variables. Finally, you can dynamically generate configuration
+by running a program using :code:`geninclude`. For example::
 
+     # Include other.conf
      include other.conf
      # Include *.conf files from all subdirs of kitty.d inside the kitty config dir
      globinclude kitty.d/**/*.conf
      # Include the *contents* of all env vars starting with KITTY_CONF_
      envinclude KITTY_CONF_*
+     # Run the script dynamic.py placed in the same directory as this config file
+     # and include its :file:`STDOUT`. Note that Python scripts are fastest
+     # as they use the embedded Python interpreter, but any executable script
+     # or program is supported, in any language. Remember to mark the script
+     # file executable.
+     geninclude dynamic.py
 
 
 .. note:: Syntax highlighting for :file:`kitty.conf` in vim is available via

kitty/conf/utils.py

@@ -198,12 +198,67 @@ class NamedLineIterator:
         return self.lines
 
 
+def pygeninclude(path: str) -> list[str]:
+    import io
+    import runpy
+    before = sys.stdout
+    buf = sys.stdout = io.StringIO()
+    try:
+        runpy.run_path(path, run_name='__main__')
+    finally:
+        sys.stdout = before
+    return buf.getvalue().splitlines()
+
+
+def geninclude(path: str) -> list[str]:
+    old = os.environ.get('KITTY_OS')
+    os.environ['KITTY_OS'] = os_name()
+    try:
+        if path.endswith('.py'):
+            return pygeninclude(path)
+        import subprocess
+        cp = subprocess.run([path], stdout=subprocess.PIPE, text=True)
+        return cp.stdout.splitlines()
+    finally:
+        if old is None:
+            os.environ.pop('KITTY_OS', None)
+        else:
+            os.environ['KITTY_OS'] = old
+
+
+
+include_keys = 'include', 'globinclude', 'envinclude', 'geninclude'
+
+
+class RecursiveInclude(Exception):
+    pass
+
+
+class Memory:
+
+    def __init__(self, accumulate_bad_lines: Optional[List[BadLine]]) -> None:
+        self.s: set[str] = set()
+        if accumulate_bad_lines is None:
+            accumulate_bad_lines = []
+        self.accumulate_bad_lines = accumulate_bad_lines
+
+    def seen(self, path: str) -> bool:
+        key = os.path.normpath(path)
+        if key in self.s:
+            self.accumulate_bad_lines.append(BadLine(currently_parsing.number, currently_parsing.line.rstrip(), RecursiveInclude(
+                f'The file {path} has already been included, ignoring'), currently_parsing.file))
+            return True
+        self.s.add(key)
+        return False
+
+
 def parse_line(
     line: str,
     parse_conf_item: ItemParser,
     ans: Dict[str, Any],
     base_path_for_includes: str,
     effective_config_lines: Callable[[str, str], None],
+    memory: Memory,
     accumulate_bad_lines: Optional[List[BadLine]] = None,
 ) -> None:
     line = line.strip()
@@ -214,7 +269,7 @@ def parse_line(
         log_error(f'Ignoring invalid config line: {line!r}')
         return
     key, val = m.groups()
-    if key in ('include', 'globinclude', 'envinclude'):
+    if key.endswith('include') and key in include_keys:
         val = expandvars(os.path.expanduser(val.strip()), {'KITTY_OS': os_name()})
         if key == 'globinclude':
             from pathlib import Path
@@ -226,7 +281,22 @@ def parse_line(
                     with currently_parsing.set_file(f'<env var: {x}>'):
                         _parse(
                             NamedLineIterator(os.path.join(base_path_for_includes, ''), iter(os.environ[x].splitlines())),
-                            parse_conf_item, ans, accumulate_bad_lines, effective_config_lines,
+                            parse_conf_item, ans, memory, accumulate_bad_lines, effective_config_lines
+                        )
+            return
+        elif key == 'geninclude':
+            if not os.path.isabs(val):
+                val = os.path.join(base_path_for_includes, val)
+            if not memory.seen(val):
+                try:
+                    lines = geninclude(val)
+                except Exception:
+                    log_error(f'Could not process geninclude {val}, ignoring')
+                else:
+                    with currently_parsing.set_file(f'<get: {val}>'):
+                        _parse(
+                            NamedLineIterator(os.path.join(base_path_for_includes, ''), iter(lines)),
+                            parse_conf_item, ans, memory, accumulate_bad_lines, effective_config_lines
                         )
             return
         else:
@@ -234,15 +304,14 @@ def parse_line(
                 val = os.path.join(base_path_for_includes, val)
             vals = (val,)
         for val in vals:
+            if memory.seen(val):
+                continue
             try:
                 with open(val, encoding='utf-8', errors='replace') as include:
                     with currently_parsing.set_file(val):
-                        _parse(include, parse_conf_item, ans, accumulate_bad_lines, effective_config_lines)
+                        _parse(include, parse_conf_item, ans, memory, accumulate_bad_lines, effective_config_lines)
             except FileNotFoundError:
-                log_error(
-                    'Could not find included config file: {}, ignoring'.
-                    format(val)
-                )
+                log_error(f'Could not find included config file: {val}, ignoring')
             except OSError:
                 log_error(
                     'Could not read from included config file: {}, ignoring'.
@@ -260,6 +329,7 @@ def _parse(
     lines: Iterable[str],
     parse_conf_item: ItemParser,
     ans: Dict[str, Any],
+    memory: Memory,
     accumulate_bad_lines: Optional[List[BadLine]] = None,
     effective_config_lines: Optional[Callable[[str, str], None]] = None,
 ) -> None:
@@ -301,7 +371,7 @@ def _parse(
                 next_line = ''
             try:
                 with currently_parsing.set_line(line, line_num):
-                    parse_line(line, parse_conf_item, ans, base_path_for_includes, effective_config_lines, accumulate_bad_lines)
+                    parse_line(line, parse_conf_item, ans, base_path_for_includes, effective_config_lines, memory, accumulate_bad_lines)
             except Exception as e:
                 if accumulate_bad_lines is None:
                     raise
@@ -317,7 +387,7 @@ def parse_config_base(
     accumulate_bad_lines: Optional[List[BadLine]] = None,
     effective_config_lines: Optional[Callable[[str, str], None]] = None,
 ) -> None:
-    _parse(lines, parse_conf_item, ans, accumulate_bad_lines, effective_config_lines)
+    _parse(lines, parse_conf_item, ans, Memory(accumulate_bad_lines), accumulate_bad_lines, effective_config_lines)
 
 
 def merge_dicts(defaults: Dict[str, Any], newvals: Dict[str, Any]) -> Dict[str, Any]:

kitty_tests/options.py

@@ -1,9 +1,12 @@
 #!/usr/bin/env python
 # License: GPL v3 Copyright: 2018, Kovid Goyal <kovid at kovidgoyal.net>
 
+import os
+import shutil
+import tempfile
 
 from kitty.fast_data_types import Color, test_cursor_blink_easing_function
-from kitty.options.utils import DELETE_ENV_VAR, EasingFunction
+from kitty.options.utils import DELETE_ENV_VAR, EasingFunction, to_color
 from kitty.utils import log_error
 
 from . import BaseTest
@@ -15,19 +18,25 @@ class TestConfParsing(BaseTest):
         super().setUp()
         self.error_messages = []
         log_error.redirect = self.error_messages.append
+        self.tdir = tempfile.mkdtemp()
 
     def tearDown(self):
         del log_error.redirect
+        shutil.rmtree(self.tdir)
         super().tearDown()
 
     def test_conf_parsing(self):
+        conf_parsing(self)
+
+
+def conf_parsing(self):
     from kitty.config import defaults, load_config
     from kitty.constants import is_macos
     from kitty.fonts import FontModification, ModificationType, ModificationUnit, ModificationValue
     from kitty.options.utils import to_modifiers
     bad_lines = []
 
-        def p(*lines, bad_line_num=0):
+    def p(*lines, bad_line_num=0, num_err=None):
         del bad_lines[:]
         del self.error_messages[:]
         ans = load_config(overrides=lines, accumulate_bad_lines=bad_lines)
@@ -35,6 +44,8 @@ class TestConfParsing(BaseTest):
             self.ae(len(bad_lines), bad_line_num)
         else:
             self.assertFalse(bad_lines)
+        if num_err is not None:
+            self.ae(len(self.error_messages), num_err, '\n'.join(self.error_messages))
         return ans
 
     def keys_for_func(opts, name):
@@ -182,3 +193,44 @@ class TestConfParsing(BaseTest):
     ef('steps(5, start)', {0: 0.2, 0.1: 0.2, 0.3: 0.4, 0.9:1})
     ef('steps(4, jump-both)', {0: 0.2, 0.1: 0.2, 0.3: 0.4, 0.9:1})
     ef('steps(6, jump-none)', {0: 0, 0.1: 0.0, 0.3: 0.2, 0.9:1})
+
+    # test various include modes
+    base = os.path.join(self.tdir, 'glob')
+    os.mkdir(base)
+    with open(os.path.join(base, 'fg'), 'w') as f:
+        print('foreground red', file=f)
+    opts = p(f'include {f.name}', num_err=0)
+    self.ae(opts.foreground, to_color('red'))
+    with open(os.path.join(self.tdir, 'bg'), 'w') as f:
+        print('background white', file=f)
+        print('globinclude glob/*', file=f)
+        print('envinclude ENVINCLUDE', file=f)
+        print('geninclude g.py', file=f)
+        print('geninclude g', file=f)
+    with open(os.path.join(self.tdir, 'g.py'), 'w') as g:
+        print('print("background_opacity .77")', file=g)
+        print('print("background_blur 77")', file=g)
+    with open(os.path.join(self.tdir, 'g'), 'w') as g:
+        print('#!/bin/sh', file=g)
+        print('echo background_image_linear y', file=g)
+        print('echo background_image_layout clamped', file=g)
+        os.chmod(g.fileno(), 0o700)
+    os.environ['ENVINCLUDE'] = 'cursor yellow'
+    opts = p(f'include {f.name}', num_err=0)
+    os.environ.pop('ENVINCLUDE')
+    self.ae(opts.foreground, to_color('red'))
+    self.ae(opts.background, to_color('white'))
+    self.ae(opts.cursor, to_color('yellow'))
+    self.ae(opts.background_opacity, .77)
+    self.ae(opts.background_blur, 77)
+    self.ae(opts.background_image_linear, True)
+    self.ae(opts.background_image_layout, 'clamped')
+    with open(os.path.join(self.tdir, 'a'), 'w') as a:
+        print('background red', file=a)
+        print('include b', file=a)
+    with open(os.path.join(self.tdir, 'b'), 'w') as a:
+        print('foreground red', file=a)
+        print('include a', file=a)
+    opts = p(f'include {a.name}', num_err=0, bad_line_num=1)
+    self.ae(opts.foreground, to_color('red'))
+    self.ae(opts.background, to_color('red'))

tools/config/api.go

@@ -13,6 +13,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"regexp"
+	"runtime"
 	"strconv"
 	"strings"
 	"sync"
@@ -61,6 +62,55 @@ var key_pat = sync.OnceValue(func() *regexp.Regexp {
 	return regexp.MustCompile(`([a-zA-Z][a-zA-Z0-9_-]*)\s+(.+)$`)
 })
 
+var kitty_os = sync.OnceValue(func() string {
+	switch runtime.GOOS {
+	case "linux":
+		return "linux"
+	case "freebsd", "netbsd", "openbsd":
+		return "bsd"
+	case "darwin":
+		return "macos"
+	}
+	return "unknown"
+})
+
+func geninclude(path string) (string, error) {
+	cmd := exec.Command(path)
+	cmd.Env = os.Environ()
+	cmd.Env = append(cmd.Env, "KITTY_OS="+kitty_os())
+	if strings.HasSuffix(path, ".py") && unix.Access(path, unix.X_OK) != nil {
+		if utils.KittyExe() == "" || strings.HasPrefix(path, ":") {
+			cmd = exec.Command("python", path)
+		} else {
+			cmd = exec.Command(utils.KittyExe(), "+launch", path)
+		}
+	}
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return "", err
+	}
+	if err = cmd.Start(); err != nil {
+		return "", err
+	}
+	data, err := io.ReadAll(stdout)
+	if err != nil {
+		return "", err
+	}
+	if err = cmd.Wait(); err != nil {
+		return "", err
+	}
+	return utils.UnsafeBytesToString(data), nil
+}
+
+func ExpandVars(x string) string {
+	return os.Expand(x, func(k string) string {
+		if k == "KITTY_OS" {
+			return kitty_os()
+		}
+		return os.Getenv(k)
+	})
+}
+
 func (self *ConfigParser) parse(scanner Scanner, name, base_path_for_includes string, depth int) error {
 	if self.seen_includes[name] { // avoid include loops
 		return nil
@@ -90,6 +140,10 @@ func (self *ConfigParser) parse(scanner Scanner, name, base_path_for_includes st
 	next_line := ""
 	var line string
 
+	add_bad_line := func(err error) {
+		self.bad_lines = append(self.bad_lines, ConfigLine{Src_file: name, Line: line, Line_number: lnum, Err: err})
+	}
+
 	for {
 		if next_line != "" {
 			line = next_line
@@ -124,16 +178,15 @@ func (self *ConfigParser) parse(scanner Scanner, name, base_path_for_includes st
 
 		if line[0] == '#' {
 			if self.CommentsHandler != nil {
-				err := self.CommentsHandler(line)
-				if err != nil {
-					self.bad_lines = append(self.bad_lines, ConfigLine{Src_file: name, Line: line, Line_number: lnum, Err: err})
+				if err := self.CommentsHandler(line); err != nil {
+					add_bad_line(err)
 				}
 			}
 			continue
 		}
 		m := key_pat().FindStringSubmatch(line)
 		if len(m) < 3 {
-			self.bad_lines = append(self.bad_lines, ConfigLine{Src_file: name, Line: line, Line_number: lnum, Err: fmt.Errorf("Invalid config line: %#v", line)})
+			add_bad_line(fmt.Errorf("Invalid config line: %#v", line))
 			continue
 		}
 		key, val := m[1], m[2]
@@ -146,17 +199,18 @@ func (self *ConfigParser) parse(scanner Scanner, name, base_path_for_includes st
 		}
 		switch key {
 		default:
-			err := self.LineHandler(key, val)
-			if err != nil {
-				self.bad_lines = append(self.bad_lines, ConfigLine{Src_file: name, Line: line, Line_number: lnum, Err: err})
+			if err := self.LineHandler(key, val); err != nil {
+				add_bad_line(err)
 			}
-		case "include", "globinclude", "envinclude":
+		case "include", "globinclude", "envinclude", "geninclude":
 			var includes []string
+			val = ExpandVars(val)
 			switch key {
 			case "include":
-				aval, err := make_absolute(val)
-				if err == nil {
+				if aval, err := make_absolute(val); err == nil {
 					includes = []string{aval}
+				} else {
+					add_bad_line(err)
 				}
 			case "globinclude":
 				aval, err := make_absolute(val)
@@ -164,13 +218,26 @@ func (self *ConfigParser) parse(scanner Scanner, name, base_path_for_includes st
 					matches, err := filepath.Glob(aval)
 					if err == nil {
 						includes = matches
+					} else {
+						add_bad_line(err)
 					}
+				} else {
+					add_bad_line(err)
+				}
+			case "geninclude":
+				if aval, err := make_absolute(val); err == nil {
+					if g, err := geninclude(aval); err == nil {
+						if err := recurse(strings.NewReader(g), "<gen: "+val+">", base_path_for_includes); err != nil {
+							return err
+						}
+					} else {
+						add_bad_line(err)
+					}
+				} else {
+					add_bad_line(err)
 				}
 			case "envinclude":
-				env := self.override_env
-				if env == nil {
-					env = os.Environ()
-				}
+				env := utils.IfElse(self.override_env == nil, os.Environ(), self.override_env)
 				for _, x := range env {
 					key, eval, _ := strings.Cut(x, "=")
 					is_match, err := filepath.Match(val, key)
@@ -184,14 +251,12 @@ func (self *ConfigParser) parse(scanner Scanner, name, base_path_for_includes st
 			}
 			if len(includes) > 0 {
 				for _, incpath := range includes {
-					raw, err := os.ReadFile(incpath)
-					if err == nil {
-						err := recurse(bytes.NewReader(raw), incpath, filepath.Dir(incpath))
-						if err != nil {
+					if raw, err := os.ReadFile(incpath); err == nil {
+						if err := recurse(bytes.NewReader(raw), incpath, filepath.Dir(incpath)); err != nil {
 							return err
 						}
 					} else if !errors.Is(err, fs.ErrNotExist) {
-						return fmt.Errorf("Failed to process include %#v with error: %w", incpath, err)
+						add_bad_line(err)
 					}
 				}
 			}

tools/config/api_test.go

@@ -24,6 +24,10 @@ func TestConfigParsing(t *testing.T) {
 			t.Fatal(err)
 		}
 	}
+	w(filepath.Join(tdir, "g.py"), []byte(`
+print('gpy 1')
+print('gpy 2')
+`))
 	w(conf_file, []byte(
 		`error main
 # igno
@@ -39,6 +43,7 @@ globin
 \clude sub/c?.c
    \onf
 badline
+geninclude g.py
 `))
 	w(filepath.Join(tdir, "sub/b.conf"), []byte("incb cool\ninclude a.conf"))
 	w(filepath.Join(tdir, "sub/c1.conf"), []byte("inc1 cool"))
@@ -62,7 +67,7 @@ badline
 	if err = p.ParseOverrides("over one", "over two"); err != nil {
 		t.Fatal(err)
 	}
-	diff := cmp.Diff([]string{"a one", "incb cool", "b x", "inc1 cool", "inc2 cool", "env cool", "inc notcool", "over one", "over two"}, parsed_lines)
+	diff := cmp.Diff([]string{"a one", "incb cool", "b x", "inc1 cool", "inc2 cool", "env cool", "inc notcool", "gpy 1", "gpy 2", "over one", "over two"}, parsed_lines)
 	if diff != "" {
 		t.Fatalf("Unexpected parsed config values:\n%s", diff)
 	}