Implement grapheme segmentation in Go

tools/wcswidth/char-props.go

@@ -2,6 +2,7 @@ package wcswidth
 
 import (
 	"fmt"
+	"iter"
 )
 
 var _ = fmt.Print
@@ -31,14 +32,44 @@ type GraphemeSegmentationState struct {
 	ri_count uint
 }
 
-func Char_props_for(ch rune) CharProps {
+func CharPropsFor(ch rune) CharProps {
 	return charprops_t2[(rune(charprops_t1[ch>>charprops_shift])<<charprops_shift)+(ch&charprops_mask)]
 }
 
+func IteratorOverGraphemes(text string) iter.Seq[string] {
+	s := GraphemeSegmentationState{}
+	start_pos := 0
+	return func(yield func(string) bool) {
+		for pos, ch := range text {
+			if !s.Step(CharPropsFor(ch)) {
+				if !yield(text[start_pos:pos]) {
+					return
+				}
+				start_pos = pos
+			}
+		}
+		if start_pos < len(text) {
+			yield(text[start_pos:len(text)])
+		}
+	}
+}
+
+func SplitIntoGraphemes(text string) []string {
+	ans := make([]string, 0, len(text))
+	for t := range IteratorOverGraphemes(text) {
+		ans = append(ans, t)
+	}
+	return ans
+}
+
 func (i IndicConjunctBreak) is_linker_or_extend() bool {
 	return i == ICB_Linker || i == ICB_Extend
 }
 
+func (s *GraphemeSegmentationState) Reset() {
+	*s = GraphemeSegmentationState{}
+}
+
 func (s *GraphemeSegmentationState) Step(ch CharProps) bool {
 	// Grapheme segmentation as per UAX29-C1-1 as defined in https://www.unicode.org/reports/tr29/
 	// Returns true iff ch should be added to the current cell based on s which

tools/wcswidth/char-props_test.go

@@ -0,0 +1,54 @@
+package wcswidth
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+	"os/exec"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+
+	"kitty/tools/utils"
+)
+
+var _ = fmt.Print
+
+type GraphemeBreakTest struct {
+	Data    []string `json:"data"`
+	Comment string   `json:"comment"`
+}
+
+func TestSplitIntoGraphemes(t *testing.T) {
+	var m = map[string][]string{
+		" \u0308 ": []string{" \u0308", " "},
+		"abc": []string{"a", "b", "c"},
+	}
+	for text, expected := range m {
+		if diff := cmp.Diff(expected, SplitIntoGraphemes(text)); diff != "" {
+			t.Fatalf("Failed to split %#v into graphemes: %s", text, diff)
+		}
+	}
+	cmd := exec.Command(utils.KittyExe(), "+runpy", `
+from kitty.constants import read_kitty_resource
+import sys
+sys.stdout.buffer.write(read_kitty_resource("GraphemeBreakTest.json", "kitty_tests"))
+sys.stdout.flush()
+`)
+	var output []byte
+	var err error
+	if output, err = cmd.Output(); err != nil {
+		t.Fatalf("Getting GraphemeBreakTest.json failed with error: %s", err)
+	}
+	tests := []GraphemeBreakTest{}
+	if err = json.Unmarshal(output, &tests); err != nil {
+		t.Fatalf("Failed to parse GraphemeBreakTest JSON with error: %s", err)
+	}
+	for i, x := range tests {
+		text := strings.Join(x.Data, "")
+		actual := SplitIntoGraphemes(text)
+		if diff := cmp.Diff(x.Data, actual); diff != "" {
+			t.Fatalf("Failed test #%d: split %#v into graphemes (%s): %s", i, text, x.Comment, diff)
+		}
+	}
+}