mirrors/kitty
1
0
Fork 0
mirror of https://github.com/kovidgoyal/kitty synced 2026-07-16 13:34:48 +02:00
Code Issues Packages Projects Releases Wiki Activity

Use a multi-stage lookup table for grapheme segmentation

Browse Source
This commit is contained in:
Kovid Goyal
2025-03-31 21:51:28 +05:30
parent 72a1aabafd
commit 66856e7b52
9 changed files with 1873 additions and 474 deletions
Download Patch File Download Diff File Expand all files Collapse all files

938
tools/wcswidth/char-props-data.go generated
View File

File diff suppressed because one or more lines are too long

103
tools/wcswidth/char-props.go
View File

@@ -7,41 +7,16 @@ import (
var _ = fmt.Print
type GraphemeSegmentationState struct {
last_char_prop GraphemeBreakProperty
/* True if the last character ends a sequence of Indic_Conjunct_Break
values: consonant {extend|linker}* */
incb_consonant_extended bool
/* True if the last character ends a sequence of Indic_Conjunct_Break
values: consonant {extend|linker}* linker */
incb_consonant_extended_linker bool
/* True if the last character ends a sequence of Indic_Conjunct_Break
values: consonant {extend|linker}* linker {extend|linker}* */
incb_consonant_extended_linker_extended bool
/* True if the last character ends an emoji modifier sequence
\p{Extended_Pictographic} Extend*. */
emoji_modifier_sequence bool
/* True if the last character was immediately preceded by an
emoji modifier sequence \p{Extended_Pictographic} Extend*. */
emoji_modifier_sequence_before_last_char bool
/* Number of consecutive regional indicator (RI) characters seen
immediately before the current point. */
ri_count uint
}
func CharPropsFor(ch rune) CharProps {
return charprops_t3[charprops_t2[(rune(charprops_t1[ch>>charprops_shift])<<charprops_shift)+(ch&charprops_mask)]]
}
func IteratorOverGraphemes(text string) iter.Seq[string] {
s := GraphemeSegmentationState{}
var s GraphemeSegmentationResult
start_pos := 0
return func(yield func(string) bool) {
for pos, ch := range text {
if !s.Step(CharPropsFor(ch)) {
if s = s.Step(CharPropsFor(ch)); s.Add_to_current_cell() == 0 {
if !yield(text[start_pos:pos]) {
return
}
@@ -62,73 +37,17 @@ func SplitIntoGraphemes(text string) []string {
return ans
}
func (i IndicConjunctBreak) is_linker_or_extend() bool {
return i == ICB_Linker || i == ICB_Extend
func (s *GraphemeSegmentationResult) Reset() {
*s = 0
}
func (s *GraphemeSegmentationState) Reset() {
*s = GraphemeSegmentationState{}
}
func (s *GraphemeSegmentationState) Step(ch CharProps) bool {
// Grapheme segmentation as per UAX29-C1-1 as defined in https://www.unicode.org/reports/tr29/
// Returns true iff ch should be added to the current cell based on s which
// must reflect the state of the current cell. s is updated by ch.
prop := GraphemeBreakProperty(ch.Grapheme_break())
incb := IndicConjunctBreak(ch.Indic_conjunct_break())
add_to_cell := false
if s.last_char_prop == GBP_AtStart {
add_to_cell = true
} else {
/* No break between CR and LF (GB3). */
if s.last_char_prop == GBP_CR && prop == GBP_LF {
add_to_cell = true
} else if
/* Break before and after newlines (GB4, GB5). */
(s.last_char_prop == GBP_CR || s.last_char_prop == GBP_LF || s.last_char_prop == GBP_Control) ||
(prop == GBP_CR || prop == GBP_LF || prop == GBP_Control) {
} else if
/* No break between Hangul syllable sequences (GB6, GB7, GB8). */
(s.last_char_prop == GBP_L && (prop == GBP_L || prop == GBP_V || prop == GBP_LV || prop == GBP_LVT)) ||
((s.last_char_prop == GBP_LV || s.last_char_prop == GBP_V) && (prop == GBP_V || prop == GBP_T)) ||
((s.last_char_prop == GBP_LVT || s.last_char_prop == GBP_T) && prop == GBP_T) {
add_to_cell = true
} else if
/* No break before: extending characters or ZWJ (GB9), SpacingMarks (GB9a), Prepend characters (GB9b) */
prop == GBP_Extend || prop == GBP_ZWJ || prop == GBP_SpacingMark || s.last_char_prop == GBP_Prepend {
add_to_cell = true
} else if
/* No break within certain combinations of Indic_Conjunct_Break values:
* Between consonant {extend|linker}* linker {extend|linker}* and consonant (GB9c). */
s.incb_consonant_extended_linker_extended && incb == ICB_Consonant {
add_to_cell = true
} else if
/* No break within emoji modifier sequences or emoji zwj sequences (GB11). */
s.last_char_prop == GBP_ZWJ && s.emoji_modifier_sequence_before_last_char && (ch.Is_extended_pictographic() == 1) {
add_to_cell = true
} else if
/* No break between RI if there is an odd number of RI characters before (GB12, GB13). */
prop == GBP_Regional_Indicator && (s.ri_count%2) != 0 {
add_to_cell = true
} else
/* Break everywhere else */
{
}
}
s.incb_consonant_extended_linker = s.incb_consonant_extended && incb == ICB_Linker
s.incb_consonant_extended_linker_extended = (s.incb_consonant_extended_linker || (s.incb_consonant_extended_linker_extended && incb.is_linker_or_extend()))
s.incb_consonant_extended = (incb == ICB_Consonant || (s.incb_consonant_extended && incb.is_linker_or_extend()))
s.emoji_modifier_sequence_before_last_char = s.emoji_modifier_sequence
s.emoji_modifier_sequence = (s.emoji_modifier_sequence && prop == GBP_Extend) || (ch.Is_extended_pictographic() == 1)
s.last_char_prop = prop
if prop == GBP_Regional_Indicator {
s.ri_count++
} else {
s.ri_count = 0
}
return add_to_cell
func (s GraphemeSegmentationResult) Step(ch CharProps) GraphemeSegmentationResult {
key := grapheme_segmentation_key(s, ch)
t1 := uint16(graphemesegmentationresult_t1[key>>graphemesegmentationresult_shift]) << graphemesegmentationresult_shift
t2 := graphemesegmentationresult_t2[t1+key&graphemesegmentationresult_mask]
ans := graphemesegmentationresult_t3[t2]
// fmt.Printf("state: %d gsp: %d -> key: %d t1: %d -> add_to_cell: %d\n", s.State(), ch.GraphemeSegmentationProperty(), key, t1, ans.Add_to_current_cell())
return ans
}
func Runewidth(code rune) int {