From 9c1c1417758cacf05d35d8ecba8566af1dc89605 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 13 Mar 2025 11:19:54 +0530 Subject: [PATCH] Start work on grapheme segmentation algorithm --- gen/wcwidth.py | 2 +- kitty/grapheme-segmentation-data.h | 1 + kitty/grapheme-segmentation.c | 83 ++++++++++++++++++++ tools/wcswidth/grapheme-segmentation-data.go | 3 +- 4 files changed, 87 insertions(+), 2 deletions(-) create mode 100644 kitty/grapheme-segmentation.c diff --git a/gen/wcwidth.py b/gen/wcwidth.py index ee3cb9852..6870996b0 100755 --- a/gen/wcwidth.py +++ b/gen/wcwidth.py @@ -483,7 +483,7 @@ def gen_grapheme_segmentation() -> None: p('') gp('') - enum('GraphemeBreakProperty', 'None', *grapheme_segmentation_maps, prefix='GBP_') + enum('GraphemeBreakProperty', 'AtStart', 'None', *grapheme_segmentation_maps, prefix='GBP_') enum('IndicConjunctBreak', 'None', *incb_map, prefix='ICB_') def get_cat(name: str, c_func_name: str, go_func_name: str, prefix: str, m: dict[str, set[int]]) -> None: diff --git a/kitty/grapheme-segmentation-data.h b/kitty/grapheme-segmentation-data.h index c556b061b..2b9fd0b01 100644 --- a/kitty/grapheme-segmentation-data.h +++ b/kitty/grapheme-segmentation-data.h @@ -7,6 +7,7 @@ START_ALLOW_CASE_RANGE typedef enum GraphemeBreakProperty { + GBP_AtStart, GBP_None, GBP_Prepend, GBP_CR, diff --git a/kitty/grapheme-segmentation.c b/kitty/grapheme-segmentation.c new file mode 100644 index 000000000..df6887404 --- /dev/null +++ b/kitty/grapheme-segmentation.c @@ -0,0 +1,83 @@ +/* + * grapheme-segmentation.c + * Copyright (C) 2025 Kovid Goyal + * + * Distributed under terms of the GPL3 license. + */ + +#include "text-cache.h" +#include "grapheme-segmentation-data.h" + +typedef struct GraphemeSegmentationState { + GraphemeBreakProperty last_char_prop; + + /* True if the last character ends a sequence of Indic_Conjunct_Break +values: consonant {extend|linker}* */ + bool incb_consonant_extended; + /* True if the last character ends a sequence of Indic_Conjunct_Break +values: consonant {extend|linker}* linker */ + bool incb_consonant_extended_linker; + /* True if the last character ends a sequence of Indic_Conjunct_Break +values: consonant {extend|linker}* linker {extend|linker}* */ + bool incb_consonant_extended_linker_extended; + + /* True if the last character ends an emoji modifier sequence + \p{Extended_Pictographic} Extend*. */ + bool emoji_modifier_sequence; + /* True if the last character was immediately preceded by an + emoji modifier sequence \p{Extended_Pictographic} Extend*. */ + bool emoji_modifier_sequence_before_last_char; + + /* Number of consecutive regional indicator (RI) characters seen + immediately before the current point. */ + size_t ri_count; +} GraphemeSegmentationState; +#define is_linker_or_extend(incb) ((incb) == ICB_Linker || (incb) == ICB_Extend) + +void +grapheme_segmentation_reset(GraphemeSegmentationState *s) { + *s = (GraphemeSegmentationState){0}; +} + +bool +grapheme_segmentation_step(GraphemeSegmentationState *s, char_type ch) { + // Grapheme segmentation as per UAX29-C1-1 as defined in https://www.unicode.org/reports/tr29/ + GraphemeBreakProperty prop = grapheme_break_property(ch); + IndicConjunctBreak incb = indic_conjunct_break(ch); + bool add_to_cell = false; + if (s->last_char_prop == GBP_AtStart) { + add_to_cell = true; + } else { + /* No break between CR and LF (GB3). */ + if (s->last_char_prop == GBP_CR && prop == GBP_LF) add_to_cell = true; + /* Break before and after newlines (GB4, GB5). */ + else if ((s->last_char_prop == GBP_CR || s->last_char_prop == GBP_LF || s->last_char_prop == GBP_Control) + || (prop == GBP_CR || prop == GBP_LF || prop == GBP_Control) + ) {} + /* No break between Hangul syllable sequences (GB6, GB7, GB8). */ + else if ((s->last_char_prop == GBP_L && (prop == GBP_L || prop == GBP_V || prop == GBP_LV || prop == GBP_LVT)) + || ((s->last_char_prop == GBP_LV || s->last_char_prop == GBP_V) && (prop == GBP_V || prop == GBP_T)) + || ((s->last_char_prop == GBP_LVT || s->last_char_prop == GBP_T) && prop == GBP_T) + ) add_to_cell = true; + /* No break before: extending characters or ZWJ (GB9), SpacingMarks (GB9a), Prepend characters (GB9b) */ + else if (prop == GBP_Extend || prop == GBP_ZWJ || prop == GBP_SpacingMark || s->last_char_prop == GBP_Prepend) add_to_cell = true; + /* No break within certain combinations of Indic_Conjunct_Break values: + * Between consonant {extend|linker}* linker {extend|linker}* and consonant (GB9c). */ + else if (s->incb_consonant_extended_linker_extended && incb == ICB_Consonant) add_to_cell = true; + /* No break within emoji modifier sequences or emoji zwj sequences (GB11). */ + else if (s->last_char_prop == GBP_ZWJ && s->emoji_modifier_sequence_before_last_char && is_extended_pictographic(ch)) add_to_cell = true; + else {} // break everywhere else + } + + s->incb_consonant_extended_linker = s->incb_consonant_extended && incb == ICB_Linker; + s->incb_consonant_extended_linker_extended = (s->incb_consonant_extended_linker || ( + s->incb_consonant_extended_linker_extended && is_linker_or_extend(incb))); + s->incb_consonant_extended = (incb == ICB_Consonant || ( + s->incb_consonant_extended && is_linker_or_extend(incb))); + s->emoji_modifier_sequence_before_last_char = s->emoji_modifier_sequence; + s->emoji_modifier_sequence = (s->emoji_modifier_sequence && prop == GBP_Extend) || is_extended_pictographic(ch); + s->last_char_prop = prop; + + if (prop == GBP_Regional_Indicator) s->ri_count++; else s->ri_count = 0; + return add_to_cell; +} diff --git a/tools/wcswidth/grapheme-segmentation-data.go b/tools/wcswidth/grapheme-segmentation-data.go index f538c150e..57bba12f5 100644 --- a/tools/wcswidth/grapheme-segmentation-data.go +++ b/tools/wcswidth/grapheme-segmentation-data.go @@ -3,7 +3,8 @@ package wcswidth type GraphemeBreakProperty uint8 const ( - GBP_None GraphemeBreakProperty = iota + GBP_AtStart GraphemeBreakProperty = iota + GBP_None GBP_Prepend GBP_CR GBP_LF