mirrors/kitty
1
0
Fork 0
mirror of https://github.com/kovidgoyal/kitty synced 2026-07-28 11:11:47 +02:00
Code Issues Packages Projects Releases Wiki Activity

Use a multi-stage lookup table for grapheme segmentation

Browse Source
This commit is contained in:
Kovid Goyal
2025-03-31 21:51:28 +05:30
parent 72a1aabafd
commit 66856e7b52
9 changed files with 1873 additions and 474 deletions
Download Patch File Download Diff File Expand all files Collapse all files

115
kitty/char-props.h
View File

@@ -9,16 +9,29 @@
#include "data-types.h"
// CharPropsDeclaration
// Uses 23 bits
// CharPropsDeclaration: uses 23 bits {{{
typedef union CharProps {
struct {
uint8_t shifted_width : 3;
struct __attribute__((packed)) {
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
uint8_t is_extended_pictographic : 1;
uint8_t grapheme_break : 4;
uint8_t indic_conjunct_break : 2;
uint8_t grapheme_break : 4;
uint8_t is_punctuation : 1;
uint8_t is_word_char : 1;
uint8_t is_combining_char : 1;
uint8_t is_symbol : 1;
uint8_t is_non_rendered : 1;
uint8_t is_invalid : 1;
uint8_t is_emoji_presentation_base : 1;
uint8_t category : 5;
uint8_t is_emoji : 1;
uint8_t shifted_width : 3;
uint16_t : 9;
#elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
uint16_t : 9;
uint8_t shifted_width : 3;
uint8_t is_emoji : 1;
uint8_t category : 5;
uint8_t is_emoji_presentation_base : 1;
uint8_t is_invalid : 1;
uint8_t is_non_rendered : 1;
@@ -26,14 +39,71 @@ typedef union CharProps {
uint8_t is_combining_char : 1;
uint8_t is_word_char : 1;
uint8_t is_punctuation : 1;
uint8_t grapheme_break : 4;
uint8_t indic_conjunct_break : 2;
uint8_t is_extended_pictographic : 1;
#else
#error "Unsupported endianness"
#endif
};
struct __attribute__((packed)) {
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
uint8_t grapheme_segmentation_property : 7;
uint32_t : 25;
#elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
uint32_t : 25;
uint8_t grapheme_segmentation_property : 7;
#else
#error "Unsupported endianness"
#endif
};
uint32_t val;
} CharProps;
static_assert(sizeof(CharProps) == sizeof(uint32_t), "Fix the ordering of CharProps");
// EndCharPropsDeclaration
// EndCharPropsDeclaration }}}
// GraphemeSegmentationResultDeclaration: uses 10 bits {{{
typedef union GraphemeSegmentationResult {
struct __attribute__((packed)) {
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
uint8_t emoji_modifier_sequence_before_last_char : 1;
uint8_t emoji_modifier_sequence : 1;
uint8_t incb_consonant_extended_linker_extended : 1;
uint8_t incb_consonant_extended_linker : 1;
uint8_t incb_consonant_extended : 1;
uint8_t grapheme_break : 4;
uint8_t add_to_current_cell : 1;
uint8_t : 6;
#elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
uint8_t : 6;
uint8_t add_to_current_cell : 1;
uint8_t grapheme_break : 4;
uint8_t incb_consonant_extended : 1;
uint8_t incb_consonant_extended_linker : 1;
uint8_t incb_consonant_extended_linker_extended : 1;
uint8_t emoji_modifier_sequence : 1;
uint8_t emoji_modifier_sequence_before_last_char : 1;
#else
#error "Unsupported endianness"
#endif
};
struct __attribute__((packed)) {
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
uint16_t state : 9;
uint8_t : 7;
#elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
uint8_t : 7;
uint16_t state : 9;
#else
#error "Unsupported endianness"
#endif
};
uint16_t val;
} GraphemeSegmentationResult;
static_assert(sizeof(GraphemeSegmentationResult) == sizeof(uint16_t), "Fix the ordering of GraphemeSegmentationResult");
// EndGraphemeSegmentationResultDeclaration }}}
// UCBDeclaration
// UCBDeclaration {{{
typedef enum UnicodeCategory {
UC_Cn,
UC_Cc,
@@ -67,37 +137,12 @@ typedef enum UnicodeCategory {
UC_Co,
} UnicodeCategory;
// EndUCBDeclaration
// EndUCBDeclaration }}}
typedef struct GraphemeSegmentationState {
int last_char_prop;
/* True if the last character ends a sequence of Indic_Conjunct_Break
values: consonant {extend|linker}* */
bool incb_consonant_extended;
/* True if the last character ends a sequence of Indic_Conjunct_Break
values: consonant {extend|linker}* linker */
bool incb_consonant_extended_linker;
/* True if the last character ends a sequence of Indic_Conjunct_Break
values: consonant {extend|linker}* linker {extend|linker}* */
bool incb_consonant_extended_linker_extended;
/* True if the last character ends an emoji modifier sequence
\p{Extended_Pictographic} Extend*. */
bool emoji_modifier_sequence;
/* True if the last character was immediately preceded by an
emoji modifier sequence \p{Extended_Pictographic} Extend*. */
bool emoji_modifier_sequence_before_last_char;
/* Number of consecutive regional indicator (RI) characters seen
immediately before the current point. */
size_t ri_count;
} GraphemeSegmentationState;
CharProps char_props_for(char_type ch);
void grapheme_segmentation_reset(GraphemeSegmentationState *s);
bool grapheme_segmentation_step(GraphemeSegmentationState *s, CharProps ch);
void grapheme_segmentation_reset(GraphemeSegmentationResult *s);
GraphemeSegmentationResult grapheme_segmentation_step(GraphemeSegmentationResult r, CharProps ch);
static inline int wcwidth_std(CharProps ch) { return (int)ch.shifted_width - 4/*=width_shift*/; }
static inline bool is_private_use(CharProps ch) { return ch.category == UC_Co; }
static inline const char* char_category(CharProps cp) {