mirrors/kitty
1
0
Fork 0
mirror of https://github.com/kovidgoyal/kitty synced 2026-07-14 04:24:52 +02:00
Code Issues Packages Projects Releases Wiki Activity

Fix soft hyphens not being preserved when round tripping text through the terminal

Browse Source 
Also roundtrip all characters in the Cf category.

Characters with the DI (Default Ignorable) property are now
preserved but not rendered and treated as zero-width
as per the unicode standard.
See https://www.unicode.org/faq/unsup_char.html
This commit is contained in:
Kovid Goyal
2021-10-07 10:26:57 +05:30
parent 1b42f69119
commit fbf47f75d5
10 changed files with 465 additions and 392 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
gen-wcwidth.py
View File

@@ -51,9 +51,26 @@ all_symbols: Set[int] = set()
name_map: Dict[int, str] = {}
word_search_map: DefaultDict[str, Set[int]] = defaultdict(set)
zwj = 0x200d
soft_hyphen = 0xad
flag_codepoints = frozenset(range(0x1F1E6, 0x1F1E6 + 26))
# See https://github.com/harfbuzz/harfbuzz/issues/169
marks = set(emoji_skin_tone_modifiers) | {zwj} | flag_codepoints
not_assigned = set(range(0, sys.maxunicode))
property_maps: Dict[str, Set[int]] = defaultdict(set)
def parse_prop_list() -> None:
global marks
for line in get_data('ucd/PropList.txt'):
if line.startswith('#'):
continue
cp_or_range, rest = line.split(';', 1)
chars = parse_range_spec(cp_or_range.strip())
name = rest.strip().split()[0]
property_maps[name] |= chars
# see https://www.unicode.org/faq/unsup_char.html#3
marks |= property_maps['Other_Default_Ignorable_Code_Point']
marks.add(soft_hyphen)
def parse_ucd() -> None:
@@ -354,16 +371,21 @@ def gen_ucd() -> None:
p('#include "unicode-data.h"')
category_test(
'is_combining_char', p,
{c for c in class_maps if c.startswith('M')},
'M category (marks)',
# See https://github.com/harfbuzz/harfbuzz/issues/169
extra_chars=emoji_skin_tone_modifiers | {zwj},
(),
'Combining and default ignored characters',
extra_chars=marks,
least_check_return='false'
)
category_test(
'is_ignored_char', p, 'Cc Cf Cs'.split(),
'is_ignored_char', p, 'Cc Cs'.split(),
'Control characters and non-characters',
extra_chars=non_characters, exclude={zwj},
extra_chars=non_characters,
ascii_range='false'
)
category_test(
'is_non_rendered_char', p, 'Cc Cs'.split(),
'Other_Default_Ignorable_Code_Point and soft hyphen',
extra_chars=property_maps['Other_Default_Ignorable_Code_Point'] | {soft_hyphen},
ascii_range='false'
)
category_test('is_word_char', p, {c for c in class_maps if c[0] in 'LN'}, 'L and N categories')
@@ -378,15 +400,19 @@ def gen_ucd() -> None:
p('combining_type mark_for_codepoint(char_type c) {')
rmap = codepoint_to_mark_map(p, mark_map)
p('}\n')
with open('kitty/unicode-data.h') as f:
unicode_data = f.read()
m = re.search(r'^#define VS15 (\d+)', unicode_data, re.M)
if m is not None:
expected = int(m.group(1))
if rmap[0xfe0e] != expected:
raise ValueError('The mark for 0xfe0e has changed, you have to update VS15 to {} and VS16 to {} in unicode-data.h'.format(
rmap[0xfe0e], rmap[0xfe0f]
))
with open('kitty/unicode-data.h', 'r+') as f:
raw = f.read()
f.seek(0)
raw, num = re.subn(
r'^// START_KNOWN_MARKS.+?^// END_KNOWN_MARKS',
'// START_KNOWN_MARKS\nstatic const combining_type '
f'VS15 = {rmap[0xfe0e]}, VS16 = {rmap[0xfe0f]};'
'\n// END_KNOWN_MARKS', raw, flags=re.MULTILINE | re.DOTALL)
if not num:
raise SystemExit('Faile dto patch mark definitions in unicode-data.h')
f.truncate()
f.write(raw)
with open('kittens/hints/url_regex.py', 'w') as f:
f.write('# generated by gen-wcwidth.py, do not edit\n\n')
f.write("url_delimiters = '{}' # noqa".format(''.join(classes_to_regex(cz, exclude='\n\r'))))
@@ -537,6 +563,7 @@ def gen_wcwidth() -> None:
parse_ucd()
parse_prop_list()
parse_emoji()
parse_eaw()
gen_ucd()