Fix soft hyphens not being preserved when round tripping text through the terminal

Also roundtrip all characters in the Cf category.

Characters with the DI (Default Ignorable) property are now
preserved but not rendered and treated as zero-width
as per the unicode standard.
See https://www.unicode.org/faq/unsup_char.html
This commit is contained in:
Kovid Goyal
2021-10-07 10:26:57 +05:30
parent 1b42f69119
commit fbf47f75d5
10 changed files with 465 additions and 392 deletions

View File

@@ -1,14 +1,16 @@
#pragma once
#include "data-types.h"
#include "state.h"
#define VS15 1320
#define VS16 1321
// START_KNOWN_MARKS
static const combining_type VS15 = 1325, VS16 = 1326;
// END_KNOWN_MARKS
bool is_combining_char(char_type ch);
bool is_ignored_char(char_type ch);
bool is_word_char(char_type ch);
bool is_CZ_category(char_type);
bool is_P_category(char_type);
bool is_non_rendered_char(char_type);
char_type codepoint_for_mark(combining_type m);
combining_type mark_for_codepoint(char_type c);