Dont use the python unicodedata module as we use libunistring

No sense in loading two hude unicode datasets into memory
This commit is contained in:
Kovid Goyal
2017-09-14 18:18:38 +05:30
parent 21accfe114
commit ed3427f349
3 changed files with 11 additions and 2 deletions

View File

@@ -1,6 +1,7 @@
#pragma once
#include <unictype.h>
#include <uninorm.h>
static inline bool
is_combining_char(uint32_t ch) {
@@ -22,3 +23,10 @@ static inline bool
is_url_char(uint32_t ch) {
return ch && !uc_is_general_category_withtable(ch, UC_CATEGORY_MASK_C | UC_CATEGORY_MASK_Z);
}
static inline uint32_t
normalize(uint32_t ch, uint32_t cc1, uint32_t cc2) {
uint32_t ans = uc_composition(ch, cc1);
if (ans && cc2) ans = uc_composition(ans, cc2);
return ans;
}