From 9bea1001f958498630b35d8fbe65a321910f37d6 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 10 Sep 2017 09:57:34 +0530 Subject: [PATCH] Speedup unicode character property lookup Use libunistring instead of building predicates from the unicode database --- README.asciidoc | 1 + count-lines-of-code | 2 +- generate-unicode-data.py | 57 ---------------------------------------- kitty/unicode-data.h | 18 +++++++++---- setup.py | 2 +- 5 files changed, 16 insertions(+), 64 deletions(-) delete mode 100755 generate-unicode-data.py diff --git a/README.asciidoc b/README.asciidoc index 7f5cddd69..b08ab5432 100644 --- a/README.asciidoc +++ b/README.asciidoc @@ -80,6 +80,7 @@ the following dependencies are installed first. * python >= 3.5 * glfw >= 3.2 +* libunistring * glew >= 2.0 (not needed on macOS) * fontconfig (not needed on macOS) * xrdb and xsel (only on X11 based systems) diff --git a/count-lines-of-code b/count-lines-of-code index 0c6800a0f..67b76e6c7 100755 --- a/count-lines-of-code +++ b/count-lines-of-code @@ -1,2 +1,2 @@ #!/bin/bash -cloc --exclude-list-file <(echo -e 'kitty/wcwidth9.h\nkitty/unicode-data.h\nkitty/gl.c\nkitty/glfw.c\nkitty/glfw.h\nkitty/charsets.c\nkitty/key_encoding.py\nkitty/rgb.py') kitty +cloc --exclude-list-file <(echo -e 'kitty/wcwidth9.h\nkitty/gl.c\nkitty/glfw.c\nkitty/glfw.h\nkitty/charsets.c\nkitty/key_encoding.py\nkitty/rgb.py') kitty diff --git a/generate-unicode-data.py b/generate-unicode-data.py deleted file mode 100755 index e39ead63f..000000000 --- a/generate-unicode-data.py +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env python3 -# vim:fileencoding=utf-8 -# License: GPL v3 Copyright: 2016, Kovid Goyal - -import unicodedata -import itertools -import sys - -IGNORED_CATEGORIES = ('Cc', 'Cf', 'Cs') - - -def ranges(i): - for a, b in itertools.groupby(enumerate(i), lambda r: r[1] - r[0]): - b = list(b) - yield b[0][1], b[-1][1] - - -def generate_data(chars): - points, cranges = [], [] - for l, r in ranges(chars): - if l == r: - points.append(l) - else: - cranges.append((l, r)) - return points, cranges - - -def generate_predicate(name, chars): - points, cranges = generate_data(chars) - cranges = ['(0x%x %s ch && ch <= 0x%x)' % (l, '<' if l == 0 else '<=', r) for l, r in cranges] - points = ['(ch == 0x%x)' % p for p in points] - return ''' -static inline bool %s(uint32_t ch) { - return %s || %s; -} - ''' % (name, '||'.join(cranges), '||'.join(points)) - - -def main(): - combining_chars = [] - igchars = [] - for c in map(chr, range(sys.maxunicode + 1)): - if unicodedata.category(c) in IGNORED_CATEGORIES: - igchars.append(ord(c)) - if unicodedata.combining(c): - combining_chars.append(ord(c)) - - cc = generate_predicate('is_combining_char', combining_chars) - ig = generate_predicate('is_ignored_char', igchars) - with open('kitty/unicode-data.h', 'w') as f: - print('#pragma once', file=f) - print(cc, file=f) - print(ig, file=f) - - -if __name__ == '__main__': - main() diff --git a/kitty/unicode-data.h b/kitty/unicode-data.h index 41244ef04..d65f96adb 100644 --- a/kitty/unicode-data.h +++ b/kitty/unicode-data.h @@ -1,11 +1,19 @@ #pragma once -static inline bool is_combining_char(uint32_t ch) { - return (0x300 <= ch && ch <= 0x34e)||(0x350 <= ch && ch <= 0x36f)||(0x483 <= ch && ch <= 0x487)||(0x591 <= ch && ch <= 0x5bd)||(0x5c1 <= ch && ch <= 0x5c2)||(0x5c4 <= ch && ch <= 0x5c5)||(0x610 <= ch && ch <= 0x61a)||(0x64b <= ch && ch <= 0x65f)||(0x6d6 <= ch && ch <= 0x6dc)||(0x6df <= ch && ch <= 0x6e4)||(0x6e7 <= ch && ch <= 0x6e8)||(0x6ea <= ch && ch <= 0x6ed)||(0x730 <= ch && ch <= 0x74a)||(0x7eb <= ch && ch <= 0x7f3)||(0x816 <= ch && ch <= 0x819)||(0x81b <= ch && ch <= 0x823)||(0x825 <= ch && ch <= 0x827)||(0x829 <= ch && ch <= 0x82d)||(0x859 <= ch && ch <= 0x85b)||(0x8e3 <= ch && ch <= 0x8ff)||(0x951 <= ch && ch <= 0x954)||(0xc55 <= ch && ch <= 0xc56)||(0xe38 <= ch && ch <= 0xe3a)||(0xe48 <= ch && ch <= 0xe4b)||(0xeb8 <= ch && ch <= 0xeb9)||(0xec8 <= ch && ch <= 0xecb)||(0xf18 <= ch && ch <= 0xf19)||(0xf71 <= ch && ch <= 0xf72)||(0xf7a <= ch && ch <= 0xf7d)||(0xf82 <= ch && ch <= 0xf84)||(0xf86 <= ch && ch <= 0xf87)||(0x1039 <= ch && ch <= 0x103a)||(0x135d <= ch && ch <= 0x135f)||(0x1939 <= ch && ch <= 0x193b)||(0x1a17 <= ch && ch <= 0x1a18)||(0x1a75 <= ch && ch <= 0x1a7c)||(0x1ab0 <= ch && ch <= 0x1abd)||(0x1b6b <= ch && ch <= 0x1b73)||(0x1baa <= ch && ch <= 0x1bab)||(0x1bf2 <= ch && ch <= 0x1bf3)||(0x1cd0 <= ch && ch <= 0x1cd2)||(0x1cd4 <= ch && ch <= 0x1ce0)||(0x1ce2 <= ch && ch <= 0x1ce8)||(0x1cf8 <= ch && ch <= 0x1cf9)||(0x1dc0 <= ch && ch <= 0x1df5)||(0x1dfc <= ch && ch <= 0x1dff)||(0x20d0 <= ch && ch <= 0x20dc)||(0x20e5 <= ch && ch <= 0x20f0)||(0x2cef <= ch && ch <= 0x2cf1)||(0x2de0 <= ch && ch <= 0x2dff)||(0x302a <= ch && ch <= 0x302f)||(0x3099 <= ch && ch <= 0x309a)||(0xa674 <= ch && ch <= 0xa67d)||(0xa69e <= ch && ch <= 0xa69f)||(0xa6f0 <= ch && ch <= 0xa6f1)||(0xa8e0 <= ch && ch <= 0xa8f1)||(0xa92b <= ch && ch <= 0xa92d)||(0xaab2 <= ch && ch <= 0xaab4)||(0xaab7 <= ch && ch <= 0xaab8)||(0xaabe <= ch && ch <= 0xaabf)||(0xfe20 <= ch && ch <= 0xfe2f)||(0x10376 <= ch && ch <= 0x1037a)||(0x10a38 <= ch && ch <= 0x10a3a)||(0x10ae5 <= ch && ch <= 0x10ae6)||(0x110b9 <= ch && ch <= 0x110ba)||(0x11100 <= ch && ch <= 0x11102)||(0x11133 <= ch && ch <= 0x11134)||(0x11235 <= ch && ch <= 0x11236)||(0x112e9 <= ch && ch <= 0x112ea)||(0x11366 <= ch && ch <= 0x1136c)||(0x11370 <= ch && ch <= 0x11374)||(0x114c2 <= ch && ch <= 0x114c3)||(0x115bf <= ch && ch <= 0x115c0)||(0x116b6 <= ch && ch <= 0x116b7)||(0x16af0 <= ch && ch <= 0x16af4)||(0x16b30 <= ch && ch <= 0x16b36)||(0x1d165 <= ch && ch <= 0x1d169)||(0x1d16d <= ch && ch <= 0x1d172)||(0x1d17b <= ch && ch <= 0x1d182)||(0x1d185 <= ch && ch <= 0x1d18b)||(0x1d1aa <= ch && ch <= 0x1d1ad)||(0x1d242 <= ch && ch <= 0x1d244)||(0x1e8d0 <= ch && ch <= 0x1e8d6) || (ch == 0x5bf)||(ch == 0x5c7)||(ch == 0x670)||(ch == 0x711)||(ch == 0x93c)||(ch == 0x94d)||(ch == 0x9bc)||(ch == 0x9cd)||(ch == 0xa3c)||(ch == 0xa4d)||(ch == 0xabc)||(ch == 0xacd)||(ch == 0xb3c)||(ch == 0xb4d)||(ch == 0xbcd)||(ch == 0xc4d)||(ch == 0xcbc)||(ch == 0xccd)||(ch == 0xd4d)||(ch == 0xdca)||(ch == 0xf35)||(ch == 0xf37)||(ch == 0xf39)||(ch == 0xf74)||(ch == 0xf80)||(ch == 0xfc6)||(ch == 0x1037)||(ch == 0x108d)||(ch == 0x1714)||(ch == 0x1734)||(ch == 0x17d2)||(ch == 0x17dd)||(ch == 0x18a9)||(ch == 0x1a60)||(ch == 0x1a7f)||(ch == 0x1b34)||(ch == 0x1b44)||(ch == 0x1be6)||(ch == 0x1c37)||(ch == 0x1ced)||(ch == 0x1cf4)||(ch == 0x20e1)||(ch == 0x2d7f)||(ch == 0xa66f)||(ch == 0xa806)||(ch == 0xa8c4)||(ch == 0xa953)||(ch == 0xa9b3)||(ch == 0xa9c0)||(ch == 0xaab0)||(ch == 0xaac1)||(ch == 0xaaf6)||(ch == 0xabed)||(ch == 0xfb1e)||(ch == 0x101fd)||(ch == 0x102e0)||(ch == 0x10a0d)||(ch == 0x10a0f)||(ch == 0x10a3f)||(ch == 0x11046)||(ch == 0x1107f)||(ch == 0x11173)||(ch == 0x111c0)||(ch == 0x111ca)||(ch == 0x1133c)||(ch == 0x1134d)||(ch == 0x1163f)||(ch == 0x1172b)||(ch == 0x1bc9e); +#include + +static inline bool +is_combining_char(uint32_t ch) { + return uc_combining_class(ch) != UC_CCC_NR; } -static inline bool is_ignored_char(uint32_t ch) { - return (0x0 < ch && ch <= 0x1f)||(0x7f <= ch && ch <= 0x9f)||(0x600 <= ch && ch <= 0x605)||(0x200b <= ch && ch <= 0x200f)||(0x202a <= ch && ch <= 0x202e)||(0x2060 <= ch && ch <= 0x2064)||(0x2066 <= ch && ch <= 0x206f)||(0xd800 <= ch && ch <= 0xdfff)||(0xfff9 <= ch && ch <= 0xfffb)||(0x1bca0 <= ch && ch <= 0x1bca3)||(0x1d173 <= ch && ch <= 0x1d17a)||(0xe0020 <= ch && ch <= 0xe007f) || (ch == 0xad)||(ch == 0x61c)||(ch == 0x6dd)||(ch == 0x70f)||(ch == 0x180e)||(ch == 0xfeff)||(ch == 0x110bd)||(ch == 0xe0001); +static inline bool +is_ignored_char(uint32_t ch) { + return uc_is_general_category_withtable(ch, UC_CATEGORY_MASK_Cc | UC_CATEGORY_MASK_Cf | UC_CATEGORY_MASK_Cs); +} + +static inline bool +is_word_char(uint32_t ch) { + return uc_is_general_category_withtable(ch, UC_CATEGORY_MASK_L | UC_CATEGORY_MASK_N); } - diff --git a/setup.py b/setup.py index 97aefc38e..9b86f8fa2 100755 --- a/setup.py +++ b/setup.py @@ -170,7 +170,7 @@ def init_env(debug=False, sanitize=False, native_optimizations=True, profile=Fal else: glfw_ldflags = pkg_config('glfw3', '--libs') glew_libs = pkg_config('glew', '--libs') - ldpaths = pylib + glew_libs + font_libs + glfw_ldflags + ldpaths = pylib + glew_libs + font_libs + glfw_ldflags + ['-lunistring'] try: os.mkdir(build_dir)