mirrors/kitty
1
0
Fork 0
mirror of https://github.com/kovidgoyal/kitty synced 2026-07-23 16:58:09 +02:00
Code Issues Packages Projects Releases Wiki Activity

Use the new SIMD utf-8 decoder

Browse Source
This commit is contained in:
Kovid Goyal
2024-01-14 14:12:19 +05:30
parent e783eccc97
commit b3ca5d51fb
2 changed files with 7 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
kitty/simd-string.c
View File

@@ -139,14 +139,14 @@ init_simd(void *x) {
if (has_avx2) { if (has_avx2) {
A(has_avx2, True); A(has_avx2, True);
find_either_of_two_bytes_impl = find_either_of_two_bytes_256; find_either_of_two_bytes_impl = find_either_of_two_bytes_256;
/* utf8_decode_to_esc_impl = utf8_decode_to_esc_256; */ utf8_decode_to_esc_impl = utf8_decode_to_esc_256;
} else { } else {
A(has_avx2, False); A(has_avx2, False);
} }
if (has_sse4_2) { if (has_sse4_2) {
A(has_sse4_2, True); A(has_sse4_2, True);
if (find_either_of_two_bytes_impl == find_either_of_two_bytes_scalar) find_either_of_two_bytes_impl = find_either_of_two_bytes_128; if (find_either_of_two_bytes_impl == find_either_of_two_bytes_scalar) find_either_of_two_bytes_impl = find_either_of_two_bytes_128;
/* if (utf8_decode_to_esc_impl == utf8_decode_to_esc_scalar) utf8_decode_to_esc_impl = utf8_decode_to_esc_128; */ if (utf8_decode_to_esc_impl == utf8_decode_to_esc_scalar) utf8_decode_to_esc_impl = utf8_decode_to_esc_128;
} else { } else {
A(has_sse4_2, False); A(has_sse4_2, False);
} }

10
kitty_tests/parser.py
View File

@@ -220,8 +220,6 @@ class TestParser(BaseTest):
x('abcd1234efgh5678') x('abcd1234efgh5678')
x('abc\x1bd1234efgh5678') x('abc\x1bd1234efgh5678')
x('abcd1234efgh5678ijklABCDmnopEFGH') x('abcd1234efgh5678ijklABCDmnopEFGH')
x('ニチ')
x('\x84\x85')
for which in (2, 3): for which in (2, 3):
x = partial(t, which=which) x = partial(t, which=which)
@@ -235,15 +233,17 @@ class TestParser(BaseTest):
x(b'abcd\xe2', b'\x89\xa41234') x(b'abcd\xe2', b'\x89\xa41234')
x(b'abcd\xe2\x89', b'\xa41234') x(b'abcd\xe2\x89', b'\xa41234')
def test_invalid(src, expected, which=2): def test_expected(src, expected, which=2):
reset_state() reset_state()
_, actual = test_utf8_decode_to_sentinel(b'filler' + asbytes(src), which) _, actual = test_utf8_decode_to_sentinel(b'filler' + asbytes(src), which)
expected = 'filler' + expected expected = 'filler' + expected
self.ae(expected, actual, f'Failed for: {src!r} with {which=}') self.ae(expected, actual, f'Failed for: {src!r} with {which=}')
# various invalid input
for which in (1, 2, 3): for which in (1, 2, 3):
pb = partial(test_invalid, which=which) pb = partial(test_expected, which=which)
pb('ニチ', 'ニチ')
pb('\x84\x85', '\x84\x85')
# various invalid input
pb(b'abcd\xf51234', 'abcd\ufffd1234') # bytes > 0xf4 pb(b'abcd\xf51234', 'abcd\ufffd1234') # bytes > 0xf4
pb(b'abcd\xff1234', 'abcd\ufffd1234') # bytes > 0xf4 pb(b'abcd\xff1234', 'abcd\ufffd1234') # bytes > 0xf4
pb(b'"\xbf"', '"\ufffd"') pb(b'"\xbf"', '"\ufffd"')