From f1fe0bf40a834787826f2be72591e11749e9ae73 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Fri, 2 Feb 2024 11:35:51 +0530
Subject: [PATCH] Code to easily compare SIMD and scalar decode in a live
 instance

Also remove -mtune=intel as it fails with clang
---
 kitty/simd-string-impl.h | 24 ++++++++++++++++++++++++
 kitty/simd-string.c      |  2 +-
 kitty/simd-string.h      |  1 +
 setup.py                 |  4 ----
 4 files changed, 26 insertions(+), 5 deletions(-)

diff --git a/kitty/simd-string-impl.h b/kitty/simd-string-impl.h
index f499181cc..636a870ba 100644
--- a/kitty/simd-string-impl.h
+++ b/kitty/simd-string-impl.h
@@ -423,6 +423,11 @@ scalar_decode_all(UTF8Decoder *d, const uint8_t *src, size_t src_sz) {
 bool
 FUNC(utf8_decode_to_esc)(UTF8Decoder *d, const uint8_t *src_data, size_t src_len) {
     // Based on the algorithm described in: https://woboq.com/blog/utf-8-processing-using-simd.html
+#ifdef compare_with_scalar
+    UTF8Decoder debugdec ={0};
+    memcpy(&debugdec.state, &d->state, sizeof(debugdec.state));
+    bool scalar_sentinel_found = utf8_decode_to_esc_scalar(&debugdec, src_data, src_len);
+#endif
     d->output.pos = 0; d->num_consumed = 0;
     if (d->state.cur != UTF8_ACCEPT) {
         // Finish the trailing sequence only
@@ -618,6 +623,25 @@ start_classification:
         FUNC(output_unicode)(d, output1, output2, output3, num_codepoints);
         handle_trailing_bytes();
     }
+#ifdef compare_with_scalar
+    if (debugdec.output.pos != d->output.pos || debugdec.num_consumed != d->num_consumed ||
+        memcmp(d->output.storage, debugdec.output.storage, d->output.pos * sizeof(d->output.storage[0])) != 0 ||
+        sentinel_found != scalar_sentinel_found || debugdec.state.cur != d->state.cur
+    ) {
+        fprintf(stderr, "vector decode output differs from scalar: input_sz=%zu consumed=(%u %u) output_sz=(%u %u) sentinel=(%d %d) state_changed: %d output_different: %d\n",
+                src_len, debugdec.num_consumed, d->num_consumed, debugdec.output.pos, d->output.pos, scalar_sentinel_found, sentinel_found,
+                debugdec.state.cur != d->state.cur,
+                memcmp(d->output.storage, debugdec.output.storage, MIN(d->output.pos, debugdec.output.pos) * sizeof(d->output.storage[0]))
+        );
+        fprintf(stderr, "\"");
+        for (unsigned i = 0; i < src_len; i++) {
+            if (32 <= src_data[i] && src_data[i] < 0x7f && src_data[i] != '"') fprintf(stderr, "%c", src_data[i]);
+            else fprintf(stderr, "\\x%x", src_data[i]);
+        }
+        fprintf(stderr, "\"\n");
+    }
+    utf8_decoder_free(&debugdec);
+#endif
     zero_upper();
     return sentinel_found;
 #undef abort_with_invalid_utf8
diff --git a/kitty/simd-string.c b/kitty/simd-string.c
index 9c8d779c2..851909e1d 100644
--- a/kitty/simd-string.c
+++ b/kitty/simd-string.c
@@ -29,7 +29,7 @@ find_either_of_two_bytes(const uint8_t *haystack, const size_t sz, const uint8_t
 
 // UTF-8 {{{
 
-static bool
+bool
 utf8_decode_to_esc_scalar(UTF8Decoder *d, const uint8_t *src, const size_t src_sz) {
     d->output.pos = 0; d->num_consumed = 0;
     utf8_decoder_ensure_capacity(d, src_sz);
diff --git a/kitty/simd-string.h b/kitty/simd-string.h
index 789e68b06..606846ac4 100644
--- a/kitty/simd-string.h
+++ b/kitty/simd-string.h
@@ -22,6 +22,7 @@ typedef struct UTF8Decoder {
 static inline void utf8_decoder_reset(UTF8Decoder *self) { zero_at_ptr(&self->state); }
 
 bool utf8_decode_to_esc(UTF8Decoder *d, const uint8_t *src, size_t src_sz);
+bool utf8_decode_to_esc_scalar(UTF8Decoder *d, const uint8_t *src, const size_t src_sz);
 
 static inline void utf8_decoder_ensure_capacity(UTF8Decoder *d, unsigned sz) {
     if (d->output.pos + sz > d->output.capacity) {
diff --git a/setup.py b/setup.py
index 3a4d7c3d1..de077b304 100755
--- a/setup.py
+++ b/setup.py
@@ -712,15 +712,11 @@ def get_source_specific_cflags(env: Env, src: str) -> List[str]:
     if src in ('kitty/simd-string-128.c', 'kitty/simd-string-256.c'):
         if env.binary_arch.isa in (ISA.AMD64, ISA.X86):
             ans.append('-msse4.2' if '128' in src else '-mavx2')
-            if not env.native_optimizations:
-                ans.append('-mtune=intel')
         elif env.binary_arch.isa != ISA.ARM64:
             ans.append('-DKITTY_NO_SIMD')
     elif src.startswith('3rdparty/base64/lib/arch/'):
         if env.binary_arch.isa in (ISA.AMD64, ISA.X86):
             q = src.split(os.path.sep)
-            if not env.native_optimizations:
-                ans.append('-mtune=intel')
             if 'sse3' in q:
                 ans.append('-msse3')
             elif 'sse41' in q: