From 16c7681c7ccf71364256268b05863a61f833a913 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 6 Mar 2023 09:55:55 +0530 Subject: [PATCH] diff kitten: Speedup patch parsing by working with bytes rather than unicode Also change the line split algorithm to only split on \n, \r and \r\n. This is hopefully closer to what git/diff generate in their patch files. I cant find any documentation specifying this however. Fixes #6052 Fixes #6092 --- kittens/diff/patch.py | 45 ++++++++++++++++++++++----------------- kitty/fast_data_types.pyi | 1 + kitty/kittens.c | 33 ++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 19 deletions(-) diff --git a/kittens/diff/patch.py b/kittens/diff/patch.py index 32efffc26..9ca1465fe 100644 --- a/kittens/diff/patch.py +++ b/kittens/diff/patch.py @@ -8,6 +8,8 @@ import shutil import subprocess from typing import Dict, Iterator, List, Optional, Sequence, Tuple, Union +from kitty.fast_data_types import splitlines_like_git + from . import global_data from .collect import lines_for_path from .diff_speedup import changed_center @@ -37,7 +39,7 @@ def set_diff_command(opt: str) -> None: global_data.cmd = cmd -def run_diff(file1: str, file2: str, context: int = 3) -> Tuple[bool, Union[int, bool], str]: +def run_diff(file1: str, file2: str, context: int = 3) -> Tuple[bool, Union[int, bool], bytes]: # returns: ok, is_different, patch cmd = shlex.split(global_data.cmd.replace('_CONTEXT_', str(context))) # we resolve symlinks because git diff does not follow symlinks, while diff @@ -53,8 +55,8 @@ def run_diff(file1: str, file2: str, context: int = 3) -> Tuple[bool, Union[int, returncode = p.wait() worker_processes.remove(p.pid) if returncode in (0, 1): - return True, returncode == 1, stdout.decode('utf-8') - return False, returncode, stderr.decode('utf-8') + return True, returncode == 1, stdout + return False, returncode, stderr class Chunk: @@ -158,19 +160,19 @@ class Hunk: c.finalize() -def parse_range(x: str) -> Tuple[int, int]: - parts = x[1:].split(',', 1) +def parse_range(x: bytes) -> Tuple[int, int]: + parts = x[1:].split(b',', 1) start = abs(int(parts[0])) count = 1 if len(parts) < 2 else int(parts[1]) return start, count -def parse_hunk_header(line: str) -> Hunk: - parts: Tuple[str, ...] = tuple(filter(None, line.split('@@', 2))) +def parse_hunk_header(line: bytes) -> Hunk: + parts: Tuple[bytes, ...] = tuple(filter(None, line.split(b'@@', 2))) linespec = parts[0].strip() title = '' if len(parts) == 2: - title = parts[1].strip() + title = parts[1].strip().decode('utf-8', 'replace') left, right = map(parse_range, linespec.split()) return Hunk(title, left, right) @@ -190,25 +192,30 @@ class Patch: return len(self.all_hunks) -def parse_patch(raw: str) -> Patch: +def parse_patch(raw: bytes) -> Patch: all_hunks = [] current_hunk = None - for line in raw.splitlines(): - if line.startswith('@@ '): - current_hunk = parse_hunk_header(line) + plus, minus, backslash = map(ord, '+-\\') + + def parse_line(line: memoryview) -> None: + nonlocal current_hunk + if line[:3] == b'@@ ': + current_hunk = parse_hunk_header(bytes(line)) all_hunks.append(current_hunk) else: if current_hunk is None: - continue - q = line[0] if line else '' - if q == '+': + return + q:int = line[0] if len(line) > 0 else 0 + if q == plus: current_hunk.add_line() - elif q == '-': + elif q == minus: current_hunk.remove_line() - elif q == '\\': - continue + elif q == backslash: + return else: current_hunk.context_line() + + splitlines_like_git(raw, parse_line) for h in all_hunks: h.finalize() return Patch(all_hunks) @@ -244,7 +251,7 @@ class Differ: except Exception as e: return f'Running git diff for {left_path} vs. {right_path} generated an exception: {e}' if not ok: - return f'{output}\nRunning git diff for {left_path} vs. {right_path} failed' + return f'{output.decode("utf-8", "replace")}\nRunning git diff for {left_path} vs. {right_path} failed' left_lines = lines_for_path(left_path) right_lines = lines_for_path(right_path) try: diff --git a/kitty/fast_data_types.pyi b/kitty/fast_data_types.pyi index 2bb3e6bce..a32675dd8 100644 --- a/kitty/fast_data_types.pyi +++ b/kitty/fast_data_types.pyi @@ -1536,3 +1536,4 @@ def expand_ansi_c_escapes(test: str) -> str: ... def update_tab_bar_edge_colors(os_window_id: int) -> bool: ... def mask_kitty_signals_process_wide() -> None: ... def is_modifier_key(key: int) -> bool: ... +def splitlines_like_git(raw: bytes, callback: Callable[[memoryview], None]) -> None: ... diff --git a/kitty/kittens.c b/kitty/kittens.c index 29b4461f4..9dcae4e2c 100644 --- a/kitty/kittens.c +++ b/kitty/kittens.c @@ -196,9 +196,42 @@ parse_input_from_terminal(PyObject *self UNUSED, PyObject *args) { #undef CALL } +static PyObject* +splitlines_like_git(PyObject *self UNUSED, PyObject *args) { + char *raw; Py_ssize_t sz; + PyObject *callback; + if (!PyArg_ParseTuple(args, "y#O", &raw, &sz, &callback)) return NULL; + while (sz > 0 && (raw[sz-1] == '\n' || raw[sz-1] == '\r')) sz--; + PyObject *mv, *ret; +#define CALLBACK \ + mv = PyMemoryView_FromMemory(raw + start, i - start, PyBUF_READ); \ + if (mv == NULL) return NULL; \ + ret = PyObject_CallFunctionObjArgs(callback, mv, NULL); \ + Py_DECREF(mv); \ + if (ret == NULL) return NULL; \ + Py_DECREF(ret); start = i + 1; + + Py_ssize_t i = 0, start = 0; + for (; i < sz; i++) { + switch (raw[i]) { + case '\n': + CALLBACK; break; + case '\r': + CALLBACK; + if (i + 1 < sz && raw[i+1] == '\n') { i++; start++; } + break; + } + } + if (start < sz) { + i = sz; CALLBACK; + } + Py_RETURN_NONE; +} + static PyMethodDef module_methods[] = { METHODB(parse_input_from_terminal, METH_VARARGS), METHODB(read_command_response, METH_VARARGS), + METHODB(splitlines_like_git, METH_VARARGS), {NULL, NULL, 0, NULL} /* Sentinel */ };