Use a faster result collection type for rendering sorted results

This commit is contained in:
Kovid Goyal
2025-06-29 22:22:20 +05:30
parent 2cde543a7b
commit a8bb24e1c0
6 changed files with 265 additions and 119 deletions

View File

@@ -20,6 +20,10 @@ func (c CollectionIndex) Compare(o CollectionIndex) int {
return c.Slice - o.Slice
}
func (c CollectionIndex) Less(o CollectionIndex) bool {
return c.Slice < o.Slice || (c.Slice == o.Slice && c.Pos < o.Pos)
}
func (c *CollectionIndex) NextSlice() {
c.Slice++
c.Pos = 0
@@ -99,6 +103,13 @@ func (s *SortedResults) Len() int {
return s.len
}
func (s *SortedResults) Clear() {
s.lock()
defer s.unlock()
s.slices = nil
s.len = 0
}
func (s *SortedResults) At(pos CollectionIndex) (ans *ResultItem) {
s.lock()
defer s.unlock()
@@ -117,6 +128,9 @@ func (s *SortedResults) RenderedMatches(pos CollectionIndex, max_num int) (ans [
if pos.Slice >= len(s.slices) {
return
}
if max_num < 0 {
max_num = s.len
}
ans = make([]*ResultItem, 0, max_num)
for ; pos.Slice < len(s.slices) && max_num > 0; pos.NextSlice() {
sl := s.slices[pos.Slice]
@@ -195,3 +209,103 @@ func (s *SortedResults) AddSortedSlice(sl []*ResultItem) {
}
s.slices = append(s.slices, sl)
}
func (s *SortedResults) IncrementIndexWithWrapAround(idx CollectionIndex, amt int) CollectionIndex {
s.lock()
defer s.unlock()
ans, _ := s.increment_with_wrap_around(idx, amt)
return ans
}
func (s *SortedResults) increment_with_wrap_around(idx CollectionIndex, amt int) (CollectionIndex, bool) {
did_wrap := false
if amt > 0 {
for amt > 0 {
if delta := min(amt, len(s.slices[idx.Slice])-1-idx.Pos); delta > 0 {
idx.Pos += delta
amt -= delta
} else {
idx.NextSlice()
if idx.Slice >= len(s.slices) {
idx = CollectionIndex{} // wraparound
did_wrap = true
}
amt--
}
}
} else {
// we use separate code for negative increment instead of doing
// increment = len - increment as it is faster in the common case of
// increment much smaller than len
amt *= -1
for amt > 0 {
if idx.Pos > 0 {
delta := min(amt, idx.Pos)
amt -= delta
idx.Pos -= delta
} else {
if idx.Slice == 0 {
idx = CollectionIndex{Slice: len(s.slices) - 1, Pos: len(s.slices[len(s.slices)-1]) - 1}
did_wrap = true
} else {
idx.Slice--
idx.Pos = len(s.slices[idx.Slice]) - 1
}
amt--
}
}
}
return idx, did_wrap
}
// Return |a - b|
func (s *SortedResults) distance(a, b CollectionIndex) (ans int) {
if b.Less(a) {
a, b = b, a
}
for ; a.Slice < b.Slice; a.NextSlice() {
ans += len(s.slices[a.Slice]) - a.Pos
}
return ans + (b.Pos - a.Pos)
}
func (s *SortedResults) SplitIntoColumns(calc_num_cols func(int) int, num_per_column, num_before_current int, current CollectionIndex) (ans [][]*ResultItem, num_before int) {
s.lock()
defer s.unlock()
num_cols := calc_num_cols(s.len)
total := num_cols * num_per_column
if total < 1 {
return nil, 0
}
num_before = min(total-1, num_before_current)
idx, did_wrap := s.increment_with_wrap_around(current, -num_before)
last_slice := s.slices[len(s.slices)-1]
last := CollectionIndex{Slice: len(s.slices) - 1, Pos: len(last_slice) - 1}
if did_wrap {
idx = CollectionIndex{}
} else if s.distance(idx, last) < total-1 {
if idx, did_wrap = s.increment_with_wrap_around(last, 1-total); did_wrap {
idx = CollectionIndex{}
}
}
num_before = s.distance(idx, current)
// fmt.Printf("111111 idx: %v current: %v num_before: %d\n", idx, current, num_before)
ans = make([][]*ResultItem, num_cols)
for colidx := range len(ans) {
col := make([]*ResultItem, 0, num_per_column)
for len(col) < num_per_column && idx.Slice < len(s.slices) {
ss := s.slices[idx.Slice]
limit := min(len(ss), idx.Pos+num_per_column-len(col))
col = append(col, ss[idx.Pos:limit]...)
idx.Pos = limit
if idx.Pos >= len(ss) {
idx.NextSlice()
if idx.Slice >= len(s.slices) {
break
}
}
}
ans[colidx] = col
}
return
}

View File

@@ -91,6 +91,10 @@ func (m Mode) WindowTitle() string {
return ""
}
type render_state struct {
num_matches, num_of_slots, num_before int
}
type State struct {
base_dir string
current_dir string
@@ -102,11 +106,10 @@ type State struct {
window_title string
screen Screen
save_file_cdir string
selections []string
current_idx int
num_of_matches_at_last_render int
num_of_slots_per_column_at_last_render int
save_file_cdir string
selections []string
current_idx CollectionIndex
last_render render_state
}
func (s State) BaseDir() string { return utils.IfElse(s.base_dir == "", default_cwd, s.base_dir) }
@@ -118,7 +121,7 @@ func (s State) OnlyDirs() bool { return s.mode.OnlyDirs() }
func (s *State) SetSearchText(val string) {
if s.search_text != val {
s.search_text = val
s.current_idx = 0
s.current_idx = CollectionIndex{}
}
}
func (s *State) SetCurrentDir(val string) {
@@ -127,12 +130,12 @@ func (s *State) SetCurrentDir(val string) {
}
if s.CurrentDir() != val {
s.search_text = ""
s.current_idx = 0
s.current_idx = CollectionIndex{}
s.current_dir = val
}
}
func (s State) CurrentIndex() int { return s.current_idx }
func (s *State) SetCurrentIndex(val int) { s.current_idx = max(0, val) }
func (s State) CurrentIndex() CollectionIndex { return s.current_idx }
func (s *State) SetCurrentIndex(val CollectionIndex) { s.current_idx = val }
func (s State) CurrentDir() string {
return utils.IfElse(s.current_dir == "", s.BaseDir(), s.current_dir)
}
@@ -228,10 +231,8 @@ func (h *Handler) OnInitialize() (ans string, err error) {
func (h *Handler) current_abspath() string {
matches, _ := h.get_results()
if len(matches) > 0 {
if idx := h.state.CurrentIndex(); idx < len(matches) {
return filepath.Join(h.state.CurrentDir(), matches[idx].text)
}
if r := matches.At(h.state.CurrentIndex()); r != nil {
return filepath.Join(h.state.CurrentDir(), r.text)
}
return ""
@@ -254,16 +255,31 @@ func (h *Handler) toggle_selection() bool {
return false
}
func (h *Handler) change_current_dir(dir string) {
if dir != h.state.CurrentDir() {
h.state.SetCurrentDir(dir)
h.result_manager.set_root_dir(h.state.CurrentDir())
h.state.last_render = render_state{}
}
}
func (h *Handler) set_query(q string) {
if q != h.state.SearchText() {
h.state.SetSearchText(q)
h.result_manager.set_query(h.state.SearchText())
h.state.last_render = render_state{}
}
}
func (h *Handler) change_to_current_dir_if_possible() error {
matches, _ := h.get_results()
if len(matches) > 0 {
if matches.Len() > 0 {
m := h.current_abspath()
if st, err := os.Stat(m); err == nil {
if !st.IsDir() {
m = filepath.Dir(m)
h.change_current_dir(m)
}
h.state.SetCurrentDir(m)
h.result_manager.set_root_dir(h.state.CurrentDir())
return h.draw_screen()
}
}
@@ -296,13 +312,11 @@ func (h *Handler) OnKeyEvent(ev *loop.KeyEvent) (err error) {
case "/":
case ".":
if curr, err = os.Getwd(); err == nil && curr != "/" {
h.state.SetCurrentDir(filepath.Dir(curr))
h.result_manager.set_root_dir(h.state.CurrentDir())
h.change_current_dir(filepath.Dir(curr))
return h.draw_screen()
}
default:
h.state.SetCurrentDir(filepath.Dir(curr))
h.result_manager.set_root_dir(h.state.CurrentDir())
h.change_current_dir(filepath.Dir(curr))
return h.draw_screen()
}
h.lp.Beep()
@@ -345,8 +359,7 @@ func (h *Handler) OnKeyEvent(ev *loop.KeyEvent) (err error) {
func (h *Handler) OnText(text string, from_key_event, in_bracketed_paste bool) (err error) {
switch h.state.screen {
case NORMAL:
h.state.SetSearchText(h.state.SearchText() + text)
h.result_manager.set_query(h.state.SearchText())
h.set_query(h.state.SearchText() + text)
return h.draw_screen()
case SAVE_FILE:
if err = h.rl.OnText(text, from_key_event, in_bracketed_paste); err == nil {
@@ -356,7 +369,7 @@ func (h *Handler) OnText(text string, from_key_event, in_bracketed_paste bool) (
return
}
func (h *Handler) set_state_from_config(conf *Config, opts *Options) (err error) {
func (h *Handler) set_state_from_config(_ *Config, opts *Options) (err error) {
h.state = State{}
switch opts.Mode {
case "file":

View File

@@ -145,49 +145,43 @@ func (h *Handler) draw_column_of_matches(matches ResultsType, current_idx int, x
}
}
func (h *Handler) draw_list_of_results(matches ResultsType, y, height int) int {
if len(matches) == 0 || height < 2 {
return 0
}
func (h *Handler) draw_list_of_results(matches *SortedResults, y, height int) int {
available_width := h.screen_size.width - 2
col_width := available_width
num_cols := 1
if len(matches) > height {
col_width = 40
num_cols = available_width / col_width
for num_cols > 0 && height*(num_cols-1) >= len(matches) {
num_cols--
calc_num_cols := func(num_matches int) int {
if num_matches == 0 || height < 2 {
return 0
}
col_width = available_width / num_cols
if num_matches > height {
col_width = 40
num_cols = available_width / col_width
for num_cols > 0 && height*(num_cols-1) >= num_matches {
num_cols--
}
col_width = available_width / num_cols
}
return num_cols
}
num_of_slots := num_cols * height
idx := min(h.state.CurrentIndex(), len(matches)-1)
pos := 0
for pos+num_of_slots <= idx {
pos += height
}
x, limit, total := 1, 0, 0
for range num_cols {
columns, num_before := matches.SplitIntoColumns(calc_num_cols, height, h.state.last_render.num_before, h.state.CurrentIndex())
h.state.last_render.num_before = num_before
x := 1
for _, col := range columns {
h.lp.MoveCursorTo(x, y)
limit = min(len(matches), pos+height)
total += limit - pos
h.draw_column_of_matches(matches[pos:limit], idx-pos, x, col_width-1)
h.draw_column_of_matches(col, num_before, x, col_width-1)
num_before -= height
x += col_width
pos += height
if pos >= len(matches) {
break
}
}
return num_cols
return len(columns)
}
func (h *Handler) draw_num_of_matches(num_shown, y int) {
m := ""
switch h.state.num_of_matches_at_last_render {
switch h.state.last_render.num_matches {
case 0:
m = " no matches "
default:
m = fmt.Sprintf(" %d of %d matches ", min(num_shown, h.state.num_of_matches_at_last_render), h.state.num_of_matches_at_last_render)
m = fmt.Sprintf(" %d of %d matches ", min(num_shown, h.state.last_render.num_matches), h.state.last_render.num_matches)
}
w := int(math.Ceil(float64(wcswidth.Stringwidth(m)) / 2.0))
h.lp.MoveCursorTo(h.screen_size.width-w-2, y)
@@ -204,7 +198,7 @@ func (h *Handler) draw_num_of_matches(num_shown, y int) {
}
}
func (h *Handler) draw_results(y, bottom_margin int, matches ResultsType, in_progress bool) (height int) {
func (h *Handler) draw_results(y, bottom_margin int, matches *SortedResults, in_progress bool) (height int) {
height = h.screen_size.height - y - bottom_margin
h.lp.MoveCursorTo(1, 1+y)
h.draw_frame(h.screen_size.width, height)
@@ -212,44 +206,42 @@ func (h *Handler) draw_results(y, bottom_margin int, matches ResultsType, in_pro
h.draw_results_title()
y += 2
h.lp.MoveCursorTo(1, y)
h.state.num_of_slots_per_column_at_last_render = height - 2
h.state.last_render.num_of_slots = height - 2
num_cols := 0
switch len(matches) {
num := matches.Len()
switch num {
case 0:
h.draw_no_matches_message(in_progress)
default:
num_cols = h.draw_list_of_results(matches, y, h.state.num_of_slots_per_column_at_last_render)
num_cols = h.draw_list_of_results(matches, y, h.state.last_render.num_of_slots)
}
h.state.num_of_matches_at_last_render = len(matches)
h.draw_num_of_matches(h.state.num_of_slots_per_column_at_last_render*num_cols, y+height-2)
h.state.last_render.num_matches = num
h.draw_num_of_matches(h.state.last_render.num_of_slots*num_cols, y+height-2)
return
}
func (h *Handler) next_result(amt int) {
if h.state.num_of_matches_at_last_render > 0 {
if h.state.last_render.num_matches > 0 {
idx := h.state.CurrentIndex()
idx += amt
for idx < 0 {
idx += h.state.num_of_matches_at_last_render
}
idx %= h.state.num_of_matches_at_last_render
idx = h.result_manager.scorer.sorted_results.IncrementIndexWithWrapAround(idx, amt)
h.state.SetCurrentIndex(idx)
}
}
func (h *Handler) move_sideways(leftwards bool) {
if h.state.num_of_matches_at_last_render > 0 {
idx := h.state.CurrentIndex()
slots := h.state.num_of_slots_per_column_at_last_render
if h.state.last_render.num_matches > 0 {
cidx := h.state.CurrentIndex()
slots := h.state.last_render.num_of_slots
if leftwards {
if idx >= slots {
idx -= slots
idx := h.result_manager.scorer.sorted_results.IncrementIndexWithWrapAround(cidx, -slots)
if idx.Less(cidx) {
h.state.SetCurrentIndex(idx)
}
} else {
idx = min(h.state.num_of_matches_at_last_render-1, idx+slots)
}
if idx != h.state.CurrentIndex() {
h.state.SetCurrentIndex(idx)
idx := h.result_manager.scorer.sorted_results.IncrementIndexWithWrapAround(cidx, slots)
if cidx.Less(idx) {
h.state.SetCurrentIndex(idx)
}
}
}
}

View File

@@ -280,7 +280,7 @@ type FileSystemScorer struct {
root_dir, query string
only_dirs bool
mutex sync.Mutex
renderable_results []*ResultItem
sorted_results *SortedResults
on_results func(error, bool)
current_worker_wait *sync.WaitGroup
scorer *fzf.FuzzyMatcher
@@ -289,7 +289,7 @@ type FileSystemScorer struct {
func NewFileSystemScorer(root_dir, query string, only_dirs bool, on_results func(error, bool)) (ans *FileSystemScorer) {
return &FileSystemScorer{
query: query, root_dir: root_dir, only_dirs: only_dirs, on_results: on_results,
scorer: fzf.NewFuzzyMatcher(fzf.PATH_SCHEME)}
scorer: fzf.NewFuzzyMatcher(fzf.PATH_SCHEME), sorted_results: NewSortedResults()}
}
func (fss *FileSystemScorer) lock() { fss.mutex.Lock() }
@@ -320,7 +320,7 @@ func (fss *FileSystemScorer) Change_query(query string) {
}
fss.lock()
fss.query = query
fss.renderable_results = nil
fss.sorted_results.Clear()
fss.unlock()
fss.Start()
}
@@ -340,7 +340,6 @@ func (fss *FileSystemScorer) worker(on_results chan bool, worker_wait *sync.Wait
}
}
}()
global_min_score, global_max_score := CombinedScore(math.MaxUint64), CombinedScore(0)
handle_batch := func(results []ResultItem) (err error) {
if err = fss.scanner.Error(); err != nil {
return
@@ -377,30 +376,10 @@ func (fss *FileSystemScorer) worker(on_results chan bool, worker_wait *sync.Wait
}
}
}
min_score, max_score := CombinedScore(math.MaxUint64), CombinedScore(0)
if len(rp) > 0 {
slices.SortFunc(rp, func(a, b *ResultItem) int { return cmp.Compare(a.score, b.score) })
min_score, max_score = rp[0].score, rp[len(rp)-1].score
}
var rr []*ResultItem
fss.lock()
existing := fss.renderable_results
fss.unlock()
switch {
case min_score >= global_max_score:
rr = append(existing, rp...)
case max_score < global_min_score:
rr = make([]*ResultItem, len(existing)+len(rp), max(16*1024, len(existing)+len(rp), 2*cap(existing)))
copy(rr, rp)
copy(rr[len(rp):], existing)
default:
rr = merge_sorted_slices(existing, rp)
}
global_min_score = min(global_min_score, min_score)
global_max_score = max(global_max_score, max_score)
fss.lock()
fss.renderable_results = rr
fss.unlock()
fss.sorted_results.AddSortedSlice(rp)
return
}
@@ -423,10 +402,10 @@ func (fss *FileSystemScorer) worker(on_results chan bool, worker_wait *sync.Wait
}
}
func (fss *FileSystemScorer) Results() (ans ResultsType, is_finished bool) {
func (fss *FileSystemScorer) Results() (ans *SortedResults, is_finished bool) {
fss.lock()
defer fss.unlock()
return fss.renderable_results, fss.is_complete.Load()
return fss.sorted_results, fss.is_complete.Load()
}
func (fss *FileSystemScorer) Cancel() {
@@ -473,22 +452,6 @@ func (m *ResultManager) on_results(err error, is_finished bool) {
}
}
func merge_sorted_slices(a, b []*ResultItem) []*ResultItem {
result := make([]*ResultItem, 0, 2*(len(a)+len(b)))
i, j := 0, 0
for i < len(a) && j < len(b) {
if a[i].score <= b[j].score {
result = append(result, a[i])
i++
} else {
result = append(result, b[j])
j++
}
}
result = append(result, a[i:]...)
return append(result, b[j:]...)
}
func (m *ResultManager) set_root_dir(root_dir string) {
var err error
if root_dir == "" || root_dir == "." {
@@ -504,10 +467,16 @@ func (m *ResultManager) set_root_dir(root_dir string) {
m.scorer.Cancel()
}
m.scorer = NewFileSystemScorer(root_dir, "", m.settings.OnlyDirs(), m.on_results)
m.mutex.Lock()
m.last_wakeup_at = time.Time{}
m.mutex.Unlock()
m.scorer.Start()
}
func (m *ResultManager) set_query(query string) {
m.mutex.Lock()
m.last_wakeup_at = time.Time{}
m.mutex.Unlock()
if m.scorer == nil {
m.scorer = NewFileSystemScorer(".", "", m.settings.OnlyDirs(), m.on_results)
m.scorer.Start()
@@ -516,7 +485,7 @@ func (m *ResultManager) set_query(query string) {
}
}
func (h *Handler) get_results() (ans ResultsType, is_complete bool) {
func (h *Handler) get_results() (ans *SortedResults, is_complete bool) {
if h.result_manager.scorer == nil {
return
}

View File

@@ -135,7 +135,7 @@ func TestChooseFilesScoring(t *testing.T) {
wg.Wait()
results := func() (ans []string) {
rr, _ := s.Results()
for _, r := range rr {
for _, r := range rr.RenderedMatches(CollectionIndex{}, -1) {
ans = append(ans, r.text)
}
return
@@ -159,6 +159,7 @@ func TestChooseFilesScoring(t *testing.T) {
func TestSortedResults(t *testing.T) {
r := NewSortedResults()
idx := CollectionIndex{}
m := func(items ...int) []*ResultItem {
ans := make([]*ResultItem, len(items))
for i, x := range items {
@@ -177,17 +178,75 @@ func TestSortedResults(t *testing.T) {
t.Fatalf("view failed for %v num:%d\n%s", CollectionIndex{slice, pos}, num, diff)
}
}
tci := func(increment int, expected int) {
orig := idx
idx = r.IncrementIndexWithWrapAround(idx, increment)
actual := int(r.At(idx).score)
if actual != expected {
t.Fatalf("increment: %d on %v failed\nexpected: %d actual: %d idx: %v", increment, orig, expected, actual, idx)
}
}
dt := func(a, b CollectionIndex, expected int) {
actual := r.distance(a, b)
if expected != actual {
t.Fatalf("distance on %v and %v failed\nexpected: %d actual: %d ", a, b, expected, actual)
}
if r.distance(b, a) != actual {
t.Fatalf("distance on %v and %v not commutative %d != %d", a, b, actual, r.distance(b, a))
}
}
tc := func(num_before, expected_new_before int, ci CollectionIndex, expected ...[]int) {
ac, new_num_before := r.SplitIntoColumns(func(int) int { return 2 }, 2, num_before, ci)
actual := make([][]int, len(ac))
for i, x := range ac {
actual[i] = utils.Map(func(r *ResultItem) int { return int(r.score) }, x)
}
if expected_new_before != new_num_before {
t.Fatalf("new_num_before not as expected for num_before: %d ci: %v\n%d != %d", num_before, ci, expected_new_before, new_num_before)
}
if diff := cmp.Diff(expected, actual); diff != "" {
t.Fatalf("wrong columns for num_before: %d ci: %v\n%s", num_before, ci, diff)
}
}
r.AddSortedSlice(m(10, 20, 30))
r.AddSortedSlice(m(40, 50, 60))
r.AddSortedSlice(m(70, 80, 90))
tc(0, 0, CollectionIndex{}, []int{10, 20}, []int{30, 40})
tc(1, 1, CollectionIndex{Pos: 1}, []int{10, 20}, []int{30, 40})
tc(1, 1, CollectionIndex{Pos: 2}, []int{20, 30}, []int{40, 50})
tc(2, 2, CollectionIndex{Pos: 2}, []int{10, 20}, []int{30, 40})
tc(20, 2, CollectionIndex{Pos: 2}, []int{10, 20}, []int{30, 40})
for num_before := range 4 {
tc(num_before, 3, CollectionIndex{2, 2}, []int{60, 70}, []int{80, 90})
}
tc(1, 1, CollectionIndex{1, 1}, []int{40, 50}, []int{60, 70})
dt(CollectionIndex{Pos: 0}, CollectionIndex{Pos: 2}, 2)
dt(CollectionIndex{Pos: 0}, CollectionIndex{Slice: 1}, 3)
dt(CollectionIndex{Pos: 0}, CollectionIndex{Slice: 1, Pos: 1}, 4)
tv(0, 0, 0, 10, 20, 30, 40, 50, 60, 70, 80, 90)
tv(0, 2, 3, 30, 40, 50)
tv(0, 3, 3, 40, 50, 60)
tv(1, 0, 4, 40, 50, 60, 70)
tci(1, 20)
tci(3, 50)
tci(-1, 40)
tci(-3, 10)
tci(-2, 80)
tci(3, 20)
tci(9, 20)
tci(-9, 20)
r.AddSortedSlice(m(100, 110, 120))
r.AddSortedSlice(m(41, 61, 71, 99))
tv(0, 0, 0, 10, 20, 30, 40, 41, 50, 60, 61, 70, 71, 80, 90, 99, 100, 110, 120)
r.AddSortedSlice(m(1, 2, 3))
tv(0, 0, 0, 1, 2, 3, 10, 20, 30, 40, 41, 50, 60, 61, 70, 71, 80, 90, 99, 100, 110, 120)
r.AddSortedSlice(m(1000, 2000))
tv(0, 0, 0, 1, 2, 3, 10, 20, 30, 40, 41, 50, 60, 61, 70, 71, 80, 90, 99, 100, 110, 120, 1000, 2000)
}
func run_scoring(b *testing.B, depth, breadth int, query string) {

View File

@@ -64,8 +64,7 @@ func (h *Handler) handle_edit_keys(ev *loop.KeyEvent) bool {
h.lp.Beep()
} else {
g := wcswidth.SplitIntoGraphemes(h.state.search_text)
h.state.SetSearchText(strings.Join(g[:len(g)-1], ""))
h.result_manager.set_query(h.state.SearchText())
h.set_query(strings.Join(g[:len(g)-1], ""))
return true
}
}