From a6335777d9b6625ba5bff5d1e5b86779d247a8d0 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 8 Oct 2025 06:41:12 +0530 Subject: [PATCH] disk cache: add a path based API This allows maintaining only a single cache entry per path when the path's contents change. --- kittens/choose_files/image_preview.go | 14 ++- tools/disk_cache/api.go | 27 ++++- tools/disk_cache/implementation.go | 136 ++++++++++++++++++------ tools/disk_cache/implementation_test.go | 75 +++++++++++-- 4 files changed, 198 insertions(+), 54 deletions(-) diff --git a/kittens/choose_files/image_preview.go b/kittens/choose_files/image_preview.go index bbd01c3ca..9b905aae9 100644 --- a/kittens/choose_files/image_preview.go +++ b/kittens/choose_files/image_preview.go @@ -32,7 +32,7 @@ type render_data struct { } type ImagePreview struct { - abspath, cache_key string + abspath string metadata fs.FileInfo disk_cache *disk_cache.DiskCache cached_data map[string]string @@ -73,7 +73,10 @@ func (p *ImagePreview) start_rendering() { defer func() { p.WakeupMainThread() }() - ans := p.disk_cache.Get(p.cache_key) + key, ans, err := p.disk_cache.GetPath(p.abspath) + if err != nil { + p.render_channel <- render_data{nil, err} + } if len(ans) > 0 { p.render_channel <- render_data{ans, nil} return @@ -82,7 +85,7 @@ func (p *ImagePreview) start_rendering() { if err != nil { p.render_channel <- render_data{nil, err} } else { - ans, err = p.disk_cache.Add(p.cache_key, rdata) + ans, err = p.disk_cache.AddPath(p.abspath, key, rdata) p.render_channel <- render_data{utils.IfElse(err == nil, ans, nil), err} } } @@ -110,11 +113,6 @@ func NewImagePreview( } else { ans.disk_cache = dc } - if key, err := disk_cache.KeyForPath(abspath); err != nil { - return nil, err - } else { - ans.cache_key = key - } go ans.start_rendering() return ans, nil } diff --git a/tools/disk_cache/api.go b/tools/disk_cache/api.go index e27249571..ac828a502 100644 --- a/tools/disk_cache/api.go +++ b/tools/disk_cache/api.go @@ -17,6 +17,7 @@ type Entry struct { type Metadata struct { TotalSize int64 + PathMap map[string]string SortedEntries []*Entry } @@ -29,29 +30,45 @@ type DiskCache struct { entries Metadata entry_map map[string]*Entry entries_mod_time time.Time + entries_dirty bool get_dir string + read_count int } func NewDiskCache(path string, max_size int64) (dc *DiskCache, err error) { return new_disk_cache(path, max_size) } -func KeyForPath(path string) (key string, err error) { - return key_for_path(path) -} - -func (dc *DiskCache) Get(key string, items ...string) map[string]string { +func (dc *DiskCache) Get(key string, items ...string) (map[string]string, error) { dc.lock() defer dc.unlock() return dc.get(key, items) } +func (dc *DiskCache) GetPath(path string, items ...string) (string, map[string]string, error) { + dc.lock() + defer dc.unlock() + key, err := key_for_path(path) + if err != nil { + return "", nil, err + } + ans, err := dc.get(key, items) + return key, ans, err +} + func (dc *DiskCache) Remove(key string) (err error) { dc.lock() defer dc.unlock() return dc.remove(key) } +func (dc *DiskCache) AddPath(path, key string, items map[string][]byte) (ans map[string]string, err error) { + dc.lock() + defer dc.unlock() + ans, err = dc.add_path(path, key, items) + return +} + func (dc *DiskCache) Add(key string, items map[string][]byte) (ans map[string]string, err error) { dc.lock() defer dc.unlock() diff --git a/tools/disk_cache/implementation.go b/tools/disk_cache/implementation.go index 37eeb6315..3a55dc620 100644 --- a/tools/disk_cache/implementation.go +++ b/tools/disk_cache/implementation.go @@ -5,6 +5,8 @@ import ( "encoding/hex" "encoding/json" "fmt" + "io/fs" + "maps" "os" "path/filepath" "slices" @@ -28,12 +30,13 @@ func new_disk_cache(path string, max_size int64) (dc *DiskCache, err error) { if err = ans.ensure_entries(); err != nil { return } - if pruned, err := ans.prune(); err != nil { - return nil, err - } else if pruned { - if err = ans.write_entries(); err != nil { - return nil, err + defer func() { + if we := ans.write_entries_if_dirty(); we != nil && err == nil { + err = we } + }() + if _, err := ans.prune(); err != nil { + return nil, err } if ans.get_dir, err = os.MkdirTemp(ans.Path, "getdir-*"); err != nil { return @@ -85,7 +88,16 @@ func (dc *DiskCache) unlock() { func (dc *DiskCache) entries_path() string { return filepath.Join(dc.Path, "entries.json") } -func (dc *DiskCache) write_entries() (err error) { +func (dc *DiskCache) write_entries_if_dirty() (err error) { + if !dc.entries_dirty { + return + } + defer func() { + if err == nil { + dc.entries_dirty = false + dc.entries_mod_time = time.Now() + } + }() if d, err := json.Marshal(dc.entries); err != nil { return err } else { @@ -93,11 +105,15 @@ func (dc *DiskCache) write_entries() (err error) { } } -func (dc *DiskCache) rebuild_entries() error { +func (e Entry) String() string { + return fmt.Sprintf("Entry{Key: %s, Size: %d, LastUsed: %s}", e.Key, e.Size, e.LastUsed) +} + +func (dc *DiskCache) entries_from_folders() (total_size int64, ans map[string]*Entry, sorted []*Entry, err error) { if entries, err := os.ReadDir(dc.Path); err != nil { - return err + return 0, nil, nil, err } else { - ans := make(map[string]*Entry) + ans = make(map[string]*Entry) var total int64 for _, x := range entries { if x.IsDir() { @@ -105,7 +121,7 @@ func (dc *DiskCache) rebuild_entries() error { key := sub_entries[0].Name() path := dc.folder_for_key(key) if file_entries, err := os.ReadDir(path); err == nil { - e := Entry{} + e := Entry{Key: key} for _, f := range file_entries { if fi, err := f.Info(); err == nil { e.Size += fi.Size() @@ -120,37 +136,56 @@ func (dc *DiskCache) rebuild_entries() error { } } } - sorted := utils.Values(ans) + sorted = utils.Values(ans) slices.SortFunc(sorted, func(a, b *Entry) int { return a.LastUsed.Compare(b.LastUsed) }) - dc.entries = Metadata{TotalSize: total, SortedEntries: sorted} - dc.entry_map = ans + return total, ans, sorted, nil } - return nil +} + +func (dc *DiskCache) rebuild_entries() error { + total, ans, sorted, err := dc.entries_from_folders() + if err != nil { + return err + } + dc.entries = Metadata{TotalSize: total, SortedEntries: sorted, PathMap: make(map[string]string)} + dc.entry_map = ans + dc.entries_dirty = true + return dc.write_entries_if_dirty() } func (dc *DiskCache) ensure_entries() error { needed := dc.entry_map == nil path := dc.entries_path() + var stat_result fs.FileInfo if !needed { if s, err := os.Stat(path); err == nil && s.ModTime().After(dc.entries_mod_time) { needed = true + stat_result = s + dc.entries_mod_time = s.ModTime() } } if needed { if data, err := os.ReadFile(path); err != nil { if os.IsNotExist(err) { dc.entry_map = make(map[string]*Entry) - dc.entries = Metadata{SortedEntries: make([]*Entry, 0)} + dc.entries = Metadata{SortedEntries: make([]*Entry, 0), PathMap: make(map[string]string)} } else { return err } } else { - dc.entries = Metadata{SortedEntries: make([]*Entry, 0)} + dc.read_count += 1 + dc.entries = Metadata{SortedEntries: make([]*Entry, 0), PathMap: make(map[string]string)} if err := json.Unmarshal(data, &dc.entries); err != nil { // corrupted data dc.rebuild_entries() + } else { + if stat_result == nil { + if s, err := os.Stat(path); err == nil { + dc.entries_mod_time = s.ModTime() + } + } } dc.entry_map = make(map[string]*Entry) for _, e := range dc.entries.SortedEntries { @@ -170,13 +205,6 @@ func (dc *DiskCache) folder_for_key(key string) (ans string) { return filepath.Join(dc.Path, ans) } -func (dc *DiskCache) update_last_used(key string) { - if dc.ensure_entries() == nil { - dc.update_timestamp(key) - } - -} - func (dc *DiskCache) export_to_get_dir(key, path string) (string, error) { dest := filepath.Join(dc.get_dir, key+"-"+filepath.Base(path)) if err := os.Link(path, dest); err != nil { @@ -189,11 +217,17 @@ func (dc *DiskCache) export_to_get_dir(key, path string) (string, error) { } -func (dc *DiskCache) get(key string, items []string) map[string]string { +func (dc *DiskCache) get(key string, items []string) (map[string]string, error) { + if err := dc.ensure_entries(); err != nil { + return nil, err + } base := dc.folder_for_key(key) if len(items) == 0 { if entries, err := os.ReadDir(base); err != nil { - return make(map[string]string) + if os.IsNotExist(err) { + err = nil + } + return nil, err } else { for _, e := range entries { items = append(items, e.Name()) @@ -201,7 +235,10 @@ func (dc *DiskCache) get(key string, items []string) map[string]string { } } else { if s, err := os.Stat(base); err != nil || !s.IsDir() { - return make(map[string]string) + if os.IsNotExist(err) { + err = nil + } + return nil, err } } ans := make(map[string]string, len(items)) @@ -215,14 +252,21 @@ func (dc *DiskCache) get(key string, items []string) map[string]string { ans[x] = dest } } - dc.update_last_used(key) - return ans + if len(items) > 0 { + dc.update_timestamp(key) + } + return ans, dc.write_entries_if_dirty() } func (dc *DiskCache) remove(key string) (err error) { if err = dc.ensure_entries(); err != nil { return } + defer func() { + if we := dc.write_entries_if_dirty(); we != nil && err == nil { + err = we + } + }() base := dc.folder_for_key(key) if err = os.RemoveAll(base); err == nil { t := dc.entry_map[key] @@ -230,7 +274,7 @@ func (dc *DiskCache) remove(key string) (err error) { delete(dc.entry_map, key) dc.entries.TotalSize = max(0, dc.entries.TotalSize-t.Size) dc.entries.SortedEntries = utils.Filter(dc.entries.SortedEntries, func(x *Entry) bool { return x.Key != key }) - return dc.write_entries() + dc.entries_dirty = true } } return @@ -245,8 +289,10 @@ func (dc *DiskCache) prune() (bool, error) { if err := os.RemoveAll(base); err == nil { t := dc.entries.SortedEntries[0] delete(dc.entry_map, t.Key) + maps.DeleteFunc(dc.entries.PathMap, func(path, key string) bool { return key == t.Key }) dc.entries.TotalSize = max(0, dc.entries.TotalSize-t.Size) dc.entries.SortedEntries = dc.entries.SortedEntries[1:] + dc.entries_dirty = true } else { return false, err } @@ -260,6 +306,7 @@ func (dc *DiskCache) update_timestamp(key string) { idx := slices.Index(dc.entries.SortedEntries, t) copy(dc.entries.SortedEntries[idx:], dc.entries.SortedEntries[idx+1:]) dc.entries.SortedEntries[len(dc.entries.SortedEntries)-1] = t + dc.entries_dirty = true } func (dc *DiskCache) update_accounting(key string, changed int64) (err error) { @@ -273,16 +320,43 @@ func (dc *DiskCache) update_accounting(key string, changed int64) (err error) { t.Size += changed t.Size = max(0, t.Size) dc.entries.TotalSize += t.Size - old_size + dc.entries_dirty = true dc.update_timestamp(key) dc.prune() - return dc.write_entries() + return dc.write_entries_if_dirty() } func (dc *DiskCache) keys() (ans []string, err error) { if err = dc.ensure_entries(); err != nil { return } - return utils.Keys(dc.entry_map), nil + ans = make([]string, len(dc.entries.SortedEntries)) + for i, e := range dc.entries.SortedEntries { + ans[i] = e.Key + } + return +} + +func (dc *DiskCache) add_path(path, key string, items map[string][]byte) (ans map[string]string, err error) { + if err = dc.ensure_entries(); err != nil { + return + } + defer func() { + if we := dc.write_entries_if_dirty(); we != nil && err == nil { + err = we + } + }() + + if existing := dc.entries.PathMap[path]; existing != "" && existing != key { + delete(dc.entries.PathMap, path) + dc.entries_dirty = true + if err = dc.remove(existing); err != nil { + return + } + } + dc.entries.PathMap[path] = key + dc.entries_dirty = true + return dc.add(key, items) } func (dc *DiskCache) add(key string, items map[string][]byte) (ans map[string]string, err error) { diff --git a/tools/disk_cache/implementation_test.go b/tools/disk_cache/implementation_test.go index 8105fc142..0bf0187de 100644 --- a/tools/disk_cache/implementation_test.go +++ b/tools/disk_cache/implementation_test.go @@ -3,7 +3,8 @@ package disk_cache import ( "fmt" "os" - "slices" + "path/filepath" + "runtime/debug" "strings" "testing" @@ -23,16 +24,28 @@ func TestDiskCache(t *testing.T) { if err != nil { t.Fatal(err) } + arc := func(dc *DiskCache, expected int) { + if expected != dc.read_count { + t.Fatalf("disk cache has unexpected read count: %d != %d\n%s", expected, dc.read_count, debug.Stack()) + } + } - m := dc.Get("missing", "one", "two") - if diff := cmp.Diff(m, make(map[string]string)); diff != "" { - t.Fatalf("Unexpected return from missing: %s", diff) + m, err := dc.Get("missing", "one", "two") + if err != nil { + t.Fatal(err) + } + if len(m) > 0 { + t.Fatalf("Unexpected return from missing: %s", m) } dc.Add("k1", map[string][]byte{"1": []byte("abcd"), "2": []byte("efgh")}) + arc(dc, 0) ad := func(key string, expected map[string]string) { for _, x := range []*DiskCache{dc, dc2} { - actual := x.Get(key, utils.Keys(expected)...) + actual, err := x.Get(key, utils.Keys(expected)...) + if err != nil { + t.Fatal(err) + } for k, path := range actual { d, err := os.ReadFile(path) @@ -47,32 +60,74 @@ func TestDiskCache(t *testing.T) { } } ak := func(keys ...string) { - for _, x := range []*DiskCache{dc, dc2} { + for i, x := range []*DiskCache{dc, dc2} { kk, err := x.keys() if err != nil { t.Fatal(err) } - slices.Sort(kk) - slices.Sort(keys) if diff := cmp.Diff(keys, kk); diff != "" { - t.Fatalf("Unexpected keys: %s", diff) + t.Fatalf("wrong keys in %d: %s", i+1, diff) } } } ad("k1", map[string]string{"1": "abcd", "2": "efgh"}) + arc(dc, 0) + arc(dc2, 1) dc.Add("k1", map[string][]byte{"3": []byte("ijk"), "4": []byte("lmo")}) + arc(dc, 1) // because dc2.Get() will have updated the file + arc(dc2, 1) + ak("k1") dc2.Add("k2", map[string][]byte{"1": []byte("123456789")}) + arc(dc, 1) + arc(dc2, 2) + ak("k1", "k2") ad("k1", map[string]string{"1": "abcd", "2": "efgh", "3": "ijk"}) if dc.entries.TotalSize != 14+9 { t.Fatalf("TotalSize: %d != %d", dc.entries.TotalSize, 14+9) } + arc(dc, 2) // dc2.Add() + arc(dc2, 3) // dc.Get() + ak("k2", "k1") + dc.Get("k2") + arc(dc, 3) // dc2.Get() + arc(dc2, 3) ak("k1", "k2") dc.Add("k3", map[string][]byte{"1": []byte(strings.Repeat("a", int(dc.MaxSize)-10))}) - ak("k3", "k2") + arc(dc, 3) + ak("k2", "k3") // check that creating a new disk cache prunes _, err = NewDiskCache(tdir, dc.MaxSize-8) if err != nil { t.Fatal(err) } ak("k3") + arc(dc, 4) // NewDiskCache() + + // test the path api + path := filepath.Join(tdir, "source") + if err = os.WriteFile(path, []byte("abcdfjrof"), 0o600); err != nil { + t.Fatal(err) + } + key, _, err := dc.GetPath(path) + if _, err = dc.AddPath(path, key, map[string][]byte{"1": []byte("1")}); err != nil { + t.Fatal(err) + } + _, _, entries_before, _ := dc.entries_from_folders() + if diff := cmp.Diff(1, len(dc.entries.PathMap)); diff != "" { + t.Fatalf("unexpected pathmap count: %s", diff) + } + if err = os.WriteFile(path, []byte("changed contents"), 0o600); err != nil { + t.Fatal(err) + } + if _, err = dc.AddPath(path, key, map[string][]byte{"1": []byte("2")}); err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(1, len(dc.entries.PathMap)); diff != "" { + t.Fatalf("unexpected pathmap count: %s", diff) + } + _, _, entries_after, _ := dc.entries_from_folders() + if len(entries_before) != len(entries_after) { + t.Fatalf("unexpected entries: %s", entries_after) + } + arc(dc, 4) }