disk cache: add a path based API

This allows maintaining only a single cache entry per path when the
path's contents change.

kittens/choose_files/image_preview.go

@@ -32,7 +32,7 @@ type render_data struct {
 }
 
 type ImagePreview struct {
-	abspath, cache_key    string
+	abspath               string
 	metadata              fs.FileInfo
 	disk_cache            *disk_cache.DiskCache
 	cached_data           map[string]string
@@ -73,7 +73,10 @@ func (p *ImagePreview) start_rendering() {
 	defer func() {
 		p.WakeupMainThread()
 	}()
-	ans := p.disk_cache.Get(p.cache_key)
+	key, ans, err := p.disk_cache.GetPath(p.abspath)
+	if err != nil {
+		p.render_channel <- render_data{nil, err}
+	}
 	if len(ans) > 0 {
 		p.render_channel <- render_data{ans, nil}
 		return
@@ -82,7 +85,7 @@ func (p *ImagePreview) start_rendering() {
 	if err != nil {
 		p.render_channel <- render_data{nil, err}
 	} else {
-		ans, err = p.disk_cache.Add(p.cache_key, rdata)
+		ans, err = p.disk_cache.AddPath(p.abspath, key, rdata)
 		p.render_channel <- render_data{utils.IfElse(err == nil, ans, nil), err}
 	}
 }
@@ -110,11 +113,6 @@ func NewImagePreview(
 	} else {
 		ans.disk_cache = dc
 	}
-	if key, err := disk_cache.KeyForPath(abspath); err != nil {
-		return nil, err
-	} else {
-		ans.cache_key = key
-	}
 	go ans.start_rendering()
 	return ans, nil
 }

tools/disk_cache/api.go

@@ -17,6 +17,7 @@ type Entry struct {
 
 type Metadata struct {
 	TotalSize     int64
+	PathMap       map[string]string
 	SortedEntries []*Entry
 }
 
@@ -29,29 +30,45 @@ type DiskCache struct {
 	entries          Metadata
 	entry_map        map[string]*Entry
 	entries_mod_time time.Time
+	entries_dirty    bool
 	get_dir          string
+	read_count       int
 }
 
 func NewDiskCache(path string, max_size int64) (dc *DiskCache, err error) {
 	return new_disk_cache(path, max_size)
 }
 
-func KeyForPath(path string) (key string, err error) {
-	return key_for_path(path)
-}
-
-func (dc *DiskCache) Get(key string, items ...string) map[string]string {
+func (dc *DiskCache) Get(key string, items ...string) (map[string]string, error) {
 	dc.lock()
 	defer dc.unlock()
 	return dc.get(key, items)
 }
 
+func (dc *DiskCache) GetPath(path string, items ...string) (string, map[string]string, error) {
+	dc.lock()
+	defer dc.unlock()
+	key, err := key_for_path(path)
+	if err != nil {
+		return "", nil, err
+	}
+	ans, err := dc.get(key, items)
+	return key, ans, err
+}
+
 func (dc *DiskCache) Remove(key string) (err error) {
 	dc.lock()
 	defer dc.unlock()
 	return dc.remove(key)
 }
 
+func (dc *DiskCache) AddPath(path, key string, items map[string][]byte) (ans map[string]string, err error) {
+	dc.lock()
+	defer dc.unlock()
+	ans, err = dc.add_path(path, key, items)
+	return
+}
+
 func (dc *DiskCache) Add(key string, items map[string][]byte) (ans map[string]string, err error) {
 	dc.lock()
 	defer dc.unlock()

tools/disk_cache/implementation.go

@@ -5,6 +5,8 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
+	"io/fs"
+	"maps"
 	"os"
 	"path/filepath"
 	"slices"
@@ -28,12 +30,13 @@ func new_disk_cache(path string, max_size int64) (dc *DiskCache, err error) {
 	if err = ans.ensure_entries(); err != nil {
 		return
 	}
-	if pruned, err := ans.prune(); err != nil {
-		return nil, err
-	} else if pruned {
-		if err = ans.write_entries(); err != nil {
-			return nil, err
+	defer func() {
+		if we := ans.write_entries_if_dirty(); we != nil && err == nil {
+			err = we
 		}
+	}()
+	if _, err := ans.prune(); err != nil {
+		return nil, err
 	}
 	if ans.get_dir, err = os.MkdirTemp(ans.Path, "getdir-*"); err != nil {
 		return
@@ -85,7 +88,16 @@ func (dc *DiskCache) unlock() {
 
 func (dc *DiskCache) entries_path() string { return filepath.Join(dc.Path, "entries.json") }
 
-func (dc *DiskCache) write_entries() (err error) {
+func (dc *DiskCache) write_entries_if_dirty() (err error) {
+	if !dc.entries_dirty {
+		return
+	}
+	defer func() {
+		if err == nil {
+			dc.entries_dirty = false
+			dc.entries_mod_time = time.Now()
+		}
+	}()
 	if d, err := json.Marshal(dc.entries); err != nil {
 		return err
 	} else {
@@ -93,11 +105,15 @@ func (dc *DiskCache) write_entries() (err error) {
 	}
 }
 
-func (dc *DiskCache) rebuild_entries() error {
+func (e Entry) String() string {
+	return fmt.Sprintf("Entry{Key: %s, Size: %d, LastUsed: %s}", e.Key, e.Size, e.LastUsed)
+}
+
+func (dc *DiskCache) entries_from_folders() (total_size int64, ans map[string]*Entry, sorted []*Entry, err error) {
 	if entries, err := os.ReadDir(dc.Path); err != nil {
-		return err
+		return 0, nil, nil, err
 	} else {
-		ans := make(map[string]*Entry)
+		ans = make(map[string]*Entry)
 		var total int64
 		for _, x := range entries {
 			if x.IsDir() {
@@ -105,7 +121,7 @@ func (dc *DiskCache) rebuild_entries() error {
 					key := sub_entries[0].Name()
 					path := dc.folder_for_key(key)
 					if file_entries, err := os.ReadDir(path); err == nil {
-						e := Entry{}
+						e := Entry{Key: key}
 						for _, f := range file_entries {
 							if fi, err := f.Info(); err == nil {
 								e.Size += fi.Size()
@@ -120,37 +136,56 @@ func (dc *DiskCache) rebuild_entries() error {
 				}
 			}
 		}
-		sorted := utils.Values(ans)
+		sorted = utils.Values(ans)
 		slices.SortFunc(sorted, func(a, b *Entry) int {
 			return a.LastUsed.Compare(b.LastUsed)
 		})
-		dc.entries = Metadata{TotalSize: total, SortedEntries: sorted}
-		dc.entry_map = ans
+		return total, ans, sorted, nil
 	}
-	return nil
+}
+
+func (dc *DiskCache) rebuild_entries() error {
+	total, ans, sorted, err := dc.entries_from_folders()
+	if err != nil {
+		return err
+	}
+	dc.entries = Metadata{TotalSize: total, SortedEntries: sorted, PathMap: make(map[string]string)}
+	dc.entry_map = ans
+	dc.entries_dirty = true
+	return dc.write_entries_if_dirty()
 }
 
 func (dc *DiskCache) ensure_entries() error {
 	needed := dc.entry_map == nil
 	path := dc.entries_path()
+	var stat_result fs.FileInfo
 	if !needed {
 		if s, err := os.Stat(path); err == nil && s.ModTime().After(dc.entries_mod_time) {
 			needed = true
+			stat_result = s
+			dc.entries_mod_time = s.ModTime()
 		}
 	}
 	if needed {
 		if data, err := os.ReadFile(path); err != nil {
 			if os.IsNotExist(err) {
 				dc.entry_map = make(map[string]*Entry)
-				dc.entries = Metadata{SortedEntries: make([]*Entry, 0)}
+				dc.entries = Metadata{SortedEntries: make([]*Entry, 0), PathMap: make(map[string]string)}
 			} else {
 				return err
 			}
 		} else {
-			dc.entries = Metadata{SortedEntries: make([]*Entry, 0)}
+			dc.read_count += 1
+			dc.entries = Metadata{SortedEntries: make([]*Entry, 0), PathMap: make(map[string]string)}
 			if err := json.Unmarshal(data, &dc.entries); err != nil {
 				// corrupted data
 				dc.rebuild_entries()
+			} else {
+				if stat_result == nil {
+					if s, err := os.Stat(path); err == nil {
+						dc.entries_mod_time = s.ModTime()
+					}
+				}
 			}
 			dc.entry_map = make(map[string]*Entry)
 			for _, e := range dc.entries.SortedEntries {
@@ -170,13 +205,6 @@ func (dc *DiskCache) folder_for_key(key string) (ans string) {
 	return filepath.Join(dc.Path, ans)
 }
 
-func (dc *DiskCache) update_last_used(key string) {
-	if dc.ensure_entries() == nil {
-		dc.update_timestamp(key)
-	}
-
-}
-
 func (dc *DiskCache) export_to_get_dir(key, path string) (string, error) {
 	dest := filepath.Join(dc.get_dir, key+"-"+filepath.Base(path))
 	if err := os.Link(path, dest); err != nil {
@@ -189,11 +217,17 @@ func (dc *DiskCache) export_to_get_dir(key, path string) (string, error) {
 
 }
 
-func (dc *DiskCache) get(key string, items []string) map[string]string {
+func (dc *DiskCache) get(key string, items []string) (map[string]string, error) {
+	if err := dc.ensure_entries(); err != nil {
+		return nil, err
+	}
 	base := dc.folder_for_key(key)
 	if len(items) == 0 {
 		if entries, err := os.ReadDir(base); err != nil {
-			return make(map[string]string)
+			if os.IsNotExist(err) {
+				err = nil
+			}
+			return nil, err
 		} else {
 			for _, e := range entries {
 				items = append(items, e.Name())
@@ -201,7 +235,10 @@ func (dc *DiskCache) get(key string, items []string) map[string]string {
 		}
 	} else {
 		if s, err := os.Stat(base); err != nil || !s.IsDir() {
-			return make(map[string]string)
+			if os.IsNotExist(err) {
+				err = nil
+			}
+			return nil, err
 		}
 	}
 	ans := make(map[string]string, len(items))
@@ -215,14 +252,21 @@ func (dc *DiskCache) get(key string, items []string) map[string]string {
 			ans[x] = dest
 		}
 	}
-	dc.update_last_used(key)
-	return ans
+	if len(items) > 0 {
+		dc.update_timestamp(key)
+	}
+	return ans, dc.write_entries_if_dirty()
 }
 
 func (dc *DiskCache) remove(key string) (err error) {
 	if err = dc.ensure_entries(); err != nil {
 		return
 	}
+	defer func() {
+		if we := dc.write_entries_if_dirty(); we != nil && err == nil {
+			err = we
+		}
+	}()
 	base := dc.folder_for_key(key)
 	if err = os.RemoveAll(base); err == nil {
 		t := dc.entry_map[key]
@@ -230,7 +274,7 @@ func (dc *DiskCache) remove(key string) (err error) {
 			delete(dc.entry_map, key)
 			dc.entries.TotalSize = max(0, dc.entries.TotalSize-t.Size)
 			dc.entries.SortedEntries = utils.Filter(dc.entries.SortedEntries, func(x *Entry) bool { return x.Key != key })
-			return dc.write_entries()
+			dc.entries_dirty = true
 		}
 	}
 	return
@@ -245,8 +289,10 @@ func (dc *DiskCache) prune() (bool, error) {
 		if err := os.RemoveAll(base); err == nil {
 			t := dc.entries.SortedEntries[0]
 			delete(dc.entry_map, t.Key)
+			maps.DeleteFunc(dc.entries.PathMap, func(path, key string) bool { return key == t.Key })
 			dc.entries.TotalSize = max(0, dc.entries.TotalSize-t.Size)
 			dc.entries.SortedEntries = dc.entries.SortedEntries[1:]
+			dc.entries_dirty = true
 		} else {
 			return false, err
 		}
@@ -260,6 +306,7 @@ func (dc *DiskCache) update_timestamp(key string) {
 	idx := slices.Index(dc.entries.SortedEntries, t)
 	copy(dc.entries.SortedEntries[idx:], dc.entries.SortedEntries[idx+1:])
 	dc.entries.SortedEntries[len(dc.entries.SortedEntries)-1] = t
+	dc.entries_dirty = true
 }
 
 func (dc *DiskCache) update_accounting(key string, changed int64) (err error) {
@@ -273,16 +320,43 @@ func (dc *DiskCache) update_accounting(key string, changed int64) (err error) {
 	t.Size += changed
 	t.Size = max(0, t.Size)
 	dc.entries.TotalSize += t.Size - old_size
+	dc.entries_dirty = true
 	dc.update_timestamp(key)
 	dc.prune()
-	return dc.write_entries()
+	return dc.write_entries_if_dirty()
 }
 
 func (dc *DiskCache) keys() (ans []string, err error) {
 	if err = dc.ensure_entries(); err != nil {
 		return
 	}
-	return utils.Keys(dc.entry_map), nil
+	ans = make([]string, len(dc.entries.SortedEntries))
+	for i, e := range dc.entries.SortedEntries {
+		ans[i] = e.Key
+	}
+	return
+}
+
+func (dc *DiskCache) add_path(path, key string, items map[string][]byte) (ans map[string]string, err error) {
+	if err = dc.ensure_entries(); err != nil {
+		return
+	}
+	defer func() {
+		if we := dc.write_entries_if_dirty(); we != nil && err == nil {
+			err = we
+		}
+	}()
+
+	if existing := dc.entries.PathMap[path]; existing != "" && existing != key {
+		delete(dc.entries.PathMap, path)
+		dc.entries_dirty = true
+		if err = dc.remove(existing); err != nil {
+			return
+		}
+	}
+	dc.entries.PathMap[path] = key
+	dc.entries_dirty = true
+	return dc.add(key, items)
 }
 
 func (dc *DiskCache) add(key string, items map[string][]byte) (ans map[string]string, err error) {

tools/disk_cache/implementation_test.go

@@ -3,7 +3,8 @@ package disk_cache
 import (
 	"fmt"
 	"os"
-	"slices"
+	"path/filepath"
+	"runtime/debug"
 	"strings"
 	"testing"
 
@@ -23,16 +24,28 @@ func TestDiskCache(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	arc := func(dc *DiskCache, expected int) {
+		if expected != dc.read_count {
+			t.Fatalf("disk cache has unexpected read count: %d != %d\n%s", expected, dc.read_count, debug.Stack())
+		}
+	}
 
-	m := dc.Get("missing", "one", "two")
-	if diff := cmp.Diff(m, make(map[string]string)); diff != "" {
-		t.Fatalf("Unexpected return from missing: %s", diff)
+	m, err := dc.Get("missing", "one", "two")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(m) > 0 {
+		t.Fatalf("Unexpected return from missing: %s", m)
 	}
 	dc.Add("k1", map[string][]byte{"1": []byte("abcd"), "2": []byte("efgh")})
+	arc(dc, 0)
 
 	ad := func(key string, expected map[string]string) {
 		for _, x := range []*DiskCache{dc, dc2} {
-			actual := x.Get(key, utils.Keys(expected)...)
+			actual, err := x.Get(key, utils.Keys(expected)...)
+			if err != nil {
+				t.Fatal(err)
+			}
 
 			for k, path := range actual {
 				d, err := os.ReadFile(path)
@@ -47,32 +60,74 @@ func TestDiskCache(t *testing.T) {
 		}
 	}
 	ak := func(keys ...string) {
-		for _, x := range []*DiskCache{dc, dc2} {
+		for i, x := range []*DiskCache{dc, dc2} {
 			kk, err := x.keys()
 			if err != nil {
 				t.Fatal(err)
 			}
-			slices.Sort(kk)
-			slices.Sort(keys)
 			if diff := cmp.Diff(keys, kk); diff != "" {
-				t.Fatalf("Unexpected keys: %s", diff)
+				t.Fatalf("wrong keys in %d: %s", i+1, diff)
 			}
 		}
 	}
 	ad("k1", map[string]string{"1": "abcd", "2": "efgh"})
+	arc(dc, 0)
+	arc(dc2, 1)
 	dc.Add("k1", map[string][]byte{"3": []byte("ijk"), "4": []byte("lmo")})
+	arc(dc, 1) // because dc2.Get() will have updated the file
+	arc(dc2, 1)
+	ak("k1")
 	dc2.Add("k2", map[string][]byte{"1": []byte("123456789")})
+	arc(dc, 1)
+	arc(dc2, 2)
+	ak("k1", "k2")
 	ad("k1", map[string]string{"1": "abcd", "2": "efgh", "3": "ijk"})
 	if dc.entries.TotalSize != 14+9 {
 		t.Fatalf("TotalSize: %d != %d", dc.entries.TotalSize, 14+9)
 	}
+	arc(dc, 2)  // dc2.Add()
+	arc(dc2, 3) // dc.Get()
+	ak("k2", "k1")
+	dc.Get("k2")
+	arc(dc, 3) // dc2.Get()
+	arc(dc2, 3)
 	ak("k1", "k2")
 	dc.Add("k3", map[string][]byte{"1": []byte(strings.Repeat("a", int(dc.MaxSize)-10))})
-	ak("k3", "k2")
+	arc(dc, 3)
+	ak("k2", "k3")
 	// check that creating a new disk cache prunes
 	_, err = NewDiskCache(tdir, dc.MaxSize-8)
 	if err != nil {
 		t.Fatal(err)
 	}
 	ak("k3")
+	arc(dc, 4) // NewDiskCache()
+
+	// test the path api
+	path := filepath.Join(tdir, "source")
+	if err = os.WriteFile(path, []byte("abcdfjrof"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	key, _, err := dc.GetPath(path)
+	if _, err = dc.AddPath(path, key, map[string][]byte{"1": []byte("1")}); err != nil {
+		t.Fatal(err)
+	}
+	_, _, entries_before, _ := dc.entries_from_folders()
+	if diff := cmp.Diff(1, len(dc.entries.PathMap)); diff != "" {
+		t.Fatalf("unexpected pathmap count: %s", diff)
+	}
+	if err = os.WriteFile(path, []byte("changed contents"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	if _, err = dc.AddPath(path, key, map[string][]byte{"1": []byte("2")}); err != nil {
+		t.Fatal(err)
+	}
+	if diff := cmp.Diff(1, len(dc.entries.PathMap)); diff != "" {
+		t.Fatalf("unexpected pathmap count: %s", diff)
+	}
+	_, _, entries_after, _ := dc.entries_from_folders()
+	if len(entries_before) != len(entries_after) {
+		t.Fatalf("unexpected entries: %s", entries_after)
+	}
+	arc(dc, 4)
 }