feat: add filecache prometheus metrics (#18089)

Dynamic parameters has an in memory file cache. This adds prometheus
metrics to monitor said cache.

coderd/files/cache.go

@@ -7,6 +7,8 @@ import (
 	"sync"
 
 	"github.com/google/uuid"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	"golang.org/x/xerrors"
 
 	archivefs "github.com/coder/coder/v2/archive/fs"
@@ -16,22 +18,78 @@ import (
 
 // NewFromStore returns a file cache that will fetch files from the provided
 // database.
-func NewFromStore(store database.Store) *Cache {
-	fetcher := func(ctx context.Context, fileID uuid.UUID) (fs.FS, error) {
+func NewFromStore(store database.Store, registerer prometheus.Registerer) *Cache {
+	fetch := func(ctx context.Context, fileID uuid.UUID) (cacheEntryValue, error) {
 		file, err := store.GetFileByID(ctx, fileID)
 		if err != nil {
-			return nil, xerrors.Errorf("failed to read file from database: %w", err)
+			return cacheEntryValue{}, xerrors.Errorf("failed to read file from database: %w", err)
 		}
 
 		content := bytes.NewBuffer(file.Data)
-		return archivefs.FromTarReader(content), nil
+		return cacheEntryValue{
+			FS:   archivefs.FromTarReader(content),
+			size: int64(content.Len()),
+		}, nil
 	}
 
-	return &Cache{
+	return New(fetch, registerer)
+}
+
+func New(fetch fetcher, registerer prometheus.Registerer) *Cache {
+	return (&Cache{
 		lock:    sync.Mutex{},
 		data:    make(map[uuid.UUID]*cacheEntry),
-		fetcher: fetcher,
-	}
+		fetcher: fetch,
+	}).registerMetrics(registerer)
+}
+
+func (c *Cache) registerMetrics(registerer prometheus.Registerer) *Cache {
+	subsystem := "file_cache"
+	f := promauto.With(registerer)
+
+	c.currentCacheSize = f.NewGauge(prometheus.GaugeOpts{
+		Namespace: "coderd",
+		Subsystem: subsystem,
+		Name:      "open_files_size_bytes_current",
+		Help:      "The current amount of memory of all files currently open in the file cache.",
+	})
+
+	c.totalCacheSize = f.NewCounter(prometheus.CounterOpts{
+		Namespace: "coderd",
+		Subsystem: subsystem,
+		Name:      "open_files_size_bytes_total",
+		Help:      "The total amount of memory ever opened in the file cache. This number never decrements.",
+	})
+
+	c.currentOpenFiles = f.NewGauge(prometheus.GaugeOpts{
+		Namespace: "coderd",
+		Subsystem: subsystem,
+		Name:      "open_files_current",
+		Help:      "The count of unique files currently open in the file cache.",
+	})
+
+	c.totalOpenedFiles = f.NewCounter(prometheus.CounterOpts{
+		Namespace: "coderd",
+		Subsystem: subsystem,
+		Name:      "open_files_total",
+		Help:      "The total count of unique files ever opened in the file cache.",
+	})
+
+	c.currentOpenFileReferences = f.NewGauge(prometheus.GaugeOpts{
+		Namespace: "coderd",
+		Subsystem: subsystem,
+		Name:      "open_file_refs_current",
+		Help:      "The count of file references currently open in the file cache. Multiple references can be held for the same file.",
+	})
+
+	c.totalOpenFileReferences = f.NewCounter(prometheus.CounterOpts{
+		Namespace: "coderd",
+		Subsystem: subsystem,
+		Name:      "open_file_refs_total",
+		Help:      "The total number of file references ever opened in the file cache.",
+	})
+
+	return c
 }
 
 // Cache persists the files for template versions, and is used by dynamic
@@ -43,15 +101,34 @@ type Cache struct {
 	lock sync.Mutex
 	data map[uuid.UUID]*cacheEntry
 	fetcher
+
+	// metrics
+	cacheMetrics
+}
+
+type cacheMetrics struct {
+	currentOpenFileReferences prometheus.Gauge
+	totalOpenFileReferences   prometheus.Counter
+
+	currentOpenFiles prometheus.Gauge
+	totalOpenedFiles prometheus.Counter
+
+	currentCacheSize prometheus.Gauge
+	totalCacheSize   prometheus.Counter
+}
+
+type cacheEntryValue struct {
+	fs.FS
+	size int64
 }
 
 type cacheEntry struct {
 	// refCount must only be accessed while the Cache lock is held.
 	refCount int
-	value    *lazy.ValueWithError[fs.FS]
+	value    *lazy.ValueWithError[cacheEntryValue]
 }
 
-type fetcher func(context.Context, uuid.UUID) (fs.FS, error)
+type fetcher func(context.Context, uuid.UUID) (cacheEntryValue, error)
 
 // Acquire will load the fs.FS for the given file. It guarantees that parallel
 // calls for the same fileID will only result in one fetch, and that parallel
@@ -66,18 +143,27 @@ func (c *Cache) Acquire(ctx context.Context, fileID uuid.UUID) (fs.FS, error) {
 	it, err := c.prepare(ctx, fileID).Load()
 	if err != nil {
 		c.Release(fileID)
+		return nil, err
 	}
-	return it, err
+	return it.FS, err
 }
 
-func (c *Cache) prepare(ctx context.Context, fileID uuid.UUID) *lazy.ValueWithError[fs.FS] {
+func (c *Cache) prepare(ctx context.Context, fileID uuid.UUID) *lazy.ValueWithError[cacheEntryValue] {
 	c.lock.Lock()
 	defer c.lock.Unlock()
 
 	entry, ok := c.data[fileID]
 	if !ok {
-		value := lazy.NewWithError(func() (fs.FS, error) {
-			return c.fetcher(ctx, fileID)
+		value := lazy.NewWithError(func() (cacheEntryValue, error) {
+			val, err := c.fetcher(ctx, fileID)
+
+			// Always add to the cache size the bytes of the file loaded.
+			if err == nil {
+				c.currentCacheSize.Add(float64(val.size))
+				c.totalCacheSize.Add(float64(val.size))
+			}
+
+			return val, err
 		})
 
 		entry = &cacheEntry{
@@ -85,8 +171,12 @@ func (c *Cache) prepare(ctx context.Context, fileID uuid.UUID) *lazy.ValueWithEr
 			refCount: 0,
 		}
 		c.data[fileID] = entry
+		c.currentOpenFiles.Inc()
+		c.totalOpenedFiles.Inc()
 	}
 
+	c.currentOpenFileReferences.Inc()
+	c.totalOpenFileReferences.Inc()
 	entry.refCount++
 	return entry.value
 }
@@ -105,11 +195,19 @@ func (c *Cache) Release(fileID uuid.UUID) {
 		return
 	}
 
+	c.currentOpenFileReferences.Dec()
 	entry.refCount--
 	if entry.refCount > 0 {
 		return
 	}
 
+	c.currentOpenFiles.Dec()
+
+	ev, err := entry.value.Load()
+	if err == nil {
+		c.currentCacheSize.Add(-1 * float64(ev.size))
+	}
+
 	delete(c.data, fileID)
 }