Add burst-proof admission control and fast resource accounting

lib/images/disk_usage.go

@@ -0,0 +1,119 @@
+package images
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+)
+
+// totalReadyImageBytesFromMetadata sums ready image sizes directly from metadata.json files.
+// This is conservative for admission control and disk accounting: if metadata says an
+// image is ready, we count its recorded size without re-validating the rootfs path.
+func totalReadyImageBytesFromMetadata(imagesDir string) (int64, error) {
+	var total int64
+
+	err := filepath.Walk(imagesDir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return nil
+		}
+		if info.IsDir() || info.Name() != "metadata.json" {
+			return nil
+		}
+
+		data, err := os.ReadFile(path)
+		if err != nil {
+			return nil
+		}
+
+		var meta imageMetadata
+		if err := json.Unmarshal(data, &meta); err != nil {
+			return nil
+		}
+		if meta.Status == StatusReady && meta.SizeBytes > 0 {
+			total += meta.SizeBytes
+		}
+		return nil
+	})
+	if err != nil && !os.IsNotExist(err) {
+		return 0, fmt.Errorf("walk images directory: %w", err)
+	}
+
+	return total, nil
+}
+
+// totalOCICacheBlobBytesFromFilesystem sums blob sizes directly from the OCI cache blob store.
+// This counts the actual bytes on disk, including any blob files that are currently
+// present but no longer referenced by the OCI layout index.
+func totalOCICacheBlobBytesFromFilesystem(blobDir string) (int64, error) {
+	var total int64
+
+	err := filepath.Walk(blobDir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return nil
+		}
+		if info.IsDir() {
+			return nil
+		}
+		total += info.Size()
+		return nil
+	})
+	if err != nil && !os.IsNotExist(err) {
+		return 0, fmt.Errorf("walk OCI cache blobs: %w", err)
+	}
+
+	return total, nil
+}
+
+func (m *manager) getDiskUsageTotals() (int64, int64, error) {
+	m.diskUsageMu.RLock()
+	if m.diskUsageLoaded {
+		readyImageBytes := m.readyImageBytes
+		ociCacheBytes := m.ociCacheBytes
+		m.diskUsageMu.RUnlock()
+		return readyImageBytes, ociCacheBytes, nil
+	}
+	m.diskUsageMu.RUnlock()
+
+	readyImageBytes, ociCacheBytes, err := m.computeDiskUsageTotals()
+	if err != nil {
+		return 0, 0, err
+	}
+
+	m.diskUsageMu.Lock()
+	if !m.diskUsageLoaded {
+		m.readyImageBytes = readyImageBytes
+		m.ociCacheBytes = ociCacheBytes
+		m.diskUsageLoaded = true
+	}
+	readyImageBytes = m.readyImageBytes
+	ociCacheBytes = m.ociCacheBytes
+	m.diskUsageMu.Unlock()
+
+	return readyImageBytes, ociCacheBytes, nil
+}
+
+func (m *manager) refreshDiskUsageTotals() {
+	readyImageBytes, ociCacheBytes, err := m.computeDiskUsageTotals()
+	if err != nil {
+		return
+	}
+
+	m.diskUsageMu.Lock()
+	m.readyImageBytes = readyImageBytes
+	m.ociCacheBytes = ociCacheBytes
+	m.diskUsageLoaded = true
+	m.diskUsageMu.Unlock()
+}
+
+func (m *manager) computeDiskUsageTotals() (int64, int64, error) {
+	readyImageBytes, err := totalReadyImageBytesFromMetadata(m.paths.ImagesDir())
+	if err != nil {
+		return 0, 0, err
+	}
+	ociCacheBytes, err := totalOCICacheBlobBytesFromFilesystem(m.paths.OCICacheBlobDir())
+	if err != nil {
+		return 0, 0, err
+	}
+	return readyImageBytes, ociCacheBytes, nil
+}

lib/images/manager.go

@@ -10,7 +10,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/google/go-containerregistry/pkg/v1/layout"
 	"github.com/kernel/hypeman/lib/paths"
 	"github.com/kernel/hypeman/lib/tags"
 	"go.opentelemetry.io/otel/metric"
@@ -55,6 +54,10 @@ type manager struct {
 	ociClient        *ociClient
 	queue            *BuildQueue
 	createMu         sync.Mutex
+	diskUsageMu      sync.RWMutex
+	diskUsageLoaded  bool
+	readyImageBytes  int64
+	ociCacheBytes    int64
 	metrics          *Metrics
 	readySubscribers map[string][]chan StatusEvent // keyed by digestHex
 	subscriberMu     sync.RWMutex
@@ -319,6 +322,8 @@ func (m *manager) buildImage(ctx context.Context, ref *ResolvedRef) {
 		}
 	}
 
+	m.refreshDiskUsageTotals()
+
 	m.recordBuildMetrics(ctx, buildStart, "success")
 }
 
@@ -438,7 +443,11 @@ func (m *manager) DeleteImage(ctx context.Context, name string) error {
 		if err := deleteTagsForDigest(m.paths, repository, digestHex); err != nil {
 			return err
 		}
-		return deleteDigest(m.paths, repository, digestHex)
+		if err := deleteDigest(m.paths, repository, digestHex); err != nil {
+			return err
+		}
+		m.refreshDiskUsageTotals()
+		return nil
 	}
 
 	tag := ref.Tag()
@@ -467,82 +476,28 @@ func (m *manager) DeleteImage(ctx context.Context, name string) error {
 			fmt.Fprintf(os.Stderr, "Warning: failed to delete orphaned digest %s: %v\n", digestHex, err)
 			return nil
 		}
+		m.refreshDiskUsageTotals()
 	}
 
 	return nil
 }
 
 // TotalImageBytes returns the total size of all ready images on disk.
 func (m *manager) TotalImageBytes(ctx context.Context) (int64, error) {
-	images, err := m.ListImages(ctx)
+	readyImageBytes, _, err := m.getDiskUsageTotals()
 	if err != nil {
 		return 0, err
 	}
-
-	var total int64
-	for _, img := range images {
-		if img.Status == StatusReady && img.SizeBytes != nil {
-			total += *img.SizeBytes
-		}
-	}
-	return total, nil
+	return readyImageBytes, nil
 }
 
 // TotalOCICacheBytes returns the total size of the OCI layer cache.
-// Uses OCI layout metadata instead of walking the filesystem for efficiency.
 func (m *manager) TotalOCICacheBytes(ctx context.Context) (int64, error) {
-	path, err := layout.FromPath(m.paths.SystemOCICache())
+	_, ociCacheBytes, err := m.getDiskUsageTotals()
 	if err != nil {
-		return 0, nil // No cache yet
-	}
-
-	index, err := path.ImageIndex()
-	if err != nil {
-		return 0, nil // Empty or invalid cache
-	}
-
-	manifest, err := index.IndexManifest()
-	if err != nil {
-		return 0, nil
-	}
-
-	// Collect unique blob digests and sizes (layers are shared/deduplicated)
-	blobSizes := make(map[string]int64)
-
-	for _, desc := range manifest.Manifests {
-		// Count the manifest blob itself
-		blobSizes[desc.Digest.String()] = desc.Size
-
-		// Get image to access layers and config
-		img, err := path.Image(desc.Digest)
-		if err != nil {
-			continue
-		}
-
-		// Count config blob
-		if configDigest, err := img.ConfigName(); err == nil {
-			if configFile, err := img.RawConfigFile(); err == nil {
-				blobSizes[configDigest.String()] = int64(len(configFile))
-			}
-		}
-
-		// Count layer blobs
-		if layers, err := img.Layers(); err == nil {
-			for _, layer := range layers {
-				if digest, err := layer.Digest(); err == nil {
-					if size, err := layer.Size(); err == nil {
-						blobSizes[digest.String()] = size
-					}
-				}
-			}
-		}
-	}
-
-	var total int64
-	for _, size := range blobSizes {
-		total += size
+		return 0, err
 	}
-	return total, nil
+	return ociCacheBytes, nil
 }
 
 // WaitForReady blocks until the image reaches a terminal state (ready or failed)

lib/instances/admission.go

@@ -0,0 +1,22 @@
+package instances
+
+func volumeOverlayReservationBytes(volumes []VolumeAttachment) int64 {
+	var total int64
+	for _, vol := range volumes {
+		if vol.Overlay {
+			total += vol.OverlaySize
+		}
+	}
+	return total
+}
+
+func requestedDiskReservationBytes(overlaySize int64, volumes []VolumeAttachment) int64 {
+	return overlaySize + volumeOverlayReservationBytes(volumes)
+}
+
+func storedDiskReservationBytes(stored *StoredMetadata) int64 {
+	if stored == nil {
+		return 0
+	}
+	return requestedDiskReservationBytes(stored.OverlaySize, stored.Volumes)
+}