Fix stats overflow (#451)

cmd/agent/daemon/state/stats_pipeline.go

@@ -2,11 +2,13 @@ package state
 
 import (
 	"context"
+	"errors"
 	"log/slog"
 	"time"
 
 	castaipb "github.com/castai/kvisor/api/v1/runtime"
 	"github.com/castai/kvisor/cmd/agent/daemon/metrics"
+	"github.com/castai/kvisor/pkg/cgroup"
 	"github.com/castai/kvisor/pkg/containers"
 )
 
@@ -63,6 +65,9 @@ func (c *Controller) scrapeContainersResourceStats(batch *castaipb.StatsBatch) {
 func (c *Controller) scrapeContainerResourcesStats(cont *containers.Container, batch *castaipb.StatsBatch) {
 	cgStats, err := c.containersClient.GetCgroupStats(cont)
 	if err != nil {
+		if errors.Is(err, cgroup.ErrStatsNotFound) {
+			return
+		}
 		if c.log.IsEnabled(slog.LevelDebug) {
 			c.log.Errorf("getting cgroup stats for container %q: %v", cont.Name, err)
 		}

pkg/cgroup/cgroup.go

@@ -1,6 +1,7 @@
 package cgroup
 
 import (
+	"errors"
 	"os"
 	"path"
 	"strings"
@@ -10,6 +11,8 @@ import (
 
 const UnifiedMountpoint = "/sys/fs/cgroup"
 
+var ErrStatsNotFound = errors.New("stats not found")
+
 type Stats struct {
 	CpuStats    *castaipb.CpuStats
 	MemoryStats *castaipb.MemoryStats
@@ -34,7 +37,8 @@ func (cg *Cgroup) GetStats() (Stats, error) {
 	}
 	if err := cg.statsGetterFunc(&res); err != nil {
 		if os.IsNotExist(err) {
-			return res, nil
+			// Most likely container was deleted.
+			return res, ErrStatsNotFound
 		}
 		return res, err
 	}