Adding mem monitoring measurement

go.work.sum

@@ -219,31 +219,41 @@ golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0
 golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0=
 golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw=
 golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
+golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc=
+golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ=
 golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI=
 golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
+golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo=
+golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
 golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE=
 golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
+golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
+golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
+golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
 golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
 golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
+golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
@@ -252,6 +262,8 @@ golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM=
+golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
+golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps=
 golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
 google.golang.org/api v0.128.0/go.mod h1:Y611qgqaE92On/7g65MQgxYul3c0rEB894kniWLY750=
 google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64=

launcher/container_runner.go

@@ -281,6 +281,23 @@ func appendTokenMounts(mounts []specs.Mount) []specs.Mount {
 	return append(mounts, m)
 }
 
+func (r *ContainerRunner) measureCELEvents(ctx context.Context) error {
+	if err := r.measureContainerClaims(ctx); err != nil {
+		return fmt.Errorf("failed to measure container claims: %v", err)
+	}
+	if r.launchSpec.Experiments.EnableMeasureMemoryMonitor {
+		if err := r.measureMemoryMonitor(); err != nil {
+			return fmt.Errorf("failed to measure memory monitoring state: %v", err)
+		}
+	}
+
+	separator := cel.CosTlv{
+		EventType:    cel.LaunchSeparatorType,
+		EventContent: nil, // Success
+	}
+	return r.attestAgent.MeasureEvent(separator)
+}
+
 // measureContainerClaims will measure various container claims into the COS
 // eventlog in the AttestationAgent.
 func (r *ContainerRunner) measureContainerClaims(ctx context.Context) error {
@@ -334,11 +351,21 @@ func (r *ContainerRunner) measureContainerClaims(ctx context.Context) error {
 		}
 	}
 
-	separator := cel.CosTlv{
-		EventType:    cel.LaunchSeparatorType,
-		EventContent: nil, // Success
+	return nil
+}
+
+// measureMemoryMonitor will measure memory monitoring claims into the COS
+// eventlog in the AttestationAgent.
+func (r *ContainerRunner) measureMemoryMonitor() error {
+	var enabled uint8
+	if r.launchSpec.MemoryMonitoringEnabled {
+		enabled = 1
 	}
-	return r.attestAgent.MeasureEvent(separator)
+	if err := r.attestAgent.MeasureEvent(cel.CosTlv{EventType: cel.MemoryMonitorType, EventContent: []byte{enabled}}); err != nil {
+		return err
+	}
+	r.logger.Println("Successfully measured memory monitoring event")
+	return nil
 }
 
 // Retrieves the default OIDC token from the attestation service, and returns how long
@@ -494,13 +521,6 @@ func (r *ContainerRunner) Run(ctx context.Context) error {
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
-	if err := r.measureContainerClaims(ctx); err != nil {
-		return fmt.Errorf("failed to measure container claims: %v", err)
-	}
-	if err := r.fetchAndWriteToken(ctx); err != nil {
-		return fmt.Errorf("failed to fetch and write OIDC token: %v", err)
-	}
-
 	r.logger.Printf("EnableTestFeatureForImage is set to %v\n", r.launchSpec.Experiments.EnableTestFeatureForImage)
 	// create and start the TEE server behind the experiment
 	if r.launchSpec.Experiments.EnableOnDemandAttestation {
@@ -513,24 +533,30 @@ func (r *ContainerRunner) Run(ctx context.Context) error {
 		defer teeServer.Shutdown(ctx)
 	}
 
-	if r.launchSpec.Experiments.EnableMemoryMonitoring {
-		// start node-problem-detector.service to collect memory related metrics.
-		if r.launchSpec.MemoryMonitoringEnabled {
-			r.logger.Println("MemoryMonitoring is enabled by the VM operator")
-			s, err := systemctl.New()
-			if err != nil {
-				return fmt.Errorf("failed to create systemctl client: %v", err)
-			}
-			defer s.Close()
+	// start node-problem-detector.service to collect memory related metrics.
+	if r.launchSpec.MemoryMonitoringEnabled {
+		r.logger.Println("MemoryMonitoring is enabled by the VM operator")
+		s, err := systemctl.New()
+		if err != nil {
+			return fmt.Errorf("failed to create systemctl client: %v", err)
+		}
+		defer s.Close()
 
-			r.logger.Println("Starting a systemctl operation: systemctl start node-problem-detector.service")
-			if err := s.Start("node-problem-detector.service"); err != nil {
-				return fmt.Errorf("failed to start node-problem-detector.service: %v", err)
-			}
-			r.logger.Println("node-problem-detector.service successfully started.")
-		} else {
-			r.logger.Println("MemoryMonitoring is disabled by the VM operator")
+		r.logger.Println("Starting a systemctl operation: systemctl start node-problem-detector.service")
+		if err := s.Start("node-problem-detector.service"); err != nil {
+			return fmt.Errorf("failed to start node-problem-detector.service: %v", err)
 		}
+		r.logger.Println("node-problem-detector.service successfully started.")
+	} else {
+		r.logger.Println("MemoryMonitoring is disabled by the VM operator")
+	}
+
+	if err := r.measureCELEvents(ctx); err != nil {
+		return fmt.Errorf("failed to measure CEL events: %v", err)
+	}
+
+	if err := r.fetchAndWriteToken(ctx); err != nil {
+		return fmt.Errorf("failed to fetch and write OIDC token: %v", err)
 	}
 
 	var streamOpt cio.Opt

launcher/container_runner_test.go

@@ -18,13 +18,18 @@ import (
 	"github.com/containerd/containerd"
 	"github.com/containerd/containerd/defaults"
 	"github.com/containerd/containerd/namespaces"
+	"github.com/containerd/containerd/oci"
 	"github.com/golang-jwt/jwt/v4"
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-tpm-tools/cel"
 	"github.com/google/go-tpm-tools/launcher/agent"
 	"github.com/google/go-tpm-tools/launcher/internal/experiments"
 	"github.com/google/go-tpm-tools/launcher/launcherfile"
 	"github.com/google/go-tpm-tools/launcher/spec"
+	"github.com/opencontainers/go-digest"
+	v1 "github.com/opencontainers/image-spec/specs-go/v1"
+	specs "github.com/opencontainers/runtime-spec/specs-go"
+	"golang.org/x/exp/slices"
 	"golang.org/x/oauth2"
 )
 
@@ -540,3 +545,106 @@ func TestInitImageDockerPublic(t *testing.T) {
 		}
 	}
 }
+
+func TestMeasureCELEvents(t *testing.T) {
+	ctx := context.Background()
+	fakeContainer := &fakeContainer{
+		image: &fakeImage{
+			name:   "fake image name",
+			digest: "fake digest",
+			id:     "fake id",
+		},
+		args: []string{"fake", "args"},
+		env:  []string{"fake", "env"},
+	}
+
+	testCases := []struct {
+		name          string
+		wantCELEvents []cel.CosType
+		launchSpec    spec.LaunchSpec
+	}{
+		{
+			name:          "measure container events and launch separator event",
+			wantCELEvents: []cel.CosType{cel.LaunchSeparatorType, cel.ImageIDType, cel.ImageDigestType},
+			launchSpec:    spec.LaunchSpec{},
+		},
+		{
+			name:          "measure memory monitoring event and launch separator event",
+			wantCELEvents: []cel.CosType{cel.LaunchSeparatorType, cel.MemoryMonitorType},
+			launchSpec:    spec.LaunchSpec{Experiments: experiments.Experiments{EnableMeasureMemoryMonitor: true}},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			gotEvents := []cel.CosType{}
+
+			fakeAgent := &fakeAttestationAgent{
+				measureEventFunc: func(content cel.Content) error {
+					got, _ := content.GetTLV()
+					tlv := &cel.TLV{}
+					tlv.UnmarshalBinary(got.Value)
+					gotEvents = append(gotEvents, cel.CosType(tlv.Type))
+					return nil
+				},
+			}
+
+			r := ContainerRunner{
+				attestAgent: fakeAgent,
+				container:   fakeContainer,
+				launchSpec:  tc.launchSpec,
+				logger:      log.Default(),
+			}
+
+			if err := r.measureCELEvents(ctx); err != nil {
+				t.Errorf("failed to measureCELEvents: %v", err)
+			}
+
+			for _, wantEvent := range tc.wantCELEvents {
+				if !slices.Contains(gotEvents, cel.CosType(wantEvent)) {
+					t.Errorf("failed to measure CEL event %v", cel.CosType(wantEvent))
+				}
+			}
+		})
+	}
+}
+
+// This ensures fakeContainer implements containerd.Container interface.
+var _ containerd.Container = &fakeContainer{}
+
+// This ensures fakeImage implements containerd.Image interface.
+var _ containerd.Image = &fakeImage{}
+
+type fakeContainer struct {
+	containerd.Container
+	image containerd.Image
+	args  []string
+	env   []string
+}
+
+func (c *fakeContainer) Image(context.Context) (containerd.Image, error) {
+	return c.image, nil
+}
+
+func (c *fakeContainer) Spec(context.Context) (*oci.Spec, error) {
+	return &oci.Spec{Process: &specs.Process{Args: c.args, Env: c.env}}, nil
+}
+
+type fakeImage struct {
+	containerd.Image
+	name   string
+	digest digest.Digest
+	id     digest.Digest
+}
+
+func (i *fakeImage) Name() string {
+	return i.name
+}
+
+func (i *fakeImage) Target() v1.Descriptor {
+	return v1.Descriptor{Digest: i.digest}
+}
+
+func (i *fakeImage) Config(_ context.Context) (v1.Descriptor, error) {
+	return v1.Descriptor{Digest: i.id}, nil
+}

launcher/go.mod

@@ -58,6 +58,7 @@ require (
 	github.com/stretchr/testify v1.8.3 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	golang.org/x/crypto v0.14.0 // indirect
+	golang.org/x/exp v0.0.0-20240205201215-2c58cdc269a3 // indirect
 	golang.org/x/net v0.17.0 // indirect
 	golang.org/x/sync v0.4.0 // indirect
 	golang.org/x/sys v0.13.0 // indirect

launcher/go.sum

@@ -1181,6 +1181,8 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
 golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw=
+golang.org/x/exp v0.0.0-20240205201215-2c58cdc269a3 h1:/RIbNt/Zr7rVhIkQhooTxCxFcdWLGIKnZA4IXNFSrvo=
+golang.org/x/exp v0.0.0-20240205201215-2c58cdc269a3/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=

launcher/image/test/test_memory_monitoring.yaml

@@ -22,7 +22,8 @@ steps:
 - name: 'gcr.io/cloud-builders/gcloud'
   id: CheckMemoryMonitoringEnabled
   entrypoint: 'bash'
-  args: ['scripts/test_memory_monitoring.sh', '${_VM_NAME_PREFIX}-enable-${BUILD_ID}', '${_ZONE}', 'node-problem-detector.service successfully started']
+  # Search a regex pattern that ensures memory monitoring is enabled and measured into COS event logs.
+  args: ['scripts/test_memory_monitoring.sh', '${_VM_NAME_PREFIX}-enable-${BUILD_ID}', '${_ZONE}', 'node-problem-detector.service successfully started.*Successfully measured memory monitoring event']
   waitFor: ['CreateVMMemoryMemonitorEnabled']
 - name: 'gcr.io/cloud-builders/gcloud'
   id: CleanUpVMMemoryMonitorEnabled
@@ -47,7 +48,8 @@ steps:
 - name: 'gcr.io/cloud-builders/gcloud'
   id: CheckMemoryMonitoringDisabled
   entrypoint: 'bash'
-  args: ['scripts/test_memory_monitoring.sh', '${_VM_NAME_PREFIX}-disable-${BUILD_ID}', '${_ZONE}', 'MemoryMonitoring is disabled by the VM operator']
+  # Search a regex pattern that ensures memory monitoring is disabled and measured into COS event logs.
+  args: ['scripts/test_memory_monitoring.sh', '${_VM_NAME_PREFIX}-disable-${BUILD_ID}', '${_ZONE}', 'MemoryMonitoring is disabled by the VM operator.*Successfully measured memory monitoring event']
   waitFor: ['CreateVMMemoryMemonitorDisabled']
 - name: 'gcr.io/cloud-builders/gcloud'
   id: CleanUpVMMemoryMonitorDisabled

launcher/internal/experiments/experiments.go

@@ -16,6 +16,7 @@ type Experiments struct {
 	EnableOnDemandAttestation  bool
 	EnableMemoryMonitoring     bool
 	EnableSignedContainerCache bool
+	EnableMeasureMemoryMonitor bool
 }
 
 // New takes a filepath, opens the file, and calls ReadJsonInput with the contents

launcher/internal/systemctl/systemctl.go

@@ -13,6 +13,7 @@ import (
 type Systemd interface {
 	Start(string) error
 	Stop(string) error
+	IsActive(context.Context, string) (string, error)
 	Close()
 }
 
@@ -42,6 +43,19 @@ func (s *Systemctl) Stop(unit string) error {
 	return runSystemdCmd(s.dbus.StopUnitContext, "stop", unit)
 }
 
+// IsActive is the equivalent of `systemctl is-active $unit`.
+// The status can be "active", "activating", "deactivating", "inactive" or "failed".
+func (s *Systemctl) IsActive(ctx context.Context, unit string) (string, error) {
+	status, err := s.dbus.ListUnitsByNamesContext(ctx, []string{unit})
+	if err != nil {
+		return "", err
+	}
+	if len(status) != 1 {
+		return "", fmt.Errorf("want 1 unit from ListUnitsByNames, got %d", len(status))
+	}
+	return status[0].ActiveState, nil
+}
+
 // Close disconnects from dbus.
 func (s *Systemctl) Close() { s.dbus.Close() }