Merge pull request containerd#9882 from abel-von/sandbox-controller-v2

sandbox: make podsandbox controller plugin type of PodSandboxPlugin
adamlaska · Oct 17, 2024 · 72e4db7 · 72e4db7
2 parents 6c386c3 + fc5086a
commit 72e4db7
Show file tree

Hide file tree

Showing 9 changed files with 64 additions and 37 deletions.
diff --git a/client/client.go b/client/client.go
@@ -753,7 +753,7 @@ func (c *Client) SandboxController(name string) sandbox.Controller {
 	}
 	c.connMu.Lock()
 	defer c.connMu.Unlock()
-	return sandboxproxy.NewSandboxController(sandboxsapi.NewControllerClient(c.conn))
+	return sandboxproxy.NewSandboxController(sandboxsapi.NewControllerClient(c.conn), name)
 }
 
 // VersionService returns the underlying VersionClient

diff --git a/client/services.go b/client/services.go
@@ -230,18 +230,3 @@ func WithInMemoryServices(ic *plugin.InitContext) Opt {
 		return nil
 	}
 }
-
-func WithInMemorySandboxControllers(ic *plugin.InitContext) Opt {
-	return func(c *clientOpts) error {
-		sandboxers, err := ic.GetByType(plugins.SandboxControllerPlugin)
-		if err != nil {
-			return err
-		}
-		sc := make(map[string]sandbox.Controller)
-		for name, p := range sandboxers {
-			sc[name] = p.(sandbox.Controller)
-		}
-		c.services.sandboxers = sc
-		return nil
-	}
-}
diff --git a/cmd/containerd/server/server.go b/cmd/containerd/server/server.go
@@ -510,7 +510,7 @@ func LoadPlugins(ctx context.Context, config *srvconfig.Config) ([]plugin.Regist
 		case string(plugins.SandboxControllerPlugin), "sandbox":
 			t = plugins.SandboxControllerPlugin
 			f = func(conn *grpc.ClientConn) interface{} {
-				return sbproxy.NewSandboxController(sbapi.NewControllerClient(conn))
+				return sbproxy.NewSandboxController(sbapi.NewControllerClient(conn), name)
 			}
 		case string(plugins.DiffPlugin), "diff":
 			t = plugins.DiffPlugin

diff --git a/core/sandbox/proxy/controller.go b/core/sandbox/proxy/controller.go
@@ -31,14 +31,15 @@ import (
 
 // remoteSandboxController is a low level GRPC client for containerd's sandbox controller service
 type remoteSandboxController struct {
-	client api.ControllerClient
+	client        api.ControllerClient
+	sandboxerName string
 }
 
 var _ sandbox.Controller = (*remoteSandboxController)(nil)
 
 // NewSandboxController creates a client for a sandbox controller
-func NewSandboxController(client api.ControllerClient) sandbox.Controller {
-	return &remoteSandboxController{client: client}
+func NewSandboxController(client api.ControllerClient, name string) sandbox.Controller {
+	return &remoteSandboxController{client: client, sandboxerName: name}
 }
 
 func (s *remoteSandboxController) Create(ctx context.Context, sandboxInfo sandbox.Sandbox, opts ...sandbox.CreateOpt) error {
@@ -54,6 +55,7 @@ func (s *remoteSandboxController) Create(ctx context.Context, sandboxInfo sandbo
 		NetnsPath:   options.NetNSPath,
 		Annotations: options.Annotations,
 		Sandbox:     apiSandbox,
+		Sandboxer:   s.sandboxerName,
 	})
 	if err != nil {
 		return errdefs.FromGRPC(err)
@@ -63,7 +65,10 @@ func (s *remoteSandboxController) Create(ctx context.Context, sandboxInfo sandbo
 }
 
 func (s *remoteSandboxController) Start(ctx context.Context, sandboxID string) (sandbox.ControllerInstance, error) {
-	resp, err := s.client.Start(ctx, &api.ControllerStartRequest{SandboxID: sandboxID})
+	resp, err := s.client.Start(ctx, &api.ControllerStartRequest{
+		SandboxID: sandboxID,
+		Sandboxer: s.sandboxerName,
+	})
 	if err != nil {
 		return sandbox.ControllerInstance{}, errdefs.FromGRPC(err)
 	}
@@ -79,7 +84,10 @@ func (s *remoteSandboxController) Start(ctx context.Context, sandboxID string) (
 }
 
 func (s *remoteSandboxController) Platform(ctx context.Context, sandboxID string) (imagespec.Platform, error) {
-	resp, err := s.client.Platform(ctx, &api.ControllerPlatformRequest{SandboxID: sandboxID})
+	resp, err := s.client.Platform(ctx, &api.ControllerPlatformRequest{
+		SandboxID: sandboxID,
+		Sandboxer: s.sandboxerName,
+	})
 	if err != nil {
 		return imagespec.Platform{}, errdefs.FromGRPC(err)
 	}
@@ -97,7 +105,10 @@ func (s *remoteSandboxController) Stop(ctx context.Context, sandboxID string, op
 	for _, opt := range opts {
 		opt(&soptions)
 	}
-	req := &api.ControllerStopRequest{SandboxID: sandboxID}
+	req := &api.ControllerStopRequest{
+		SandboxID: sandboxID,
+		Sandboxer: s.sandboxerName,
+	}
 	if soptions.Timeout != nil {
 		req.TimeoutSecs = uint32(soptions.Timeout.Seconds())
 	}
@@ -110,7 +121,10 @@ func (s *remoteSandboxController) Stop(ctx context.Context, sandboxID string, op
 }
 
 func (s *remoteSandboxController) Shutdown(ctx context.Context, sandboxID string) error {
-	_, err := s.client.Shutdown(ctx, &api.ControllerShutdownRequest{SandboxID: sandboxID})
+	_, err := s.client.Shutdown(ctx, &api.ControllerShutdownRequest{
+		SandboxID: sandboxID,
+		Sandboxer: s.sandboxerName,
+	})
 	if err != nil {
 		return errdefs.FromGRPC(err)
 	}
@@ -127,7 +141,10 @@ func (s *remoteSandboxController) Wait(ctx context.Context, sandboxID string) (s
 		retryInterval time.Duration = 128
 	)
 	for {
-		resp, err = s.client.Wait(ctx, &api.ControllerWaitRequest{SandboxID: sandboxID})
+		resp, err = s.client.Wait(ctx, &api.ControllerWaitRequest{
+			SandboxID: sandboxID,
+			Sandboxer: s.sandboxerName,
+		})
 		if err != nil {
 			grpcErr := errdefs.FromGRPC(err)
 			if !errdefs.IsUnavailable(grpcErr) {
@@ -153,7 +170,11 @@ func (s *remoteSandboxController) Wait(ctx context.Context, sandboxID string) (s
 }
 
 func (s *remoteSandboxController) Status(ctx context.Context, sandboxID string, verbose bool) (sandbox.ControllerStatus, error) {
-	resp, err := s.client.Status(ctx, &api.ControllerStatusRequest{SandboxID: sandboxID, Verbose: verbose})
+	resp, err := s.client.Status(ctx, &api.ControllerStatusRequest{
+		SandboxID: sandboxID,
+		Verbose:   verbose,
+		Sandboxer: s.sandboxerName,
+	})
 	if err != nil {
 		return sandbox.ControllerStatus{}, errdefs.FromGRPC(err)
 	}
@@ -171,7 +192,10 @@ func (s *remoteSandboxController) Status(ctx context.Context, sandboxID string,
 }
 
 func (s *remoteSandboxController) Metrics(ctx context.Context, sandboxID string) (*types.Metric, error) {
-	resp, err := s.client.Metrics(ctx, &api.ControllerMetricsRequest{SandboxID: sandboxID})
+	resp, err := s.client.Metrics(ctx, &api.ControllerMetricsRequest{
+		SandboxID: sandboxID,
+		Sandboxer: s.sandboxerName,
+	})
 	if err != nil {
 		return nil, errdefs.FromGRPC(err)
 	}

diff --git a/internal/cri/server/podsandbox/controller.go b/internal/cri/server/podsandbox/controller.go
@@ -46,7 +46,7 @@ import (
 
 func init() {
 	registry.Register(&plugin.Registration{
-		Type: plugins.SandboxControllerPlugin,
+		Type: plugins.PodSandboxPlugin,
 		ID:   "podsandbox",
 		Requires: []plugin.Type{
 			plugins.EventPlugin,

diff --git a/internal/cri/server/restart.go b/internal/cri/server/restart.go
@@ -111,12 +111,11 @@ func (c *criService) recover(ctx context.Context) error {
 		}
 
 		var (
-			state      = sandboxstore.StateUnknown
-			controller = c.client.SandboxController(sbx.Sandboxer)
-			endpoint   sandboxstore.Endpoint
+			state    = sandboxstore.StateUnknown
+			endpoint sandboxstore.Endpoint
 		)
 
-		status, err := controller.Status(ctx, sbx.ID, false)
+		status, err := c.sandboxService.SandboxStatus(ctx, sbx.Sandboxer, sbx.ID, false)
 		if err != nil {
 			log.G(ctx).
 				WithError(err).

diff --git a/plugins/cri/cri.go b/plugins/cri/cri.go
@@ -48,6 +48,7 @@ func init() {
 		ID:   "cri",
 		Requires: []plugin.Type{
 			plugins.CRIServicePlugin,
+			plugins.PodSandboxPlugin,
 			plugins.SandboxControllerPlugin,
 			plugins.NRIApiPlugin,
 			plugins.EventPlugin,
@@ -118,7 +119,6 @@ func initCRIService(ic *plugin.InitContext) (interface{}, error) {
 		containerd.WithDefaultNamespace(constants.K8sContainerdNamespace),
 		containerd.WithDefaultPlatform(platforms.Default()),
 		containerd.WithInMemoryServices(ic),
-		containerd.WithInMemorySandboxControllers(ic),
 	)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create containerd client: %w", err)
@@ -237,13 +237,21 @@ func getNRIAPI(ic *plugin.InitContext) nriservice.API {
 }
 
 func getSandboxControllers(ic *plugin.InitContext) (map[string]sandbox.Controller, error) {
+	sc := make(map[string]sandbox.Controller)
 	sandboxers, err := ic.GetByType(plugins.SandboxControllerPlugin)
 	if err != nil {
 		return nil, err
 	}
-	sc := make(map[string]sandbox.Controller)
 	for name, p := range sandboxers {
 		sc[name] = p.(sandbox.Controller)
 	}
+
+	podSandboxers, err := ic.GetByType(plugins.PodSandboxPlugin)
+	if err != nil {
+		return nil, err
+	}
+	for name, p := range podSandboxers {
+		sc[name] = p.(sandbox.Controller)
+	}
 	return sc, nil
 }
diff --git a/plugins/services/sandbox/controller_service.go b/plugins/services/sandbox/controller_service.go
@@ -41,20 +41,29 @@ func init() {
 		Type: plugins.GRPCPlugin,
 		ID:   "sandbox-controllers",
 		Requires: []plugin.Type{
+			plugins.PodSandboxPlugin,
 			plugins.SandboxControllerPlugin,
 			plugins.EventPlugin,
 		},
 		InitFn: func(ic *plugin.InitContext) (interface{}, error) {
-			sandboxers, err := ic.GetByType(plugins.SandboxControllerPlugin)
+			sc := make(map[string]sandbox.Controller)
+
+			sandboxers, err := ic.GetByType(plugins.PodSandboxPlugin)
 			if err != nil {
 				return nil, err
 			}
-
-			sc := make(map[string]sandbox.Controller)
 			for name, p := range sandboxers {
 				sc[name] = p.(sandbox.Controller)
 			}
 
+			sandboxersV2, err := ic.GetByType(plugins.SandboxControllerPlugin)
+			if err != nil {
+				return nil, err
+			}
+			for name, p := range sandboxersV2 {
+				sc[name] = p.(sandbox.Controller)
+			}
+
 			ep, err := ic.GetSingle(plugins.EventPlugin)
 			if err != nil {
 				return nil, err

diff --git a/plugins/types.go b/plugins/types.go
@@ -63,6 +63,8 @@ const (
 	TransferPlugin plugin.Type = "io.containerd.transfer.v1"
 	// SandboxStorePlugin implements a sandbox store
 	SandboxStorePlugin plugin.Type = "io.containerd.sandbox.store.v1"
+	// PodSandboxPlugin is a special sandbox controller which use pause container as a sandbox.
+	PodSandboxPlugin plugin.Type = "io.containerd.podsandbox.controller.v1"
 	// SandboxControllerPlugin implements a sandbox controller
 	SandboxControllerPlugin plugin.Type = "io.containerd.sandbox.controller.v1"
 	// ImageVerifierPlugin implements an image verifier service