From ce5614e7a9fc566dc95bbaa0e8698fc1fb30fabf Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 2 Feb 2024 02:37:15 +0000 Subject: [PATCH 1/2] Unified nghttpx secret This commit introduces new nghttpx secret. The Secret previously specified by quic-keying-materials-secret flag is integrated to this new Secret. The name of new Secret defaults to nghttpx-secret and can be configured by nghttpx-secret flag. quic-keying-materials-secret flag has been removed. Refer to HTTP/3 section of README.md for the migration from the previous release. --- README.md | 9 ++ cmd/nghttpx-ingress-controller/main.go | 66 ++++++------ pkg/controller/controller.go | 138 ++++++++++++------------- pkg/controller/controller_test.go | 96 +++++++++++------ pkg/nghttpx/nghttpx.tmpl | 2 + pkg/nghttpx/template_test.go | 74 +++++++++++++ 6 files changed, 253 insertions(+), 132 deletions(-) diff --git a/README.md b/README.md index 585723f7..b1042571 100644 --- a/README.md +++ b/README.md @@ -188,6 +188,15 @@ Pod. The controller maintains the secret as a whole, and it should not be altered by an external tool or user. nghttpx listens on UDP port specified by `--nghttpx-https-port` flag. +> [!WARNING] +> As of v0.66.0, Secret is integrated to the one specified by +> `--nghttpx-secret` flag, and `--quic-keying-materials-secret` flag +> has been removed. The default value is also changed. Previously, +> it is `nghttpx-quic-km` but now `nghttpx-km`. To migrate from the +> previous release, before upgrading nghttpx-ingress-controller to +> v0.66.0, copy Secret `nghttpx-quic-km` to `nghttpx-km`, and upgrade +> nghttpx-ingress-controller. + HTTP/3 requires writing Secret and extra capabilities to load eBPF program. For writing Secret, you might need to add the following entry to ClusterRole: diff --git a/cmd/nghttpx-ingress-controller/main.go b/cmd/nghttpx-ingress-controller/main.go index 7e6f6512..8a6e034a 100644 --- a/cmd/nghttpx-ingress-controller/main.go +++ b/cmd/nghttpx-ingress-controller/main.go @@ -62,37 +62,37 @@ var ( gitRepo = "" // Command-line flags - defaultSvc string - ngxConfigMap string - kubeconfig string - watchNamespace = metav1.NamespaceAll - healthzPort = int32(11249) - nghttpxHealthPort = int32(10901) - nghttpxAPIPort = int32(10902) - profiling = true - allowInternalIP = false - defaultTLSSecret string - ingressClass = "nghttpx" - ingressClassController = "zlab.co.jp/nghttpx" - nghttpxConfDir = "/etc/nghttpx" - nghttpxExecPath = "/usr/local/bin/nghttpx" - nghttpxHTTPPort = int32(80) - nghttpxHTTPSPort = int32(443) - fetchOCSPRespFromSecret = false - proxyProto = false - ocspRespKey = "tls.ocsp-resp" - publishSvc string - endpointSlices = true - reloadRate = 1.0 - reloadBurst = 1 - noDefaultBackendOverride = false - deferredShutdownPeriod time.Duration - configOverrides clientcmd.ConfigOverrides - internalDefaultBackend = false - http3 = false - quicKeyingMaterialsSecret = "nghttpx-quic-km" - reconcileTimeout = 10 * time.Minute - leaderElectionConfig = componentbaseconfig.LeaderElectionConfiguration{ + defaultSvc string + ngxConfigMap string + kubeconfig string + watchNamespace = metav1.NamespaceAll + healthzPort = int32(11249) + nghttpxHealthPort = int32(10901) + nghttpxAPIPort = int32(10902) + profiling = true + allowInternalIP = false + defaultTLSSecret string + ingressClass = "nghttpx" + ingressClassController = "zlab.co.jp/nghttpx" + nghttpxConfDir = "/etc/nghttpx" + nghttpxExecPath = "/usr/local/bin/nghttpx" + nghttpxHTTPPort = int32(80) + nghttpxHTTPSPort = int32(443) + fetchOCSPRespFromSecret = false + proxyProto = false + ocspRespKey = "tls.ocsp-resp" + publishSvc string + endpointSlices = true + reloadRate = 1.0 + reloadBurst = 1 + noDefaultBackendOverride = false + deferredShutdownPeriod time.Duration + configOverrides clientcmd.ConfigOverrides + internalDefaultBackend = false + http3 = false + nghttpxSecret = "nghttpx-km" + reconcileTimeout = 10 * time.Minute + leaderElectionConfig = componentbaseconfig.LeaderElectionConfiguration{ LeaseDuration: metav1.Duration{Duration: 15 * time.Second}, RenewDeadline: metav1.Duration{Duration: 10 * time.Second}, RetryPeriod: metav1.Duration{Duration: 2 * time.Second}, @@ -189,7 +189,7 @@ func main() { rootCmd.Flags().BoolVar(&http3, "http3", http3, `Enable HTTP/3. This makes nghttpx listen to UDP port specified by nghttpx-https-port for HTTP/3 traffic.`) - rootCmd.Flags().StringVar(&quicKeyingMaterialsSecret, "quic-keying-materials-secret", quicKeyingMaterialsSecret, `The name of Secret resource which contains QUIC keying materials for nghttpx. The resource must belong to the same namespace as the controller Pod.`) + rootCmd.Flags().StringVar(&nghttpxSecret, "nghttpx-secret", nghttpxSecret, `The name of Secret resource which contains the keying materials for nghttpx. The resource must belong to the same namespace as the controller Pod. If it is not found, the controller will create new one.`) rootCmd.Flags().DurationVar(&reconcileTimeout, "reconcile-timeout", reconcileTimeout, `A timeout for a single reconciliation. It is a safe guard to prevent a reconciliation from getting stuck indefinitely.`) @@ -369,6 +369,7 @@ func run(ctx context.Context, _ *cobra.Command, _ []string) { NghttpxWorkers: nghttpxWorkers, NghttpxWorkerProcessGraceShutdownPeriod: nghttpxWorkerProcessGraceShutdownPeriod, NghttpxMaxWorkerProcesses: nghttpxMaxWorkerProcesses, + NghttpxSecret: types.NamespacedName{Name: nghttpxSecret, Namespace: thisPod.Namespace}, DefaultTLSSecret: defaultTLSSecretKey, IngressClassController: ingressClassController, AllowInternalIP: allowInternalIP, @@ -384,7 +385,6 @@ func run(ctx context.Context, _ *cobra.Command, _ []string) { HealthzPort: healthzPort, InternalDefaultBackend: internalDefaultBackend, HTTP3: http3, - QUICKeyingMaterialsSecret: &types.NamespacedName{Name: quicKeyingMaterialsSecret, Namespace: thisPod.Namespace}, ReconcileTimeout: reconcileTimeout, LeaderElectionConfig: leaderElectionConfig, RequireIngressClass: requireIngressClass, diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 4c9054f4..ea45b2c3 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -70,8 +70,6 @@ const ( // syncKey is a key to put into the queue. Since we create load balancer configuration using all available information, it is // suffice to queue only one item. Further, queue is somewhat overkill here, but we just keep using it for simplicity. syncKey = "ingress" - // quicSecretKey is a key to put into quicSecretQueue. - quicSecretKey = "quic-secret" // quicSecretTimeout is the timeout for the last QUIC keying material. quicSecretTimeout = time.Hour @@ -139,7 +137,7 @@ type LoadBalancerController struct { healthzPort int32 internalDefaultBackend bool http3 bool - quicKeyingMaterialsSecret *types.NamespacedName + nghttpxSecret types.NamespacedName reconcileTimeout time.Duration leaderElectionConfig componentbaseconfig.LeaderElectionConfiguration requireIngressClass bool @@ -181,6 +179,8 @@ type Config struct { NghttpxWorkerProcessGraceShutdownPeriod time.Duration // NghttpxMaxWorkerProcesses is the maximum number of nghttpx worker processes which are spawned in every configuration reload. NghttpxMaxWorkerProcesses int32 + // NghttpxSecret is the Secret resource which contains secrets used by nghttpx. + NghttpxSecret types.NamespacedName // DefaultTLSSecret is the default TLS Secret to enable TLS by default. DefaultTLSSecret *types.NamespacedName // IngressClassController is the name of IngressClass controller for this controller. @@ -212,8 +212,6 @@ type Config struct { // ReconcileTimeout is a timeout for a single reconciliation. It is a safe guard to prevent a reconciliation from getting stuck // indefinitely. ReconcileTimeout time.Duration - // QUICKeyingMaterialsSecret is the Secret resource which contains QUIC keying materials used by nghttpx. - QUICKeyingMaterialsSecret *types.NamespacedName // LeaderElectionConfig is the configuration of leader election. LeaderElectionConfig componentbaseconfig.LeaderElectionConfiguration // RequireIngressClass, if set to true, ignores Ingress resource which does not specify .spec.ingressClassName. @@ -246,6 +244,7 @@ func NewLoadBalancerController(ctx context.Context, clientset clientset.Interfac nghttpxWorkers: config.NghttpxWorkers, nghttpxWorkerProcessGraceShutdownPeriod: config.NghttpxWorkerProcessGraceShutdownPeriod, nghttpxMaxWorkerProcesses: config.NghttpxMaxWorkerProcesses, + nghttpxSecret: config.NghttpxSecret, defaultSvc: config.DefaultBackendService, defaultTLSSecret: config.DefaultTLSSecret, watchNamespace: config.WatchNamespace, @@ -260,7 +259,6 @@ func NewLoadBalancerController(ctx context.Context, clientset clientset.Interfac healthzPort: config.HealthzPort, internalDefaultBackend: config.InternalDefaultBackend, http3: config.HTTP3, - quicKeyingMaterialsSecret: config.QUICKeyingMaterialsSecret, reconcileTimeout: config.ReconcileTimeout, leaderElectionConfig: config.LeaderElectionConfig, requireIngressClass: config.RequireIngressClass, @@ -963,20 +961,6 @@ func (lbc *LoadBalancerController) getConfigMap(ctx context.Context, cmKey *type return cm, nil } -func (lbc *LoadBalancerController) getQUICKeyingMaterials(key types.NamespacedName) ([]byte, error) { - secret, err := lbc.secretLister.Secrets(key.Namespace).Get(key.Name) - if err != nil { - return nil, err - } - - km, ok := secret.Data[nghttpxQUICKeyingMaterialsSecretKey] - if !ok { - return nil, fmt.Errorf("Secret %v does not contain %v key", key, nghttpxQUICKeyingMaterialsSecretKey) - } - - return km, nil -} - func (lbc *LoadBalancerController) sync(ctx context.Context, key string) error { log := klog.FromContext(ctx) @@ -1008,13 +992,20 @@ func (lbc *LoadBalancerController) sync(ctx context.Context, key string) error { nghttpx.ReadConfig(ingConfig, cm) - if lbc.http3 { - quicKM, err := lbc.getQUICKeyingMaterials(*lbc.quicKeyingMaterialsSecret) - if err != nil { - return err - } + secret, err := lbc.secretLister.Secrets(lbc.nghttpxSecret.Namespace).Get(lbc.nghttpxSecret.Name) + if err != nil { + log.Error(err, "nghttpx secret not found") + + // Continue to processing so that missing Secret does not prevent the controller from reconciling new configuration. + } - ingConfig.QUICSecretFile = nghttpx.CreateQUICSecretFile(ingConfig.ConfDir, quicKM) + if secret != nil && lbc.http3 { + quicKM, ok := secret.Data[nghttpxQUICKeyingMaterialsSecretKey] + if !ok { + log.Error(nil, "Secret does not contain QUIC keying materials") + } else { + ingConfig.QUICSecretFile = nghttpx.CreateQUICSecretFile(ingConfig.ConfDir, quicKM) + } } reloaded, err := lbc.nghttpx.CheckAndReload(ctx, ingConfig) @@ -1755,7 +1746,8 @@ func (lbc *LoadBalancerController) garbageCollectCertificate(ctx context.Context func (lbc *LoadBalancerController) secretReferenced(ctx context.Context, s *corev1.Secret) bool { log := klog.FromContext(ctx) - if lbc.defaultTLSSecret != nil && s.Namespace == lbc.defaultTLSSecret.Namespace && s.Name == lbc.defaultTLSSecret.Name { + if (lbc.defaultTLSSecret != nil && s.Namespace == lbc.defaultTLSSecret.Namespace && s.Name == lbc.defaultTLSSecret.Name) || + (s.Namespace == lbc.nghttpxSecret.Namespace && s.Name == lbc.nghttpxSecret.Name) { return true } @@ -2326,8 +2318,8 @@ type LeaderController struct { secretLister listerscorev1.SecretLister nodeLister listerscorev1.NodeLister - ingQueue workqueue.RateLimitingInterface - quicSecretQueue workqueue.RateLimitingInterface + ingQueue workqueue.RateLimitingInterface + secretQueue workqueue.RateLimitingInterface } func NewLeaderController(ctx context.Context, lbc *LoadBalancerController) (*LeaderController, error) { @@ -2345,11 +2337,17 @@ func NewLeaderController(ctx context.Context, lbc *LoadBalancerController) (*Lea }), ), allNSInformers: informers.NewSharedInformerFactory(lbc.clientset, noResyncPeriod), + secretInformers: informers.NewSharedInformerFactoryWithOptions(lbc.clientset, noResyncPeriod, + informers.WithNamespace(lbc.nghttpxSecret.Namespace), + informers.WithTweakListOptions(func(opts *metav1.ListOptions) { + opts.FieldSelector = "metadata.name=" + lbc.nghttpxSecret.Name + }), + ), ingQueue: workqueue.NewRateLimitingQueue(workqueue.NewMaxOfRateLimiter( workqueue.NewItemExponentialFailureRateLimiter(5*time.Millisecond, 30*time.Second), &workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(10), 100)}, )), - quicSecretQueue: workqueue.NewRateLimitingQueue(workqueue.NewMaxOfRateLimiter( + secretQueue: workqueue.NewRateLimitingQueue(workqueue.NewMaxOfRateLimiter( workqueue.NewItemExponentialFailureRateLimiter(5*time.Millisecond, 30*time.Second), &workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(10), 100)}, )), @@ -2384,13 +2382,7 @@ func NewLeaderController(ctx context.Context, lbc *LoadBalancerController) (*Lea lc.podIndexer = inf.GetIndexer() } - if lbc.http3 { - lc.secretInformers = informers.NewSharedInformerFactoryWithOptions(lbc.clientset, noResyncPeriod, - informers.WithNamespace(lbc.quicKeyingMaterialsSecret.Namespace), - informers.WithTweakListOptions(func(opts *metav1.ListOptions) { - opts.FieldSelector = "metadata.name=" + lbc.quicKeyingMaterialsSecret.Name - }), - ) + { f := lc.secretInformers.Core().V1().Secrets() lc.secretLister = f.Lister() inf := f.Informer() @@ -2481,17 +2473,16 @@ func (lc *LeaderController) Run(ctx context.Context) error { } } - if lc.lbc.http3 { - lc.quicSecretQueue.Add(quicSecretKey) + // Add Secret to queue so that we can create it if missing. + lc.secretQueue.Add(lc.lbc.nghttpxSecret.String()) - wg.Add(1) + wg.Add(1) - go func() { - defer wg.Done() + go func() { + defer wg.Done() - lc.quicSecretWorker(ctx) - }() - } + lc.secretWorker(ctx) + }() wg.Add(1) @@ -2507,7 +2498,7 @@ func (lc *LeaderController) Run(ctx context.Context) error { lc.ingQueue.ShutDown() if lc.lbc.http3 { - lc.quicSecretQueue.ShutDown() + lc.secretQueue.ShutDown() } wg.Wait() @@ -2628,7 +2619,7 @@ func (lc *LeaderController) addSecretNotification(ctx context.Context, obj any) s := obj.(*corev1.Secret) log.V(4).Info("Secret added", "secret", klog.KObj(s)) - lc.enqueueQUICSecret() + lc.enqueueSecret(s) } func (lc *LeaderController) updateSecretNotification(ctx context.Context, _, cur any) { @@ -2637,7 +2628,7 @@ func (lc *LeaderController) updateSecretNotification(ctx context.Context, _, cur curS := cur.(*corev1.Secret) log.V(4).Info("Secret updated", "secret", klog.KObj(curS)) - lc.enqueueQUICSecret() + lc.enqueueSecret(curS) } func (lc *LeaderController) deleteSecretNotification(ctx context.Context, obj any) { @@ -2658,7 +2649,7 @@ func (lc *LeaderController) deleteSecretNotification(ctx context.Context, obj an } log.V(4).Info("Secret deleted", "secret", klog.KObj(s)) - lc.enqueueQUICSecret() + lc.enqueueSecret(s) } func (lc *LeaderController) addIngressClassNotification(ctx context.Context, obj any) { @@ -2756,31 +2747,31 @@ func (lc *LeaderController) validateIngressClass(ctx context.Context, ing *netwo return validateIngressClass(ctx, ing, lc.lbc.ingressClassController, lc.ingClassLister, lc.lbc.requireIngressClass) } -func (lc *LeaderController) enqueueQUICSecret() { - lc.quicSecretQueue.Add(quicSecretKey) +func (lc *LeaderController) enqueueSecret(s *corev1.Secret) { + lc.secretQueue.Add(namespacedName(s).String()) } -func (lc *LeaderController) quicSecretWorker(ctx context.Context) { +func (lc *LeaderController) secretWorker(ctx context.Context) { log := klog.FromContext(ctx) work := func() bool { - key, quit := lc.quicSecretQueue.Get() + key, quit := lc.secretQueue.Get() if quit { return true } - defer lc.quicSecretQueue.Done(key) + defer lc.secretQueue.Done(key) - log := klog.LoggerWithValues(log, "secret", lc.lbc.quicKeyingMaterialsSecret, "reconcileID", uuid.NewUUID()) + log := klog.LoggerWithValues(log, "secret", key, "reconcileID", uuid.NewUUID()) ctx, cancel := context.WithTimeout(klog.NewContext(ctx, log), lc.lbc.reconcileTimeout) defer cancel() - if err := lc.syncQUICKeyingMaterials(ctx, time.Now()); err != nil { + if err := lc.syncSecret(ctx, key.(string), time.Now()); err != nil { log.Error(err, "Unable to sync QUIC keying materials") - lc.quicSecretQueue.AddRateLimited(key) + lc.secretQueue.AddRateLimited(key) } else { - lc.quicSecretQueue.Forget(key) + lc.secretQueue.Forget(key) } return false @@ -2793,12 +2784,19 @@ func (lc *LeaderController) quicSecretWorker(ctx context.Context) { } } -func (lc *LeaderController) syncQUICKeyingMaterials(ctx context.Context, now time.Time) error { +func (lc *LeaderController) syncSecret(ctx context.Context, key string, now time.Time) error { log := klog.FromContext(ctx) log.V(2).Info("Syncing Secret") - secret, err := lc.secretLister.Secrets(lc.lbc.quicKeyingMaterialsSecret.Namespace).Get(lc.lbc.quicKeyingMaterialsSecret.Name) + ns, name, err := cache.SplitMetaNamespaceKey(key) + if err != nil { + log.Error(err, "Unable to split namespace and name from key", "key", key) + // Since key is broken, we do not retry. + return nil + } + + secret, err := lc.secretLister.Secrets(ns).Get(name) if err != nil { if !apierrors.IsNotFound(err) { log.Error(err, "Unable to get Secret") @@ -2807,8 +2805,8 @@ func (lc *LeaderController) syncQUICKeyingMaterials(ctx context.Context, now tim secret = &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: lc.lbc.quicKeyingMaterialsSecret.Name, - Namespace: lc.lbc.quicKeyingMaterialsSecret.Namespace, + Name: name, + Namespace: ns, Annotations: map[string]string{ quicKeyingMaterialsUpdateTimestampKey: now.Format(time.RFC3339), }, @@ -2823,10 +2821,10 @@ func (lc *LeaderController) syncQUICKeyingMaterials(ctx context.Context, now tim return err } - log.Info("QUIC keying materials Secret was created") + log.Info("Secret was created") - // If quicSecretKey has been added to the queue and is waiting, this effectively overrides it. - lc.quicSecretQueue.AddAfter(quicSecretKey, quicSecretTimeout) + // If Secret has been added to the queue and is waiting, this effectively overrides it. + lc.secretQueue.AddAfter(key, quicSecretTimeout) return nil } @@ -2844,8 +2842,8 @@ func (lc *LeaderController) syncQUICKeyingMaterials(ctx context.Context, now tim d := t.Add(quicSecretTimeout).Sub(now) if d > 0 { log.Info("QUIC keying materials are not expired and in a good shape", "retryAfter", d) - // If quicSecretKey has been added to the queue and is waiting, this effectively overrides it. - lc.quicSecretQueue.AddAfter(quicSecretKey, d) + // If Secret has been added to the queue and is waiting, this effectively overrides it. + lc.secretQueue.AddAfter(key, d) return nil } @@ -2871,10 +2869,10 @@ func (lc *LeaderController) syncQUICKeyingMaterials(ctx context.Context, now tim return err } - log.Info("QUIC keying materials Secret was updated") + log.Info("Secret was updated") - // If quicSecretKey has been added to the queue and is waiting, this effectively overrides it. - lc.quicSecretQueue.AddAfter(quicSecretKey, quicSecretTimeout) + // If Secret has been added to the queue and is waiting, this effectively overrides it. + lc.secretQueue.AddAfter(key, quicSecretTimeout) return nil } diff --git a/pkg/controller/controller_test.go b/pkg/controller/controller_test.go index eac290bc..abf6acf0 100644 --- a/pkg/controller/controller_test.go +++ b/pkg/controller/controller_test.go @@ -128,8 +128,8 @@ var ( "k8s-app": "ingress", } - defaultQUICSecret = types.NamespacedName{ - Name: "nghttpx-quic-secret", + defaultNghttpxSecret = types.NamespacedName{ + Name: "nghttpx-km", Namespace: "kube-system", } ) @@ -142,10 +142,14 @@ func (f *fixture) prepare() { func (f *fixture) preparePod(pod *corev1.Pod) { f.clientset = fake.NewSimpleClientset(f.objects...) config := Config{ - DefaultBackendService: &types.NamespacedName{Namespace: defaultBackendNamespace, Name: defaultBackendName}, - WatchNamespace: defaultIngNamespace, - NghttpxConfigMap: &types.NamespacedName{Namespace: defaultConfigMapNamespace, Name: defaultConfigMapName}, - NghttpxConfDir: defaultConfDir, + DefaultBackendService: &types.NamespacedName{Namespace: defaultBackendNamespace, Name: defaultBackendName}, + WatchNamespace: defaultIngNamespace, + NghttpxConfigMap: &types.NamespacedName{Namespace: defaultConfigMapNamespace, Name: defaultConfigMapName}, + NghttpxConfDir: defaultConfDir, + NghttpxSecret: types.NamespacedName{ + Name: defaultNghttpxSecret.Name, + Namespace: defaultNghttpxSecret.Namespace, + }, IngressClassController: defaultIngressClassController, EnableEndpointSlice: f.enableEndpointSlice, ReloadRate: 1.0, @@ -157,13 +161,6 @@ func (f *fixture) preparePod(pod *corev1.Pod) { EventRecorder: &events.FakeRecorder{}, } - if f.http3 { - config.QUICKeyingMaterialsSecret = &types.NamespacedName{ - Name: defaultQUICSecret.Name, - Namespace: defaultQUICSecret.Namespace, - } - } - lbc, err := NewLoadBalancerController(context.Background(), f.clientset, newFakeLoadBalancer(), config) if err != nil { f.t.Fatalf("NewLoadBalancerController: %v", err) @@ -901,6 +898,15 @@ func newTLSSecret(namespace, name string, tlsCrt, tlsKey []byte) *corev1.Secret } } +func newNghttpxSecret() *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultNghttpxSecret.Name, + Namespace: defaultNghttpxSecret.Namespace, + }, + } +} + func newChecksumFile(path string) *nghttpx.ChecksumFile { return &nghttpx.ChecksumFile{ Path: path, @@ -946,10 +952,12 @@ func TestSyncDefaultBackend(t *testing.T) { } else { svc, eps, ess = newDefaultBackend() } + nghttpxSecret := newNghttpxSecret() f.cmStore = append(f.cmStore, cm) f.svcStore = append(f.svcStore, svc) f.epStore = append(f.epStore, eps) + f.secretStore = append(f.secretStore, nghttpxSecret) f.epSliceStore = append(f.epSliceStore, ess...) f.objects = append(f.objects, cm, svc, eps) @@ -1024,9 +1032,10 @@ func TestSyncDefaultSecret(t *testing.T) { dCrt := []byte(tlsCrt) dKey := []byte(tlsKey) tlsSecret := newTLSSecret("kube-system", "default-tls", dCrt, dKey) + nghttpxSecret := newNghttpxSecret() svc, eps, _ := newDefaultBackend() - f.secretStore = append(f.secretStore, tlsSecret) + f.secretStore = append(f.secretStore, tlsSecret, nghttpxSecret) f.svcStore = append(f.svcStore, svc) f.epStore = append(f.epStore, eps) @@ -1074,6 +1083,7 @@ func TestSyncDupDefaultSecret(t *testing.T) { dCrt := []byte(tlsCrt) dKey := []byte(tlsKey) tlsSecret := newTLSSecret("kube-system", "default-tls", dCrt, dKey) + nghttpxSecret := newNghttpxSecret() svc, eps, _ := newDefaultBackend() bs1, be1, _ := newBackend("alpha", []string{"192.168.10.1"}) @@ -1082,7 +1092,7 @@ func TestSyncDupDefaultSecret(t *testing.T) { WithTLS(tlsSecret.Name). Complete() - f.secretStore = append(f.secretStore, tlsSecret) + f.secretStore = append(f.secretStore, tlsSecret, nghttpxSecret) f.ingStore = append(f.ingStore, ing1) f.svcStore = append(f.svcStore, svc, bs1) f.epStore = append(f.epStore, eps, be1) @@ -1150,6 +1160,7 @@ Qu6PQqBCMaMh3xbmq1M9OwKwW/NwU0GW7w== dCrt := []byte(badlyFormattedTLSCrt) dKey := []byte(badlyFormattedTLSKey) tlsSecret := newTLSSecret(metav1.NamespaceDefault, "tls", dCrt, dKey) + nghttpxSecret := newNghttpxSecret() svc, eps, _ := newDefaultBackend() bs1, be1, _ := newBackend("alpha", []string{"192.168.10.1"}) @@ -1158,7 +1169,7 @@ Qu6PQqBCMaMh3xbmq1M9OwKwW/NwU0GW7w== WithTLS(tlsSecret.Name). Complete() - f.secretStore = append(f.secretStore, tlsSecret) + f.secretStore = append(f.secretStore, tlsSecret, nghttpxSecret) f.ingStore = append(f.ingStore, ing1) f.svcStore = append(f.svcStore, svc, bs1) f.epStore = append(f.epStore, eps, be1) @@ -1274,12 +1285,15 @@ func TestSyncStringNamedPort(t *testing.T) { }, } + nghttpxSecret := newNghttpxSecret() + f.svcStore = append(f.svcStore, svc, bs1) f.epStore = append(f.epStore, eps, be1) f.epSliceStore = append(f.epSliceStore, ess...) f.epSliceStore = append(f.epSliceStore, bes1) f.ingStore = append(f.ingStore, ing1) f.podStore = append(f.podStore, bp1, bp2) + f.secretStore = append(f.secretStore, nghttpxSecret) f.objects = append(f.objects, svc, eps, bs1, be1, ing1, bp1, bp2) @@ -1326,10 +1340,12 @@ func TestSyncEmptyTargetPort(t *testing.T) { ing1 := newIngressBuilder(bs1.Namespace, "alpha-ing"). WithRule("/", bs1.Name, serviceBackendPortNumber(bs1.Spec.Ports[0].Port)). Complete() + nghttpxSecret := newNghttpxSecret() f.svcStore = append(f.svcStore, svc, bs1) f.epStore = append(f.epStore, eps, be1) f.ingStore = append(f.ingStore, ing1) + f.secretStore = append(f.secretStore, nghttpxSecret) f.objects = append(f.objects, svc, eps, bs1, be1, ing1) @@ -1379,12 +1395,14 @@ func TestSyncWithoutSelectors(t *testing.T) { ing1 := newIngressBuilder(bs1.Namespace, "alpha-ing"). WithRule("/", bs1.Name, serviceBackendPortNumber(bs1.Spec.Ports[0].Port)). Complete() + nghttpxSecret := newNghttpxSecret() f.svcStore = append(f.svcStore, svc, bs1) f.epStore = append(f.epStore, eps, be1) f.epSliceStore = append(f.epSliceStore, ess...) f.epSliceStore = append(f.epSliceStore, bes1) f.ingStore = append(f.ingStore, ing1) + f.secretStore = append(f.secretStore, nghttpxSecret) f.objects = append(f.objects, svc, eps, bs1, be1, ing1) @@ -1523,10 +1541,12 @@ func TestSyncIngressDefaultBackend(t *testing.T) { WithRule("/", bs1.Name, serviceBackendPortNumber(bs1.Spec.Ports[0].Port)). WithDefaultBackend("bravo", serviceBackendPortNumber(bs2.Spec.Ports[0].Port)). Complete() + nghttpxSecret := newNghttpxSecret() f.svcStore = append(f.svcStore, svc, bs1, bs2) f.epStore = append(f.epStore, eps, be1, be2) f.ingStore = append(f.ingStore, ing1) + f.secretStore = append(f.secretStore, nghttpxSecret) f.objects = append(f.objects, svc, eps, bs1, be1, ing1, bs2, be2) @@ -1584,11 +1604,13 @@ func TestSyncIngressNoDefaultBackendOverride(t *testing.T) { f := newFixture(t) svc, eps, ess := newDefaultBackend() + nghttpxSecret := newNghttpxSecret() f.svcStore = append(f.svcStore, svc, bs1.DeepCopyObject().(*corev1.Service), bs2.DeepCopyObject().(*corev1.Service)) f.epStore = append(f.epStore, eps, be1.DeepCopyObject().(*corev1.Endpoints), be2.DeepCopyObject().(*corev1.Endpoints)) f.epSliceStore = append(f.epSliceStore, ess...) f.ingStore = append(f.ingStore, tt.ing) + f.secretStore = append(f.secretStore, nghttpxSecret) f.prepare() f.lbc.noDefaultBackendOverride = true @@ -1812,10 +1834,12 @@ func TestSyncNamedServicePort(t *testing.T) { ing1 := newIngressBuilder(bs1.Namespace, "alpha-ing"). WithRule("/", bs1.Name, serviceBackendPortName(bs1.Spec.Ports[0].Name)). Complete() + nghttpxSecret := newNghttpxSecret() f.svcStore = append(f.svcStore, svc, bs1) f.epStore = append(f.epStore, eps, be1) f.ingStore = append(f.ingStore, ing1) + f.secretStore = append(f.secretStore, nghttpxSecret) f.objects = append(f.objects, svc, eps, bs1, be1, ing1) @@ -1853,6 +1877,10 @@ func TestSyncInternalDefaultBackend(t *testing.T) { t.Run(tt.desc, func(t *testing.T) { f := newFixture(t) + nghttpxSecret := newNghttpxSecret() + + f.secretStore = append(f.secretStore, nghttpxSecret) + f.prepare() f.lbc.internalDefaultBackend = true @@ -1903,9 +1931,12 @@ func TestSyncDoNotForward(t *testing.T) { `}). Complete() + nghttpxSecret := newNghttpxSecret() + f.svcStore = append(f.svcStore, svc) f.epStore = append(f.epStore, eps) f.ingStore = append(f.ingStore, ing1) + f.secretStore = append(f.secretStore, nghttpxSecret) f.objects = append(f.objects, svc, eps, ing1) @@ -1974,10 +2005,12 @@ func TestSyncNormalizePath(t *testing.T) { ing1 := newIngressBuilder(metav1.NamespaceDefault, "alpha-ing"). WithRule(tt.path, bs1.Name, serviceBackendPortNumber(bs1.Spec.Ports[0].Port)). Complete() + nghttpxSecret := newNghttpxSecret() f.svcStore = append(f.svcStore, svc, bs1) f.epStore = append(f.epStore, eps, be1) f.ingStore = append(f.ingStore, ing1) + f.secretStore = append(f.secretStore, nghttpxSecret) f.objects = append(f.objects, svc, eps, bs1, be1, ing1) @@ -2004,8 +2037,8 @@ func TestSyncNormalizePath(t *testing.T) { } } -// TestSyncQUICKeyingMaterials verifies syncQUICKeyingMaterials. -func TestSyncQUICKeyingMaterials(t *testing.T) { +// TestSyncSecret verifies syncSecret. +func TestSyncSecret(t *testing.T) { now := time.Now().Round(time.Second) expiredTimestamp := now.Add(-quicSecretTimeout) notExpiredTimestamp := now.Add(-quicSecretTimeout + time.Second) @@ -2022,8 +2055,8 @@ func TestSyncQUICKeyingMaterials(t *testing.T) { desc: "QUIC secret is up to date", secret: &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: defaultQUICSecret.Name, - Namespace: defaultQUICSecret.Namespace, + Name: defaultNghttpxSecret.Name, + Namespace: defaultNghttpxSecret.Namespace, Annotations: map[string]string{ quicKeyingMaterialsUpdateTimestampKey: notExpiredTimestamp.Format(time.RFC3339), }, @@ -2038,8 +2071,8 @@ func TestSyncQUICKeyingMaterials(t *testing.T) { desc: "QUIC secret has been expired", secret: &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: defaultQUICSecret.Name, - Namespace: defaultQUICSecret.Namespace, + Name: defaultNghttpxSecret.Name, + Namespace: defaultNghttpxSecret.Namespace, Annotations: map[string]string{ quicKeyingMaterialsUpdateTimestampKey: expiredTimestamp.Format(time.RFC3339), }, @@ -2053,8 +2086,8 @@ func TestSyncQUICKeyingMaterials(t *testing.T) { desc: "QUIC secret timestamp is not expired, but data is malformed", secret: &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: defaultQUICSecret.Name, - Namespace: defaultQUICSecret.Namespace, + Name: defaultNghttpxSecret.Name, + Namespace: defaultNghttpxSecret.Namespace, Annotations: map[string]string{ quicKeyingMaterialsUpdateTimestampKey: notExpiredTimestamp.Format(time.RFC3339), }, @@ -2080,16 +2113,17 @@ func TestSyncQUICKeyingMaterials(t *testing.T) { f.prepare() f.setupStore() - f.lbc.quicKeyingMaterialsSecret = &defaultQUICSecret + f.lbc.nghttpxSecret = defaultNghttpxSecret - err := f.lc.syncQUICKeyingMaterials(context.Background(), now) + err := f.lc.syncSecret(context.Background(), defaultNghttpxSecret.String(), now) if err != nil { - t.Fatalf("f.lc.syncQUICKeyingMaterials(...): %v", err) + t.Fatalf("f.lc.syncSecret(...): %v", err) } - updatedSecret, err := f.clientset.CoreV1().Secrets(defaultQUICSecret.Namespace).Get(context.Background(), defaultQUICSecret.Name, metav1.GetOptions{}) + updatedSecret, err := f.clientset.CoreV1().Secrets(defaultNghttpxSecret.Namespace).Get(context.Background(), + defaultNghttpxSecret.Name, metav1.GetOptions{}) if err != nil { - t.Fatalf("Unable to get Secret %v/%v: %v", defaultQUICSecret.Namespace, defaultQUICSecret.Name, err) + t.Fatalf("Unable to get Secret %v/%v: %v", defaultNghttpxSecret.Namespace, defaultNghttpxSecret.Name, err) } if tt.wantKeepTimestamp { @@ -2628,10 +2662,12 @@ func TestSyncIgnoreUpstreamsWithInconsistentBackendParams(t *testing.T) { mruby: foo `}). Complete() + nghttpxSecret := newNghttpxSecret() f.svcStore = append(f.svcStore, svc, bs1, bs2, bs3) f.epStore = append(f.epStore, eps, be1, be2, be3) f.ingStore = append(f.ingStore, ing1, ing2, ing3) + f.secretStore = append(f.secretStore, nghttpxSecret) f.objects = append(f.objects, svc, bs1, bs2, bs3, eps, be1, be2, be3, ing1, ing2, ing3) @@ -2690,10 +2726,12 @@ func TestSyncEmptyAffinityCookieName(t *testing.T) { affinityCookieName: "foo" `}). Complete() + nghttpxSecret := newNghttpxSecret() f.svcStore = append(f.svcStore, svc, bs1, bs2, bs3) f.epStore = append(f.epStore, eps, be1, be2, be3) f.ingStore = append(f.ingStore, ing1, ing2, ing3) + f.secretStore = append(f.secretStore, nghttpxSecret) f.objects = append(f.objects, svc, bs1, bs2, bs3, eps, be1, be2, be3, ing1, ing2, ing3) diff --git a/pkg/nghttpx/nghttpx.tmpl b/pkg/nghttpx/nghttpx.tmpl index 6f189f5c..e3c47824 100644 --- a/pkg/nghttpx/nghttpx.tmpl +++ b/pkg/nghttpx/nghttpx.tmpl @@ -12,8 +12,10 @@ frontend=*,{{ .HTTPSPort }}{{ if .ProxyProto }};proxyproto{{ end }} {{ if .HTTP3 -}} # HTTP/3 frontend=*,{{ .HTTPSPort }};quic +{{ if .QUICSecretFile -}} # checksum: {{ encodeHex .QUICSecretFile.Checksum }} frontend-quic-secret-file={{ .QUICSecretFile.Path }} +{{ end -}} altsvc=h3,{{ .HTTPSPort }},,,ma=3600 altsvc=h3-29,{{ .HTTPSPort }},,,ma=3600 http2-altsvc=h3,{{ .HTTPSPort }},,,ma=3600 diff --git a/pkg/nghttpx/template_test.go b/pkg/nghttpx/template_test.go index 811d89c6..847a2aa6 100644 --- a/pkg/nghttpx/template_test.go +++ b/pkg/nghttpx/template_test.go @@ -532,6 +532,80 @@ fetch-ocsp-response-file=/fetch-ocsp-response wantBackendConfig: `# foo backend=192.168.0.1,8080;example.com/;proto=h2;affinity=none backend=192.168.0.2,80;example.com/;proto=h2;affinity=none +`, + }, + { + desc: "With HTTP/3 but missing QUICSecretFile", + ingConfig: &IngressConfig{ + HTTPPort: 80, + HTTPSPort: 443, + TLS: true, + DefaultTLSCred: &TLSCred{ + Key: PrivateChecksumFile{ + Path: "/tls/server.key", + Content: []byte("key"), + Checksum: hexMustDecodeString("2c70e12b7a0646f92279f427c7b38e7334d8e5389cff167a1dc30e73f826b683"), + }, + Cert: ChecksumFile{ + Path: "/tls/server.crt", + Content: []byte("cert"), + Checksum: hexMustDecodeString("06298432e8066b29e2223bcc23aa9504b56ae508fabf3435508869b9c3190e22"), + }, + }, + HTTP3: true, + Upstreams: []*Upstream{ + { + Name: "foo", + Host: "example.com", + Path: "/", + Affinity: AffinityNone, + Backends: []Backend{ + { + Address: "192.168.0.1", + Port: "8080", + Protocol: ProtocolH2, + }, + { + Address: "192.168.0.2", + Port: "80", + Protocol: ProtocolH2, + }, + }, + }, + }, + Workers: 8, + WorkerProcessGraceShutdownPeriod: 30 * time.Second, + MaxWorkerProcesses: 111, + }, + wantMainConfig: `accesslog-file=/dev/stdout +include=/nghttpx-backend.conf +# HTTP port +frontend=*,80;no-tls +# API endpoint +frontend=127.0.0.1,0;api;no-tls +# HTTPS port +frontend=*,443 +# HTTP/3 +frontend=*,443;quic +altsvc=h3,443,,,ma=3600 +altsvc=h3-29,443,,,ma=3600 +http2-altsvc=h3,443,,,ma=3600 +http2-altsvc=h3-29,443,,,ma=3600 +# Default TLS credential +private-key-file=/tls/server.key +certificate-file=/tls/server.crt +# for health check +frontend=127.0.0.1,0;healthmon;no-tls +# default configuration by controller +workers=8 +worker-process-grace-shutdown-period=30 +max-worker-processes=111 +# OCSP +fetch-ocsp-response-file=/fetch-ocsp-response +`, + wantBackendConfig: `# foo +backend=192.168.0.1,8080;example.com/;proto=h2;affinity=none +backend=192.168.0.2,80;example.com/;proto=h2;affinity=none `, }, { From 9d01f45d658280aac290298fb004ace4fb55eb19 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 2 Feb 2024 03:18:33 +0000 Subject: [PATCH 2/2] Add Role and RoleBinding Add Role and RoleBinding for nghttpx-secret. Move permissions for events and leases to Role. --- README.md | 19 ++------------ examples/default/service-account.yaml | 28 ++++++++++++++++++-- examples/proxyproto/02-nghttpx-rbac.yaml | 22 +++++++++++----- rc.yaml | 33 +++++++++++++++++++----- 4 files changed, 69 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index b1042571..6788a4ef 100644 --- a/README.md +++ b/README.md @@ -197,23 +197,8 @@ port specified by `--nghttpx-https-port` flag. > v0.66.0, copy Secret `nghttpx-quic-km` to `nghttpx-km`, and upgrade > nghttpx-ingress-controller. -HTTP/3 requires writing Secret and extra capabilities to load eBPF -program. For writing Secret, you might need to add the following -entry to ClusterRole: - -```yaml -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -... -rules: -- apiGroups: [""] - resources: ["secrets"] - verbs: ["create", "update", "patch"] -... -``` - -Add the following capabilities to the nghttpx-ingress-controller -container: +HTTP/3 requires the extra capabilities to load eBPF program. Add the +following capabilities to the nghttpx-ingress-controller container: ```yaml apiVersion: apps/v1 diff --git a/examples/default/service-account.yaml b/examples/default/service-account.yaml index c83f50d8..46116ab2 100644 --- a/examples/default/service-account.yaml +++ b/examples/default/service-account.yaml @@ -21,6 +21,29 @@ rules: - apiGroups: ["discovery.k8s.io"] resources: ["endpointslices"] verbs: ["get", "list", "watch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ingress +subjects: +- kind: ServiceAccount + name: ingress + namespace: kube-system +roleRef: + kind: ClusterRole + name: ingress + apiGroup: rbac.authorization.k8s.io +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ingress + namespace: kube-system +rules: +- apiGroups: [""] + resources: ["secrets"] + verbs: ["create", "update", "patch"] - apiGroups: ["events.k8s.io"] resources: ["events"] verbs: ["create", "patch"] @@ -28,15 +51,16 @@ rules: resources: ["leases"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] --- -kind: ClusterRoleBinding +kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: ingress + namespace: kube-system subjects: - kind: ServiceAccount name: ingress namespace: kube-system roleRef: - kind: ClusterRole + kind: Role name: ingress apiGroup: rbac.authorization.k8s.io diff --git a/examples/proxyproto/02-nghttpx-rbac.yaml b/examples/proxyproto/02-nghttpx-rbac.yaml index c0a56a76..cf88b47d 100644 --- a/examples/proxyproto/02-nghttpx-rbac.yaml +++ b/examples/proxyproto/02-nghttpx-rbac.yaml @@ -43,13 +43,6 @@ rules: - get - list - watch - - apiGroups: - - "events.k8s.io" - resources: - - events - verbs: - - create - - patch - apiGroups: - "networking.k8s.io" resources: @@ -107,6 +100,21 @@ rules: - get - create - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - create + - update + - patch + - apiGroups: + - "events.k8s.io" + resources: + - events + verbs: + - create + - patch - apiGroups: - "coordination.k8s.io" resources: diff --git a/rc.yaml b/rc.yaml index 32c580bf..4d77e462 100644 --- a/rc.yaml +++ b/rc.yaml @@ -77,28 +77,47 @@ rules: - apiGroups: ["discovery.k8s.io"] resources: ["endpointslices"] verbs: ["get", "list", "watch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ingress +subjects: +- kind: ServiceAccount + name: ingress + namespace: kube-system +roleRef: + kind: ClusterRole + name: ingress + apiGroup: rbac.authorization.k8s.io +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ingress + namespace: kube-system +rules: +- apiGroups: [""] + resources: ["secrets"] + verbs: ["create", "update", "patch"] - apiGroups: ["events.k8s.io"] resources: ["events"] verbs: ["create", "patch"] - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] -# Permission to write Secret is required for HTTP/3. -# You can remove it if HTTP/3 is not used. -- apiGroups: [""] - resources: ["secrets"] - verbs: ["create", "update", "patch"] --- -kind: ClusterRoleBinding +kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: ingress + namespace: kube-system subjects: - kind: ServiceAccount name: ingress namespace: kube-system roleRef: - kind: ClusterRole + kind: Role name: ingress apiGroup: rbac.authorization.k8s.io ---