Skip to content

Commit

Permalink
Merge pull request #207 from weinimo/cert-management
Browse files Browse the repository at this point in the history
Cert management
openshift-merge-bot[bot] authored Dec 12, 2023

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
2 parents cdbdd42 + 99c281e commit c143deb
Showing 11 changed files with 405 additions and 24 deletions.
10 changes: 8 additions & 2 deletions api/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml
Original file line number Diff line number Diff line change
@@ -50,9 +50,15 @@ spec:
description: OctaviaAmphoraControllerSpec defines common state for all
Octavia Amphora Controllers
properties:
certspassphrasesecret:
default: octavia-ca-passphrase
description: Name of secret containing passphrase for the CA private
keys
type: string
certssecret:
description: '*kubebuilder:validation:Required Secret containing certs
for securing communication with amphora based Load Balancers'
default: octavia-certs-secret
description: LoadBalancerCerts - Secret containing certs for securing
communication with amphora based Load Balancers
type: string
containerImage:
description: ContainerImage - Amphora Controller Container Image URL
30 changes: 24 additions & 6 deletions api/bases/octavia.openstack.org_octavias.yaml
Original file line number Diff line number Diff line change
@@ -456,9 +456,15 @@ spec:
description: OctaviaHousekeeping - Spec definition for the Octavia
Housekeeping agent for the Octavia deployment
properties:
certspassphrasesecret:
default: octavia-ca-passphrase
description: Name of secret containing passphrase for the CA private
keys
type: string
certssecret:
description: '*kubebuilder:validation:Required Secret containing
certs for securing communication with amphora based Load Balancers'
default: octavia-certs-secret
description: LoadBalancerCerts - Secret containing certs for securing
communication with amphora based Load Balancers
type: string
containerImage:
description: ContainerImage - Amphora Controller Container Image
@@ -632,9 +638,15 @@ spec:
description: OctaviaHousekeeping - Spec definition for the Octavia
Housekeeping agent for the Octavia deployment
properties:
certspassphrasesecret:
default: octavia-ca-passphrase
description: Name of secret containing passphrase for the CA private
keys
type: string
certssecret:
description: '*kubebuilder:validation:Required Secret containing
certs for securing communication with amphora based Load Balancers'
default: octavia-certs-secret
description: LoadBalancerCerts - Secret containing certs for securing
communication with amphora based Load Balancers
type: string
containerImage:
description: ContainerImage - Amphora Controller Container Image
@@ -808,9 +820,15 @@ spec:
description: OctaviaHousekeeping - Spec definition for the Octavia
Housekeeping agent for the Octavia deployment
properties:
certspassphrasesecret:
default: octavia-ca-passphrase
description: Name of secret containing passphrase for the CA private
keys
type: string
certssecret:
description: '*kubebuilder:validation:Required Secret containing
certs for securing communication with amphora based Load Balancers'
default: octavia-certs-secret
description: LoadBalancerCerts - Secret containing certs for securing
communication with amphora based Load Balancers
type: string
containerImage:
description: ContainerImage - Amphora Controller Container Image
10 changes: 8 additions & 2 deletions api/v1beta1/amphoracontroller_types.go
Original file line number Diff line number Diff line change
@@ -77,10 +77,16 @@ type OctaviaAmphoraControllerSpec struct {
// Secret containing OpenStack password information for octavia OctaviaDatabasePassword, AdminPassword
Secret string `json:"secret"`

// *kubebuilder:validation:Required
// Secret containing certs for securing communication with amphora based Load Balancers
// +kubebuilder:validation:Required
// +kubebuilder:default=octavia-certs-secret
// LoadBalancerCerts - Secret containing certs for securing communication with amphora based Load Balancers
LoadBalancerCerts string `json:"certssecret"`

// +kubebuilder:validation:Optional
// +kubebuilder:default=octavia-ca-passphrase
// Name of secret containing passphrase for the CA private keys
CAKeyPassphraseSecret string `json:"certspassphrasesecret"`

// +kubebuilder:validation:Optional
// +kubebuilder:default={database: OctaviaDatabasePassword, service: OctaviaPassword}
// PasswordSelectors - Selectors to identify the DB and AdminUser password from the Secret
Original file line number Diff line number Diff line change
@@ -50,9 +50,15 @@ spec:
description: OctaviaAmphoraControllerSpec defines common state for all
Octavia Amphora Controllers
properties:
certspassphrasesecret:
default: octavia-ca-passphrase
description: Name of secret containing passphrase for the CA private
keys
type: string
certssecret:
description: '*kubebuilder:validation:Required Secret containing certs
for securing communication with amphora based Load Balancers'
default: octavia-certs-secret
description: LoadBalancerCerts - Secret containing certs for securing
communication with amphora based Load Balancers
type: string
containerImage:
description: ContainerImage - Amphora Controller Container Image URL
30 changes: 24 additions & 6 deletions config/crd/bases/octavia.openstack.org_octavias.yaml
Original file line number Diff line number Diff line change
@@ -456,9 +456,15 @@ spec:
description: OctaviaHousekeeping - Spec definition for the Octavia
Housekeeping agent for the Octavia deployment
properties:
certspassphrasesecret:
default: octavia-ca-passphrase
description: Name of secret containing passphrase for the CA private
keys
type: string
certssecret:
description: '*kubebuilder:validation:Required Secret containing
certs for securing communication with amphora based Load Balancers'
default: octavia-certs-secret
description: LoadBalancerCerts - Secret containing certs for securing
communication with amphora based Load Balancers
type: string
containerImage:
description: ContainerImage - Amphora Controller Container Image
@@ -632,9 +638,15 @@ spec:
description: OctaviaHousekeeping - Spec definition for the Octavia
Housekeeping agent for the Octavia deployment
properties:
certspassphrasesecret:
default: octavia-ca-passphrase
description: Name of secret containing passphrase for the CA private
keys
type: string
certssecret:
description: '*kubebuilder:validation:Required Secret containing
certs for securing communication with amphora based Load Balancers'
default: octavia-certs-secret
description: LoadBalancerCerts - Secret containing certs for securing
communication with amphora based Load Balancers
type: string
containerImage:
description: ContainerImage - Amphora Controller Container Image
@@ -808,9 +820,15 @@ spec:
description: OctaviaHousekeeping - Spec definition for the Octavia
Housekeeping agent for the Octavia deployment
properties:
certspassphrasesecret:
default: octavia-ca-passphrase
description: Name of secret containing passphrase for the CA private
keys
type: string
certssecret:
description: '*kubebuilder:validation:Required Secret containing
certs for securing communication with amphora based Load Balancers'
default: octavia-certs-secret
description: LoadBalancerCerts - Secret containing certs for securing
communication with amphora based Load Balancers
type: string
containerImage:
description: ContainerImage - Amphora Controller Container Image
9 changes: 6 additions & 3 deletions config/samples/octavia_v1beta1_octavia.yaml
Original file line number Diff line number Diff line change
@@ -21,7 +21,8 @@ spec:
serviceUser: octavia
serviceAccount: octavia
role: housekeeping
certssecret: todo
certssecret: octavia-amp-cert-data
certspassphrasesecret: octavia-ca-passphrase
secret: osp-secret
preserveJobs: false
customServiceConfig: |
@@ -33,7 +34,8 @@ spec:
serviceUser: octavia
serviceAccount: octavia
role: healthmanager
certssecret: todo
certssecret: octavia-amp-cert-data
certspassphrasesecret: octavia-ca-passphrase
secret: osp-secret
preserveJobs: false
customServiceConfig: |
@@ -45,7 +47,8 @@ spec:
serviceUser: octavia
serviceAccount: octavia
role: worker
certssecret: todo
certssecret: octavia-amp-cert-data
certspassphrasesecret: octavia-ca-passphrase
secret: osp-secret
preserveJobs: false
customServiceConfig: |
29 changes: 27 additions & 2 deletions controllers/amphoracontroller_controller.go
Original file line number Diff line number Diff line change
@@ -31,6 +31,7 @@ import (
"github.com/openstack-k8s-operators/lib-common/modules/common/helper"
"github.com/openstack-k8s-operators/lib-common/modules/common/labels"
nad "github.com/openstack-k8s-operators/lib-common/modules/common/networkattachment"
"github.com/openstack-k8s-operators/lib-common/modules/common/secret"
"github.com/openstack-k8s-operators/lib-common/modules/common/util"

keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1"
@@ -252,6 +253,17 @@ func (r *OctaviaAmphoraControllerReconciler) reconcileNormal(ctx context.Context
return ctrl.Result{}, err
}

err = amphoracontrollers.EnsureAmphoraCerts(ctx, instance, helper, &Log)
if err != nil {
instance.Status.Conditions.Set(condition.FalseCondition(
condition.ServiceConfigReadyCondition,
condition.ErrorReason,
condition.SeverityWarning,
condition.ServiceConfigReadyErrorMessage,
err.Error()))
return ctrl.Result{}, err
}

instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage)

//
@@ -416,12 +428,25 @@ func (r *OctaviaAmphoraControllerReconciler) generateServiceConfigMaps(
if err != nil {
return err
}
templateParameters["ServiceUser"] = instance.Spec.ServiceUser
caPassSecret, _, err := secret.GetSecret(
ctx, helper, instance.Spec.CAKeyPassphraseSecret, instance.Namespace)
if err != nil {
return err
}
spec := instance.Spec
templateParameters["ServiceUser"] = spec.ServiceUser
templateParameters["KeystoneInternalURL"] = keystoneInternalURL
templateParameters["KeystonePublicURL"] = keystonePublicURL
templateParameters["ServiceRoleName"] = instance.Spec.Role
templateParameters["ServiceRoleName"] = spec.Role
templateParameters["LbMgmtNetworkId"] = templateVars.LbMgmtNetworkID
templateParameters["AmpFlavorId"] = templateVars.AmphoraDefaultFlavorID
serverCAPassphrase := caPassSecret.Data["server-ca-passphrase"]
if serverCAPassphrase != nil {
templateParameters["ServerCAKeyPassphrase"] = string(serverCAPassphrase)
} else {
// Can't do string(nil)
templateParameters["ServerCAKeyPassphrase"] = ""
}

// TODO(beagles): populate the template parameters
cms := []util.Template{
233 changes: 233 additions & 0 deletions pkg/amphoracontrollers/amphora_certs.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,233 @@
/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package amphoracontrollers

import (
"bytes"
"context"
"crypto/rand"
"crypto/rsa"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"fmt"
"math/big"
"time"

"github.com/go-logr/logr"
"github.com/openstack-k8s-operators/lib-common/modules/common/helper"
"github.com/openstack-k8s-operators/lib-common/modules/common/secret"
octaviav1 "github.com/openstack-k8s-operators/octavia-operator/api/v1beta1"
corev1 "k8s.io/api/core/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
)

var (
subjectDefault = pkix.Name{
Organization: []string{"Dis"},
Country: []string{"US"},
Province: []string{"Oregon"},
Locality: []string{"Springfield"},
StreetAddress: []string{"Denial"},
PostalCode: []string{""},
CommonName: "www.example.com",
}
)

// generateKey generates a PEM encoded private RSA key and applies PEM
// encryption if given passphrase is not an empty string.
func generateKey(passphrase []byte) (*rsa.PrivateKey, []byte, error) {
priv, err := rsa.GenerateKey(rand.Reader, 4096)
if err != nil {
return nil, nil, err
}
pkcs8Key, err := x509.MarshalPKCS8PrivateKey(priv)
if err != nil {
err = fmt.Errorf("Error private key to PKCS #8 form: %w", err)
return priv, nil, err
}

var pemBlock *pem.Block
if passphrase != nil {
pemBlock, err = x509.EncryptPEMBlock( //nolint:staticcheck
rand.Reader,
"PRIVATE KEY",
pkcs8Key,
passphrase,
x509.PEMCipherAES128)
if err != nil {
err = fmt.Errorf("Error encrypting private key: %w", err)
return priv, nil, err
}
} else {
pemBlock = &pem.Block{Type: "PRIVATE KEY", Bytes: pkcs8Key}
}

privPEM := new(bytes.Buffer)
err = pem.Encode(privPEM, pemBlock)
if err != nil {
return priv, nil, err
}

return priv, privPEM.Bytes(), nil
}

func generateCACert(caPrivKey *rsa.PrivateKey, commonName string) ([]byte, error) {
caTemplate := &x509.Certificate{
SerialNumber: big.NewInt(2019),
Subject: subjectDefault,
NotBefore: time.Now(),
NotAfter: time.Now().AddDate(10, 0, 0),
IsCA: true,
BasicConstraintsValid: true,
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCRLSign | x509.KeyUsageCertSign,
}
caTemplate.Subject.CommonName = commonName

caBytes, err := x509.CreateCertificate(
rand.Reader, caTemplate, caTemplate, &caPrivKey.PublicKey, caPrivKey)
if err != nil {
return nil, err
}
caCertPEM := new(bytes.Buffer)
err = pem.Encode(caCertPEM, &pem.Block{
Type: "CERTIFICATE",
Bytes: caBytes,
})
if err != nil {
return nil, err
}
return caCertPEM.Bytes(), nil
}

// Create a certificate and key for the client and sign it with the CA
func generateClientCert(caCertPEM []byte, caPrivKey *rsa.PrivateKey) ([]byte, error) {

certTemplate := &x509.Certificate{
SerialNumber: big.NewInt(2019),
Subject: subjectDefault,
NotBefore: time.Now(),
NotAfter: time.Now().AddDate(0, 1, 0),
IsCA: false,
BasicConstraintsValid: false,
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageEmailProtection},
}

certBytes, err := x509.CreateCertificate(
rand.Reader, certTemplate, certTemplate, &caPrivKey.PublicKey, caPrivKey)
if err != nil {
return nil, err
}

certPEM := new(bytes.Buffer)
err = pem.Encode(certPEM, &pem.Block{
Type: "CERTIFICATE",
Bytes: certBytes,
})
if err != nil {
return nil, err
}

return certPEM.Bytes(), nil
}

// EnsureAmphoraCerts ensures Amphora certificates exist in the secret store
func EnsureAmphoraCerts(ctx context.Context, instance *octaviav1.OctaviaAmphoraController, h *helper.Helper, log *logr.Logger) error {
var oAmpSecret *corev1.Secret
var serverCAPass []byte = nil

_, _, err := secret.GetSecret(ctx, h, instance.Spec.LoadBalancerCerts, instance.Namespace)
if err != nil {
if !k8serrors.IsNotFound(err) {
err = fmt.Errorf("Error retrieving secret %s - %w", instance.Spec.LoadBalancerCerts, err)
return err
}

cAPassSecret, _, err := secret.GetSecret(
ctx, h, instance.Spec.CAKeyPassphraseSecret, instance.Namespace)
if err != nil {
log.Info("Could not read server CA passphrase. No encryption will be applied to the generated key.")
} else {
serverCAPass = cAPassSecret.Data["server-ca-passphrase"]
}

serverCAKey, serverCAKeyPEM, err := generateKey(serverCAPass)
if err != nil {
err = fmt.Errorf("Error while generating server CA key: %w", err)
return err
}
serverCACert, err := generateCACert(serverCAKey, "Octavia server CA")
if err != nil {
err = fmt.Errorf("Error while generating server CA certificate: %w", err)
return err
}

clientCAKey, _, err := generateKey(nil)
if err != nil {
err = fmt.Errorf("Error while generating client CA key: %w", err)
return err
}
clientCACert, err := generateCACert(clientCAKey, "Octavia client CA")
if err != nil {
err = fmt.Errorf("Error while generating amphora client CA certificate: %w", err)
return err
}

clientKey, clientKeyPEM, err := generateKey(nil)
if err != nil {
err = fmt.Errorf("Error while generating amphora client key: %w", err)
return err
}
clientCert, err := generateClientCert(clientCACert, clientKey)
if err != nil {
err = fmt.Errorf("Error while generating amphora client certificate: %w", err)
return err
}
clientKeyAndCert := append(clientKeyPEM, clientCert...)

oAmpSecret = &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: instance.Spec.LoadBalancerCerts,
Namespace: instance.Namespace,
},

// note: the client CA key seem to be needed only for generating the
// client CA cert and should not get mounted to the pods
Data: map[string][]byte{
"server_ca.key.pem": serverCAKeyPEM,
"server_ca.cert.pem": serverCACert,
"client_ca.cert.pem": clientCACert,
// Unencrypted client key and cert
"client.cert-and-key.pem": clientKeyAndCert,
},
}

// err = h.GetClient().Create(ctx, oAmpSecret)
_, result, err := secret.CreateOrPatchSecret(ctx, h, instance, oAmpSecret)

if err != nil {
err = fmt.Errorf("Error creating certs secret %s - %w",
instance.Spec.LoadBalancerCerts, err)
return err
} else if result != controllerutil.OperationResultNone {
return nil
}
}

return nil
}
6 changes: 5 additions & 1 deletion pkg/amphoracontrollers/deployment.go
Original file line number Diff line number Diff line change
@@ -41,6 +41,10 @@ func Deployment(
// The API pod has an extra volume so the API and the provider agent can
// communicate with each other.
volumes := octavia.GetVolumes(instance.Name)
volumes = append(volumes, GetCertVolume(instance.Spec.LoadBalancerCerts)...)

volumeMounts := octavia.GetVolumeMounts(serviceName)
volumeMounts = append(volumeMounts, GetCertVolumeMount()...)

// TODO(beagles): service debugging

@@ -100,7 +104,7 @@ func Deployment(
Name: serviceName,
Image: instance.Spec.ContainerImage,
Env: env.MergeEnvs([]corev1.EnvVar{}, envVars),
VolumeMounts: octavia.GetVolumeMounts(serviceName),
VolumeMounts: volumeMounts,
Resources: instance.Spec.Resources,
ReadinessProbe: readinessProbe,
LivenessProbe: livenessProbe,
55 changes: 55 additions & 0 deletions pkg/amphoracontrollers/volumes.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package amphoracontrollers

import (
corev1 "k8s.io/api/core/v1"
)

const (
configVolume = "amphora-certs"
)

var (
// Files get mounted as root:root, but process is running as octavia
configMode int32 = 0644
)

// GetCertVolume - service volumes
func GetCertVolume(certSecretName string) []corev1.Volume {
return []corev1.Volume{
{
Name: configVolume,
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
DefaultMode: &configMode,
SecretName: certSecretName,
},
},
},
}
}

// GetCertVolumeMount - certificate VolumeMount
func GetCertVolumeMount() []corev1.VolumeMount {
return []corev1.VolumeMount{
{
Name: configVolume,
MountPath: "/etc/octavia/certs",
ReadOnly: true,
},
}
}
7 changes: 7 additions & 0 deletions templates/octaviaamphoracontroller/config/octavia.conf
Original file line number Diff line number Diff line change
@@ -22,13 +22,20 @@ auth_type=password
# region_name=regionOne
interface=internal
[certificates]
cert_generator = local_cert_generator
ca_certificate = /etc/octavia/certs/server_ca.cert.pem
ca_private_key = /etc/octavia/certs/server_ca.key.pem
ca_private_key_passphrase = {{ .ServerCAKeyPassphrase }}
[compute]
[networking]
port_detach_timeout=300
[haproxy_amphora]
client_cert = /etc/octavia/certs/client.cert-and-key.pem
server_ca = /etc/octavia/certs/server_ca.cert.pem
[controller_worker]
amp_boot_network_list={{ .LbMgmtNetworkId }}
amp_flavor_id={{ .AmpFlavorId }}
client_ca = /etc/octavia/certs/client_ca.cert.pem
[task_flow]
[oslo_messaging]
# topic=octavia-rpc

0 comments on commit c143deb

Please sign in to comment.