Skip to content

Commit

Permalink
fix: support to change config (#17)
Browse files Browse the repository at this point in the history
Signed-off-by: shilinlee <[email protected]>
  • Loading branch information
shilinlee authored Aug 31, 2023
1 parent 6555659 commit 3e0b59f
Show file tree
Hide file tree
Showing 8 changed files with 289 additions and 38 deletions.
1 change: 1 addition & 0 deletions config/samples/opengemini-operator_v1_geminicluster.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ spec:
affinity:
enablePodAntiAffinity: false
enableHttpAuth: false
customConfigMapName: custom-config
monitoring:
type: string
paused: false
Expand Down
68 changes: 41 additions & 27 deletions controllers/geminicluster_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,24 +32,30 @@ import (
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/equality"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
"sigs.k8s.io/controller-runtime/pkg/log"
)

const (
ControllerManagerName = "openminicluster-controller"
)

type K8sController interface {
SetControllerReference(owner, controlled metav1.Object, scheme *runtime.Scheme) error
}

// GeminiClusterReconciler reconciles a GeminiCluster object
type GeminiClusterReconciler struct {
client.Client
Owner client.FieldOwner
Scheme *runtime.Scheme

Controller K8sController
}

//+kubebuilder:rbac:groups=opengemini-operator.opengemini.org,resources=geminiclusters,verbs=get;list;watch;create;update;patch;delete
Expand Down Expand Up @@ -216,41 +222,25 @@ func (r *GeminiClusterReconciler) reconcileClusterConfigMap(
ctx context.Context,
cluster *opengeminiv1.GeminiCluster,
) error {
confData, err := configfile.NewBaseConfiguration(cluster)
confdata, err := configfile.NewBaseConfiguration(cluster)
if err != nil {
return fmt.Errorf("cannot generate cluster configruation: %w", err)
}

if cluster.Spec.CustomConfigMapName != "" {
var customCM corev1.ConfigMap
err := r.Get(
ctx,
client.ObjectKey{Namespace: cluster.Namespace, Name: cluster.Spec.CustomConfigMapName},
&customCM)
if err != nil {
return fmt.Errorf("fetch custom conf ConfigMap failed, err: %w", err)
}

confFiles := make([]string, 0)
for _, v := range customCM.Data {
if v != "" {
confFiles = append(confFiles, v)
}
}
confFiles = append(confFiles, confData)

confData, err = configfile.Merge(confFiles...)
if err != nil {
return fmt.Errorf("merge custom conf failed, err: %w", err)
}
updatedConfig, err := r.getUpdatedConfig(ctx, cluster, confdata)
if err != nil {
return err
}
if updatedConfig != "" {
confdata = updatedConfig
}

clusterConfigMap := &corev1.ConfigMap{ObjectMeta: naming.ClusterConfigMap(cluster)}
clusterConfigMap.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("ConfigMap"))
clusterConfigMap.Data = map[string]string{
naming.ConfigurationFile: confData,
naming.ConfigurationFile: confdata,
}
confHash := utils.CalcMd5Hash(confData)
confHash := utils.CalcMd5Hash(confdata)

cluster.SetInheritedMetadata(&clusterConfigMap.ObjectMeta)
if err := r.setControllerReference(cluster, clusterConfigMap); err != nil {
Expand All @@ -265,6 +255,29 @@ func (r *GeminiClusterReconciler) reconcileClusterConfigMap(
return nil
}

// getUpdatedConfig returns the updated config items if set config map
func (r *GeminiClusterReconciler) getUpdatedConfig(ctx context.Context, cluster *opengeminiv1.GeminiCluster, tmplConf string) (string, error) {
var customCM corev1.ConfigMap
err := r.Get(
ctx,
client.ObjectKey{Namespace: cluster.Namespace, Name: cluster.Spec.CustomConfigMapName},
&customCM)
if errors.IsNotFound(err) {
return "", nil
}
if err != nil {
return "", fmt.Errorf("fetch custom conf ConfigMap failed, err: %w", err)
}

var newConfigData string
var ok bool
if newConfigData, ok = customCM.Data[naming.ConfigurationFile]; !ok {
return "", nil
}

return configfile.UpdateConfig(tmplConf, newConfigData)
}

// +kubebuilder:rbac:groups="",resources="services",verbs={get,create}

func (r *GeminiClusterReconciler) reconcileClusterServices(
Expand Down Expand Up @@ -448,7 +461,8 @@ func (r *GeminiClusterReconciler) reconcileClusterInstances(
func (r *GeminiClusterReconciler) setControllerReference(
owner *opengeminiv1.GeminiCluster, controlled client.Object,
) error {
return controllerutil.SetControllerReference(owner, controlled, r.Client.Scheme())
return r.Controller.SetControllerReference(owner, controlled, r.Client.Scheme())
//return controllerutil.SetControllerReference(owner, controlled, r.Client.Scheme())
}

// func (r *GeminiClusterReconciler) setOwnerReference(
Expand Down
87 changes: 87 additions & 0 deletions controllers/geminicluster_controller_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
/*
Copyright 2023.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controllers

import (
"context"
"testing"

opengeminiv1 "github.com/openGemini/openGemini-operator/api/v1"
"github.com/stretchr/testify/assert"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
)

type MockK8sClient struct {
client.Client
}

func (MockK8sClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
obj.(*corev1.ConfigMap).Data = map[string]string{
"opengemini.conf": `
[common]
meta-join = ["opengemini-meta-01.test.svc.cluster.local:8092", "opengemini-meta-02.test.svc.cluster.local:8092", "opengemini-meta-03.test.svc.cluster.local:8092"]
executor-memory-size-limit= "16G"
executor-memory-wait-time = "0s"
cpu-num = 8
memory-size = "32G"`,
}

return nil
}

func (MockK8sClient) Scheme() *runtime.Scheme {
return nil
}

func (MockK8sClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
return nil
}

type MockK8sController struct {
K8sController
}

func (MockK8sController) SetControllerReference(owner, controlled metav1.Object, scheme *runtime.Scheme) error {
return nil
}

func Test_ReconcileClusterConfigMap(t *testing.T) {
var r = &GeminiClusterReconciler{
Client: &MockK8sClient{},
Controller: &MockK8sController{},
}

var metaReplicas int32 = 3

cluster := &opengeminiv1.GeminiCluster{
Spec: opengeminiv1.GeminiClusterSpec{
Meta: opengeminiv1.MetaSpec{
Replicas: &metaReplicas,
},
CustomConfigMapName: "config.conf",
},
}
cluster.Name = "test"
cluster.Namespace = "opengemini"

err := r.reconcileClusterConfigMap(context.Background(), cluster)
assert.NoError(t, err)
assert.Equal(t, cluster.Status.AppliedConfigHash, "1e94d9f395144eaf69571fb83161d278")
}
6 changes: 4 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@ require (
github.com/influxdata/influxdb1-client v0.0.0-20220302092344-a9ab5670611c
github.com/onsi/ginkgo/v2 v2.1.4
github.com/onsi/gomega v1.19.0
github.com/sethvargo/go-password v0.2.0
github.com/stretchr/testify v1.7.0
golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63
k8s.io/api v0.25.0
k8s.io/apimachinery v0.25.0
k8s.io/client-go v0.25.0
Expand Down Expand Up @@ -52,6 +53,7 @@ require (
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_golang v1.12.2 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.32.1 // indirect
Expand All @@ -63,7 +65,7 @@ require (
golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd // indirect
golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect
golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect
golang.org/x/sys v0.11.0 // indirect
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
golang.org/x/text v0.3.7 // indirect
golang.org/x/time v0.0.0-20220609170525-579cf78fd858 // indirect
Expand Down
8 changes: 4 additions & 4 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -315,8 +315,6 @@ github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0
github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/sethvargo/go-password v0.2.0 h1:BTDl4CC/gjf/axHMaDQtw507ogrXLci6XRiLc7i/UHI=
github.com/sethvargo/go-password v0.2.0/go.mod h1:Ym4Mr9JXLBycr02MFuVQ/0JHidNetSgbzutTr3zsYXE=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
Expand Down Expand Up @@ -377,6 +375,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 h1:m64FZMko/V45gv0bNmrNYoDEq8U5YUhetc9cBWKS1TQ=
golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63/go.mod h1:0v4NqG35kSWCMzLaMeX+IQrlSnVE/bqGSyC2cz/9Le8=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
Expand Down Expand Up @@ -528,8 +528,8 @@ golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
Expand Down
10 changes: 10 additions & 0 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,13 @@ import (

opengeminioperatorv1 "github.com/openGemini/openGemini-operator/api/v1"
"github.com/openGemini/openGemini-operator/controllers"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
_ "k8s.io/client-go/plugin/pkg/client/auth"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
"sigs.k8s.io/controller-runtime/pkg/healthz"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
)
Expand All @@ -43,6 +45,12 @@ func init() {
//+kubebuilder:scaffold:scheme
}

type k8sController struct{}

func (k8sController) SetControllerReference(owner, controlled metav1.Object, scheme *runtime.Scheme) error {
return controllerutil.SetControllerReference(owner, controlled, scheme)
}

func main() {
var metricsAddr string
var enableLeaderElection bool
Expand Down Expand Up @@ -88,6 +96,8 @@ func main() {
Client: mgr.GetClient(),
Owner: controllers.ControllerManagerName,
Scheme: mgr.GetScheme(),

Controller: &k8sController{},
}).SetupWithManager(mgr); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "GeminiCluster")
os.Exit(1)
Expand Down
48 changes: 43 additions & 5 deletions pkg/configfile/configfile.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,18 +76,18 @@ func NewBaseConfiguration(cluster *opengeminiv1.GeminiCluster) (string, error) {
BindAddress: "<HOST_IP>:8088",
HttpBindAddress: "<HOST_IP>:8091",
RpcBindAddress: "<HOST_IP>:8092",
Dir: "/ogdata/meta",
Dir: naming.DataMountPath + "/meta",
Domain: "<META_DOMAIN>",
},
Data: DataConfig{
StoreIngestAddr: "<HOST_IP>:8400",
StoreSelectAddr: "<HOST_IP>:8401",
StoreDataDir: "/ogdata/data",
StoreWalDir: "/ogdata/wal",
StoreMetaDir: "/ogdata/meta",
StoreDataDir: naming.DataMountPath + "/data",
StoreWalDir: naming.DataMountPath + "/wal",
StoreMetaDir: naming.DataMountPath + "/meta",
},
Logging: LoggingConfig{
Path: "/ogdata/logs",
Path: naming.DataMountPath + "/logs",
},
Gossip: GossipConfig{
Enabled: true,
Expand All @@ -106,6 +106,44 @@ func NewBaseConfiguration(cluster *opengeminiv1.GeminiCluster) (string, error) {
return buf.String(), nil
}

func UpdateConfig(tmpl, newConf string) (string, error) {
var tmplToml map[string]interface{}
var newToml map[string]interface{}

_, err := toml.Decode(tmpl, &tmplToml)
if err != nil {
return "", err
}

_, err = toml.Decode(newConf, &newToml)
if err != nil {
return "", err
}

mapDeepCopy(newToml, tmplToml)

var buf bytes.Buffer
err = toml.NewEncoder(&buf).Encode(newToml)
if err != nil {
return "", err
}
return buf.String(), nil
}

func mapDeepCopy(dst, src map[string]interface{}) {
for k, v := range src {
if inv, ok := v.(map[string]interface{}); !ok {
dst[k] = v
} else {
if _, ok := dst[k]; !ok {
dst[k] = make(map[string]interface{})
}
mapDeepCopy(dst[k].(map[string]interface{}), inv)
}
}
}

// TODO: delete these codes below
func Merge(data ...string) (string, error) {
output := make(map[string]interface{})
for _, dt := range data {
Expand Down
Loading

0 comments on commit 3e0b59f

Please sign in to comment.