From d09b00e7d9ca46ba16b3ece04edf1e21423947d5 Mon Sep 17 00:00:00 2001 From: sh2 Date: Fri, 27 Oct 2023 10:37:29 +0800 Subject: [PATCH 1/3] refactor: implement cluster create on k8s Signed-off-by: sh2 --- cmd/gtctl/cluster.go | 3 +- cmd/gtctl/cluster_create.go | 285 ++++++++++++++++++ .../gtctl/cluster_create_test.go | 2 +- cmd/gtctl/playground.go | 10 +- pkg/cluster/baremetal/not_implemented.go | 5 + pkg/cluster/kubernetes/cluster.go | 14 + pkg/cluster/kubernetes/create.go | 185 ++++++++++++ pkg/cluster/types.go | 53 +++- pkg/cmd/gtctl/cluster/common/common.go | 23 -- pkg/cmd/gtctl/cluster/create/create.go | 9 +- pkg/cmd/gtctl/cluster/delete/delete.go | 7 +- 11 files changed, 557 insertions(+), 39 deletions(-) create mode 100644 cmd/gtctl/cluster_create.go rename pkg/cmd/gtctl/cluster/create/config_test.go => cmd/gtctl/cluster_create_test.go (99%) create mode 100644 pkg/cluster/kubernetes/create.go delete mode 100644 pkg/cmd/gtctl/cluster/common/common.go diff --git a/cmd/gtctl/cluster.go b/cmd/gtctl/cluster.go index a0603169..1c88f02c 100644 --- a/cmd/gtctl/cluster.go +++ b/cmd/gtctl/cluster.go @@ -19,7 +19,6 @@ import ( "github.com/spf13/cobra" - "github.com/GreptimeTeam/gtctl/pkg/cmd/gtctl/cluster/create" "github.com/GreptimeTeam/gtctl/pkg/cmd/gtctl/cluster/delete" "github.com/GreptimeTeam/gtctl/pkg/logger" ) @@ -40,7 +39,7 @@ func NewClusterCommand(l logger.Logger) *cobra.Command { } // TODO(sh2): will refactor them in the following PR. - cmd.AddCommand(create.NewCreateClusterCommand(l)) + cmd.AddCommand(NewCreateClusterCommand(l)) cmd.AddCommand(delete.NewDeleteClusterCommand(l)) cmd.AddCommand(NewScaleClusterCommand(l)) cmd.AddCommand(NewGetClusterCommand(l)) diff --git a/cmd/gtctl/cluster_create.go b/cmd/gtctl/cluster_create.go new file mode 100644 index 00000000..cb53a7e4 --- /dev/null +++ b/cmd/gtctl/cluster_create.go @@ -0,0 +1,285 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "fmt" + "os/signal" + "strings" + "syscall" + "time" + + "github.com/spf13/cobra" + + opt "github.com/GreptimeTeam/gtctl/pkg/cluster" + "github.com/GreptimeTeam/gtctl/pkg/cluster/kubernetes" + "github.com/GreptimeTeam/gtctl/pkg/logger" + "github.com/GreptimeTeam/gtctl/pkg/status" +) + +const ( + // Various of support config type + configOperator = "operator" + configCluster = "cluster" + configEtcd = "etcd" +) + +type clusterCreateCliOptions struct { + // The options for deploying GreptimeDBCluster in K8s. + Namespace string + OperatorNamespace string + EtcdNamespace string + StorageClassName string + StorageSize string + StorageRetainPolicy string + GreptimeDBChartVersion string + GreptimeDBOperatorChartVersion string + ImageRegistry string + EtcdEndpoint string + EtcdChartVersion string + EtcdStorageClassName string + EtcdStorageSize string + EtcdClusterSize string + + // Values files that set in command line. + GreptimeDBClusterValuesFile string + EtcdClusterValuesFile string + GreptimeDBOperatorValuesFile string + + // The options for deploying GreptimeDBCluster in bare-metal. + BareMetal bool + Config string + GreptimeBinVersion string + EnableCache bool + RetainLogs bool + + // Common options. + Timeout int + DryRun bool + Set configValues + + // If UseGreptimeCNArtifacts is true, the creation will download the artifacts(charts and binaries) from 'downloads.greptime.cn'. + // Also, it will use ACR registry for charts images. + UseGreptimeCNArtifacts bool +} + +type configValues struct { + rawConfig []string + + operatorConfig string + clusterConfig string + etcdConfig string +} + +// parseConfig parse raw config values and classify it to different +// categories of config type by its prefix. +func (c *configValues) parseConfig() error { + var ( + operatorConfig []string + clusterConfig []string + etcdConfig []string + ) + + for _, raw := range c.rawConfig { + if len(raw) == 0 { + return fmt.Errorf("cannot parse empty config values") + } + + var configPrefix, configValue string + values := strings.Split(raw, ",") + + for _, value := range values { + value = strings.Trim(value, " ") + config := strings.SplitN(value, ".", 2) + configPrefix = config[0] + if len(config) == 2 { + configValue = config[1] + } else { + configValue = configPrefix + } + + switch configPrefix { + case configOperator: + operatorConfig = append(operatorConfig, configValue) + case configCluster: + clusterConfig = append(clusterConfig, configValue) + case configEtcd: + etcdConfig = append(etcdConfig, configValue) + default: + clusterConfig = append(clusterConfig, value) + } + } + } + + if len(operatorConfig) > 0 { + c.operatorConfig = strings.Join(operatorConfig, ",") + } + + if len(clusterConfig) > 0 { + c.clusterConfig = strings.Join(clusterConfig, ",") + } + + if len(etcdConfig) > 0 { + c.etcdConfig = strings.Join(etcdConfig, ",") + } + + return nil +} + +func NewCreateClusterCommand(l logger.Logger) *cobra.Command { + var options clusterCreateCliOptions + + cmd := &cobra.Command{ + Use: "create", + Short: "Create a GreptimeDB cluster", + Long: `Create a GreptimeDB cluster`, + RunE: func(cmd *cobra.Command, args []string) error { + return NewCluster(args, &options, l) + }, + } + + cmd.Flags().StringVar(&options.OperatorNamespace, "operator-namespace", "default", "The namespace of deploying greptimedb-operator.") + cmd.Flags().StringVar(&options.StorageClassName, "storage-class-name", "null", "Datanode storage class name.") + cmd.Flags().StringVar(&options.StorageSize, "storage-size", "10Gi", "Datanode persistent volume size.") + cmd.Flags().StringVar(&options.StorageRetainPolicy, "retain-policy", "Retain", "Datanode pvc retain policy.") + cmd.Flags().StringVarP(&options.Namespace, "namespace", "n", "default", "Namespace of GreptimeDB cluster.") + cmd.Flags().BoolVar(&options.DryRun, "dry-run", false, "Output the manifests without applying them.") + cmd.Flags().IntVar(&options.Timeout, "timeout", 600, "Timeout in seconds for the command to complete, -1 means no timeout, default is 10 min.") + cmd.Flags().StringArrayVar(&options.Set.rawConfig, "set", []string{}, "set values on the command line for greptimedb cluster, etcd and operator (can specify multiple or separate values with commas: eg. cluster.key1=val1,etcd.key2=val2).") + cmd.Flags().StringVar(&options.GreptimeDBChartVersion, "greptimedb-chart-version", "", "The greptimedb helm chart version, use latest version if not specified.") + cmd.Flags().StringVar(&options.GreptimeDBOperatorChartVersion, "greptimedb-operator-chart-version", "", "The greptimedb-operator helm chart version, use latest version if not specified.") + cmd.Flags().StringVar(&options.EtcdChartVersion, "etcd-chart-version", "", "The greptimedb-etcd helm chart version, use latest version if not specified.") + cmd.Flags().StringVar(&options.ImageRegistry, "image-registry", "", "The image registry.") + cmd.Flags().StringVar(&options.EtcdNamespace, "etcd-namespace", "default", "The namespace of etcd cluster.") + cmd.Flags().StringVar(&options.EtcdStorageClassName, "etcd-storage-class-name", "null", "The etcd storage class name.") + cmd.Flags().StringVar(&options.EtcdStorageSize, "etcd-storage-size", "10Gi", "the etcd persistent volume size.") + cmd.Flags().StringVar(&options.EtcdClusterSize, "etcd-cluster-size", "1", "the etcd cluster size.") + cmd.Flags().BoolVar(&options.BareMetal, "bare-metal", false, "Deploy the greptimedb cluster on bare-metal environment.") + cmd.Flags().StringVar(&options.GreptimeBinVersion, "greptime-bin-version", "", "The version of greptime binary(can be override by config file).") + cmd.Flags().StringVar(&options.Config, "config", "", "Configuration to deploy the greptimedb cluster on bare-metal environment.") + cmd.Flags().BoolVar(&options.EnableCache, "enable-cache", true, "If true, enable cache for downloading artifacts(charts and binaries).") + cmd.Flags().BoolVar(&options.RetainLogs, "retain-logs", true, "If true, always retain the logs of binary.") + cmd.Flags().BoolVar(&options.UseGreptimeCNArtifacts, "use-greptime-cn-artifacts", false, "If true, use greptime-cn artifacts(charts and binaries).") + cmd.Flags().StringVar(&options.GreptimeDBClusterValuesFile, "greptimedb-cluster-values-file", "", "The values file for greptimedb cluster.") + cmd.Flags().StringVar(&options.EtcdClusterValuesFile, "etcd-cluster-values-file", "", "The values file for etcd cluster.") + cmd.Flags().StringVar(&options.GreptimeDBOperatorValuesFile, "greptimedb-operator-values-file", "", "The values file for greptimedb operator.") + + return cmd +} + +// NewCluster creates a new cluster. +func NewCluster(args []string, options *clusterCreateCliOptions, l logger.Logger) error { + if len(args) == 0 { + return fmt.Errorf("cluster name should be set") + } + + var ( + clusterName = args[0] + ctx = context.Background() + cancel context.CancelFunc + ) + + if options.Timeout > 0 { + ctx, cancel = context.WithTimeout(ctx, time.Duration(options.Timeout)*time.Second) + defer cancel() + } + ctx, stop := signal.NotifyContext(ctx, syscall.SIGINT, syscall.SIGTERM) + defer stop() + + spinner, err := status.NewSpinner() + if err != nil { + return err + } + + // Parse config values that set in command line. + if err = options.Set.parseConfig(); err != nil { + return err + } + + createOptions := &opt.CreateOptions{ + Namespace: options.Namespace, + Name: clusterName, + Etcd: &opt.CreateEtcdOptions{ + ImageRegistry: options.ImageRegistry, + EtcdChartVersion: options.EtcdChartVersion, + EtcdStorageClassName: options.EtcdStorageClassName, + EtcdStorageSize: options.EtcdStorageSize, + EtcdClusterSize: options.EtcdClusterSize, + ConfigValues: options.Set.etcdConfig, + UseGreptimeCNArtifacts: options.UseGreptimeCNArtifacts, + ValuesFile: options.EtcdClusterValuesFile, + }, + Operator: &opt.CreateOperatorOptions{ + GreptimeDBOperatorChartVersion: options.GreptimeDBOperatorChartVersion, + ImageRegistry: options.ImageRegistry, + ConfigValues: options.Set.operatorConfig, + UseGreptimeCNArtifacts: options.UseGreptimeCNArtifacts, + ValuesFile: options.GreptimeDBOperatorValuesFile, + }, + Cluster: &opt.CreateClusterOptions{ + GreptimeDBChartVersion: options.GreptimeDBChartVersion, + ImageRegistry: options.ImageRegistry, + InitializerImageRegistry: options.ImageRegistry, + DatanodeStorageClassName: options.StorageClassName, + DatanodeStorageSize: options.StorageSize, + DatanodeStorageRetainPolicy: options.StorageRetainPolicy, + EtcdEndPoints: fmt.Sprintf("%s.%s:2379", kubernetes.EtcdClusterName(clusterName), options.EtcdNamespace), + ConfigValues: options.Set.clusterConfig, + UseGreptimeCNArtifacts: options.UseGreptimeCNArtifacts, + ValuesFile: options.GreptimeDBClusterValuesFile, + }, + } + + if options.BareMetal { + // TODO(sh2): implement later + l.V(0).Infof("Creating GreptimeDB cluster '%s' on bare-metal", logger.Bold(clusterName)) + + } else { + l.V(0).Infof("Creating GreptimeDB cluster '%s' in namespace '%s'", logger.Bold(clusterName), logger.Bold(options.Namespace)) + + cluster, err := kubernetes.NewCluster(l, + kubernetes.WithDryRun(options.DryRun), + kubernetes.WithTimeout(time.Duration(options.Timeout)*time.Second)) + if err != nil { + return err + } + if err = cluster.Create(ctx, createOptions, spinner); err != nil { + return err + } + } + + if !options.DryRun { + printTips(l, clusterName, options) + } + + return nil +} + +func printTips(l logger.Logger, clusterName string, options *clusterCreateCliOptions) { + l.V(0).Infof("\nNow you can use the following commands to access the GreptimeDB cluster:") + l.V(0).Infof("\n%s", logger.Bold("MySQL >")) + if !options.BareMetal { + l.V(0).Infof("%s", fmt.Sprintf("%s kubectl port-forward svc/%s-frontend -n %s 4002:4002 > connections-mysql.out &", logger.Bold("$"), clusterName, options.Namespace)) + } + l.V(0).Infof("%s", fmt.Sprintf("%s mysql -h 127.0.0.1 -P 4002", logger.Bold("$"))) + l.V(0).Infof("\n%s", logger.Bold("PostgreSQL >")) + if !options.BareMetal { + l.V(0).Infof("%s", fmt.Sprintf("%s kubectl port-forward svc/%s-frontend -n %s 4003:4003 > connections-pg.out &", logger.Bold("$"), clusterName, options.Namespace)) + } + l.V(0).Infof("%s", fmt.Sprintf("%s psql -h 127.0.0.1 -p 4003 -d public", logger.Bold("$"))) + l.V(0).Infof("\nThank you for using %s! Check for more information on %s. 😊", logger.Bold("GreptimeDB"), logger.Bold("https://greptime.com")) + l.V(0).Infof("\n%s 🔑", logger.Bold("Invest in Data, Harvest over Time.")) +} diff --git a/pkg/cmd/gtctl/cluster/create/config_test.go b/cmd/gtctl/cluster_create_test.go similarity index 99% rename from pkg/cmd/gtctl/cluster/create/config_test.go rename to cmd/gtctl/cluster_create_test.go index 6549c3a0..7a7031b1 100644 --- a/pkg/cmd/gtctl/cluster/create/config_test.go +++ b/cmd/gtctl/cluster_create_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package create +package main import ( "testing" diff --git a/cmd/gtctl/playground.go b/cmd/gtctl/playground.go index 8fb965fd..7d407cfd 100644 --- a/cmd/gtctl/playground.go +++ b/cmd/gtctl/playground.go @@ -18,7 +18,6 @@ import ( "github.com/lucasepe/codename" "github.com/spf13/cobra" - "github.com/GreptimeTeam/gtctl/pkg/cmd/gtctl/cluster/create" "github.com/GreptimeTeam/gtctl/pkg/logger" ) @@ -26,7 +25,7 @@ func NewPlaygroundCommand(l logger.Logger) *cobra.Command { return &cobra.Command{ Use: "playground", Short: "Starts a GreptimeDB cluster playground", - Long: "Starts a GreptimeDB cluster playground in bare-metal", + Long: "Starts a GreptimeDB cluster playground in bare-metal mode", RunE: func(cmd *cobra.Command, args []string) error { rng, err := codename.DefaultRNG() if err != nil { @@ -34,17 +33,14 @@ func NewPlaygroundCommand(l logger.Logger) *cobra.Command { } playgroundName := codename.Generate(rng, 0) - playgroundOptions := create.ClusterCliOptions{ + playgroundOptions := &clusterCreateCliOptions{ BareMetal: true, RetainLogs: false, Timeout: 900, // 15min EnableCache: false, } - if err = create.NewCluster([]string{playgroundName}, playgroundOptions, l); err != nil { - return err - } - return nil + return NewCluster([]string{playgroundName}, playgroundOptions, l) }, } } diff --git a/pkg/cluster/baremetal/not_implemented.go b/pkg/cluster/baremetal/not_implemented.go index 24af6684..8f7e42d8 100644 --- a/pkg/cluster/baremetal/not_implemented.go +++ b/pkg/cluster/baremetal/not_implemented.go @@ -19,6 +19,7 @@ import ( "fmt" opt "github.com/GreptimeTeam/gtctl/pkg/cluster" + "github.com/GreptimeTeam/gtctl/pkg/status" ) func (c *Cluster) List(ctx context.Context, options *opt.ListOptions) error { @@ -28,3 +29,7 @@ func (c *Cluster) List(ctx context.Context, options *opt.ListOptions) error { func (c *Cluster) Scale(ctx context.Context, options *opt.ScaleOptions) error { return fmt.Errorf("do not support") } + +func (c *Cluster) Create(ctx context.Context, options *opt.CreateOptions, spinner *status.Spinner) error { + return fmt.Errorf("do not support") +} diff --git a/pkg/cluster/kubernetes/cluster.go b/pkg/cluster/kubernetes/cluster.go index 63f4445c..8e11f390 100644 --- a/pkg/cluster/kubernetes/cluster.go +++ b/pkg/cluster/kubernetes/cluster.go @@ -34,6 +34,20 @@ type Cluster struct { type Option func(cluster *Cluster) +// WithDryRun enables Cluster to dry run. +func WithDryRun(dryRun bool) Option { + return func(d *Cluster) { + d.dryRun = dryRun + } +} + +// WithTimeout enables Cluster to have a timeout. +func WithTimeout(timeout time.Duration) Option { + return func(d *Cluster) { + d.timeout = timeout + } +} + func NewCluster(l logger.Logger, opts ...Option) (cluster.Operations, error) { hl, err := helm.NewLoader(l) if err != nil { diff --git a/pkg/cluster/kubernetes/create.go b/pkg/cluster/kubernetes/create.go new file mode 100644 index 00000000..dbe4c463 --- /dev/null +++ b/pkg/cluster/kubernetes/create.go @@ -0,0 +1,185 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kubernetes + +import ( + "context" + "fmt" + + "github.com/GreptimeTeam/gtctl/pkg/artifacts" + opt "github.com/GreptimeTeam/gtctl/pkg/cluster" + "github.com/GreptimeTeam/gtctl/pkg/helm" + "github.com/GreptimeTeam/gtctl/pkg/status" +) + +const ( + AliCloudRegistry = "greptime-registry.cn-hangzhou.cr.aliyuncs.com" + + disableRBACConfig = "auth.rbac.create=false,auth.rbac.token.enabled=false," +) + +func (c *Cluster) Create(ctx context.Context, options *opt.CreateOptions, spinner *status.Spinner) error { + withSpinner := func(target string, f func(context.Context, *opt.CreateOptions) error) error { + if !c.dryRun { + spinner.Start(fmt.Sprintf("Installing %s...", target)) + } + if err := f(ctx, options); err != nil { + spinner.Stop(false, fmt.Sprintf("Installing %s failed", target)) + return err + } + if !c.dryRun { + spinner.Stop(true, fmt.Sprintf("Installing %s successfully 🎉", target)) + } + return nil + } + + if err := withSpinner("Greptime Operator", c.createOperator); err != nil { + return err + } + if err := withSpinner("Etcd cluster", c.createEtcdCluster); err != nil { + return err + } + if err := withSpinner("Greptime cluster", c.createCluster); err != nil { + return err + } + + return nil +} + +// createOperator creates GreptimeDB Operator. +func (c *Cluster) createOperator(ctx context.Context, options *opt.CreateOptions) error { + if options.Operator == nil { + return fmt.Errorf("missing create greptime operator options") + } + operatorOpt := options.Operator + resourceName, resourceNamespace := OperatorName(), options.Namespace + + if operatorOpt.UseGreptimeCNArtifacts && len(operatorOpt.ImageRegistry) == 0 { + operatorOpt.ConfigValues += fmt.Sprintf("image.registry=%s,", AliCloudRegistry) + } + + opts := &helm.LoadOptions{ + ReleaseName: resourceName, + Namespace: resourceNamespace, + ChartName: artifacts.GreptimeDBOperatorChartName, + ChartVersion: operatorOpt.GreptimeDBOperatorChartVersion, + FromCNRegion: operatorOpt.UseGreptimeCNArtifacts, + ValuesOptions: *operatorOpt, + EnableCache: true, + ValuesFile: operatorOpt.ValuesFile, + } + manifests, err := c.helmLoader.LoadAndRenderChart(ctx, opts) + if err != nil { + return err + } + + if c.dryRun { + c.logger.V(0).Info(string(manifests)) + return nil + } + + if err = c.client.Apply(ctx, manifests); err != nil { + return err + } + + return c.client.WaitForDeploymentReady(ctx, resourceName, resourceNamespace, c.timeout) +} + +// createCluster creates GreptimeDB cluster. +func (c *Cluster) createCluster(ctx context.Context, options *opt.CreateOptions) error { + if options.Cluster == nil { + return fmt.Errorf("missing create greptime cluster options") + } + clusterOpt := options.Cluster + resourceName, resourceNamespace := options.Name, options.Namespace + + if clusterOpt.UseGreptimeCNArtifacts && len(clusterOpt.ImageRegistry) == 0 { + clusterOpt.ConfigValues += fmt.Sprintf("image.registry=%s,initializer.registry=%s,", AliCloudRegistry, AliCloudRegistry) + } + + opts := &helm.LoadOptions{ + ReleaseName: resourceName, + Namespace: resourceNamespace, + ChartName: artifacts.GreptimeDBClusterChartName, + ChartVersion: clusterOpt.GreptimeDBChartVersion, + FromCNRegion: clusterOpt.UseGreptimeCNArtifacts, + ValuesOptions: *clusterOpt, + EnableCache: true, + ValuesFile: clusterOpt.ValuesFile, + } + manifests, err := c.helmLoader.LoadAndRenderChart(ctx, opts) + if err != nil { + return err + } + + if c.dryRun { + c.logger.V(0).Info(string(manifests)) + return nil + } + + if err = c.client.Apply(ctx, manifests); err != nil { + return err + } + + return c.client.WaitForClusterReady(ctx, resourceName, resourceNamespace, c.timeout) +} + +// createEtcdCluster creates Etcd cluster. +func (c *Cluster) createEtcdCluster(ctx context.Context, options *opt.CreateOptions) error { + if options.Etcd == nil { + return fmt.Errorf("missing create etcd cluster options") + } + etcdOpt := options.Etcd + resourceName, resourceNamespace := EtcdClusterName(options.Name), options.Namespace + + etcdOpt.ConfigValues += disableRBACConfig + if etcdOpt.UseGreptimeCNArtifacts && len(etcdOpt.ImageRegistry) == 0 { + etcdOpt.ConfigValues += fmt.Sprintf("image.registry=%s,", AliCloudRegistry) + } + + opts := &helm.LoadOptions{ + ReleaseName: resourceName, + Namespace: resourceNamespace, + ChartName: artifacts.EtcdChartName, + ChartVersion: artifacts.DefaultEtcdChartVersion, + FromCNRegion: etcdOpt.UseGreptimeCNArtifacts, + ValuesOptions: *etcdOpt, + EnableCache: true, + ValuesFile: etcdOpt.ValuesFile, + } + manifests, err := c.helmLoader.LoadAndRenderChart(ctx, opts) + if err != nil { + return fmt.Errorf("error while loading helm chart: %v", err) + } + + if c.dryRun { + c.logger.V(0).Info(string(manifests)) + return nil + } + + if err = c.client.Apply(ctx, manifests); err != nil { + return fmt.Errorf("error while applying helm chart: %v", err) + } + + return c.client.WaitForEtcdReady(ctx, resourceName, resourceNamespace, c.timeout) +} + +func EtcdClusterName(clusterName string) string { + return fmt.Sprintf("%s-etcd", clusterName) +} + +func OperatorName() string { + return "greptimedb-operator" +} diff --git a/pkg/cluster/types.go b/pkg/cluster/types.go index 327e6ed9..198fe506 100644 --- a/pkg/cluster/types.go +++ b/pkg/cluster/types.go @@ -19,6 +19,8 @@ import ( greptimedbclusterv1alpha1 "github.com/GreptimeTeam/greptimedb-operator/apis/v1alpha1" "github.com/olekukonko/tablewriter" + + "github.com/GreptimeTeam/gtctl/pkg/status" ) type Operations interface { @@ -32,7 +34,8 @@ type Operations interface { // and refill the OldReplicas in ScaleOptions. Scale(ctx context.Context, options *ScaleOptions) error - // TODO(sh2): implementing Create and Delete + // Create creates a new cluster. + Create(ctx context.Context, options *CreateOptions, spinner *status.Spinner) error } type GetOptions struct { @@ -54,3 +57,51 @@ type ScaleOptions struct { Name string ComponentType greptimedbclusterv1alpha1.ComponentKind } + +type CreateOptions struct { + Namespace string + Name string + + Cluster *CreateClusterOptions + Operator *CreateOperatorOptions + Etcd *CreateEtcdOptions +} + +// CreateClusterOptions is the options to create a GreptimeDB cluster. +type CreateClusterOptions struct { + GreptimeDBChartVersion string + UseGreptimeCNArtifacts bool + ValuesFile string + + ImageRegistry string `helm:"image.registry"` + InitializerImageRegistry string `helm:"initializer.registry"` + DatanodeStorageClassName string `helm:"datanode.storage.storageClassName"` + DatanodeStorageSize string `helm:"datanode.storage.storageSize"` + DatanodeStorageRetainPolicy string `helm:"datanode.storage.storageRetainPolicy"` + EtcdEndPoints string `helm:"meta.etcdEndpoints"` + ConfigValues string `helm:"*"` +} + +// CreateOperatorOptions is the options to create a GreptimeDB operator. +type CreateOperatorOptions struct { + GreptimeDBOperatorChartVersion string + UseGreptimeCNArtifacts bool + ValuesFile string + + ImageRegistry string `helm:"image.registry"` + ConfigValues string `helm:"*"` +} + +// CreateEtcdOptions is the options to create an etcd cluster. +type CreateEtcdOptions struct { + EtcdChartVersion string + UseGreptimeCNArtifacts bool + ValuesFile string + + // The parameters reference: https://artifacthub.io/packages/helm/bitnami/etcd. + EtcdClusterSize string `helm:"replicaCount"` + ImageRegistry string `helm:"image.registry"` + EtcdStorageClassName string `helm:"persistence.storageClass"` + EtcdStorageSize string `helm:"persistence.size"` + ConfigValues string `helm:"*"` +} diff --git a/pkg/cmd/gtctl/cluster/common/common.go b/pkg/cmd/gtctl/cluster/common/common.go deleted file mode 100644 index dd3a23ac..00000000 --- a/pkg/cmd/gtctl/cluster/common/common.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2023 Greptime Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package common - -import ( - "fmt" -) - -func EtcdClusterName(clusterName string) string { - return fmt.Sprintf("%s-etcd", clusterName) -} diff --git a/pkg/cmd/gtctl/cluster/create/create.go b/pkg/cmd/gtctl/cluster/create/create.go index 9c5c0c5d..33d41c38 100644 --- a/pkg/cmd/gtctl/cluster/create/create.go +++ b/pkg/cmd/gtctl/cluster/create/create.go @@ -26,7 +26,6 @@ import ( "gopkg.in/yaml.v3" "k8s.io/apimachinery/pkg/types" - "github.com/GreptimeTeam/gtctl/pkg/cmd/gtctl/cluster/common" "github.com/GreptimeTeam/gtctl/pkg/deployer" "github.com/GreptimeTeam/gtctl/pkg/deployer/baremetal" "github.com/GreptimeTeam/gtctl/pkg/deployer/baremetal/component" @@ -286,7 +285,7 @@ func deployEtcdCluster(ctx context.Context, l logger.Logger, options *ClusterCli if options.BareMetal { name = clusterName } else { - name = types.NamespacedName{Namespace: options.EtcdNamespace, Name: common.EtcdClusterName(clusterName)}.String() + name = types.NamespacedName{Namespace: options.EtcdNamespace, Name: etcdClusterName(clusterName)}.String() } if err := clusterDeployer.CreateEtcdCluster(ctx, name, createEtcdClusterOptions); err != nil { @@ -315,7 +314,7 @@ func deployGreptimeDBCluster(ctx context.Context, l logger.Logger, options *Clus DatanodeStorageClassName: options.StorageClassName, DatanodeStorageSize: options.StorageSize, DatanodeStorageRetainPolicy: options.StorageRetainPolicy, - EtcdEndPoints: fmt.Sprintf("%s.%s:2379", common.EtcdClusterName(clusterName), options.EtcdNamespace), + EtcdEndPoints: fmt.Sprintf("%s.%s:2379", etcdClusterName(clusterName), options.EtcdNamespace), ConfigValues: options.Set.clusterConfig, UseGreptimeCNArtifacts: options.UseGreptimeCNArtifacts, ValuesFile: options.GreptimeDBClusterValuesFile, @@ -379,3 +378,7 @@ func waitChildProcess(ctx context.Context, deployer deployer.Interface, close bo } return nil } + +func etcdClusterName(clusterName string) string { + return fmt.Sprintf("%s-etcd", clusterName) +} diff --git a/pkg/cmd/gtctl/cluster/delete/delete.go b/pkg/cmd/gtctl/cluster/delete/delete.go index 5fe21d43..1be40752 100644 --- a/pkg/cmd/gtctl/cluster/delete/delete.go +++ b/pkg/cmd/gtctl/cluster/delete/delete.go @@ -24,7 +24,6 @@ import ( "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" - "github.com/GreptimeTeam/gtctl/pkg/cmd/gtctl/cluster/common" "github.com/GreptimeTeam/gtctl/pkg/deployer/k8s" "github.com/GreptimeTeam/gtctl/pkg/logger" ) @@ -81,7 +80,7 @@ func NewDeleteClusterCommand(l logger.Logger) *cobra.Command { if options.TearDownEtcd { etcdNamespace := strings.Split(strings.Split(rawCluster.Spec.Meta.EtcdEndpoints[0], ".")[1], ":")[0] l.V(0).Infof("Deleting etcd cluster in namespace '%s'...\n", logger.Bold(etcdNamespace)) - name = types.NamespacedName{Namespace: etcdNamespace, Name: common.EtcdClusterName(clusterName)}.String() + name = types.NamespacedName{Namespace: etcdNamespace, Name: EtcdClusterName(clusterName)}.String() if err := k8sDeployer.DeleteEtcdCluster(ctx, name, nil); err != nil { return err } @@ -97,3 +96,7 @@ func NewDeleteClusterCommand(l logger.Logger) *cobra.Command { return cmd } + +func EtcdClusterName(clusterName string) string { + return fmt.Sprintf("%s-etcd", clusterName) +} From db1716fe6f438fe110a8408aae6e577dcb56f493 Mon Sep 17 00:00:00 2001 From: sh2 Date: Fri, 27 Oct 2023 16:19:52 +0800 Subject: [PATCH 2/3] refactor: implement cluster create on bare-metal Signed-off-by: sh2 --- cmd/gtctl/cluster_create.go | 56 ++- cmd/gtctl/playground.go | 1 - pkg/cluster/baremetal/cluster.go | 76 +++- pkg/cluster/baremetal/create.go | 237 +++++++++++ pkg/cluster/baremetal/get.go | 26 +- pkg/cluster/baremetal/not_implemented.go | 5 - pkg/cluster/types.go | 4 + pkg/cmd/gtctl/cluster/create/config.go | 90 ---- pkg/cmd/gtctl/cluster/create/create.go | 384 ----------------- pkg/cmd/gtctl/cluster/delete/delete.go | 2 + .../component => components}/datanode.go | 46 +- .../component => components}/etcd.go | 21 +- .../component => components}/frontend.go | 29 +- .../component => components}/metasrv.go | 27 +- pkg/components/run.go | 96 +++++ pkg/components/types.go | 79 ++++ .../config.go => config/baremetal.go} | 48 +-- .../testdata/validate/invalid_artifact.yaml | 0 .../validate/invalid_hostname_port.yaml | 0 .../testdata/validate/invalid_replicas.yaml | 0 .../testdata/validate/valid_config.yaml | 0 pkg/{cluster/baremetal => config}/validate.go | 4 +- .../baremetal => config}/validate_test.go | 6 +- pkg/deployer/baremetal/component/cluster.go | 178 -------- pkg/deployer/baremetal/config/common.go | 95 ----- pkg/deployer/baremetal/config/datanode.go | 30 -- pkg/deployer/baremetal/config/etcd.go | 19 - pkg/deployer/baremetal/config/frontend.go | 28 -- pkg/deployer/baremetal/config/metasrv.go | 28 -- pkg/deployer/baremetal/deployer.go | 394 ------------------ pkg/deployer/baremetal/validate.go | 51 --- pkg/deployer/k8s/deployer.go | 2 + pkg/deployer/types.go | 2 + pkg/metadata/manager.go | 21 +- pkg/metadata/manager_test.go | 6 +- 35 files changed, 646 insertions(+), 1445 deletions(-) create mode 100644 pkg/cluster/baremetal/create.go delete mode 100644 pkg/cmd/gtctl/cluster/create/config.go delete mode 100644 pkg/cmd/gtctl/cluster/create/create.go rename pkg/{deployer/baremetal/component => components}/datanode.go (75%) rename pkg/{deployer/baremetal/component => components}/etcd.go (74%) rename pkg/{deployer/baremetal/component => components}/frontend.go (71%) rename pkg/{deployer/baremetal/component => components}/metasrv.go (83%) create mode 100644 pkg/components/run.go create mode 100644 pkg/components/types.go rename pkg/{cluster/baremetal/config.go => config/baremetal.go} (76%) rename pkg/{cluster/baremetal => config}/testdata/validate/invalid_artifact.yaml (100%) rename pkg/{cluster/baremetal => config}/testdata/validate/invalid_hostname_port.yaml (100%) rename pkg/{cluster/baremetal => config}/testdata/validate/invalid_replicas.yaml (100%) rename pkg/{cluster/baremetal => config}/testdata/validate/valid_config.yaml (100%) rename pkg/{cluster/baremetal => config}/validate.go (94%) rename pkg/{cluster/baremetal => config}/validate_test.go (94%) delete mode 100644 pkg/deployer/baremetal/component/cluster.go delete mode 100644 pkg/deployer/baremetal/config/common.go delete mode 100644 pkg/deployer/baremetal/config/datanode.go delete mode 100644 pkg/deployer/baremetal/config/etcd.go delete mode 100644 pkg/deployer/baremetal/config/frontend.go delete mode 100644 pkg/deployer/baremetal/config/metasrv.go delete mode 100644 pkg/deployer/baremetal/deployer.go delete mode 100644 pkg/deployer/baremetal/validate.go diff --git a/cmd/gtctl/cluster_create.go b/cmd/gtctl/cluster_create.go index cb53a7e4..5dfb57b4 100644 --- a/cmd/gtctl/cluster_create.go +++ b/cmd/gtctl/cluster_create.go @@ -17,15 +17,19 @@ package main import ( "context" "fmt" + "os" "os/signal" "strings" "syscall" "time" "github.com/spf13/cobra" + "gopkg.in/yaml.v3" opt "github.com/GreptimeTeam/gtctl/pkg/cluster" + "github.com/GreptimeTeam/gtctl/pkg/cluster/baremetal" "github.com/GreptimeTeam/gtctl/pkg/cluster/kubernetes" + "github.com/GreptimeTeam/gtctl/pkg/config" "github.com/GreptimeTeam/gtctl/pkg/logger" "github.com/GreptimeTeam/gtctl/pkg/status" ) @@ -64,7 +68,6 @@ type clusterCreateCliOptions struct { Config string GreptimeBinVersion string EnableCache bool - RetainLogs bool // Common options. Timeout int @@ -103,10 +106,10 @@ func (c *configValues) parseConfig() error { for _, value := range values { value = strings.Trim(value, " ") - config := strings.SplitN(value, ".", 2) - configPrefix = config[0] - if len(config) == 2 { - configValue = config[1] + cfg := strings.SplitN(value, ".", 2) + configPrefix = cfg[0] + if len(cfg) == 2 { + configValue = cfg[1] } else { configValue = configPrefix } @@ -127,11 +130,9 @@ func (c *configValues) parseConfig() error { if len(operatorConfig) > 0 { c.operatorConfig = strings.Join(operatorConfig, ",") } - if len(clusterConfig) > 0 { c.clusterConfig = strings.Join(clusterConfig, ",") } - if len(etcdConfig) > 0 { c.etcdConfig = strings.Join(etcdConfig, ",") } @@ -171,7 +172,6 @@ func NewCreateClusterCommand(l logger.Logger) *cobra.Command { cmd.Flags().StringVar(&options.GreptimeBinVersion, "greptime-bin-version", "", "The version of greptime binary(can be override by config file).") cmd.Flags().StringVar(&options.Config, "config", "", "Configuration to deploy the greptimedb cluster on bare-metal environment.") cmd.Flags().BoolVar(&options.EnableCache, "enable-cache", true, "If true, enable cache for downloading artifacts(charts and binaries).") - cmd.Flags().BoolVar(&options.RetainLogs, "retain-logs", true, "If true, always retain the logs of binary.") cmd.Flags().BoolVar(&options.UseGreptimeCNArtifacts, "use-greptime-cn-artifacts", false, "If true, use greptime-cn artifacts(charts and binaries).") cmd.Flags().StringVar(&options.GreptimeDBClusterValuesFile, "greptimedb-cluster-values-file", "", "The values file for greptimedb cluster.") cmd.Flags().StringVar(&options.EtcdClusterValuesFile, "etcd-cluster-values-file", "", "The values file for etcd cluster.") @@ -243,28 +243,58 @@ func NewCluster(args []string, options *clusterCreateCliOptions, l logger.Logger }, } + var cluster opt.Operations if options.BareMetal { - // TODO(sh2): implement later l.V(0).Infof("Creating GreptimeDB cluster '%s' on bare-metal", logger.Bold(clusterName)) + var opts []baremetal.Option + opts = append(opts, baremetal.WithEnableCache(options.EnableCache)) + if len(options.GreptimeBinVersion) > 0 { + opts = append(opts, baremetal.WithGreptimeVersion(options.GreptimeBinVersion)) + } + if len(options.Config) > 0 { + var cfg config.BareMetalClusterConfig + raw, err := os.ReadFile(options.Config) + if err != nil { + return err + } + if err = yaml.Unmarshal(raw, &cfg); err != nil { + return err + } + + opts = append(opts, baremetal.WithReplaceConfig(&cfg)) + } + + cluster, err = baremetal.NewCluster(l, clusterName, opts...) + if err != nil { + return err + } } else { l.V(0).Infof("Creating GreptimeDB cluster '%s' in namespace '%s'", logger.Bold(clusterName), logger.Bold(options.Namespace)) - cluster, err := kubernetes.NewCluster(l, + cluster, err = kubernetes.NewCluster(l, kubernetes.WithDryRun(options.DryRun), kubernetes.WithTimeout(time.Duration(options.Timeout)*time.Second)) if err != nil { return err } - if err = cluster.Create(ctx, createOptions, spinner); err != nil { - return err - } + } + + if err = cluster.Create(ctx, createOptions, spinner); err != nil { + return err } if !options.DryRun { printTips(l, clusterName, options) } + if options.BareMetal { + bm, _ := cluster.(*baremetal.Cluster) + if err = bm.Wait(ctx, false); err != nil { + return err + } + } + return nil } diff --git a/cmd/gtctl/playground.go b/cmd/gtctl/playground.go index 7d407cfd..a2e82602 100644 --- a/cmd/gtctl/playground.go +++ b/cmd/gtctl/playground.go @@ -35,7 +35,6 @@ func NewPlaygroundCommand(l logger.Logger) *cobra.Command { playgroundName := codename.Generate(rng, 0) playgroundOptions := &clusterCreateCliOptions{ BareMetal: true, - RetainLogs: false, Timeout: 900, // 15min EnableCache: false, } diff --git a/pkg/cluster/baremetal/cluster.go b/pkg/cluster/baremetal/cluster.go index 2805dfd7..990c7ebb 100644 --- a/pkg/cluster/baremetal/cluster.go +++ b/pkg/cluster/baremetal/cluster.go @@ -22,34 +22,74 @@ import ( "github.com/GreptimeTeam/gtctl/pkg/artifacts" "github.com/GreptimeTeam/gtctl/pkg/cluster" - "github.com/GreptimeTeam/gtctl/pkg/deployer/baremetal/component" + "github.com/GreptimeTeam/gtctl/pkg/components" + "github.com/GreptimeTeam/gtctl/pkg/config" "github.com/GreptimeTeam/gtctl/pkg/logger" "github.com/GreptimeTeam/gtctl/pkg/metadata" ) type Cluster struct { - logger logger.Logger - config *Config - wg sync.WaitGroup - ctx context.Context - + config *config.BareMetalClusterConfig createNoDirs bool enableCache bool am artifacts.Manager mm metadata.Manager - bm *component.BareMetalCluster + cc *ClusterComponents + + logger logger.Logger + stop context.CancelFunc + ctx context.Context + wg sync.WaitGroup +} + +// ClusterComponents describes all the components need to be deployed under bare-metal mode. +type ClusterComponents struct { + MetaSrv components.ClusterComponent + Datanode components.ClusterComponent + Frontend components.ClusterComponent + Etcd components.ClusterComponent +} + +func NewClusterComponents(config *config.BareMetalClusterComponentsConfig, workingDirs components.WorkingDirs, + wg *sync.WaitGroup, logger logger.Logger) *ClusterComponents { + return &ClusterComponents{ + MetaSrv: components.NewMetaSrv(config.MetaSrv, workingDirs, wg, logger), + Datanode: components.NewDataNode(config.Datanode, config.MetaSrv.ServerAddr, workingDirs, wg, logger), + Frontend: components.NewFrontend(config.Frontend, config.MetaSrv.ServerAddr, workingDirs, wg, logger), + Etcd: components.NewEtcd(workingDirs, wg, logger), + } } type Option func(cluster *Cluster) +// WithReplaceConfig replaces current cluster config with given config. +func WithReplaceConfig(cfg *config.BareMetalClusterConfig) Option { + return func(d *Cluster) { + d.config = cfg + } +} + +func WithGreptimeVersion(version string) Option { + return func(d *Cluster) { + d.config.Cluster.Artifact.Version = version + } +} + +func WithEnableCache(enableCache bool) Option { + return func(d *Cluster) { + d.enableCache = enableCache + } +} + func NewCluster(l logger.Logger, clusterName string, opts ...Option) (cluster.Operations, error) { - ctx, _ := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) c := &Cluster{ logger: l, - config: DefaultConfig(), + config: config.DefaultBareMetalConfig(), ctx: ctx, + stop: stop, } for _, opt := range opts { @@ -58,7 +98,7 @@ func NewCluster(l logger.Logger, clusterName string, opts ...Option) (cluster.Op } } - if err := ValidateConfig(c.config); err != nil { + if err := config.ValidateConfig(c.config); err != nil { return nil, err } @@ -76,8 +116,20 @@ func NewCluster(l logger.Logger, clusterName string, opts ...Option) (cluster.Op } c.am = am - // TODO(sh2): implement it in the following PR - // if !cluster.createNoDirs {} + // Configure Cluster Components. + if !c.createNoDirs { + mm.AllocateClusterScopeDirs(clusterName) + if err = mm.CreateClusterScopeDirs(c.config); err != nil { + return nil, err + } + + csd := mm.GetClusterScopeDirs() + c.cc = NewClusterComponents(c.config.Cluster, components.WorkingDirs{ + DataDir: csd.DataDir, + LogsDir: csd.LogsDir, + PidsDir: csd.PidsDir, + }, &c.wg, c.logger) + } return c, nil } diff --git a/pkg/cluster/baremetal/create.go b/pkg/cluster/baremetal/create.go new file mode 100644 index 00000000..c5279cbe --- /dev/null +++ b/pkg/cluster/baremetal/create.go @@ -0,0 +1,237 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package baremetal + +import ( + "context" + "fmt" + "os" + "os/exec" + "path" + "strings" + "time" + + "github.com/GreptimeTeam/gtctl/pkg/artifacts" + opt "github.com/GreptimeTeam/gtctl/pkg/cluster" + "github.com/GreptimeTeam/gtctl/pkg/logger" + "github.com/GreptimeTeam/gtctl/pkg/status" + fileutils "github.com/GreptimeTeam/gtctl/pkg/utils/file" +) + +func (c *Cluster) Create(ctx context.Context, options *opt.CreateOptions, spinner *status.Spinner) error { + withSpinner := func(target string, f func(context.Context, *opt.CreateOptions) error) error { + spinner.Start(fmt.Sprintf("Installing %s...", target)) + if err := f(ctx, options); err != nil { + spinner.Stop(false, fmt.Sprintf("Installing %s failed", target)) + return err + } + spinner.Stop(true, fmt.Sprintf("Installing %s successfully 🎉", target)) + return nil + } + + if err := withSpinner("Etcd Cluster", c.createEtcdCluster); err != nil { + return err + } + if err := withSpinner("Greptime Cluster", c.createCluster); err != nil { + if err := c.Wait(ctx, true); err != nil { + return err + } + return err + } + + return nil +} + +func (c *Cluster) createCluster(ctx context.Context, options *opt.CreateOptions) error { + if options.Cluster == nil { + return fmt.Errorf("missing create greptime cluster options") + } + clusterOpt := options.Cluster + + var binPath string + if c.config.Cluster.Artifact != nil { + if c.config.Cluster.Artifact.Local != "" { + binPath = c.config.Cluster.Artifact.Local + } else { + src, err := c.am.NewSource(artifacts.GreptimeBinName, c.config.Cluster.Artifact.Version, + artifacts.ArtifactTypeBinary, clusterOpt.UseGreptimeCNArtifacts) + if err != nil { + return err + } + + destDir, err := c.mm.AllocateArtifactFilePath(src, false) + if err != nil { + return err + } + + installDir, err := c.mm.AllocateArtifactFilePath(src, true) + if err != nil { + return err + } + + artifactFile, err := c.am.DownloadTo(ctx, src, destDir, &artifacts.DownloadOptions{ + EnableCache: c.enableCache, + BinaryInstallDir: installDir, + }) + if err != nil { + return err + } + binPath = artifactFile + } + } + + if err := c.cc.MetaSrv.Start(c.ctx, c.stop, binPath); err != nil { + return err + } + if err := c.cc.Datanode.Start(c.ctx, c.stop, binPath); err != nil { + return err + } + if err := c.cc.Frontend.Start(c.ctx, c.stop, binPath); err != nil { + return err + } + + return nil +} + +func (c *Cluster) createEtcdCluster(ctx context.Context, options *opt.CreateOptions) error { + if options.Etcd == nil { + return fmt.Errorf("missing create etcd cluster options") + } + etcdOpt := options.Etcd + + var binPath string + if c.config.Etcd.Artifact != nil { + if c.config.Etcd.Artifact.Local != "" { + binPath = c.config.Etcd.Artifact.Local + } else { + src, err := c.am.NewSource(artifacts.EtcdBinName, c.config.Etcd.Artifact.Version, + artifacts.ArtifactTypeBinary, etcdOpt.UseGreptimeCNArtifacts) + if err != nil { + return err + } + + destDir, err := c.mm.AllocateArtifactFilePath(src, false) + if err != nil { + return err + } + + installDir, err := c.mm.AllocateArtifactFilePath(src, true) + if err != nil { + return err + } + + artifactFile, err := c.am.DownloadTo(ctx, src, destDir, &artifacts.DownloadOptions{ + EnableCache: c.enableCache, + BinaryInstallDir: installDir, + }) + if err != nil { + return err + } + binPath = artifactFile + } + } + + if err := c.cc.Etcd.Start(c.ctx, c.stop, binPath); err != nil { + return err + } + if err := c.checkEtcdHealth(binPath); err != nil { + return err + } + + return nil +} + +func (c *Cluster) checkEtcdHealth(etcdBin string) error { + // It's very likely that "etcdctl" is under the same directory of "etcd". + etcdctlBin := path.Join(etcdBin, "../etcdctl") + exists, err := fileutils.IsFileExists(etcdctlBin) + if err != nil { + return err + } + if !exists { + c.logger.V(3).Infof("'etcdctl' is not found under the same directory of 'etcd', skip checking the healthy of Etcd.") + return nil + } + + for retry := 0; retry < 10; retry++ { + outputRaw, err := exec.Command(etcdctlBin, "endpoint", "status").Output() + if err != nil { + return err + } + output := string(outputRaw) + statuses := strings.Split(output, "\n") + + hasLeader := false + for i := 0; i < len(statuses); i++ { + fields := strings.Split(statuses[i], ",") + + // We are checking Etcd status with default output format("--write-out=simple"), example output: + // 127.0.0.1:2379, 8e9e05c52164694d, 3.5.0, 131 kB, true, false, 3, 75, 75, + // + // The output fields are corresponding to the following table's columns (with format "--write-out=table"): + // +----------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+ + // | ENDPOINT | ID | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS | + // +----------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+ + // | 127.0.0.1:2379 | 8e9e05c52164694d | 3.5.0 | 131 kB | true | false | 3 | 72 | 72 | | + // +----------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+ + // + // So we can just check the "IS LEADER" field. + if strings.TrimSpace(fields[4]) == "true" { + hasLeader = true + break + } + } + if hasLeader { + return nil + } + + time.Sleep(1 * time.Second) + } + + csd := c.mm.GetClusterScopeDirs() + return fmt.Errorf("etcd is not ready in 10 second! You can find its logs in %s", path.Join(csd.LogsDir, "etcd")) +} + +func (c *Cluster) Wait(ctx context.Context, close bool) error { + v := c.config.Cluster.Artifact.Version + if len(v) == 0 { + v = "unknown" + } + + if !close { + c.logger.V(0).Infof("The cluster(pid=%d, version=%s) is running in bare-metal mode now...\n", os.Getpid(), v) + c.logger.V(0).Infof("To view dashboard by accessing: %s\n", logger.Bold("http://localhost:4000/dashboard/")) + } else { + c.logger.Warnf("The cluster(pid=%d, version=%s) run in bare-metal has been shutting down...\n", os.Getpid(), v) + return nil + } + + // Wait for all the sub-processes to exit. + if err := c.wait(ctx); err != nil { + return err + } + return nil +} + +func (c *Cluster) wait(_ context.Context) error { + c.wg.Wait() + + // We ignore the context from input params, since + // it is not the context of current cluster. + <-c.ctx.Done() + + c.logger.V(3).Info("Cluster is shutting down...") + return nil +} diff --git a/pkg/cluster/baremetal/get.go b/pkg/cluster/baremetal/get.go index 82620b80..f45fee86 100644 --- a/pkg/cluster/baremetal/get.go +++ b/pkg/cluster/baremetal/get.go @@ -27,6 +27,8 @@ import ( "gopkg.in/yaml.v3" opt "github.com/GreptimeTeam/gtctl/pkg/cluster" + cfg "github.com/GreptimeTeam/gtctl/pkg/config" + "github.com/GreptimeTeam/gtctl/pkg/metadata" fileutils "github.com/GreptimeTeam/gtctl/pkg/utils/file" ) @@ -41,7 +43,7 @@ func (c *Cluster) Get(ctx context.Context, options *opt.GetOptions) error { return nil } -func (c *Cluster) get(_ context.Context, options *opt.GetOptions) (*ClusterMetadata, error) { +func (c *Cluster) get(_ context.Context, options *opt.GetOptions) (*cfg.BareMetalClusterMetadata, error) { csd := c.mm.GetClusterScopeDirs() _, err := os.Stat(csd.BaseDir) if os.IsNotExist(err) { @@ -59,7 +61,7 @@ func (c *Cluster) get(_ context.Context, options *opt.GetOptions) (*ClusterMetad return nil, fmt.Errorf("cluster %s is not exist", options.Name) } - var cluster ClusterMetadata + var cluster cfg.BareMetalClusterMetadata in, err := os.ReadFile(csd.ConfigPath) if err != nil { return nil, err @@ -76,7 +78,7 @@ func (c *Cluster) configGetView(table *tablewriter.Table) { table.SetRowLine(true) } -func (c *Cluster) renderGetView(table *tablewriter.Table, data *ClusterMetadata) { +func (c *Cluster) renderGetView(table *tablewriter.Table, data *cfg.BareMetalClusterMetadata) { c.configGetView(table) headers, footers, bulk := collectClusterInfoFromBareMetal(data) @@ -89,11 +91,11 @@ func (c *Cluster) renderGetView(table *tablewriter.Table, data *ClusterMetadata) } } -func collectClusterInfoFromBareMetal(data *ClusterMetadata) ( +func collectClusterInfoFromBareMetal(data *cfg.BareMetalClusterMetadata) ( headers, footers []string, bulk [][]string) { headers = []string{"COMPONENT", "PID"} - pidsDir := path.Join(data.ClusterDir, PidsDir) + pidsDir := path.Join(data.ClusterDir, metadata.ClusterPidsDir) pidsMap := collectPidsForBareMetal(pidsDir) var ( @@ -110,18 +112,18 @@ func collectClusterInfoFromBareMetal(data *ClusterMetadata) ( } ) - rows(string(greptimedbclusterv1alpha1.FrontendComponentKind), data.Cluster.Frontend.Replicas) - rows(string(greptimedbclusterv1alpha1.DatanodeComponentKind), data.Cluster.Datanode.Replicas) - rows(string(greptimedbclusterv1alpha1.MetaComponentKind), data.Cluster.MetaSrv.Replicas) + rows(string(greptimedbclusterv1alpha1.FrontendComponentKind), data.Config.Cluster.Frontend.Replicas) + rows(string(greptimedbclusterv1alpha1.DatanodeComponentKind), data.Config.Cluster.Datanode.Replicas) + rows(string(greptimedbclusterv1alpha1.MetaComponentKind), data.Config.Cluster.MetaSrv.Replicas) - // TODO(sh2): make "etcd" a const + // TODO(sh2): make "etcd" a const? bulk = append(bulk, []string{"etcd", pidsMap["etcd"]}) config, err := yaml.Marshal(data.Config) footers = []string{ fmt.Sprintf("CREATION-DATE: %s", date), - fmt.Sprintf("GREPTIMEDB-VERSION: %s", data.Cluster.Artifact.Version), - fmt.Sprintf("ETCD-VERSION: %s", data.Etcd.Artifact.Version), + fmt.Sprintf("GREPTIMEDB-VERSION: %s", data.Config.Cluster.Artifact.Version), + fmt.Sprintf("ETCD-VERSION: %s", data.Config.Etcd.Artifact.Version), fmt.Sprintf("CLUSTER-DIR: %s", data.ClusterDir), } if err != nil { @@ -139,7 +141,7 @@ func collectPidsForBareMetal(pidsDir string) map[string]string { if err := filepath.WalkDir(pidsDir, func(path string, d fs.DirEntry, err error) error { if d.IsDir() { - if d.Name() == PidsDir { + if d.Name() == metadata.ClusterPidsDir { return nil } diff --git a/pkg/cluster/baremetal/not_implemented.go b/pkg/cluster/baremetal/not_implemented.go index 8f7e42d8..24af6684 100644 --- a/pkg/cluster/baremetal/not_implemented.go +++ b/pkg/cluster/baremetal/not_implemented.go @@ -19,7 +19,6 @@ import ( "fmt" opt "github.com/GreptimeTeam/gtctl/pkg/cluster" - "github.com/GreptimeTeam/gtctl/pkg/status" ) func (c *Cluster) List(ctx context.Context, options *opt.ListOptions) error { @@ -29,7 +28,3 @@ func (c *Cluster) List(ctx context.Context, options *opt.ListOptions) error { func (c *Cluster) Scale(ctx context.Context, options *opt.ScaleOptions) error { return fmt.Errorf("do not support") } - -func (c *Cluster) Create(ctx context.Context, options *opt.CreateOptions, spinner *status.Spinner) error { - return fmt.Errorf("do not support") -} diff --git a/pkg/cluster/types.go b/pkg/cluster/types.go index 198fe506..9e80957d 100644 --- a/pkg/cluster/types.go +++ b/pkg/cluster/types.go @@ -36,6 +36,8 @@ type Operations interface { // Create creates a new cluster. Create(ctx context.Context, options *CreateOptions, spinner *status.Spinner) error + + // TODO(sh2): Delete API } type GetOptions struct { @@ -58,6 +60,8 @@ type ScaleOptions struct { ComponentType greptimedbclusterv1alpha1.ComponentKind } +type DeleteOptions struct{} + type CreateOptions struct { Namespace string Name string diff --git a/pkg/cmd/gtctl/cluster/create/config.go b/pkg/cmd/gtctl/cluster/create/config.go deleted file mode 100644 index f94c0f8b..00000000 --- a/pkg/cmd/gtctl/cluster/create/config.go +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright 2023 Greptime Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package create - -import ( - "fmt" - "strings" -) - -const ( - // Various of support config type - configOperator = "operator" - configCluster = "cluster" - configEtcd = "etcd" -) - -type configValues struct { - rawConfig []string - - operatorConfig string - clusterConfig string - etcdConfig string -} - -// parseConfig parse raw config values and classify it to different -// categories of config type by its prefix. -func (c *configValues) parseConfig() error { - var ( - operatorConfig []string - clusterConfig []string - etcdConfig []string - ) - - for _, raw := range c.rawConfig { - if len(raw) == 0 { - return fmt.Errorf("cannot parse empty config values") - } - - var configPrefix, configValue string - values := strings.Split(raw, ",") - - for _, value := range values { - value = strings.Trim(value, " ") - config := strings.SplitN(value, ".", 2) - configPrefix = config[0] - if len(config) == 2 { - configValue = config[1] - } else { - configValue = configPrefix - } - - switch configPrefix { - case configOperator: - operatorConfig = append(operatorConfig, configValue) - case configCluster: - clusterConfig = append(clusterConfig, configValue) - case configEtcd: - etcdConfig = append(etcdConfig, configValue) - default: - clusterConfig = append(clusterConfig, value) - } - } - } - - if len(operatorConfig) > 0 { - c.operatorConfig = strings.Join(operatorConfig, ",") - } - - if len(clusterConfig) > 0 { - c.clusterConfig = strings.Join(clusterConfig, ",") - } - - if len(etcdConfig) > 0 { - c.etcdConfig = strings.Join(etcdConfig, ",") - } - - return nil -} diff --git a/pkg/cmd/gtctl/cluster/create/create.go b/pkg/cmd/gtctl/cluster/create/create.go deleted file mode 100644 index 33d41c38..00000000 --- a/pkg/cmd/gtctl/cluster/create/create.go +++ /dev/null @@ -1,384 +0,0 @@ -// Copyright 2023 Greptime Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package create - -import ( - "context" - "fmt" - "os" - "os/signal" - "syscall" - "time" - - "github.com/spf13/cobra" - "gopkg.in/yaml.v3" - "k8s.io/apimachinery/pkg/types" - - "github.com/GreptimeTeam/gtctl/pkg/deployer" - "github.com/GreptimeTeam/gtctl/pkg/deployer/baremetal" - "github.com/GreptimeTeam/gtctl/pkg/deployer/baremetal/component" - bmconfig "github.com/GreptimeTeam/gtctl/pkg/deployer/baremetal/config" - "github.com/GreptimeTeam/gtctl/pkg/deployer/k8s" - "github.com/GreptimeTeam/gtctl/pkg/logger" - "github.com/GreptimeTeam/gtctl/pkg/status" -) - -type ClusterCliOptions struct { - // The options for deploying GreptimeDBCluster in K8s. - Namespace string - OperatorNamespace string - EtcdNamespace string - StorageClassName string - StorageSize string - StorageRetainPolicy string - GreptimeDBChartVersion string - GreptimeDBOperatorChartVersion string - ImageRegistry string - EtcdEndpoint string - EtcdChartVersion string - EtcdStorageClassName string - EtcdStorageSize string - EtcdClusterSize string - - // Values files that set in command line. - GreptimeDBClusterValuesFile string - EtcdClusterValuesFile string - GreptimeDBOperatorValuesFile string - - // The options for deploying GreptimeDBCluster in bare-metal. - BareMetal bool - Config string - GreptimeBinVersion string - EnableCache bool - RetainLogs bool - - // Common options. - Timeout int - DryRun bool - Set configValues - - // If UseGreptimeCNArtifacts is true, the creation will download the artifacts(charts and binaries) from 'downloads.greptime.cn'. - // Also, it will use ACR registry for charts images. - UseGreptimeCNArtifacts bool -} - -func NewCreateClusterCommand(l logger.Logger) *cobra.Command { - var options ClusterCliOptions - - cmd := &cobra.Command{ - Use: "create", - Short: "Create a GreptimeDB cluster", - Long: `Create a GreptimeDB cluster`, - RunE: func(cmd *cobra.Command, args []string) error { - if err := NewCluster(args, options, l); err != nil { - return err - } - return nil - }, - } - - cmd.Flags().StringVar(&options.OperatorNamespace, "operator-namespace", "default", "The namespace of deploying greptimedb-operator.") - cmd.Flags().StringVar(&options.StorageClassName, "storage-class-name", "null", "Datanode storage class name.") - cmd.Flags().StringVar(&options.StorageSize, "storage-size", "10Gi", "Datanode persistent volume size.") - cmd.Flags().StringVar(&options.StorageRetainPolicy, "retain-policy", "Retain", "Datanode pvc retain policy.") - cmd.Flags().StringVarP(&options.Namespace, "namespace", "n", "default", "Namespace of GreptimeDB cluster.") - cmd.Flags().BoolVar(&options.DryRun, "dry-run", false, "Output the manifests without applying them.") - cmd.Flags().IntVar(&options.Timeout, "timeout", 600, "Timeout in seconds for the command to complete, -1 means no timeout, default is 10 min.") - cmd.Flags().StringArrayVar(&options.Set.rawConfig, "set", []string{}, "set values on the command line for greptimedb cluster, etcd and operator (can specify multiple or separate values with commas: eg. cluster.key1=val1,etcd.key2=val2).") - cmd.Flags().StringVar(&options.GreptimeDBChartVersion, "greptimedb-chart-version", "", "The greptimedb helm chart version, use latest version if not specified.") - cmd.Flags().StringVar(&options.GreptimeDBOperatorChartVersion, "greptimedb-operator-chart-version", "", "The greptimedb-operator helm chart version, use latest version if not specified.") - cmd.Flags().StringVar(&options.EtcdChartVersion, "etcd-chart-version", "", "The greptimedb-etcd helm chart version, use latest version if not specified.") - cmd.Flags().StringVar(&options.ImageRegistry, "image-registry", "", "The image registry.") - cmd.Flags().StringVar(&options.EtcdNamespace, "etcd-namespace", "default", "The namespace of etcd cluster.") - cmd.Flags().StringVar(&options.EtcdStorageClassName, "etcd-storage-class-name", "null", "The etcd storage class name.") - cmd.Flags().StringVar(&options.EtcdStorageSize, "etcd-storage-size", "10Gi", "the etcd persistent volume size.") - cmd.Flags().StringVar(&options.EtcdClusterSize, "etcd-cluster-size", "1", "the etcd cluster size.") - cmd.Flags().BoolVar(&options.BareMetal, "bare-metal", false, "Deploy the greptimedb cluster on bare-metal environment.") - cmd.Flags().StringVar(&options.GreptimeBinVersion, "greptime-bin-version", "", "The version of greptime binary(can be override by config file).") - cmd.Flags().StringVar(&options.Config, "config", "", "Configuration to deploy the greptimedb cluster on bare-metal environment.") - cmd.Flags().BoolVar(&options.EnableCache, "enable-cache", true, "If true, enable cache for downloading artifacts(charts and binaries).") - cmd.Flags().BoolVar(&options.RetainLogs, "retain-logs", true, "If true, always retain the logs of binary.") - cmd.Flags().BoolVar(&options.UseGreptimeCNArtifacts, "use-greptime-cn-artifacts", false, "If true, use greptime-cn artifacts(charts and binaries).") - cmd.Flags().StringVar(&options.GreptimeDBClusterValuesFile, "greptimedb-cluster-values-file", "", "The values file for greptimedb cluster.") - cmd.Flags().StringVar(&options.EtcdClusterValuesFile, "etcd-cluster-values-file", "", "The values file for etcd cluster.") - cmd.Flags().StringVar(&options.GreptimeDBOperatorValuesFile, "greptimedb-operator-values-file", "", "The values file for greptimedb operator.") - - return cmd -} - -// NewCluster creates a new cluster. -func NewCluster(args []string, options ClusterCliOptions, l logger.Logger) error { - if len(args) == 0 { - return fmt.Errorf("cluster name should be set") - } - - var ( - clusterName = args[0] - ctx = context.Background() - cancel context.CancelFunc - deleteOpts component.DeleteOptions - ) - - if options.Timeout > 0 { - ctx, cancel = context.WithTimeout(ctx, time.Duration(options.Timeout)*time.Second) - defer cancel() - } - ctx, stop := signal.NotifyContext(ctx, syscall.SIGINT, syscall.SIGTERM) - defer stop() - - clusterDeployer, err := newDeployer(l, clusterName, &options) - if err != nil { - return err - } - - spinner, err := status.NewSpinner() - if err != nil { - return err - } - - if !options.BareMetal { - l.V(0).Infof("Creating GreptimeDB cluster '%s' in namespace '%s' ...", logger.Bold(clusterName), logger.Bold(options.Namespace)) - } else { - l.V(0).Infof("Creating GreptimeDB cluster '%s' on bare-metal environment...", logger.Bold(clusterName)) - } - - deleteOpts.RetainLogs = options.RetainLogs - - // Parse config values that set in command line - if err = options.Set.parseConfig(); err != nil { - return err - } - - if !options.BareMetal { - if err = deployGreptimeDBOperator(ctx, l, &options, spinner, clusterDeployer); err != nil { - return err - } - } - - if err = deployEtcdCluster(ctx, l, &options, spinner, clusterDeployer, clusterName); err != nil { - spinner.Stop(false, "Installing etcd cluster failed") - return err - } - - if err = deployGreptimeDBCluster(ctx, l, &options, spinner, clusterDeployer, clusterName); err != nil { - // Wait the cluster closing if deploy fails in bare-metal mode. - if options.BareMetal { - if err := waitChildProcess(ctx, clusterDeployer, true, deleteOpts); err != nil { - return err - } - } - return err - } - - if !options.DryRun { - printTips(l, clusterName, &options) - } - - if options.BareMetal { - if err = waitChildProcess(ctx, clusterDeployer, false, deleteOpts); err != nil { - return err - } - } - - return nil -} - -func newDeployer(l logger.Logger, clusterName string, options *ClusterCliOptions) (deployer.Interface, error) { - if !options.BareMetal { - k8sDeployer, err := k8s.NewDeployer(l, k8s.WithDryRun(options.DryRun), - k8s.WithTimeout(time.Duration(options.Timeout)*time.Second)) - if err != nil { - return nil, err - } - return k8sDeployer, nil - } - - var opts []baremetal.Option - - if options.GreptimeBinVersion != "" { - opts = append(opts, baremetal.WithGreptimeVersion(options.GreptimeBinVersion)) - } - - if options.Config != "" { - var cfg bmconfig.Config - raw, err := os.ReadFile(options.Config) - if err != nil { - return nil, err - } - - if err = yaml.Unmarshal(raw, &cfg); err != nil { - return nil, err - } - - opts = append(opts, baremetal.WithMergeConfig(&cfg, raw)) - } - - opts = append(opts, baremetal.WithEnableCache(options.EnableCache)) - - baremetalDeployer, err := baremetal.NewDeployer(l, clusterName, opts...) - if err != nil { - return nil, err - } - - return baremetalDeployer, nil -} - -func deployGreptimeDBOperator(ctx context.Context, l logger.Logger, options *ClusterCliOptions, - spinner *status.Spinner, clusterDeployer deployer.Interface) error { - - if !options.DryRun { - spinner.Start("Installing greptimedb-operator...") - } - - createGreptimeDBOperatorOptions := &deployer.CreateGreptimeDBOperatorOptions{ - GreptimeDBOperatorChartVersion: options.GreptimeDBOperatorChartVersion, - ImageRegistry: options.ImageRegistry, - ConfigValues: options.Set.operatorConfig, - UseGreptimeCNArtifacts: options.UseGreptimeCNArtifacts, - ValuesFile: options.GreptimeDBOperatorValuesFile, - } - - name := types.NamespacedName{Namespace: options.OperatorNamespace, Name: "greptimedb-operator"}.String() - if err := clusterDeployer.CreateGreptimeDBOperator(ctx, name, createGreptimeDBOperatorOptions); err != nil { - spinner.Stop(false, "Installing greptimedb-operator failed") - return err - } - - if !options.DryRun { - spinner.Stop(true, "Installing greptimedb-operator successfully 🎉") - } - - return nil -} - -func deployEtcdCluster(ctx context.Context, l logger.Logger, options *ClusterCliOptions, - spinner *status.Spinner, clusterDeployer deployer.Interface, clusterName string) error { - - if !options.DryRun { - spinner.Start("Installing etcd cluster...") - } - - createEtcdClusterOptions := &deployer.CreateEtcdClusterOptions{ - ImageRegistry: options.ImageRegistry, - EtcdChartVersion: options.EtcdChartVersion, - EtcdStorageClassName: options.EtcdStorageClassName, - EtcdStorageSize: options.EtcdStorageSize, - EtcdClusterSize: options.EtcdClusterSize, - ConfigValues: options.Set.etcdConfig, - UseGreptimeCNArtifacts: options.UseGreptimeCNArtifacts, - ValuesFile: options.EtcdClusterValuesFile, - } - - var name string - if options.BareMetal { - name = clusterName - } else { - name = types.NamespacedName{Namespace: options.EtcdNamespace, Name: etcdClusterName(clusterName)}.String() - } - - if err := clusterDeployer.CreateEtcdCluster(ctx, name, createEtcdClusterOptions); err != nil { - spinner.Stop(false, "Installing etcd cluster failed") - return err - } - - if !options.DryRun { - spinner.Stop(true, "Installing etcd cluster successfully 🎉") - } - - return nil -} - -func deployGreptimeDBCluster(ctx context.Context, l logger.Logger, options *ClusterCliOptions, - spinner *status.Spinner, clusterDeployer deployer.Interface, clusterName string) error { - - if !options.DryRun { - spinner.Start("Installing GreptimeDB cluster...") - } - - createGreptimeDBClusterOptions := &deployer.CreateGreptimeDBClusterOptions{ - GreptimeDBChartVersion: options.GreptimeDBChartVersion, - ImageRegistry: options.ImageRegistry, - InitializerImageRegistry: options.ImageRegistry, - DatanodeStorageClassName: options.StorageClassName, - DatanodeStorageSize: options.StorageSize, - DatanodeStorageRetainPolicy: options.StorageRetainPolicy, - EtcdEndPoints: fmt.Sprintf("%s.%s:2379", etcdClusterName(clusterName), options.EtcdNamespace), - ConfigValues: options.Set.clusterConfig, - UseGreptimeCNArtifacts: options.UseGreptimeCNArtifacts, - ValuesFile: options.GreptimeDBClusterValuesFile, - } - - var name string - if options.BareMetal { - name = clusterName - } else { - name = types.NamespacedName{Namespace: options.Namespace, Name: clusterName}.String() - } - - if err := clusterDeployer.CreateGreptimeDBCluster(ctx, name, createGreptimeDBClusterOptions); err != nil { - spinner.Stop(false, "Installing GreptimeDB cluster failed") - return err - } - - if !options.DryRun { - spinner.Stop(true, "Installing GreptimeDB cluster successfully 🎉") - } - - return nil -} - -func printTips(l logger.Logger, clusterName string, options *ClusterCliOptions) { - l.V(0).Infof("\nNow you can use the following commands to access the GreptimeDB cluster:") - l.V(0).Infof("\n%s", logger.Bold("MySQL >")) - if !options.BareMetal { - l.V(0).Infof("%s", fmt.Sprintf("%s kubectl port-forward svc/%s-frontend -n %s 4002:4002 > connections-mysql.out &", logger.Bold("$"), clusterName, options.Namespace)) - } - l.V(0).Infof("%s", fmt.Sprintf("%s mysql -h 127.0.0.1 -P 4002", logger.Bold("$"))) - l.V(0).Infof("\n%s", logger.Bold("PostgreSQL >")) - if !options.BareMetal { - l.V(0).Infof("%s", fmt.Sprintf("%s kubectl port-forward svc/%s-frontend -n %s 4003:4003 > connections-pg.out &", logger.Bold("$"), clusterName, options.Namespace)) - } - l.V(0).Infof("%s", fmt.Sprintf("%s psql -h 127.0.0.1 -p 4003 -d public", logger.Bold("$"))) - l.V(0).Infof("\nThank you for using %s! Check for more information on %s. 😊", logger.Bold("GreptimeDB"), logger.Bold("https://greptime.com")) - l.V(0).Infof("\n%s 🔑", logger.Bold("Invest in Data, Harvest over Time.")) -} - -func waitChildProcess(ctx context.Context, deployer deployer.Interface, close bool, option component.DeleteOptions) error { - d, ok := deployer.(*baremetal.Deployer) - if ok { - v := d.Config().Cluster.Artifact.Version - if len(v) == 0 { - v = "unknown" - } - - if !close { - fmt.Printf("\x1b[32m%s\x1b[0m", fmt.Sprintf("The cluster(pid=%d, version=%s) is running in bare-metal mode now...\n", os.Getpid(), v)) - fmt.Printf("\x1b[32m%s\x1b[0m", fmt.Sprintf("To view dashboard by accessing: %s\n", logger.Bold("http://localhost:4000/dashboard/"))) - } else { - fmt.Printf("\x1b[32m%s\x1b[0m", fmt.Sprintf("The cluster(pid=%d, version=%s) run in bare-metal has been deleted now...\n", os.Getpid(), v)) - return nil - } - - // Wait for all the child processes to exit. - if err := d.Wait(ctx, option); err != nil { - return err - } - } - return nil -} - -func etcdClusterName(clusterName string) string { - return fmt.Sprintf("%s-etcd", clusterName) -} diff --git a/pkg/cmd/gtctl/cluster/delete/delete.go b/pkg/cmd/gtctl/cluster/delete/delete.go index 1be40752..a8a8fd79 100644 --- a/pkg/cmd/gtctl/cluster/delete/delete.go +++ b/pkg/cmd/gtctl/cluster/delete/delete.go @@ -28,6 +28,8 @@ import ( "github.com/GreptimeTeam/gtctl/pkg/logger" ) +// TODO(sh2): remove this file later + type deleteClusterOptions struct { Namespace string TearDownEtcd bool diff --git a/pkg/deployer/baremetal/component/datanode.go b/pkg/components/datanode.go similarity index 75% rename from pkg/deployer/baremetal/component/datanode.go rename to pkg/components/datanode.go index b76e9eaa..83af3535 100644 --- a/pkg/deployer/baremetal/component/datanode.go +++ b/pkg/components/datanode.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package component +package components import ( "context" @@ -24,11 +24,19 @@ import ( "sync" "time" - "github.com/GreptimeTeam/gtctl/pkg/deployer/baremetal/config" + greptimev1alpha1 "github.com/GreptimeTeam/greptimedb-operator/apis/v1alpha1" + + opt "github.com/GreptimeTeam/gtctl/pkg/cluster" + "github.com/GreptimeTeam/gtctl/pkg/config" "github.com/GreptimeTeam/gtctl/pkg/logger" fileutils "github.com/GreptimeTeam/gtctl/pkg/utils/file" ) +const ( + dataHomeDir = "home" + dataWalDir = "wal" +) + type datanode struct { config *config.Datanode metaSrvAddr string @@ -41,8 +49,8 @@ type datanode struct { allocatedDirs } -func newDataNode(config *config.Datanode, metaSrvAddr string, workingDirs WorkingDirs, - wg *sync.WaitGroup, logger logger.Logger) BareMetalClusterComponent { +func NewDataNode(config *config.Datanode, metaSrvAddr string, workingDirs WorkingDirs, + wg *sync.WaitGroup, logger logger.Logger) ClusterComponent { return &datanode{ config: config, metaSrvAddr: metaSrvAddr, @@ -53,18 +61,18 @@ func newDataNode(config *config.Datanode, metaSrvAddr string, workingDirs Workin } func (d *datanode) Name() string { - return DataNode + return string(greptimev1alpha1.DatanodeComponentKind) } -func (d *datanode) Start(ctx context.Context, binary string) error { +func (d *datanode) Start(ctx context.Context, stop context.CancelFunc, binary string) error { for i := 0; i < d.config.Replicas; i++ { dirName := fmt.Sprintf("%s.%d", d.Name(), i) - dataHomeDir := path.Join(d.workingDirs.DataDir, dirName, config.DataHomeDir) - if err := fileutils.EnsureDir(dataHomeDir); err != nil { + homeDir := path.Join(d.workingDirs.DataDir, dirName, dataHomeDir) + if err := fileutils.EnsureDir(homeDir); err != nil { return err } - d.dataHomeDirs = append(d.dataHomeDirs, dataHomeDir) + d.dataHomeDirs = append(d.dataHomeDirs, homeDir) datanodeLogDir := path.Join(d.workingDirs.LogsDir, dirName) if err := fileutils.EnsureDir(datanodeLogDir); err != nil { @@ -78,7 +86,7 @@ func (d *datanode) Start(ctx context.Context, binary string) error { } d.pidsDirs = append(d.pidsDirs, datanodePidDir) - walDir := path.Join(d.workingDirs.DataDir, dirName, config.DataWalDir) + walDir := path.Join(d.workingDirs.DataDir, dirName, dataWalDir) if err := fileutils.EnsureDir(walDir); err != nil { return err } @@ -89,9 +97,9 @@ func (d *datanode) Start(ctx context.Context, binary string) error { Name: dirName, logDir: datanodeLogDir, pidDir: datanodePidDir, - args: d.BuildArgs(ctx, i, walDir, dataHomeDir), + args: d.BuildArgs(i, walDir, homeDir), } - if err := runBinary(ctx, option, d.wg, d.logger); err != nil { + if err := runBinary(ctx, stop, option, d.wg, d.logger); err != nil { return err } } @@ -115,13 +123,13 @@ CHECKER: return nil } -func (d *datanode) BuildArgs(ctx context.Context, params ...interface{}) []string { +func (d *datanode) BuildArgs(params ...interface{}) []string { logLevel := d.config.LogLevel if logLevel == "" { - logLevel = config.DefaultLogLevel + logLevel = DefaultLogLevel } - nodeID_, _, dataHomeDir := params[0], params[1], params[2] + nodeID_, _, homeDir := params[0], params[1], params[2] nodeID := nodeID_.(int) args := []string{ @@ -131,7 +139,7 @@ func (d *datanode) BuildArgs(ctx context.Context, params ...interface{}) []strin fmt.Sprintf("--metasrv-addr=%s", d.metaSrvAddr), fmt.Sprintf("--rpc-addr=%s", generateDatanodeAddr(d.config.RPCAddr, nodeID)), fmt.Sprintf("--http-addr=%s", generateDatanodeAddr(d.config.HTTPAddr, nodeID)), - fmt.Sprintf("--data-home=%s", dataHomeDir), + fmt.Sprintf("--data-home=%s", homeDir), } if len(d.config.Config) > 0 { @@ -141,7 +149,7 @@ func (d *datanode) BuildArgs(ctx context.Context, params ...interface{}) []strin return args } -func (d *datanode) IsRunning(ctx context.Context) bool { +func (d *datanode) IsRunning(_ context.Context) bool { for i := 0; i < d.config.Replicas; i++ { addr := generateDatanodeAddr(d.config.HTTPAddr, i) _, httpPort, err := net.SplitHostPort(addr) @@ -168,14 +176,14 @@ func (d *datanode) IsRunning(ctx context.Context) bool { return true } -func (d *datanode) Delete(ctx context.Context, option DeleteOptions) error { +func (d *datanode) Delete(ctx context.Context, options *opt.DeleteOptions) error { for _, dir := range d.dataHomeDirs { if err := fileutils.DeleteDirIfExists(dir); err != nil { return err } } - if err := d.delete(ctx, option); err != nil { + if err := d.delete(ctx, options); err != nil { return err } diff --git a/pkg/deployer/baremetal/component/etcd.go b/pkg/components/etcd.go similarity index 74% rename from pkg/deployer/baremetal/component/etcd.go rename to pkg/components/etcd.go index 2e122f9e..65554c4c 100644 --- a/pkg/deployer/baremetal/component/etcd.go +++ b/pkg/components/etcd.go @@ -12,13 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -package component +package components import ( "context" "path" "sync" + opt "github.com/GreptimeTeam/gtctl/pkg/cluster" "github.com/GreptimeTeam/gtctl/pkg/logger" fileutils "github.com/GreptimeTeam/gtctl/pkg/utils/file" ) @@ -31,7 +32,7 @@ type etcd struct { allocatedDirs } -func newEtcd(workingDirs WorkingDirs, wg *sync.WaitGroup, logger logger.Logger) BareMetalClusterComponent { +func NewEtcd(workingDirs WorkingDirs, wg *sync.WaitGroup, logger logger.Logger) ClusterComponent { return &etcd{ workingDirs: workingDirs, wg: wg, @@ -40,10 +41,10 @@ func newEtcd(workingDirs WorkingDirs, wg *sync.WaitGroup, logger logger.Logger) } func (e *etcd) Name() string { - return Etcd + return "etcd" } -func (e *etcd) Start(ctx context.Context, binary string) error { +func (e *etcd) Start(ctx context.Context, stop context.CancelFunc, binary string) error { var ( etcdDataDir = path.Join(e.workingDirs.DataDir, e.Name()) etcdLogDir = path.Join(e.workingDirs.LogsDir, e.Name()) @@ -64,26 +65,26 @@ func (e *etcd) Start(ctx context.Context, binary string) error { Name: e.Name(), logDir: etcdLogDir, pidDir: etcdPidDir, - args: e.BuildArgs(ctx, etcdDataDir), + args: e.BuildArgs(etcdDataDir), } - if err := runBinary(ctx, option, e.wg, e.logger); err != nil { + if err := runBinary(ctx, stop, option, e.wg, e.logger); err != nil { return err } return nil } -func (e *etcd) BuildArgs(ctx context.Context, params ...interface{}) []string { +func (e *etcd) BuildArgs(params ...interface{}) []string { return []string{"--data-dir", params[0].(string)} } -func (e *etcd) IsRunning(ctx context.Context) bool { +func (e *etcd) IsRunning(_ context.Context) bool { // Have not implemented the healthy checker now. return false } -func (e *etcd) Delete(ctx context.Context, option DeleteOptions) error { - if err := e.delete(ctx, option); err != nil { +func (e *etcd) Delete(ctx context.Context, options *opt.DeleteOptions) error { + if err := e.delete(ctx, options); err != nil { return err } return nil diff --git a/pkg/deployer/baremetal/component/frontend.go b/pkg/components/frontend.go similarity index 71% rename from pkg/deployer/baremetal/component/frontend.go rename to pkg/components/frontend.go index 80956ae2..3bb0d202 100644 --- a/pkg/deployer/baremetal/component/frontend.go +++ b/pkg/components/frontend.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package component +package components import ( "context" @@ -20,7 +20,10 @@ import ( "path" "sync" - "github.com/GreptimeTeam/gtctl/pkg/deployer/baremetal/config" + greptimedbclusterv1alpha1 "github.com/GreptimeTeam/greptimedb-operator/apis/v1alpha1" + + opt "github.com/GreptimeTeam/gtctl/pkg/cluster" + "github.com/GreptimeTeam/gtctl/pkg/config" "github.com/GreptimeTeam/gtctl/pkg/logger" fileutils "github.com/GreptimeTeam/gtctl/pkg/utils/file" ) @@ -36,8 +39,8 @@ type frontend struct { allocatedDirs } -func newFrontend(config *config.Frontend, metaSrvAddr string, workingDirs WorkingDirs, - wg *sync.WaitGroup, logger logger.Logger) BareMetalClusterComponent { +func NewFrontend(config *config.Frontend, metaSrvAddr string, workingDirs WorkingDirs, + wg *sync.WaitGroup, logger logger.Logger) ClusterComponent { return &frontend{ config: config, metaSrvAddr: metaSrvAddr, @@ -48,10 +51,10 @@ func newFrontend(config *config.Frontend, metaSrvAddr string, workingDirs Workin } func (f *frontend) Name() string { - return Frontend + return string(greptimedbclusterv1alpha1.FrontendComponentKind) } -func (f *frontend) Start(ctx context.Context, binary string) error { +func (f *frontend) Start(ctx context.Context, stop context.CancelFunc, binary string) error { for i := 0; i < f.config.Replicas; i++ { dirName := fmt.Sprintf("%s.%d", f.Name(), i) @@ -72,9 +75,9 @@ func (f *frontend) Start(ctx context.Context, binary string) error { Name: dirName, logDir: frontendLogDir, pidDir: frontendPidDir, - args: f.BuildArgs(ctx), + args: f.BuildArgs(), } - if err := runBinary(ctx, option, f.wg, f.logger); err != nil { + if err := runBinary(ctx, stop, option, f.wg, f.logger); err != nil { return err } } @@ -82,10 +85,10 @@ func (f *frontend) Start(ctx context.Context, binary string) error { return nil } -func (f *frontend) BuildArgs(ctx context.Context, params ...interface{}) []string { +func (f *frontend) BuildArgs(_ ...interface{}) []string { logLevel := f.config.LogLevel if logLevel == "" { - logLevel = config.DefaultLogLevel + logLevel = DefaultLogLevel } args := []string{ @@ -101,13 +104,13 @@ func (f *frontend) BuildArgs(ctx context.Context, params ...interface{}) []strin return args } -func (f *frontend) IsRunning(ctx context.Context) bool { +func (f *frontend) IsRunning(_ context.Context) bool { // Have not implemented the healthy checker now. return false } -func (f *frontend) Delete(ctx context.Context, option DeleteOptions) error { - if err := f.delete(ctx, option); err != nil { +func (f *frontend) Delete(ctx context.Context, options *opt.DeleteOptions) error { + if err := f.delete(ctx, options); err != nil { return err } diff --git a/pkg/deployer/baremetal/component/metasrv.go b/pkg/components/metasrv.go similarity index 83% rename from pkg/deployer/baremetal/component/metasrv.go rename to pkg/components/metasrv.go index e29b62d9..ad8dc09a 100644 --- a/pkg/deployer/baremetal/component/metasrv.go +++ b/pkg/components/metasrv.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package component +package components import ( "context" @@ -24,7 +24,8 @@ import ( "sync" "time" - "github.com/GreptimeTeam/gtctl/pkg/deployer/baremetal/config" + opt "github.com/GreptimeTeam/gtctl/pkg/cluster" + "github.com/GreptimeTeam/gtctl/pkg/config" "github.com/GreptimeTeam/gtctl/pkg/logger" fileutils "github.com/GreptimeTeam/gtctl/pkg/utils/file" ) @@ -39,8 +40,8 @@ type metaSrv struct { allocatedDirs } -func newMetaSrv(config *config.MetaSrv, workingDirs WorkingDirs, - wg *sync.WaitGroup, logger logger.Logger) BareMetalClusterComponent { +func NewMetaSrv(config *config.MetaSrv, workingDirs WorkingDirs, + wg *sync.WaitGroup, logger logger.Logger) ClusterComponent { return &metaSrv{ config: config, workingDirs: workingDirs, @@ -50,10 +51,10 @@ func newMetaSrv(config *config.MetaSrv, workingDirs WorkingDirs, } func (m *metaSrv) Name() string { - return MetaSrv + return "metasrv" } -func (m *metaSrv) Start(ctx context.Context, binary string) error { +func (m *metaSrv) Start(ctx context.Context, stop context.CancelFunc, binary string) error { // Default bind address for meta srv. bindAddr := net.JoinHostPort("127.0.0.1", "3002") if len(m.config.BindAddr) > 0 { @@ -80,9 +81,9 @@ func (m *metaSrv) Start(ctx context.Context, binary string) error { Name: dirName, logDir: metaSrvLogDir, pidDir: metaSrvPidDir, - args: m.BuildArgs(ctx, i, bindAddr), + args: m.BuildArgs(i, bindAddr), } - if err := runBinary(ctx, option, m.wg, m.logger); err != nil { + if err := runBinary(ctx, stop, option, m.wg, m.logger); err != nil { return err } } @@ -106,10 +107,10 @@ CHECKER: return nil } -func (m *metaSrv) BuildArgs(ctx context.Context, params ...interface{}) []string { +func (m *metaSrv) BuildArgs(params ...interface{}) []string { logLevel := m.config.LogLevel if logLevel == "" { - logLevel = config.DefaultLogLevel + logLevel = DefaultLogLevel } nodeID_, bindAddr_ := params[0], params[1] @@ -132,7 +133,7 @@ func (m *metaSrv) BuildArgs(ctx context.Context, params ...interface{}) []string return args } -func (m *metaSrv) IsRunning(ctx context.Context) bool { +func (m *metaSrv) IsRunning(_ context.Context) bool { for i := 0; i < m.config.Replicas; i++ { addr := generateMetaSrvAddr(m.config.HTTPAddr, i) _, httpPort, err := net.SplitHostPort(addr) @@ -159,8 +160,8 @@ func (m *metaSrv) IsRunning(ctx context.Context) bool { return true } -func (m *metaSrv) Delete(ctx context.Context, option DeleteOptions) error { - if err := m.delete(ctx, option); err != nil { +func (m *metaSrv) Delete(ctx context.Context, options *opt.DeleteOptions) error { + if err := m.delete(ctx, options); err != nil { return err } return nil diff --git a/pkg/components/run.go b/pkg/components/run.go new file mode 100644 index 00000000..9dd649e3 --- /dev/null +++ b/pkg/components/run.go @@ -0,0 +1,96 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package components + +import ( + "bufio" + "context" + "os" + "os/exec" + "path" + "strconv" + "sync" + "syscall" + + "github.com/GreptimeTeam/gtctl/pkg/logger" +) + +// RunOptions contains all the options for one component to run on bare-metal. +type RunOptions struct { + Binary string + Name string + + pidDir string + logDir string + args []string +} + +func runBinary(ctx context.Context, stop context.CancelFunc, + option *RunOptions, wg *sync.WaitGroup, logger logger.Logger) error { + cmd := exec.CommandContext(ctx, option.Binary, option.args...) + + // output to binary. + logFile := path.Join(option.logDir, "log") + outputFile, err := os.Create(logFile) + if err != nil { + return err + } + + outputFileWriter := bufio.NewWriter(outputFile) + cmd.Stdout = outputFileWriter + cmd.Stderr = outputFileWriter + + if err = cmd.Start(); err != nil { + return err + } + + pid := strconv.Itoa(cmd.Process.Pid) + logger.V(3).Infof("run '%s' binary '%s' with args: '%v', log: '%s', pid: '%s'", + option.Name, option.Binary, option.args, option.logDir, pid) + + pidFile := path.Join(option.pidDir, "pid") + f, err := os.Create(pidFile) + if err != nil { + return err + } + + _, err = f.Write([]byte(pid)) + if err != nil { + return err + } + + go func() { + defer wg.Done() + wg.Add(1) + if err := cmd.Wait(); err != nil { + // Caught signal kill and interrupt error then ignore. + if exit, ok := err.(*exec.ExitError); ok { + if status, ok := exit.Sys().(syscall.WaitStatus); ok && status.Signaled() { + if status.Signal() == syscall.SIGKILL || status.Signal() == syscall.SIGINT { + return + } + } + } + logger.Errorf("component '%s' binary '%s' (pid '%s') exited with error: %v", option.Name, option.Binary, pid, err) + logger.Errorf("args: '%v'", option.args) + _ = outputFileWriter.Flush() + + // If one component has failed, stop the whole context. + stop() + } + }() + + return nil +} diff --git a/pkg/components/types.go b/pkg/components/types.go new file mode 100644 index 00000000..30529f68 --- /dev/null +++ b/pkg/components/types.go @@ -0,0 +1,79 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package components + +import ( + "context" + + opt "github.com/GreptimeTeam/gtctl/pkg/cluster" + fileutils "github.com/GreptimeTeam/gtctl/pkg/utils/file" +) + +const ( + DefaultLogLevel = "info" +) + +// WorkingDirs include all the dirs used in bare-metal mode. +type WorkingDirs struct { + DataDir string `yaml:"dataDir"` + LogsDir string `yaml:"logsDir"` + PidsDir string `yaml:"pidsDir"` +} + +type allocatedDirs struct { + dataDirs []string + logsDirs []string + pidsDirs []string +} + +func (ad *allocatedDirs) delete(_ context.Context, _ *opt.DeleteOptions) error { + for _, dir := range ad.logsDirs { + if err := fileutils.DeleteDirIfExists(dir); err != nil { + return err + } + } + + for _, dir := range ad.dataDirs { + if err := fileutils.DeleteDirIfExists(dir); err != nil { + return err + } + } + + for _, dir := range ad.pidsDirs { + if err := fileutils.DeleteDirIfExists(dir); err != nil { + return err + } + } + + return nil +} + +// ClusterComponent is the basic component of running GreptimeDB Cluster in bare-metal mode. +type ClusterComponent interface { + // Start starts cluster component by executing binary. + Start(ctx context.Context, stop context.CancelFunc, binary string) error + + // BuildArgs build up args for cluster component. + BuildArgs(params ...interface{}) []string + + // IsRunning returns the status of current cluster component. + IsRunning(ctx context.Context) bool + + // Delete deletes resources that allocated in the system for current component. + Delete(ctx context.Context, options *opt.DeleteOptions) error + + // Name return the name of component. + Name() string +} diff --git a/pkg/cluster/baremetal/config.go b/pkg/config/baremetal.go similarity index 76% rename from pkg/cluster/baremetal/config.go rename to pkg/config/baremetal.go index db6074e4..50c154db 100644 --- a/pkg/cluster/baremetal/config.go +++ b/pkg/config/baremetal.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package baremetal +package config import ( "time" @@ -20,40 +20,26 @@ import ( "github.com/GreptimeTeam/gtctl/pkg/artifacts" ) -const ( - // GtctlDir is the root directory that contains states of cluster info. - GtctlDir = ".gtctl" - LogsDir = "logs" - DataDir = "data" - PidsDir = "pids" - - DataHomeDir = "home" - DataWalDir = "wal" - - DefaultLogLevel = "info" -) - -// ClusterMetadata stores metadata of a GreptimeDB cluster with Config nested. -type ClusterMetadata struct { - *Config - - CreationDate time.Time `yaml:"creationDate"` - ClusterDir string `yaml:"clusterDir"` - ForegroundPid int `yaml:"foregroundPid"` +// BareMetalClusterMetadata stores metadata of a GreptimeDB cluster. +type BareMetalClusterMetadata struct { + Config *BareMetalClusterConfig `yaml:"config"` + CreationDate time.Time `yaml:"creationDate"` + ClusterDir string `yaml:"clusterDir"` + ForegroundPid int `yaml:"foregroundPid"` } -// Config is the desired state of a GreptimeDB cluster on bare metal. +// BareMetalClusterConfig is the desired state of a GreptimeDB cluster on bare metal. // -// The field of Config that with `validate` tag will be validated +// The field of BareMetalClusterConfig that with `validate` tag will be validated // against its requirement. Each filed has only one requirement. // -// Each field of Config can also have its own exported method `Validate`. -type Config struct { - Cluster *ClusterConfig `yaml:"cluster" validate:"required"` - Etcd *Etcd `yaml:"etcd" validate:"required"` +// Each field of BareMetalClusterConfig can also have its own exported method `Validate`. +type BareMetalClusterConfig struct { + Cluster *BareMetalClusterComponentsConfig `yaml:"cluster" validate:"required"` + Etcd *Etcd `yaml:"etcd" validate:"required"` } -type ClusterConfig struct { +type BareMetalClusterComponentsConfig struct { Artifact *Artifact `yaml:"artifact" validate:"required"` Frontend *Frontend `yaml:"frontend" validate:"required"` MetaSrv *MetaSrv `yaml:"meta" validate:"required"` @@ -108,9 +94,9 @@ type Etcd struct { Artifact *Artifact `yaml:"artifact" validate:"required"` } -func DefaultConfig() *Config { - return &Config{ - Cluster: &ClusterConfig{ +func DefaultBareMetalConfig() *BareMetalClusterConfig { + return &BareMetalClusterConfig{ + Cluster: &BareMetalClusterComponentsConfig{ Artifact: &Artifact{ Version: artifacts.LatestVersionTag, }, diff --git a/pkg/cluster/baremetal/testdata/validate/invalid_artifact.yaml b/pkg/config/testdata/validate/invalid_artifact.yaml similarity index 100% rename from pkg/cluster/baremetal/testdata/validate/invalid_artifact.yaml rename to pkg/config/testdata/validate/invalid_artifact.yaml diff --git a/pkg/cluster/baremetal/testdata/validate/invalid_hostname_port.yaml b/pkg/config/testdata/validate/invalid_hostname_port.yaml similarity index 100% rename from pkg/cluster/baremetal/testdata/validate/invalid_hostname_port.yaml rename to pkg/config/testdata/validate/invalid_hostname_port.yaml diff --git a/pkg/cluster/baremetal/testdata/validate/invalid_replicas.yaml b/pkg/config/testdata/validate/invalid_replicas.yaml similarity index 100% rename from pkg/cluster/baremetal/testdata/validate/invalid_replicas.yaml rename to pkg/config/testdata/validate/invalid_replicas.yaml diff --git a/pkg/cluster/baremetal/testdata/validate/valid_config.yaml b/pkg/config/testdata/validate/valid_config.yaml similarity index 100% rename from pkg/cluster/baremetal/testdata/validate/valid_config.yaml rename to pkg/config/testdata/validate/valid_config.yaml diff --git a/pkg/cluster/baremetal/validate.go b/pkg/config/validate.go similarity index 94% rename from pkg/cluster/baremetal/validate.go rename to pkg/config/validate.go index af4c965c..73721726 100644 --- a/pkg/cluster/baremetal/validate.go +++ b/pkg/config/validate.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package baremetal +package config import ( "fmt" @@ -23,7 +23,7 @@ import ( var validate *validator.Validate // ValidateConfig validate config in bare-metal mode. -func ValidateConfig(config *Config) error { +func ValidateConfig(config *BareMetalClusterConfig) error { if config == nil { return fmt.Errorf("no config to validate") } diff --git a/pkg/cluster/baremetal/validate_test.go b/pkg/config/validate_test.go similarity index 94% rename from pkg/cluster/baremetal/validate_test.go rename to pkg/config/validate_test.go index 1cbd7f2b..86f2142b 100644 --- a/pkg/cluster/baremetal/validate_test.go +++ b/pkg/config/validate_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package baremetal +package config import ( "fmt" @@ -61,7 +61,7 @@ func TestValidateConfig(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - var actual Config + var actual BareMetalClusterConfig if err := loadConfig(filepath.Join("testdata", "validate", fmt.Sprintf("%s.yaml", tc.name)), &actual); err != nil { t.Errorf("error while loading %s file: %v", tc.name, err) @@ -80,7 +80,7 @@ func TestValidateConfig(t *testing.T) { } } -func loadConfig(path string, ret *Config) error { +func loadConfig(path string, ret *BareMetalClusterConfig) error { configs, err := os.ReadFile(path) if err != nil { return err diff --git a/pkg/deployer/baremetal/component/cluster.go b/pkg/deployer/baremetal/component/cluster.go deleted file mode 100644 index 7f5a5f20..00000000 --- a/pkg/deployer/baremetal/component/cluster.go +++ /dev/null @@ -1,178 +0,0 @@ -// Copyright 2023 Greptime Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package component - -import ( - "bufio" - "context" - "os" - "os/exec" - "path" - "strconv" - "sync" - "syscall" - - "github.com/GreptimeTeam/gtctl/pkg/deployer/baremetal/config" - "github.com/GreptimeTeam/gtctl/pkg/logger" - fileutils "github.com/GreptimeTeam/gtctl/pkg/utils/file" -) - -const ( - DataNode = "datanode" - Frontend = "frontend" - MetaSrv = "metasrv" - Etcd = "etcd" -) - -// WorkingDirs include all the dirs used in bare-metal mode. -type WorkingDirs struct { - DataDir string `yaml:"dataDir"` - LogsDir string `yaml:"logsDir"` - PidsDir string `yaml:"pidsDir"` -} - -type allocatedDirs struct { - dataDirs []string - logsDirs []string - pidsDirs []string -} - -type RunOptions struct { - Binary string - Name string - - pidDir string - logDir string - args []string -} - -type DeleteOptions struct { - RetainLogs bool -} - -// BareMetalCluster describes all the components need to be deployed under bare-metal mode. -type BareMetalCluster struct { - MetaSrv BareMetalClusterComponent - Datanode BareMetalClusterComponent - Frontend BareMetalClusterComponent - Etcd BareMetalClusterComponent -} - -// BareMetalClusterComponent is the basic unit of running GreptimeDB Cluster in bare-metal mode. -type BareMetalClusterComponent interface { - // Start starts cluster component by executing binary. - Start(ctx context.Context, binary string) error - - // BuildArgs build up args for cluster component. - BuildArgs(ctx context.Context, params ...interface{}) []string - - // IsRunning returns the status of current cluster component. - IsRunning(ctx context.Context) bool - - // Delete deletes resources that allocated in the system for current component. - Delete(ctx context.Context, option DeleteOptions) error - - // Name return the name of component. - Name() string -} - -func NewBareMetalCluster(config *config.Cluster, workingDirs WorkingDirs, - wg *sync.WaitGroup, logger logger.Logger) *BareMetalCluster { - return &BareMetalCluster{ - MetaSrv: newMetaSrv(config.MetaSrv, workingDirs, wg, logger), - Datanode: newDataNode(config.Datanode, config.MetaSrv.ServerAddr, workingDirs, wg, logger), - Frontend: newFrontend(config.Frontend, config.MetaSrv.ServerAddr, workingDirs, wg, logger), - Etcd: newEtcd(workingDirs, wg, logger), - } -} - -func runBinary(ctx context.Context, option *RunOptions, wg *sync.WaitGroup, logger logger.Logger) error { - cmd := exec.CommandContext(ctx, option.Binary, option.args...) - - // output to binary. - logFile := path.Join(option.logDir, "log") - outputFile, err := os.Create(logFile) - if err != nil { - return err - } - - outputFileWriter := bufio.NewWriter(outputFile) - cmd.Stdout = outputFileWriter - cmd.Stderr = outputFileWriter - - if err := cmd.Start(); err != nil { - return err - } - - pid := strconv.Itoa(cmd.Process.Pid) - logger.V(3).Infof("run '%s' binary '%s' with args: '%v', log: '%s', pid: '%s'", - option.Name, option.Binary, option.args, option.logDir, pid) - - pidFile := path.Join(option.pidDir, "pid") - f, err := os.Create(pidFile) - if err != nil { - return err - } - - _, err = f.Write([]byte(pid)) - if err != nil { - return err - } - - go func() { - defer wg.Done() - wg.Add(1) - if err := cmd.Wait(); err != nil { - // Caught signal kill and interrupt error then ignore. - if exit, ok := err.(*exec.ExitError); ok { - if status, ok := exit.Sys().(syscall.WaitStatus); ok && status.Signaled() { - if status.Signal() == syscall.SIGKILL || status.Signal() == syscall.SIGINT { - return - } - } - } - logger.Errorf("component '%s' binary '%s' (pid '%s') exited with error: %v", option.Name, option.Binary, pid, err) - logger.Errorf("args: '%v'", option.args) - logger.Warnf("NOTE: if you have `--retain-logs` enabled, you can browse the logs at %s", option.logDir) - _ = outputFileWriter.Flush() - } - }() - - return nil -} - -func (ad *allocatedDirs) delete(ctx context.Context, option DeleteOptions) error { - if !option.RetainLogs { - for _, dir := range ad.logsDirs { - if err := fileutils.DeleteDirIfExists(dir); err != nil { - return err - } - } - } - - for _, dir := range ad.dataDirs { - if err := fileutils.DeleteDirIfExists(dir); err != nil { - return err - } - } - - for _, dir := range ad.pidsDirs { - if err := fileutils.DeleteDirIfExists(dir); err != nil { - return err - } - } - - return nil -} diff --git a/pkg/deployer/baremetal/config/common.go b/pkg/deployer/baremetal/config/common.go deleted file mode 100644 index fb79eb0e..00000000 --- a/pkg/deployer/baremetal/config/common.go +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright 2023 Greptime Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package config - -import ( - "time" - - "github.com/GreptimeTeam/gtctl/pkg/artifacts" -) - -const ( - PidsDir = "pids" - - DataHomeDir = "home" - DataWalDir = "wal" - - DefaultLogLevel = "info" -) - -// RuntimeConfig stores runtime metadata of a GreptimeDB cluster. -type RuntimeConfig struct { - *Config - - CreationDate time.Time `yaml:"creationDate"` - ClusterDir string `yaml:"clusterDir"` - ForegroundPid int `yaml:"foregroundPid"` -} - -// Config is the desired state of a GreptimeDB cluster on bare metal. -// -// The field of Config that with `validate` tag will be validated -// against its requirement. Each filed has only one requirement. -// -// Each field of Config can also have its own exported method `Validate`. -type Config struct { - Cluster *Cluster `yaml:"cluster" validate:"required"` - Etcd *Etcd `yaml:"etcd" validate:"required"` -} - -type Cluster struct { - Artifact *Artifact `yaml:"artifact" validate:"required"` - Frontend *Frontend `yaml:"frontend" validate:"required"` - MetaSrv *MetaSrv `yaml:"meta" validate:"required"` - Datanode *Datanode `yaml:"datanode" validate:"required"` -} - -type Artifact struct { - // Local is the local path of binary(greptime or etcd). - Local string `yaml:"local" validate:"omitempty,file"` - - // Version is the release version of binary(greptime or etcd). - // Usually, it points to the version of binary of GitHub release. - Version string `yaml:"version"` -} - -func DefaultConfig() *Config { - return &Config{ - Cluster: &Cluster{ - Artifact: &Artifact{ - Version: artifacts.LatestVersionTag, - }, - Frontend: &Frontend{ - Replicas: 1, - }, - MetaSrv: &MetaSrv{ - Replicas: 1, - StoreAddr: "127.0.0.1:2379", - ServerAddr: "0.0.0.0:3002", - HTTPAddr: "0.0.0.0:14001", - }, - Datanode: &Datanode{ - Replicas: 3, - RPCAddr: "0.0.0.0:14100", - HTTPAddr: "0.0.0.0:14300", - }, - }, - Etcd: &Etcd{ - Artifact: &Artifact{ - Version: artifacts.DefaultEtcdBinVersion, - }, - }, - } -} diff --git a/pkg/deployer/baremetal/config/datanode.go b/pkg/deployer/baremetal/config/datanode.go deleted file mode 100644 index e52ac575..00000000 --- a/pkg/deployer/baremetal/config/datanode.go +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2023 Greptime Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package config - -type Datanode struct { - Replicas int `yaml:"replicas" validate:"gt=0"` - NodeID int `yaml:"nodeID" validate:"gte=0"` - - RPCAddr string `yaml:"rpcAddr" validate:"required,hostname_port"` - HTTPAddr string `yaml:"httpAddr" validate:"required,hostname_port"` - - DataDir string `yaml:"dataDir" validate:"omitempty,dirpath"` - WalDir string `yaml:"walDir" validate:"omitempty,dirpath"` - ProcedureDir string `yaml:"procedureDir" validate:"omitempty,dirpath"` - Config string `yaml:"config" validate:"omitempty,filepath"` - - LogLevel string `yaml:"logLevel"` -} diff --git a/pkg/deployer/baremetal/config/etcd.go b/pkg/deployer/baremetal/config/etcd.go deleted file mode 100644 index 762bd48c..00000000 --- a/pkg/deployer/baremetal/config/etcd.go +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright 2023 Greptime Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package config - -type Etcd struct { - Artifact *Artifact `yaml:"artifact" validate:"required"` -} diff --git a/pkg/deployer/baremetal/config/frontend.go b/pkg/deployer/baremetal/config/frontend.go deleted file mode 100644 index 17089921..00000000 --- a/pkg/deployer/baremetal/config/frontend.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2023 Greptime Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package config - -type Frontend struct { - Replicas int `yaml:"replicas" validate:"gt=0"` - - GRPCAddr string `yaml:"grpcAddr" validate:"omitempty,hostname_port"` - HTTPAddr string `yaml:"httpAddr" validate:"omitempty,hostname_port"` - PostgresAddr string `yaml:"postgresAddr" validate:"omitempty,hostname_port"` - MetaAddr string `yaml:"metaAddr" validate:"omitempty,hostname_port"` - - Config string `yaml:"config" validate:"omitempty,filepath"` - - LogLevel string `yaml:"logLevel"` -} diff --git a/pkg/deployer/baremetal/config/metasrv.go b/pkg/deployer/baremetal/config/metasrv.go deleted file mode 100644 index 86bb05a5..00000000 --- a/pkg/deployer/baremetal/config/metasrv.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2023 Greptime Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package config - -type MetaSrv struct { - Replicas int `yaml:"replicas" validate:"gt=0"` - - StoreAddr string `yaml:"storeAddr" validate:"hostname_port"` - ServerAddr string `yaml:"serverAddr" validate:"hostname_port"` - BindAddr string `yaml:"bindAddr" validate:"omitempty,hostname_port"` - HTTPAddr string `yaml:"httpAddr" validate:"required,hostname_port"` - - Config string `yaml:"config" validate:"omitempty,filepath"` - - LogLevel string `yaml:"logLevel"` -} diff --git a/pkg/deployer/baremetal/deployer.go b/pkg/deployer/baremetal/deployer.go deleted file mode 100644 index 5dc7ac9e..00000000 --- a/pkg/deployer/baremetal/deployer.go +++ /dev/null @@ -1,394 +0,0 @@ -// Copyright 2023 Greptime Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package baremetal - -import ( - "context" - "fmt" - "os" - "os/exec" - "os/signal" - "path" - "strings" - "sync" - "syscall" - "time" - - "gopkg.in/yaml.v3" - - "github.com/GreptimeTeam/gtctl/pkg/artifacts" - . "github.com/GreptimeTeam/gtctl/pkg/deployer" - "github.com/GreptimeTeam/gtctl/pkg/deployer/baremetal/component" - "github.com/GreptimeTeam/gtctl/pkg/deployer/baremetal/config" - "github.com/GreptimeTeam/gtctl/pkg/logger" - "github.com/GreptimeTeam/gtctl/pkg/metadata" - fileutils "github.com/GreptimeTeam/gtctl/pkg/utils/file" -) - -type Deployer struct { - logger logger.Logger - config *config.Config - wg sync.WaitGroup - ctx context.Context - - createNoDirs bool - enableCache bool - - am artifacts.Manager - mm metadata.Manager - bm *component.BareMetalCluster -} - -var _ Interface = &Deployer{} - -type Option func(*Deployer) - -// TODO(sh2): remove this deployer later -func NewDeployer(l logger.Logger, clusterName string, opts ...Option) (Interface, error) { - ctx, _ := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) - - d := &Deployer{ - logger: l, - config: config.DefaultConfig(), - ctx: ctx, - } - - for _, opt := range opts { - if opt != nil { - opt(d) - } - } - - if err := ValidateConfig(d.config); err != nil { - return nil, err - } - - mm, err := metadata.New("") - if err != nil { - return nil, err - } - d.mm = mm - - am, err := artifacts.NewManager(l) - if err != nil { - return nil, err - } - d.am = am - - if !d.createNoDirs { - mm.AllocateClusterScopeDirs(clusterName) - if err = mm.CreateClusterScopeDirs(d.config); err != nil { - return nil, err - } - - csd := mm.GetClusterScopeDirs() - d.bm = component.NewBareMetalCluster(d.config.Cluster, component.WorkingDirs{ - DataDir: csd.DataDir, - LogsDir: csd.LogsDir, - PidsDir: csd.PidsDir, - }, &d.wg, d.logger) - } - - return d, nil -} - -// WithMergeConfig merges config with current deployer config. -// It will perform WithReplaceConfig if any error occurs during merging or receive nil raw config. -func WithMergeConfig(cfg *config.Config, rawConfig []byte) Option { - if len(rawConfig) == 0 { - return WithReplaceConfig(cfg) - } - - return func(d *Deployer) { - defaultConfig, err := yaml.Marshal(d.config) - if err != nil { - d.config = cfg - return - } - - out, err := fileutils.MergeYAML(defaultConfig, rawConfig) - if err != nil { - d.config = cfg - return - } - - var newConfig config.Config - if err = yaml.Unmarshal(out, &newConfig); err != nil { - d.config = cfg - return - } - - d.config = &newConfig - } -} - -// WithReplaceConfig replaces config with current deployer config. -func WithReplaceConfig(cfg *config.Config) Option { - return func(d *Deployer) { - d.config = cfg - } -} - -func WithGreptimeVersion(version string) Option { - return func(d *Deployer) { - d.config.Cluster.Artifact.Version = version - } -} - -func WithEnableCache(enableCache bool) Option { - return func(d *Deployer) { - d.enableCache = enableCache - } -} - -func WithCreateNoDirs() Option { - return func(d *Deployer) { - d.createNoDirs = true - } -} - -func (d *Deployer) GetGreptimeDBCluster(ctx context.Context, name string, options *GetGreptimeDBClusterOptions) (*GreptimeDBCluster, error) { - csd := d.mm.GetClusterScopeDirs() - _, err := os.Stat(csd.BaseDir) - if os.IsNotExist(err) { - return nil, fmt.Errorf("cluster %s is not exist", name) - } - if err != nil { - return nil, err - } - - ok, err := fileutils.IsFileExists(csd.ConfigPath) - if err != nil { - return nil, err - } - if !ok { - return nil, fmt.Errorf("cluster %s is not exist", name) - } - - var cluster config.RuntimeConfig - in, err := os.ReadFile(csd.ConfigPath) - if err != nil { - return nil, err - } - if err = yaml.Unmarshal(in, &cluster); err != nil { - return nil, err - } - - return &GreptimeDBCluster{ - Raw: &cluster, - }, nil -} - -func (d *Deployer) ListGreptimeDBClusters(ctx context.Context, options *ListGreptimeDBClustersOptions) ([]*GreptimeDBCluster, error) { - return nil, fmt.Errorf("unsupported operation") -} - -func (d *Deployer) CreateGreptimeDBCluster(ctx context.Context, clusterName string, options *CreateGreptimeDBClusterOptions) error { - var binPath string - if d.config.Cluster.Artifact != nil { - if d.config.Cluster.Artifact.Local != "" { - binPath = d.config.Cluster.Artifact.Local - } else { - src, err := d.am.NewSource(artifacts.GreptimeBinName, d.config.Cluster.Artifact.Version, artifacts.ArtifactTypeBinary, options.UseGreptimeCNArtifacts) - if err != nil { - return err - } - - destDir, err := d.mm.AllocateArtifactFilePath(src, false) - if err != nil { - return err - } - - installDir, err := d.mm.AllocateArtifactFilePath(src, true) - if err != nil { - return err - } - - artifactFile, err := d.am.DownloadTo(ctx, src, destDir, &artifacts.DownloadOptions{EnableCache: d.enableCache, BinaryInstallDir: installDir}) - if err != nil { - return err - } - binPath = artifactFile - } - } - - if err := d.bm.MetaSrv.Start(d.ctx, binPath); err != nil { - return err - } - - if err := d.bm.Datanode.Start(d.ctx, binPath); err != nil { - return err - } - - if err := d.bm.Frontend.Start(d.ctx, binPath); err != nil { - return err - } - - return nil -} - -func (d *Deployer) UpdateGreptimeDBCluster(ctx context.Context, name string, options *UpdateGreptimeDBClusterOptions) error { - return fmt.Errorf("unsupported operation") -} - -func (d *Deployer) DeleteGreptimeDBCluster(ctx context.Context, name string, options *DeleteGreptimeDBClusterOption) error { - return fmt.Errorf("unsupported operation") -} - -// deleteGreptimeDBClusterForeground delete the whole cluster if it runs in foreground. -func (d *Deployer) deleteGreptimeDBClusterForeground(ctx context.Context, option component.DeleteOptions) error { - // No matter what options are, the config file of one cluster must be deleted. - csd := d.mm.GetClusterScopeDirs() - if err := os.Remove(csd.ConfigPath); err != nil { - return err - } - - if option.RetainLogs { - // It is unnecessary to delete each component resources in cluster since it only retains the logs. - // So deleting the whole cluster resources excluding logs here would be fine. - if err := fileutils.DeleteDirIfExists(csd.DataDir); err != nil { - return err - } - if err := fileutils.DeleteDirIfExists(csd.PidsDir); err != nil { - return err - } - } else { - // It is unnecessary to delete each component resources in cluster since it has nothing to retain. - // So deleting the whole cluster resources here would be fine. - if err := fileutils.DeleteDirIfExists(csd.BaseDir); err != nil { - return err - } - } - - return nil -} - -func (d *Deployer) CreateEtcdCluster(ctx context.Context, clusterName string, options *CreateEtcdClusterOptions) error { - var binPath string - if d.config.Etcd.Artifact != nil { - if d.config.Etcd.Artifact.Local != "" { - binPath = d.config.Etcd.Artifact.Local - } else { - src, err := d.am.NewSource(artifacts.EtcdBinName, d.config.Etcd.Artifact.Version, artifacts.ArtifactTypeBinary, options.UseGreptimeCNArtifacts) - if err != nil { - return err - } - - destDir, err := d.mm.AllocateArtifactFilePath(src, false) - if err != nil { - return err - } - - installDir, err := d.mm.AllocateArtifactFilePath(src, true) - if err != nil { - return err - } - - artifactFile, err := d.am.DownloadTo(ctx, src, destDir, &artifacts.DownloadOptions{EnableCache: d.enableCache, BinaryInstallDir: installDir}) - if err != nil { - return err - } - binPath = artifactFile - } - } - - if err := d.bm.Etcd.Start(d.ctx, binPath); err != nil { - return err - } - - if err := d.checkEtcdHealth(binPath); err != nil { - return err - } - - return nil -} - -func (d *Deployer) checkEtcdHealth(etcdBin string) error { - // It's very likely that "etcdctl" is under the same directory of "etcd". - etcdctlBin := path.Join(etcdBin, "../etcdctl") - exists, err := fileutils.IsFileExists(etcdctlBin) - if err != nil { - return err - } - if !exists { - d.logger.V(3).Infof("'etcdctl' is not found under the same directory of 'etcd', skip checking the healthy of Etcd.") - return nil - } - - for retry := 0; retry < 10; retry++ { - outputRaw, err := exec.Command(etcdctlBin, "endpoint", "status").Output() - if err != nil { - return err - } - output := string(outputRaw) - statuses := strings.Split(output, "\n") - - hasLeader := false - for i := 0; i < len(statuses); i++ { - fields := strings.Split(statuses[i], ",") - - // We are checking Etcd status with default output format("--write-out=simple"), example output: - // 127.0.0.1:2379, 8e9e05c52164694d, 3.5.0, 131 kB, true, false, 3, 75, 75, - // - // The output fields are corresponding to the following table's columns (with format "--write-out=table"): - // +----------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+ - // | ENDPOINT | ID | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS | - // +----------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+ - // | 127.0.0.1:2379 | 8e9e05c52164694d | 3.5.0 | 131 kB | true | false | 3 | 72 | 72 | | - // +----------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+ - // - // So we can just check the "IS LEADER" field. - if strings.TrimSpace(fields[4]) == "true" { - hasLeader = true - break - } - } - if hasLeader { - return nil - } - - time.Sleep(1 * time.Second) - } - - csd := d.mm.GetClusterScopeDirs() - return fmt.Errorf("etcd is not ready in 10 second! You can find its logs in %s", path.Join(csd.LogsDir, "etcd")) -} - -func (d *Deployer) DeleteEtcdCluster(ctx context.Context, name string, options *DeleteEtcdClusterOption) error { - return fmt.Errorf("unsupported operation") -} - -func (d *Deployer) CreateGreptimeDBOperator(ctx context.Context, name string, options *CreateGreptimeDBOperatorOptions) error { - // We don't need to implement this method because we don't need to deploy GreptimeDB Operator. - return fmt.Errorf("only support for k8s Deployer") -} - -func (d *Deployer) Wait(ctx context.Context, option component.DeleteOptions) error { - d.wg.Wait() - - d.logger.V(3).Info("Cluster shutting down. Cleaning allocated resources.") - - <-d.ctx.Done() - // Delete cluster after closing, which can only happens in the foreground. - if err := d.deleteGreptimeDBClusterForeground(ctx, option); err != nil { - return err - } - - return nil -} - -func (d *Deployer) Config() *config.Config { - return d.config -} diff --git a/pkg/deployer/baremetal/validate.go b/pkg/deployer/baremetal/validate.go deleted file mode 100644 index 8a5dd657..00000000 --- a/pkg/deployer/baremetal/validate.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2023 Greptime Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package baremetal - -import ( - "fmt" - - "github.com/go-playground/validator/v10" - - bmconfig "github.com/GreptimeTeam/gtctl/pkg/deployer/baremetal/config" -) - -var validate *validator.Validate - -// ValidateConfig validate config in bare-metal mode. -func ValidateConfig(config *bmconfig.Config) error { - if config == nil { - return fmt.Errorf("no config to validate") - } - - validate = validator.New() - - // Register custom validation method for Artifact. - validate.RegisterStructValidation(ValidateArtifact, bmconfig.Artifact{}) - - err := validate.Struct(config) - if err != nil { - return err - } - - return nil -} - -func ValidateArtifact(sl validator.StructLevel) { - artifact := sl.Current().Interface().(bmconfig.Artifact) - if len(artifact.Version) == 0 && len(artifact.Local) == 0 { - sl.ReportError(sl.Current().Interface(), "Artifact", "Version/Local", "", "") - } -} diff --git a/pkg/deployer/k8s/deployer.go b/pkg/deployer/k8s/deployer.go index f60fe5a6..bc1c7454 100644 --- a/pkg/deployer/k8s/deployer.go +++ b/pkg/deployer/k8s/deployer.go @@ -29,6 +29,8 @@ import ( "github.com/GreptimeTeam/gtctl/pkg/logger" ) +// TODO(sh2): remove this file later + type deployer struct { helmLoader *helm.Loader client *kube.Client diff --git a/pkg/deployer/types.go b/pkg/deployer/types.go index 7dd7d21d..988420d7 100644 --- a/pkg/deployer/types.go +++ b/pkg/deployer/types.go @@ -18,6 +18,8 @@ import ( "context" ) +// TODO(sh2): remove this file later + // Interface is the general interface to handle the deployment of GreptimeDB cluster in different environment. type Interface interface { // GetGreptimeDBCluster get the current deployed GreptimeDBCluster by its name. diff --git a/pkg/metadata/manager.go b/pkg/metadata/manager.go index bcbb30d2..988f8c8f 100644 --- a/pkg/metadata/manager.go +++ b/pkg/metadata/manager.go @@ -24,7 +24,7 @@ import ( "gopkg.in/yaml.v3" "github.com/GreptimeTeam/gtctl/pkg/artifacts" - "github.com/GreptimeTeam/gtctl/pkg/deployer/baremetal/config" + "github.com/GreptimeTeam/gtctl/pkg/config" fileutils "github.com/GreptimeTeam/gtctl/pkg/utils/file" ) @@ -45,7 +45,7 @@ type Manager interface { GetWorkingDir() string // CreateClusterScopeDirs creates cluster scope directories and config path that allocated by AllocateClusterScopeDirs. - CreateClusterScopeDirs(cfg *config.Config) error + CreateClusterScopeDirs(cfg *config.BareMetalClusterConfig) error // GetClusterScopeDirs returns the cluster scope directory of current cluster. GetClusterScopeDirs() *ClusterScopeDirs @@ -59,10 +59,9 @@ const ( // all the metadata will be stored in ${HomeDir}/${BaseDir}. BaseDir = ".gtctl" - // Default cluster scope directories. - clusterLogsDir = "logs" - clusterDataDir = "data" - clusterPidsDir = "pids" + ClusterLogsDir = "logs" + ClusterDataDir = "data" + ClusterPidsDir = "pids" ) type ClusterScopeDirs struct { @@ -102,11 +101,11 @@ func (m *manager) AllocateClusterScopeDirs(clusterName string) { } // ${HomeDir}/${BaseDir}/${ClusterName}/logs - csd.LogsDir = path.Join(csd.BaseDir, clusterLogsDir) + csd.LogsDir = path.Join(csd.BaseDir, ClusterLogsDir) // ${HomeDir}/${BaseDir}/${ClusterName}/data - csd.DataDir = path.Join(csd.BaseDir, clusterDataDir) + csd.DataDir = path.Join(csd.BaseDir, ClusterDataDir) // ${HomeDir}/${BaseDir}/${ClusterName}/pids - csd.PidsDir = path.Join(csd.BaseDir, clusterPidsDir) + csd.PidsDir = path.Join(csd.BaseDir, ClusterPidsDir) // ${HomeDir}/${BaseDir}/${ClusterName}/${ClusterName}.yaml csd.ConfigPath = filepath.Join(csd.BaseDir, fmt.Sprintf("%s.yaml", clusterName)) @@ -132,7 +131,7 @@ func (m *manager) AllocateArtifactFilePath(src *artifacts.Source, installBinary return filePath, nil } -func (m *manager) CreateClusterScopeDirs(cfg *config.Config) error { +func (m *manager) CreateClusterScopeDirs(cfg *config.BareMetalClusterConfig) error { if m.clusterDir == nil { return fmt.Errorf("unallocated cluster dir, please initialize a metadata manager with cluster name provided") } @@ -155,7 +154,7 @@ func (m *manager) CreateClusterScopeDirs(cfg *config.Config) error { return err } - metaConfig := config.RuntimeConfig{ + metaConfig := config.BareMetalClusterMetadata{ Config: cfg, CreationDate: time.Now(), ClusterDir: m.clusterDir.BaseDir, diff --git a/pkg/metadata/manager_test.go b/pkg/metadata/manager_test.go index 39c7b8dc..a18b9bdd 100644 --- a/pkg/metadata/manager_test.go +++ b/pkg/metadata/manager_test.go @@ -23,7 +23,7 @@ import ( "gopkg.in/yaml.v3" "github.com/GreptimeTeam/gtctl/pkg/artifacts" - "github.com/GreptimeTeam/gtctl/pkg/deployer/baremetal/config" + "github.com/GreptimeTeam/gtctl/pkg/config" ) func TestMetadataManager(t *testing.T) { @@ -123,7 +123,7 @@ func TestMetadataManagerWithClusterConfigPath(t *testing.T) { m, err := New("/tmp") assert.NoError(t, err) - expect := config.DefaultConfig() + expect := config.DefaultBareMetalConfig() m.AllocateClusterScopeDirs("test") err = m.CreateClusterScopeDirs(expect) assert.NoError(t, err) @@ -135,7 +135,7 @@ func TestMetadataManagerWithClusterConfigPath(t *testing.T) { cnt, err := os.ReadFile(csd.ConfigPath) assert.NoError(t, err) - var actual config.RuntimeConfig + var actual config.BareMetalClusterMetadata err = yaml.Unmarshal(cnt, &actual) assert.NoError(t, err) assert.Equal(t, expect, actual.Config) From cefca9b99dec990458316d5f97996ef591e40cdb Mon Sep 17 00:00:00 2001 From: sh2 Date: Fri, 27 Oct 2023 16:32:32 +0800 Subject: [PATCH 3/3] enrich the log message Signed-off-by: sh2 --- pkg/cluster/baremetal/create.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pkg/cluster/baremetal/create.go b/pkg/cluster/baremetal/create.go index c5279cbe..16db6f54 100644 --- a/pkg/cluster/baremetal/create.go +++ b/pkg/cluster/baremetal/create.go @@ -210,11 +210,13 @@ func (c *Cluster) Wait(ctx context.Context, close bool) error { v = "unknown" } + csd := c.mm.GetClusterScopeDirs() if !close { - c.logger.V(0).Infof("The cluster(pid=%d, version=%s) is running in bare-metal mode now...\n", os.Getpid(), v) - c.logger.V(0).Infof("To view dashboard by accessing: %s\n", logger.Bold("http://localhost:4000/dashboard/")) + c.logger.V(0).Infof("The cluster(pid=%d, version=%s) is running in bare-metal mode now...", os.Getpid(), v) + c.logger.V(0).Infof("To view dashboard by accessing: %s", logger.Bold("http://localhost:4000/dashboard/")) } else { - c.logger.Warnf("The cluster(pid=%d, version=%s) run in bare-metal has been shutting down...\n", os.Getpid(), v) + c.logger.Warnf("The cluster(pid=%d, version=%s) run in bare-metal has been shutting down...", os.Getpid(), v) + c.logger.Warnf("To view the failure by browsing logs in: %s", logger.Bold(csd.LogsDir)) return nil } @@ -232,6 +234,7 @@ func (c *Cluster) wait(_ context.Context) error { // it is not the context of current cluster. <-c.ctx.Done() - c.logger.V(3).Info("Cluster is shutting down...") + csd := c.mm.GetClusterScopeDirs() + c.logger.V(0).Infof("Cluster is shutting down, don't worry, it still remain in %s", logger.Bold(csd.BaseDir)) return nil }