Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add backup and restore methods for kibana dashboard
Browse files Browse the repository at this point in the history
Signed-off-by: Md. Ishtiaq Islam <[email protected]>
ishtiaqhimel committed Jan 25, 2024
1 parent 649baaf commit 7b66378
Showing 73 changed files with 33,413 additions and 3,970 deletions.
2 changes: 2 additions & 0 deletions elasticsearchdashboard/api.go
Original file line number Diff line number Diff line change
@@ -23,4 +23,6 @@ import (
type EDClient interface {
GetHealthStatus() (*Health, error)
GetStateFromHealthResponse(health *Health) (esapi.DashboardServerState, error)
ExportSavedObjects() (*Response, error)
ImportSavedObjects(filepath string) (*Response, error)
}
17 changes: 16 additions & 1 deletion elasticsearchdashboard/client.go
Original file line number Diff line number Diff line change
@@ -29,6 +29,14 @@ import (
"sigs.k8s.io/controller-runtime/pkg/client"
)

const (
SavedObjectsReqBody = `{"type": ["dashboard", "config", "index-pattern", "url", "query", "tag", "canvas-element", "canvas-workpad", "action", "alert", "visualization",
"graph-workspace", "map", "lens", "cases", "search", "osquery-saved-query", "osquery-pack", "uptime-dynamic-settings", "infrastructure-ui-source", "metrics-explorer-view",
"inventory-view", "apm-indices"]}`
SavedObjectsExportURL = "/api/saved_objects/_export"
SavedObjectsImportURL = "/api/saved_objects/_import"
)

type Client struct {
EDClient
}
@@ -42,13 +50,20 @@ type ClientOptions struct {
Secret *core.Secret
}

type DbVersionInfo struct {
Name string
Version string
AuthPlugin catalog.ElasticsearchAuthPlugin
}

type Config struct {
host string
api string
username string
password string
connectionScheme string
transport *http.Transport
dbVersionInfo *DbVersionInfo
}

type Health struct {
@@ -60,7 +75,7 @@ type Health struct {
type Response struct {
Code int
header http.Header
body io.ReadCloser
Body io.ReadCloser
}

type ResponseBody struct {
44 changes: 41 additions & 3 deletions elasticsearchdashboard/ed_client_v7.go
Original file line number Diff line number Diff line change
@@ -47,7 +47,7 @@ func (h *EDClientV7) GetHealthStatus() (*Health, error) {
ConnectionResponse: Response{
Code: res.StatusCode(),
header: res.Header(),
body: res.RawBody(),
Body: res.RawBody(),
},
StateFailedReason: statesList,
}
@@ -69,10 +69,10 @@ func (h *EDClientV7) GetStateFromHealthResponse(health *Health) (esapi.Dashboard
}
return
}
}(resStatus.body)
}(resStatus.Body)

var responseBody ResponseBody
body, _ := io.ReadAll(resStatus.body)
body, _ := io.ReadAll(resStatus.Body)
err := json.Unmarshal(body, &responseBody)
if err != nil {
return "", errors.Wrap(err, "Failed to parse response body")
@@ -106,3 +106,41 @@ func (h *EDClientV7) GetStateFromHealthResponse(health *Health) (esapi.Dashboard

return esapi.DashboardServerState(health.OverallState), nil
}

func (h *EDClientV7) ExportSavedObjects() (*Response, error) {
req := h.Client.R().
SetDoNotParseResponse(true).
SetHeaders(map[string]string{
"Content-Type": "application/json",
"kbn-xsrf": "true",
}).
SetBody([]byte(SavedObjectsReqBody))
res, err := req.Post(SavedObjectsExportURL)
if err != nil {
klog.Error(err, "Failed to send http request")
return nil, err
}

return &Response{
Code: res.StatusCode(),
Body: res.RawBody(),
}, nil
}

func (h *EDClientV7) ImportSavedObjects(filepath string) (*Response, error) {
req := h.Client.R().
SetDoNotParseResponse(true).
SetHeader("kbn-xsrf", "true").
SetFile("file", filepath).
SetQueryParam("overwrite", "true")
res, err := req.Post(SavedObjectsImportURL)
if err != nil {
klog.Error(err, "Failed to send http request")
return nil, err
}

return &Response{
Code: res.StatusCode(),
Body: res.RawBody(),
}, nil
}
44 changes: 41 additions & 3 deletions elasticsearchdashboard/ed_client_v8.go
Original file line number Diff line number Diff line change
@@ -47,7 +47,7 @@ func (h *EDClientV8) GetHealthStatus() (*Health, error) {
ConnectionResponse: Response{
Code: res.StatusCode(),
header: res.Header(),
body: res.RawBody(),
Body: res.RawBody(),
},
StateFailedReason: statesList,
}
@@ -69,10 +69,10 @@ func (h *EDClientV8) GetStateFromHealthResponse(health *Health) (esapi.Dashboard
}
return
}
}(resStatus.body)
}(resStatus.Body)

var responseBody ResponseBody
body, _ := io.ReadAll(resStatus.body)
body, _ := io.ReadAll(resStatus.Body)
err := json.Unmarshal(body, &responseBody)
if err != nil {
return "", errors.Wrap(err, "failed to parse response body")
@@ -104,3 +104,41 @@ func (h *EDClientV8) GetStateFromHealthResponse(health *Health) (esapi.Dashboard

return esapi.DashboardServerState(health.OverallState), nil
}

func (h *EDClientV8) ExportSavedObjects() (*Response, error) {
req := h.Client.R().
SetDoNotParseResponse(true).
SetHeaders(map[string]string{
"Content-Type": "application/json",
"kbn-xsrf": "true",
}).
SetBody([]byte(SavedObjectsReqBody))
res, err := req.Post(SavedObjectsExportURL)
if err != nil {
klog.Error(err, "Failed to send http request")
return nil, err
}

return &Response{
Code: res.StatusCode(),
Body: res.RawBody(),
}, nil
}

func (h *EDClientV8) ImportSavedObjects(filepath string) (*Response, error) {
req := h.Client.R().
SetDoNotParseResponse(true).
SetHeader("kbn-xsrf", "true").
SetFile("file", filepath).
SetQueryParam("overwrite", "true")
res, err := req.Post(SavedObjectsImportURL)
if err != nil {
klog.Error(err, "Failed to send http request")
return nil, err
}

return &Response{
Code: res.StatusCode(),
Body: res.RawBody(),
}, nil
}
45 changes: 32 additions & 13 deletions elasticsearchdashboard/kubedb-client-builder.go
Original file line number Diff line number Diff line change
@@ -39,14 +39,15 @@ import (
)

type KubeDBClientBuilder struct {
kc client.Client
dashboard *esapi.ElasticsearchDashboard
db *v1alpha2.Elasticsearch
dbVersion *catalog.ElasticsearchVersion
authSecret *core.Secret
url string
podName string
ctx context.Context
kc client.Client
dashboard *esapi.ElasticsearchDashboard
db *v1alpha2.Elasticsearch
dbVersion *catalog.ElasticsearchVersion
dbVersionInfo *DbVersionInfo
authSecret *core.Secret
url string
podName string
ctx context.Context
}

func NewKubeDBClientBuilder(kc client.Client, db *esapi.ElasticsearchDashboard) *KubeDBClientBuilder {
@@ -81,6 +82,11 @@ func (o *KubeDBClientBuilder) WithDbVersion(version *catalog.ElasticsearchVersio
return o
}

func (o *KubeDBClientBuilder) WithDbVersionInfo(versionInfo *DbVersionInfo) *KubeDBClientBuilder {
o.dbVersionInfo = versionInfo
return o
}

func (o *KubeDBClientBuilder) WithContext(ctx context.Context) *KubeDBClientBuilder {
o.ctx = ctx
return o
@@ -97,6 +103,7 @@ func (o *KubeDBClientBuilder) GetElasticsearchDashboardClient() (*Client, error)
}).DialContext,
},
connectionScheme: o.dashboard.GetConnectionScheme(),
dbVersionInfo: o.getDbVersionInfo(),
}
// If EnableSSL is true set tls config,
// provide client certs and root CA
@@ -157,15 +164,15 @@ func (o *KubeDBClientBuilder) GetElasticsearchDashboardClient() (*Client, error)
}

// parse version
version, err := semver.NewVersion(o.dbVersion.Spec.Version)
version, err := semver.NewVersion(config.dbVersionInfo.Version)
if err != nil {
return nil, errors.Wrap(err, "failed to parse version")
}

switch {
// for Elasticsearch 7.x.x and OpenSearch 1.x.x
case (o.dbVersion.Spec.AuthPlugin == catalog.ElasticsearchAuthPluginXpack && version.Major() <= 7) ||
(o.dbVersion.Spec.AuthPlugin == catalog.ElasticsearchAuthPluginOpenSearch && (version.Major() == 1 || version.Major() == 2)):
case (config.dbVersionInfo.AuthPlugin == catalog.ElasticsearchAuthPluginXpack && version.Major() <= 7) ||
(config.dbVersionInfo.AuthPlugin == catalog.ElasticsearchAuthPluginOpenSearch && (version.Major() == 1 || version.Major() == 2)):
newClient := resty.New()
newClient.SetTransport(config.transport).SetScheme(config.connectionScheme).SetBaseURL(config.host)
newClient.SetHeader("Accept", "application/json")
@@ -179,7 +186,7 @@ func (o *KubeDBClientBuilder) GetElasticsearchDashboardClient() (*Client, error)
},
}, nil

case o.dbVersion.Spec.AuthPlugin == catalog.ElasticsearchAuthPluginXpack && version.Major() == 8:
case config.dbVersionInfo.AuthPlugin == catalog.ElasticsearchAuthPluginXpack && version.Major() == 8:
newClient := resty.New()
newClient.SetTransport(config.transport).SetScheme(config.connectionScheme).SetBaseURL(config.host)
newClient.SetHeader("Accept", "application/json")
@@ -194,7 +201,19 @@ func (o *KubeDBClientBuilder) GetElasticsearchDashboardClient() (*Client, error)
}, nil
}

return nil, fmt.Errorf("unknown version: %s", o.dbVersion.Name)
return nil, fmt.Errorf("unknown version: %s", config.dbVersionInfo.Name)
}

func (o *KubeDBClientBuilder) getDbVersionInfo() *DbVersionInfo {
if o.dbVersionInfo == nil {
return &DbVersionInfo{
Name: o.dbVersion.Name,
Version: o.dbVersion.Spec.Version,
AuthPlugin: o.dbVersion.Spec.AuthPlugin,
}
}

return o.dbVersionInfo
}

// return host path in
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
@@ -22,7 +22,7 @@ require (
k8s.io/klog/v2 v2.110.1
kmodules.xyz/client-go v0.29.6
kmodules.xyz/custom-resources v0.29.0
kubedb.dev/apimachinery v0.41.0-beta.0.0.20240118063916-9d6870498d68
kubedb.dev/apimachinery v0.41.0-beta.1.0.20240124061503-ce4799bb0e5c
sigs.k8s.io/controller-runtime v0.16.3
xorm.io/xorm v1.3.6
)
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
@@ -580,8 +580,8 @@ kmodules.xyz/monitoring-agent-api v0.29.0 h1:gpFl6OZrlMLb/ySMHdREI9EwGtnJ91oZBn9
kmodules.xyz/monitoring-agent-api v0.29.0/go.mod h1:iNbvaMTgVFOI5q2LJtGK91j4Dmjv4ZRiRdasGmWLKQI=
kmodules.xyz/offshoot-api v0.29.0 h1:GHLhxxT9jU1N8+FvOCCeJNyU5g0duYS46UGrs6AHNLY=
kmodules.xyz/offshoot-api v0.29.0/go.mod h1:5NxhBblXoDHWStx9HCDJR2KFTwYjEZ7i1Id3jelIunw=
kubedb.dev/apimachinery v0.41.0-beta.0.0.20240118063916-9d6870498d68 h1:2J1DMU9OiiMzQMKfv+gTGkmcN34JTARqS5CA0pjctMc=
kubedb.dev/apimachinery v0.41.0-beta.0.0.20240118063916-9d6870498d68/go.mod h1:9iHsSfX02yJ6eA9IXSCruVjckH8wWgHB93ivpNVc1EY=
kubedb.dev/apimachinery v0.41.0-beta.1.0.20240124061503-ce4799bb0e5c h1:QdnEBPdmd/Z3JOFJKMQ3nCFiJYiWxsuIAqMLgXep880=
kubedb.dev/apimachinery v0.41.0-beta.1.0.20240124061503-ce4799bb0e5c/go.mod h1:IqIwU4I/UfmcMi4X7G01M7XAEFUPehtiFAKZNKwH8Ek=
lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
lukechampine.com/uint128 v1.2.0 h1:mBi/5l91vocEN8otkC5bDLhi2KdCticRiwbdB0O+rjI=
lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
Original file line number Diff line number Diff line change
@@ -56,14 +56,9 @@ type DruidVersionSpec struct {
// Deprecated versions usable but regarded as obsolete and best avoided, typically due to having been superseded.
// +optional
Deprecated bool `json:"deprecated,omitempty"`
// PSP names
// +optional
PodSecurityPolicies DruidVersionPodSecurityPolicy `json:"podSecurityPolicies"`
// update constraints
UpdateConstraints UpdateConstraints `json:"updateConstraints,omitempty"`
// SecurityContext is for the additional security information for the Druid container
// +optional
SecurityContext DruidSecurityContext `json:"securityContext"`
SecurityContext SecurityContext `json:"securityContext"`
}

// DruidVersionDatabase is the Druid Database image
@@ -76,22 +71,6 @@ type DruidInitContainer struct {
Image string `json:"image"`
}

// SinglestoreVersionPodSecurityPolicy is the Singlestore pod security policies
type DruidVersionPodSecurityPolicy struct {
DatabasePolicyName string `json:"databasePolicyName"`
}

// DruidSecurityContext provides additional securityContext settings for the Druid Image
type DruidSecurityContext struct {
// RunAsUser is default UID for the DB container. It defaults to 1000.
RunAsUser *int64 `json:"runAsUser,omitempty"`

RunAsGroup *int64 `json:"runAsGroup,omitempty"`

// RunAsAnyNonRoot will be true if user can change the default UID to other than 1000.
RunAsAnyNonRoot bool `json:"runAsAnyNonRoot,omitempty"`
}

// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

// DruidVersionList contains a list of DruidVersion
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
/*
Copyright AppsCode Inc. and Contributors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1alpha1

import (
"fmt"

"kubedb.dev/apimachinery/apis"
"kubedb.dev/apimachinery/apis/catalog"
"kubedb.dev/apimachinery/crds"

"kmodules.xyz/client-go/apiextensions"
)

func (f FerretDBVersion) CustomResourceDefinition() *apiextensions.CustomResourceDefinition {
return crds.MustCustomResourceDefinition(SchemeGroupVersion.WithResource(ResourcePluralFerretDBVersion))
}

var _ apis.ResourceInfo = &FerretDBVersion{}

func (f FerretDBVersion) ResourceFQN() string {
return fmt.Sprintf("%s.%s", ResourcePluralFerretDBVersion, catalog.GroupName)
}

func (f FerretDBVersion) ResourceShortCode() string {
return ResourceCodeFerretDBVersion
}

func (f FerretDBVersion) ResourceKind() string {
return ResourceKindFerretDBVersion
}

func (f FerretDBVersion) ResourceSingular() string {
return ResourceSingularFerretDBVersion
}

func (f FerretDBVersion) ResourcePlural() string {
return ResourcePluralFerretDBVersion
}

func (f FerretDBVersion) ValidateSpecs() error {
if f.Spec.Version == "" ||
f.Spec.DB.Image == "" {
return fmt.Errorf(`atleast one of the following specs is not set for ferretdbVersion "%v":
spec.version,
spec.db.image`, f.Name)
}
return nil
}
Loading

0 comments on commit 7b66378

Please sign in to comment.