diff --git a/README.md b/README.md index 670e592..9154dcd 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ # Grafana Export -A small utility to download all dashboards from your Grafana Instance using the Grafana HTTP API. +A small utility to download and backup all dashboards from your Grafana Instance using the Grafana HTTP API. ## Installation @@ -34,7 +34,7 @@ $ make dist # or just `make` for a dynamically-linked binary ## Usage -The utility requires the Grafana instance URL, and an API token to access the API: +The utility requires the Grafana instance URL and an API token: ##### Command Line Arguments @@ -43,6 +43,14 @@ $ ./grafana-export --help Usage of grafana-export: -api-key string The API key to access the Grafana Instance. + -backup + Backup the current set of downloaded dashboards to S3. Requires --bucket-name. + -bucket-key string + The key to use for storing the backup inside the S3 bucket. (default "grafana-export") + -bucket-name string + Compress and upload the dashboards to the specified S3 bucket. Requires --backup. + -compress + Create an archive of the exported dashboards folder. -dashboards-dir string The directory where the Grafana dashboards are to be downloaded. (default "dashboards") -limit int @@ -53,6 +61,10 @@ Usage of grafana-export: The base URL for the Grafana instance. ``` +To create an S3 backup for the downloaded dashboards, the utility requires `awscli` to be configured on +the host. For setting a bucket region, export `AWS_REGION=` or `AWS_SDK_LOAD_CONFIG` to load the +`~/.aws/config` file. + ##### Environment Variables The utility accepts environment variables as arguments. All environment variables prefixed with `GEXPORT_` diff --git a/config.go b/config.go index 60bfc60..c0deb26 100644 --- a/config.go +++ b/config.go @@ -18,9 +18,14 @@ func getConfig() *koanf.Koanf { f := flag.NewFlagSet("grafana-export", flag.ExitOnError) f.String("url", "", "The base URL for the Grafana instance.") f.String("api-key", "", "The API key to access the Grafana Instance.") + f.String("bucket-name", "", "Compress and upload the dashboards to the specified S3 bucket. Requires --backup.") + f.String("bucket-key", "grafana-export", "The key to use for storing the backup inside the S3 bucket.") f.String("dashboards-dir", "dashboards", "The directory where the Grafana dashboards are to be downloaded.") f.Int("limit", 1000, "The limit for number of results returned by the Grafana API.") f.Bool("overwrite", false, "Overwrite existing dashboards directory.") + f.Bool("backup", false, "Backup the current set of downloaded dashboards to S3. Requires --bucket-name.") + f.Bool("compress", false, "Create an archive of the exported dashboards folder.") + f.Parse(os.Args[1:]) if err := k.Load(basicflag.Provider(f, "."), nil); err != nil { @@ -31,11 +36,5 @@ func getConfig() *koanf.Koanf { return strings.Replace(strings.ToLower(strings.TrimPrefix(s, "GEXPORT_")), "_", "-", -1) }), nil) - if k.String("url") == "" { - log.Fatal("Missing required argument: --url") - } else if k.String("api-key") == "" { - log.Fatal("Missing required argument: --api-key") - } - return k } diff --git a/go.mod b/go.mod index 1cb8164..598ce15 100644 --- a/go.mod +++ b/go.mod @@ -2,4 +2,8 @@ module github.com/thunderbottom/grafana-export go 1.15 -require github.com/knadh/koanf v0.14.0 +require ( + github.com/aws/aws-sdk-go v1.35.24 + github.com/knadh/koanf v0.14.0 + github.com/mholt/archiver/v3 v3.5.0 +) diff --git a/go.sum b/go.sum index 33dddb1..b0d9d3b 100644 --- a/go.sum +++ b/go.sum @@ -1,21 +1,47 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/andybalholm/brotli v1.0.0 h1:7UCwP93aiSfvWpapti8g88vVVGp2qqtGyePsSuDafo4= +github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= +github.com/aws/aws-sdk-go v1.35.24 h1:U3GNTg8+7xSM6OAJ8zksiSM4bRqxBWmVwwehvOSNG3A= +github.com/aws/aws-sdk-go v1.35.24/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= +github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= +github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.10.10 h1:a/y8CglcM7gLGYmlbP/stPE5sR3hbhFRUjCBfd/0B3I= +github.com/klauspost/compress v1.10.10/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/pgzip v1.2.4 h1:TQ7CNpYKovDOmqzRHKxJh0BeaBI7UdQZYc6p7pMQh1A= +github.com/klauspost/pgzip v1.2.4/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/knadh/koanf v0.14.0 h1:h9XeG4wEiEuxdxqv/SbY7TEK+7vzrg/dOaGB+S6+mPo= github.com/knadh/koanf v0.14.0/go.mod h1:H5mEFsTeWizwFXHKtsITL5ipsLTuAMQoGuQpp+1JL9U= +github.com/mholt/archiver/v3 v3.5.0 h1:nE8gZIrw66cu4osS/U7UW7YDuGMHssxKutU8IfWxwWE= +github.com/mholt/archiver/v3 v3.5.0/go.mod h1:qqTTPUK/HZPFgFQ/TJ3BzvTpF/dPtFVJXdQbCmeMxwc= github.com/mitchellh/mapstructure v1.2.2 h1:dxe5oCinTXiTIcfgmZecdCzPmAJKd46KsCWc35r0TV4= github.com/mitchellh/mapstructure v1.2.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/nwaples/rardecode v1.1.0 h1:vSxaY8vQhOcVr4mm5e8XllHWTiM4JF507A0Katqw7MQ= +github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= github.com/pelletier/go-toml v1.7.0 h1:7utD74fnzVc/cpcyy8sjrlFr5vYpypUixARcHIMIGuI= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= +github.com/pierrec/lz4/v4 v4.0.3 h1:vNQKSVZNYUEAvRY9FaUXAF1XPbSOHJtDTiP41kzDz2E= +github.com/pierrec/lz4/v4 v4.0.3/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rhnvrm/simples3 v0.5.0/go.mod h1:Y+3vYm2V7Y4VijFoJHHTrja6OgPrJ2cBti8dPGkC3sA= @@ -24,9 +50,20 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= +github.com/ulikunitz/xz v0.5.7 h1:YvTNdFzX6+W5m9msiYg/zpkSURPPtOlzbqYjrFn7Yt4= +github.com/ulikunitz/xz v0.5.7/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= +github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200331124033-c3d80250170d h1:nc5K6ox/4lTFbMVSL9WRR81ixkcwXThoiF6yf+R9scA= golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= diff --git a/main.go b/main.go index 5d7bcb9..e0cdfe9 100644 --- a/main.go +++ b/main.go @@ -21,7 +21,7 @@ var ( func check(e error) { if e != nil { - log.Panic(e) + log.Fatal(e) } } @@ -125,6 +125,11 @@ func syncDashboards(cfg *koanf.Koanf, dashboards dashboardSearch) { func main() { cfg := getConfig() + if cfg.String("url") == "" { + log.Fatal("Missing required argument: --url") + } else if cfg.String("api-key") == "" { + log.Fatal("Missing required argument: --api-key") + } resp, err := getGrafanaData(cfg, "search") check(err) @@ -134,4 +139,37 @@ func main() { check(err) syncDashboards(cfg, ds) + + var c string + cmp := cfg.Bool("compress") + if cmp { + c, err = compress(cfg.String("dashboards-dir")) + if err != nil { + log.Fatal(err) + } + log.Printf("Dashboards compressed: %v", c) + } + + // backup to s3 if --backup is passed + if cfg.Bool("backup") { + bucketName := cfg.String("bucket-name") + bucketKey := cfg.String("bucket-key") + if bucketName == "" { + log.Fatal("No S3 bucket specified for backup.") + } + // check if --compress is passed + // prevents re-compression + if !cmp { + c, err = compress(cfg.String("dashboards-dir")) + if err != nil { + log.Fatal(err) + } + log.Printf("Dashboards compressed: %v", c) + } + + if err := backup(c, bucketName, bucketKey); err != nil { + log.Fatalf("Failed to backup to S3 bucket: %v", err) + } + log.Printf("Uploaded %v to S3 Bucket %v/%v", c, bucketName, bucketKey) + } } diff --git a/model.go b/model.go index f53fa97..1398d6f 100644 --- a/model.go +++ b/model.go @@ -1,6 +1,7 @@ package main -// dashboardSearch is a struct that the grafana dashboard search data +// dashboardSearch is a struct that contains +// the search data for the grafana dashboard api type dashboardSearch []struct { ID int `json:"id"` UID string `json:"uid"` diff --git a/util.go b/util.go new file mode 100644 index 0000000..b2fbef2 --- /dev/null +++ b/util.go @@ -0,0 +1,65 @@ +package main + +import ( + "os" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/mholt/archiver/v3" +) + +// compress is a function that creates a gzipped +// archive for the specified filepath +func compress(filepath string) (string, error) { + if _, err := os.Stat(filepath); os.IsNotExist(err) { + // the filepath specified does not exist + return "", err + } + + now := time.Now() + // create a timestamped filename. + // dashboards/ => dashboards-20060102150405.tar.gz + arcFn := strings.TrimSuffix(filepath, "/") + "-" + now.Format("20060102150405") + ".tar.gz" + + tarGZ := archiver.NewTarGz() + if err := tarGZ.Archive([]string{filepath}, arcFn); err != nil { + return "", err + } + + return arcFn, nil +} + +// backup is a function that takes a file and uploads +// it to the specified s3 bucket +func backup(filepath, bucket, key string) error { + if _, err := os.Stat(filepath); os.IsNotExist(err) { + return err + } + + f, err := os.Open(filepath) + if err != nil { + return err + } + defer f.Close() + + sess, err := session.NewSession() + if err != nil { + return err + } + + uploader := s3manager.NewUploader(sess) + _, err = uploader.Upload(&s3manager.UploadInput{ + ACL: aws.String("private"), + Body: f, + Bucket: aws.String(bucket), + Key: aws.String(key), + }) + if err != nil { + return err + } + + return nil +}