[WIP] Support for installing multiple mixins and reloading Prometheus configuration

.gitignore

@@ -1,2 +1,7 @@
 /_output/
+<<<<<<< HEAD
 /mixtool
+.vscode
+=======
+/mixtool
+>>>>>>> bd0efc3... add mixtool list functionality (#28)

cmd/mixtool/generate.go

@@ -151,7 +151,6 @@ func generateRules(filename string, options mixer.GenerateOptions) error {
 	if err != nil {
 		return err
 	}
-
 	return ioutil.WriteFile(options.RulesFilename, out, 0644)
 }
 
@@ -191,6 +190,14 @@ func generateDashboards(filename string, opts mixer.GenerateOptions) error {
 	return nil
 }
 
+func generateRulesAlerts(filename string, options mixer.GenerateOptions) ([]byte, error) {
+	out, err := mixer.GenerateRulesAlerts(filename, options)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 func generateAll(filename string, opts mixer.GenerateOptions) error {
 	if err := generateAlerts(filename, opts); err != nil {
 		return err

cmd/mixtool/install.go

@@ -0,0 +1,246 @@
+// Copyright 2018 mixtool authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"github.com/monitoring-mixins/mixtool/pkg/jsonnetbundler"
+	"github.com/monitoring-mixins/mixtool/pkg/mixer"
+
+	"github.com/urfave/cli"
+)
+
+func installCommand() cli.Command {
+	return cli.Command{
+		Name:        "install",
+		Usage:       "Install a mixin",
+		Description: "Install a mixin from a repository",
+		Action:      installAction,
+		Flags: []cli.Flag{
+			cli.StringFlag{
+				Name:  "bind-address",
+				Usage: "Address to bind HTTP server to.",
+				Value: "http://127.0.0.1:8080",
+			},
+			cli.StringFlag{
+				Name:  "directory, d",
+				Usage: "Path where the downloaded mixin is saved. If it doesn't exist already it will be created",
+			},
+			cli.BoolFlag{
+				Name:  "put, p",
+				Usage: "Specify this flag when you want to send PUT request to mixtool server once the mixins are generated",
+			},
+		},
+	}
+}
+
+// Downloads a mixin from a given repository given by url and places into directory
+// by running jb init and jb install
+func downloadMixin(url string, jsonnetHome string, directory string) error {
+	// intialize the jsonnet bundler library
+	err := jsonnetbundler.InitCommand(directory)
+	if err != nil {
+		return fmt.Errorf("jsonnet bundler init failed %v", err)
+	}
+
+	// by default, set the single flag to false
+	err = jsonnetbundler.InstallCommand(directory, jsonnetHome, []string{url}, false)
+	if err != nil {
+		return fmt.Errorf("jsonnet bundler install failed %v", err)
+	}
+
+	return nil
+}
+
+// Gets mixins from default website - mostly copied from list.go
+func getMixins() ([]mixin, error) {
+	body, err := queryWebsite(defaultWebsite)
+	if err != nil {
+		return nil, err
+	}
+	mixins, err := parseMixinJSON(body)
+	if err != nil {
+		return nil, err
+	}
+	return mixins, nil
+}
+
+func generateMixin(directory string, jsonnetHome string, mixinURL string, options mixer.GenerateOptions) ([]byte, error) {
+
+	mixinBaseDirectory := filepath.Join(directory)
+
+	err := os.Chdir(mixinBaseDirectory)
+	if err != nil {
+		return nil, fmt.Errorf("Cannot cd into directory %s", err)
+	}
+
+	// generate alerts, rules, grafana dashboards
+	// empty files if not present
+
+	u, err := url.Parse(mixinURL)
+	if err != nil {
+		return nil, fmt.Errorf("url parse %v", err)
+	}
+
+	// absolute directory is the same as the download url stripped of the scheme
+	absDirectory := path.Join(u.Host, u.Path)
+	// trim http or www etc, trim repositories with trailing .git
+	// TODO: what if it's under some different kind of VCS?
+	absDirectory = strings.TrimLeft(absDirectory, "/:")
+	absDirectory = strings.TrimSuffix(absDirectory, ".git")
+
+	importFile := filepath.Join(absDirectory, "mixin.libsonnet")
+
+	// generate rules, dashboards, alerts
+	err = generateAll(importFile, options)
+	if err != nil {
+		return nil, fmt.Errorf("generateAll: %w", err)
+	}
+
+	out, err := generateRulesAlerts(importFile, options)
+	if err != nil {
+		return nil, fmt.Errorf("generateRulesAlerts %w", err)
+	}
+
+	return out, nil
+
+}
+
+func putMixin(content []byte, bindAddress string, mixinName string) error {
+	u, err := url.Parse(bindAddress)
+	if err != nil {
+		return err
+	}
+	pathName := fmt.Sprintf("/api/v1/rules/%s", mixinName)
+	u.Path = path.Join(u.Path, pathName)
+
+	r := bytes.NewReader(content)
+	req, err := http.NewRequest("PUT", u.String(), r)
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return fmt.Errorf("response from server %v", err)
+	}
+	if resp.StatusCode != 200 {
+		responseData, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return fmt.Errorf("failed to response body in putMixin, %w", err)
+		}
+		return fmt.Errorf("non 200 response code: %d, info: %s", resp.StatusCode, string(responseData))
+	}
+	return nil
+}
+
+func installAction(c *cli.Context) error {
+	directory := c.String("directory")
+	if directory == "" {
+		return fmt.Errorf("Must specify a directory to download mixin")
+	}
+
+	_, err := os.Stat(directory)
+	if os.IsNotExist(err) {
+		err = os.MkdirAll(directory, 0755)
+		if err != nil {
+			return fmt.Errorf("could not create directory %v", err)
+		}
+	}
+
+	mixinPath := c.Args().First()
+	if mixinPath == "" {
+		return fmt.Errorf("Expected the url of mixin repository or name of the mixin. Show available mixins using mixtool list")
+	}
+
+	mixinsList, err := getMixins()
+	if err != nil {
+		return fmt.Errorf("getMixins failed %v", err)
+	}
+
+	var mixinURL string
+	if _, err := url.ParseRequestURI(mixinPath); err != nil {
+		// check if the name exists in mixinsList
+		found := false
+		for _, m := range mixinsList {
+			if m.Name == mixinPath {
+				// join paths together
+				u, err := url.Parse(m.URL)
+				if err != nil {
+					return fmt.Errorf("url parse failed %v", err)
+				}
+				u.Path = path.Join(u.Path, m.Subdir)
+				mixinURL = u.String()
+				found = true
+				break
+			}
+		}
+		if !found {
+			return fmt.Errorf("Could not find mixin with name %s", mixinPath)
+		}
+	} else {
+		mixinURL = mixinPath
+	}
+
+	if mixinURL == "" {
+		return fmt.Errorf("Empty mixinURL")
+	}
+
+	// by default jsonnet packages are downloaded under vendor
+	jsonnetHome := "vendor"
+
+	err = downloadMixin(mixinURL, jsonnetHome, directory)
+	if err != nil {
+		return err
+	}
+
+	generateCfg := mixer.GenerateOptions{
+		AlertsFilename: "alerts.yaml",
+		RulesFilename:  "rules.yaml",
+		Directory:      "dashboards_out",
+		JPaths:         []string{"./vendor"},
+		YAML:           true,
+	}
+
+	rulesAlerts, err := generateMixin(directory, jsonnetHome, mixinURL, generateCfg)
+	if err != nil {
+		return err
+	}
+
+	// check if put address flag was set
+
+	if c.Bool("put") {
+		bindAddress := c.String("bind-address")
+		// run put requests onto the server
+
+		// TODO: deal with the case where mixinPath is not a URL
+		_, err := url.ParseRequestURI(mixinPath)
+		if err == nil {
+			fmt.Println("TODO: for reloading prometheus, not dealing with mixins by url yet, since name is unknown")
+		}
+
+		err = putMixin(rulesAlerts, bindAddress, mixinPath)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

cmd/mixtool/install_test.go

@@ -0,0 +1,86 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"testing"
+
+	"github.com/monitoring-mixins/mixtool/pkg/mixer"
+)
+
+// Try to install every mixin from the mixin repository
+// verify that each package generated has the yaml files
+func TestInstallMixin(t *testing.T) {
+	tmpdir, err := ioutil.TempDir("", "mixtool-install")
+	if err != nil {
+		t.Errorf("failed to make directory %v", err)
+	}
+
+	defer os.RemoveAll(tmpdir)
+
+	body, err := queryWebsite(defaultWebsite)
+	if err != nil {
+		t.Errorf("failed to query website %v", err)
+	}
+	mixins, err := parseMixinJSON(body)
+	if err != nil {
+		t.Errorf("failed to parse mixin body %v", err)
+	}
+
+	// download each mixin in turn
+	for _, m := range mixins {
+
+		generateCfg := mixer.GenerateOptions{
+			AlertsFilename: "alerts.yaml",
+			RulesFilename:  "rules.yaml",
+			Directory:      "dashboards_out",
+			JPaths:         []string{"vendor"},
+			YAML:           true,
+		}
+
+		mixinURL := path.Join(m.URL, m.Subdir)
+
+		fmt.Printf("installing %v\n", mixinURL)
+		dldir := path.Join(tmpdir, m.Name+"mixin-test")
+
+		err = os.Mkdir(dldir, 0755)
+		if err != nil {
+			t.Errorf("failed to create directory %s", dldir)
+		}
+
+		jsonnetHome := "vendor"
+
+		err = downloadMixin(mixinURL, jsonnetHome, dldir)
+		if err != nil {
+			t.Errorf("failed to download mixin at %s: %w", mixinURL, err)
+		}
+
+		err = generateMixin(dldir, jsonnetHome, mixinURL, generateCfg)
+		if err != nil {
+			t.Errorf("failed to generate mixin yaml for %s: %w", mixinURL, err)
+		}
+
+		// verify that alerts, rules, dashboards exist
+		err = os.Chdir(dldir)
+		if err != nil {
+			t.Errorf("could not cd into %s", dldir)
+		}
+
+		if _, err := os.Stat("alerts.yaml"); os.IsNotExist(err) {
+			t.Errorf("expected alerts.yaml in %s", dldir)
+		}
+
+		if _, err := os.Stat("rules.yaml"); os.IsNotExist(err) {
+			t.Errorf("expected rules.yaml in %s", dldir)
+		}
+
+		if _, err := os.Stat("dashboards_out"); os.IsNotExist(err) {
+			t.Errorf("expected dashboards_out in %s", dldir)
+		}
+
+		// verify that the output of alerts and rules matches using jsonnet
+	}
+
+}

cmd/mixtool/lint.go

@@ -79,6 +79,5 @@ func lintAction(c *cli.Context) error {
 	if err := mixer.Lint(os.Stdout, filename, options); err != nil {
 		return fmt.Errorf("failed to lint the file %s: %v", filename, err)
 	}
-
 	return nil
 }