From 55ba2172d22d8c67f16eea11ce6fb66b4602a97d Mon Sep 17 00:00:00 2001 From: Pouyan Azari Date: Mon, 9 Sep 2019 17:11:45 +0200 Subject: [PATCH] Phabricator queries are done in parallel, an ignore option is added for blackbox --- .gitmodules | 3 --- CHANGELOG | 6 ++++++ README.md | 5 +++-- a2a.go | 41 ++++++++++++++++++++++++++++------------- a2a_test.go | 16 +--------------- api-phabricator-go | 1 - 6 files changed, 38 insertions(+), 34 deletions(-) delete mode 100644 .gitmodules delete mode 160000 api-phabricator-go diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 2b40761..0000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "api-phabricator-go"] - path = api-phabricator-go - url = ssh://gitadm@git.rz.uni-wuerzburg.de:222/source/api-phabricator-go.git diff --git a/CHANGELOG b/CHANGELOG index c740534..cb83889 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,11 @@ # CHANGELOG +## [0.0.11] 2019-09-09 + +- The phabricator queries are running parallel +- Added an ignore option for blackbox groups +- Some small fixes + ## [0.0.10] 2019-08-28 - Fixed small Bug in the cache logic diff --git a/README.md b/README.md index 3db4ea0..ec63148 100644 --- a/README.md +++ b/README.md @@ -314,6 +314,9 @@ The `module` is a parameter and it should be relabeled, the configuration would replacement: 127.0.0.1:9115 # The blackbox exporter. ``` +The blackbox mode can be called with help of `-b` and if some groups +should be ignored these can be added as comma separated values with `-i`. + ## Build You can build your own binaries from the source code, using golang standard @@ -331,8 +334,6 @@ go build There are several points that should be covered in the next couple of releases: -- Parallel Queries: to make the queries to Phabricator faster parallel queries should be possible. -it should use the concurrency feature of Go. - Test case: Add test cases for the application - Native Packages: This has low priority, but will replace the binary releases at some point. diff --git a/a2a.go b/a2a.go index 56e82fd..d7c0cfd 100644 --- a/a2a.go +++ b/a2a.go @@ -5,12 +5,11 @@ package main // It supports all the parameters and options detailed there. import ( - // "./alertmanager/config" - "app-a2a/alertmanager/config" - phabricator "app-a2a/api-phabricator-go" + "./alertmanager/config" "encoding/json" "errors" "fmt" + "github.com/uniwue-rz/phabricator-go" "github.com/urfave/cli" "gopkg.in/gcfg.v1" "gopkg.in/yaml.v2" @@ -21,6 +20,7 @@ import ( "reflect" "regexp" "strconv" + "strings" "sync" "time" ) @@ -90,7 +90,7 @@ func manageAlertManager(p *phabricator.Phabricator, configPath string, jsonWrapp } func getGroupRouteReceivers(p *phabricator.Phabricator, jsonWrapper string) (routes []config.Route, receivers []config.Receiver) { - services, err := phabricator.GetServices(p) + services, err := p.GetServicesAsync() if err != nil { panic(err) @@ -377,15 +377,21 @@ func (output *Output) Augment(p *phabricator.Phabricator, PassphraseWrapper stri } // GetBlackBoxData returns the blackbox targets and data. -func GetBlackBoxData(p *phabricator.Phabricator, JsonWrapper string) (allOutputs []PrometheusOutput, err error) { - services, err := phabricator.GetServices(p) +func GetBlackBoxData(p *phabricator.Phabricator, JsonWrapper string, ignoreArray []string) (allOutputs []PrometheusOutput, err error) { + services, err := p.GetServicesAsync() allOutputs = make([]PrometheusOutput, 0) if err != nil { return allOutputs, err } for _, d := range services.Result.Data { - if len(d.Attachments.Bindings.Bindings) != 0 { + ignored := false + for _, b := range ignoreArray { + if b == d.Fields.Name && ignored == false { + ignored = true + } + } + if len(d.Attachments.Bindings.Bindings) != 0 && ignored == false { blackBoxConfig := "" for _, property := range d.Attachments.Properties.Properties { if property.Key == "blackbox-config" { @@ -437,7 +443,7 @@ func GetBlackBoxData(p *phabricator.Phabricator, JsonWrapper string) (allOutputs // prometheus-config this will be used, when not the group settings will be used. // The script will be used here to create the dynamic configuration in Prometheus func GetPrometheusData(p *phabricator.Phabricator, JsonWrapper string) (allOutputs []PrometheusOutput, err error) { - services, err := phabricator.GetServices(p) + services, err := p.GetServicesAsync() allOutputs = make([]PrometheusOutput, 0) if err != nil { @@ -511,7 +517,7 @@ func HandlePassphrase(p *phabricator.Phabricator, PassphraseWrapper string, prop if len(passPhraseRegexMatching) > 1 { isPassphrase = true passPhraseKey := passPhraseRegexMatching[1] - passphraseObj, err := phabricator.GetPassPhraseWithId(p, passPhraseKey) + passphraseObj, err := p.GetPassPhraseWithId(passPhraseKey) if err != nil { passPhrase = "" return passPhrase, false, err @@ -539,7 +545,7 @@ func ListParallel(p *phabricator.Phabricator, playBookPath string, vagrant strin groupList := make(map[string]Group) hostVars := make(map[string]map[string]interface{}) - services, err := phabricator.GetServices(p) // -> one request, not worth paralleling + services, err := p.GetServicesAsync() // -> one request, not worth paralleling // Returns the List of services if err != nil { @@ -600,7 +606,7 @@ func ListBlocking(p *phabricator.Phabricator, playBookPath string, vagrant strin groupList := make(map[string]Group) hostVars := make(map[string]map[string]interface{}) - services, err := phabricator.GetServices(p) + services, err := p.GetServicesAsync() // Returns the List of services if err != nil { @@ -666,7 +672,7 @@ func ReplaceDotsToUnderscore(key string) string { func CreateHost(p *phabricator.Phabricator, devName string) (values map[string]interface{}, err error) { values = make(map[string]interface{}) - device, err := phabricator.GetDevice(p, devName) // -> one request + device, err := p.GetDeviceAsync(devName) // -> one request if err != nil { panic(err) } @@ -754,6 +760,10 @@ func CreateCommandLine() *cli.App { Name: "prometheus, p", Usage: "Returns the list of services supported by Prometheus for the given host", }, + cli.StringFlag{ + Name: "ignore, i", + Usage: "Make the Prometheus or blackbox exporter, ignore the given group", + }, cli.BoolFlag{ Name: "no-cache, n", Usage: "Run the application in no cache mode", @@ -814,6 +824,7 @@ func main() { prometheusIsOn := c.Bool("prometheus") blackBoxIsOn := c.Bool("blackbox") cacheIsOff := c.Bool("no-cache") + ignoreGroups := c.String("ignore") if vagrant != "" { cacheIsOff = true } @@ -840,7 +851,11 @@ func main() { // Creates the blackbox settings with modules as labels. // Should use relabeling to make parameter from the label. if blackBoxIsOn { - blackBoxData, err := GetBlackBoxData(p, Config.Wrapper.Json) + var ignoreArray []string + if ignoreGroups != "" { + ignoreArray = strings.Split(ignoreGroups, ",") + } + blackBoxData, err := GetBlackBoxData(p, Config.Wrapper.Json, ignoreArray) if err == nil { jsonData, _ := json.Marshal(blackBoxData) fmt.Println(string(jsonData)) diff --git a/a2a_test.go b/a2a_test.go index 1b79dfb..f111cdf 100644 --- a/a2a_test.go +++ b/a2a_test.go @@ -1,8 +1,8 @@ package main import ( - phabricator "app-a2a/api-phabricator-go" "encoding/json" + "github.com/uniwue-rz/phabricator-go" "testing" ) @@ -16,20 +16,6 @@ func TestReadConfig(t *testing.T) { } } -// Test List function, mainly for profiling it -//func TestList(t *testing.T) { -// Config, err := ReadConfig() -// if err != nil { -// panic(err) -// } -// p := phabricator.NewPhabricator(Config.Phabricator.ApiURL, Config.Phabricator.ApiToken) -// vagrant := "" -// _, err = List(p, Config.Ansible.Playbook, vagrant) -// if err != nil { -// panic(err) -// } -//} - func TestListWithAugmentParallel(t *testing.T) { Config, err := ReadConfig() if err != nil { diff --git a/api-phabricator-go b/api-phabricator-go deleted file mode 160000 index bd8f80e..0000000 --- a/api-phabricator-go +++ /dev/null @@ -1 +0,0 @@ -Subproject commit bd8f80e0a86a0333da76d49f5797af80ada6b8a5