diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..7b2184f --- /dev/null +++ b/.dockerignore @@ -0,0 +1,11 @@ +bin +contrib +.gitignore +.dockerignore +CHANGELOG.md +DEVELOPMENT.md +Dockerfile +Jenkinfile +LICENCE +Makefile +README.md \ No newline at end of file diff --git a/.gitignore b/.gitignore index 0fe0276..dc7b6d2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ bin/ go/ *.snap +vendor \ No newline at end of file diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index e6b3b93..d171389 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -7,7 +7,7 @@ commands. ## Install Go from source ```bash -export VERSION=1.15 OS=linux ARCH=amd64 +export VERSION=1.17 OS=linux ARCH=amd64 wget https://dl.google.com/go/go$VERSION.$OS-$ARCH.tar.gz tar -xzvf go$VERSION.$OS-$ARCH.tar.gz export PATH=$PWD/go/bin:$PATH diff --git a/Makefile b/Makefile index d7d24bc..c56113f 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ go/modules/pkg/mod: go.mod .PHONY: test test: go/modules/pkg/mod $(GOFILES) - go test -v + go test -v ./... run: $(GOBIN) $(GOBIN) diff --git a/README.md b/README.md index 5bfe42a..063585e 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ Prometheus collector and exporter for metrics extracted from the [Slurm](https://slurm.schedmd.com/overview.html) resource scheduling system. +> This work use https://github.com/vpenso/prometheus-slurm-exporter as starting point + ## Exported Metrics ### State of the CPUs diff --git a/accounts.go b/accounts.go deleted file mode 100644 index 2bc4660..0000000 --- a/accounts.go +++ /dev/null @@ -1,121 +0,0 @@ -/* Copyright 2020 Victor Penso - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . */ - -package main - -import ( - "io/ioutil" - "os/exec" - "log" - "strings" - "strconv" - "regexp" - "github.com/prometheus/client_golang/prometheus" -) - -func AccountsData() []byte { - cmd := exec.Command("squeue","-a","-r","-h","-o %A|%a|%T|%C") - stdout, err := cmd.StdoutPipe() - if err != nil { - log.Fatal(err) - } - if err := cmd.Start(); err != nil { - log.Fatal(err) - } - out, _ := ioutil.ReadAll(stdout) - if err := cmd.Wait(); err != nil { - log.Fatal(err) - } - return out -} - -type JobMetrics struct { - pending float64 - running float64 - running_cpus float64 - suspended float64 -} - -func ParseAccountsMetrics(input []byte) map[string]*JobMetrics { - accounts := make(map[string]*JobMetrics) - lines := strings.Split(string(input), "\n") - for _, line := range lines { - if strings.Contains(line,"|") { - account := strings.Split(line,"|")[1] - _,key := accounts[account] - if !key { - accounts[account] = &JobMetrics{0,0,0,0} - } - state := strings.Split(line,"|")[2] - state = strings.ToLower(state) - cpus,_ := strconv.ParseFloat(strings.Split(line,"|")[3],64) - pending := regexp.MustCompile(`^pending`) - running := regexp.MustCompile(`^running`) - suspended := regexp.MustCompile(`^suspended`) - switch { - case pending.MatchString(state) == true: - accounts[account].pending++ - case running.MatchString(state) == true: - accounts[account].running++ - accounts[account].running_cpus += cpus - case suspended.MatchString(state) == true: - accounts[account].suspended++ - } - } - } - return accounts -} - -type AccountsCollector struct { - pending *prometheus.Desc - running *prometheus.Desc - running_cpus *prometheus.Desc - suspended *prometheus.Desc -} - -func NewAccountsCollector() *AccountsCollector { - labels := []string{"account"} - return &AccountsCollector{ - pending: prometheus.NewDesc("slurm_account_jobs_pending", "Pending jobs for account", labels, nil), - running: prometheus.NewDesc("slurm_account_jobs_running", "Running jobs for account", labels, nil), - running_cpus: prometheus.NewDesc("slurm_account_cpus_running", "Running cpus for account", labels, nil), - suspended: prometheus.NewDesc("slurm_account_jobs_suspended", "Suspended jobs for account", labels, nil), - } -} - -func (ac *AccountsCollector) Describe(ch chan<- *prometheus.Desc) { - ch <- ac.pending - ch <- ac.running - ch <- ac.running_cpus - ch <- ac.suspended -} - -func (ac *AccountsCollector) Collect(ch chan<- prometheus.Metric) { - am := ParseAccountsMetrics(AccountsData()) - for a := range am { - if am[a].pending > 0 { - ch <- prometheus.MustNewConstMetric(ac.pending, prometheus.GaugeValue, am[a].pending, a) - } - if am[a].running > 0 { - ch <- prometheus.MustNewConstMetric(ac.running, prometheus.GaugeValue, am[a].running, a) - } - if am[a].running_cpus > 0 { - ch <- prometheus.MustNewConstMetric(ac.running_cpus, prometheus.GaugeValue, am[a].running_cpus, a) - } - if am[a].suspended > 0 { - ch <- prometheus.MustNewConstMetric(ac.suspended, prometheus.GaugeValue, am[a].suspended, a) - } - } -} diff --git a/images/Job_Status.png b/contrib/images/Job_Status.png similarity index 100% rename from images/Job_Status.png rename to contrib/images/Job_Status.png diff --git a/images/Node_Status.png b/contrib/images/Node_Status.png similarity index 100% rename from images/Node_Status.png rename to contrib/images/Node_Status.png diff --git a/images/Scheduler_Info.png b/contrib/images/Scheduler_Info.png similarity index 100% rename from images/Scheduler_Info.png rename to contrib/images/Scheduler_Info.png diff --git a/lib/systemd/prometheus-slurm-exporter.service b/contrib/lib/systemd/prometheus-slurm-exporter.service similarity index 100% rename from lib/systemd/prometheus-slurm-exporter.service rename to contrib/lib/systemd/prometheus-slurm-exporter.service diff --git a/packages/README.md b/contrib/packages/README.md similarity index 100% rename from packages/README.md rename to contrib/packages/README.md diff --git a/packages/rpm/README.md b/contrib/packages/rpm/README.md similarity index 100% rename from packages/rpm/README.md rename to contrib/packages/rpm/README.md diff --git a/packages/rpm/prometheus-slurm-exporter.spec b/contrib/packages/rpm/prometheus-slurm-exporter.spec similarity index 100% rename from packages/rpm/prometheus-slurm-exporter.spec rename to contrib/packages/rpm/prometheus-slurm-exporter.spec diff --git a/packages/snap/README.md b/contrib/packages/snap/README.md similarity index 100% rename from packages/snap/README.md rename to contrib/packages/snap/README.md diff --git a/snap/snapcraft.yaml b/contrib/snap/snapcraft.yaml similarity index 100% rename from snap/snapcraft.yaml rename to contrib/snap/snapcraft.yaml diff --git a/go.mod b/go.mod index fb8cd67..53aab93 100644 --- a/go.mod +++ b/go.mod @@ -1,9 +1,26 @@ module github.com/vpenso/prometheus-slurm-exporter -go 1.12 +go 1.17 require ( - github.com/prometheus/client_golang v1.2.1 - github.com/prometheus/common v0.7.0 - github.com/stretchr/testify v1.3.0 + github.com/go-ldap/ldif v0.0.0-20200320164324-fd88d9b715b3 + github.com/prometheus/client_golang v1.13.0 + github.com/stretchr/testify v1.4.0 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/go-asn1-ber/asn1-ber v1.4.1 // indirect + github.com/go-ldap/ldap/v3 v3.1.7 // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/common v0.37.0 // indirect + github.com/prometheus/procfs v0.8.0 // indirect + golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab // indirect + google.golang.org/protobuf v1.28.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index 53b2af9..cd2e56f 100644 --- a/go.sum +++ b/go.sum @@ -1,86 +1,493 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/cespare/xxhash/v2 v2.1.0 h1:yTUvW7Vhb89inJ+8irsUqiWjh8iT6sQPZiQzI6ReGkA= -github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/go-asn1-ber/asn1-ber v1.3.1/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= +github.com/go-asn1-ber/asn1-ber v1.4.1 h1:qP/QDxOtmMoJVgXHCXNzDpA0+wkgYB2x5QoLMVOciyw= +github.com/go-asn1-ber/asn1-ber v1.4.1/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= +github.com/go-ldap/ldap/v3 v3.1.7 h1:aHjuWTgZsnxjMgqzx0JHwNqz4jBYZTcNarbPFkW1Oww= +github.com/go-ldap/ldap/v3 v3.1.7/go.mod h1:5Zun81jBTabRaI8lzN7E1JjyEl1g6zI6u9pd8luAK4Q= +github.com/go-ldap/ldif v0.0.0-20200320164324-fd88d9b715b3 h1:sfz1YppV05y4sYaW7kXZtrocU/+vimnIWt4cxAYh7+o= +github.com/go-ldap/ldif v0.0.0-20200320164324-fd88d9b715b3/go.mod h1:ZXFhGda43Z2TVbfGZefXyMJzsDHhCh0go3bZUcwTx7o= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.2.1 h1:JnMpQc6ppsNgw9QPAGF6Dod479itz7lvlsMzzNayLOI= -github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_golang v1.13.0 h1:b71QUfeo5M8gq2+evJdTPfZhYMAU0uKPkyPJ7TPsloU= +github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY= -github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= +github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8= -github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= +github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY= -golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab h1:2QkjZIsXupsJbJIdSjjUOgWK3aEtzyuh2mPt3l/CkeU= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/main.go b/main.go index 48291fc..ec5bc6a 100644 --- a/main.go +++ b/main.go @@ -17,47 +17,72 @@ package main import ( "flag" + "net/http" + + "log" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" "github.com/prometheus/client_golang/prometheus/promhttp" - "github.com/prometheus/common/log" - "net/http" + "github.com/vpenso/prometheus-slurm-exporter/pkg/slurm" ) -func init() { - // Metrics have to be registered to be exposed - prometheus.MustRegister(NewAccountsCollector()) // from accounts.go - prometheus.MustRegister(NewCPUsCollector()) // from cpus.go - prometheus.MustRegister(NewNodesCollector()) // from nodes.go - prometheus.MustRegister(NewNodeCollector()) // from node.go - prometheus.MustRegister(NewPartitionsCollector()) // from partitions.go - prometheus.MustRegister(NewQueueCollector()) // from queue.go - prometheus.MustRegister(NewSchedulerCollector()) // from scheduler.go - prometheus.MustRegister(NewFairShareCollector()) // from sshare.go - prometheus.MustRegister(NewUsersCollector()) // from users.go -} - var listenAddress = flag.String( "listen-address", ":8080", "The address to listen on for HTTP requests.") +// Turn on GPUs accounting only if the corresponding command line option is set to true. var gpuAcct = flag.Bool( "gpus-acct", false, "Enable GPUs accounting") +var execTimeoutSeconds = flag.Int( + "exec-timeout", + 10, + "Timeout when executing shell commands") + +var nodeAddressSuffix = flag.String( + "address-suffix", + "", + "Suffix to add the node address when reporting metrics") + +var ldapServer = flag.String( + "ldap-address", + "", + "Address to contact ldap server. if this is not set, this feature will be disable. if configured please configure --ldap-base-search as well") + +var ldapBaseSearch = flag.String( + "ldap-base-search", + "", + "Base search for the ldap server (e.g. dc=example,dc=com)") + func main() { flag.Parse() - // Turn on GPUs accounting only if the corresponding command line option is set to true. if *gpuAcct { - prometheus.MustRegister(NewGPUsCollector()) // from gpus.go + prometheus.MustRegister(slurm.NewGPUsCollector()) // from gpus.go + } + if *ldapServer != "" && *ldapBaseSearch == "" { + log.Fatalln("--ldap-address is configured but --ldap-base-search is not. please configure --ldap-base-search (e.g. dc=example,dc=com) ") } + reg, err := slurm.NewRegistry(*gpuAcct, *execTimeoutSeconds, *nodeAddressSuffix, *ldapServer, *ldapBaseSearch) + if err != nil { + log.Fatalln(err) + } + + // Adding more collectors to the registry + reg.MustRegister( + collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), + collectors.NewGoCollector(), + ) + // The Handler function provides a default handler to expose metrics // via an HTTP server. "/metrics" is the usual endpoint for that. - log.Infof("Starting Server: %s", *listenAddress) - log.Infof("GPUs Accounting: %t", *gpuAcct) - http.Handle("/metrics", promhttp.Handler()) + log.Printf("Starting Server: %s", *listenAddress) + log.Printf("GPUs Accounting: %t", *gpuAcct) + http.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{})) log.Fatal(http.ListenAndServe(*listenAddress, nil)) } diff --git a/node.go b/node.go deleted file mode 100644 index bf2f759..0000000 --- a/node.go +++ /dev/null @@ -1,137 +0,0 @@ -/* Copyright 2021 Chris Read - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . */ - -package main - -import ( - "log" - "os/exec" - "sort" - "strconv" - "strings" - - "github.com/prometheus/client_golang/prometheus" -) - -// NodeMetrics stores metrics for each node -type NodeMetrics struct { - memAlloc uint64 - memTotal uint64 - cpuAlloc uint64 - cpuIdle uint64 - cpuOther uint64 - cpuTotal uint64 - nodeStatus string -} - -func NodeGetMetrics() map[string]*NodeMetrics { - return ParseNodeMetrics(NodeData()) -} - -// ParseNodeMetrics takes the output of sinfo with node data -// It returns a map of metrics per node -func ParseNodeMetrics(input []byte) map[string]*NodeMetrics { - nodes := make(map[string]*NodeMetrics) - lines := strings.Split(string(input), "\n") - - // Sort and remove all the duplicates from the 'sinfo' output - sort.Strings(lines) - linesUniq := RemoveDuplicates(lines) - - for _, line := range linesUniq { - node := strings.Fields(line) - nodeName := node[0] - nodeStatus := node[4] // mixed, allocated, etc. - - nodes[nodeName] = &NodeMetrics{0, 0, 0, 0, 0, 0, ""} - - memAlloc, _ := strconv.ParseUint(node[1], 10, 64) - memTotal, _ := strconv.ParseUint(node[2], 10, 64) - - - cpuInfo := strings.Split(node[3], "/") - cpuAlloc, _ := strconv.ParseUint(cpuInfo[0], 10, 64) - cpuIdle, _ := strconv.ParseUint(cpuInfo[1], 10, 64) - cpuOther, _ := strconv.ParseUint(cpuInfo[2], 10, 64) - cpuTotal, _ := strconv.ParseUint(cpuInfo[3], 10, 64) - - nodes[nodeName].memAlloc = memAlloc - nodes[nodeName].memTotal = memTotal - nodes[nodeName].cpuAlloc = cpuAlloc - nodes[nodeName].cpuIdle = cpuIdle - nodes[nodeName].cpuOther = cpuOther - nodes[nodeName].cpuTotal = cpuTotal - nodes[nodeName].nodeStatus = nodeStatus - } - - return nodes -} - -// NodeData executes the sinfo command to get data for each node -// It returns the output of the sinfo command -func NodeData() []byte { - cmd := exec.Command("sinfo", "-h", "-N", "-O", "NodeList,AllocMem,Memory,CPUsState,StateLong") - out, err := cmd.Output() - if err != nil { - log.Fatal(err) - } - return out -} - -type NodeCollector struct { - cpuAlloc *prometheus.Desc - cpuIdle *prometheus.Desc - cpuOther *prometheus.Desc - cpuTotal *prometheus.Desc - memAlloc *prometheus.Desc - memTotal *prometheus.Desc -} - -// NewNodeCollector creates a Prometheus collector to keep all our stats in -// It returns a set of collections for consumption -func NewNodeCollector() *NodeCollector { - labels := []string{"node","status"} - - return &NodeCollector{ - cpuAlloc: prometheus.NewDesc("slurm_node_cpu_alloc", "Allocated CPUs per node", labels, nil), - cpuIdle: prometheus.NewDesc("slurm_node_cpu_idle", "Idle CPUs per node", labels, nil), - cpuOther: prometheus.NewDesc("slurm_node_cpu_other", "Other CPUs per node", labels, nil), - cpuTotal: prometheus.NewDesc("slurm_node_cpu_total", "Total CPUs per node", labels, nil), - memAlloc: prometheus.NewDesc("slurm_node_mem_alloc", "Allocated memory per node", labels, nil), - memTotal: prometheus.NewDesc("slurm_node_mem_total", "Total memory per node", labels, nil), - } -} - -// Send all metric descriptions -func (nc *NodeCollector) Describe(ch chan<- *prometheus.Desc) { - ch <- nc.cpuAlloc - ch <- nc.cpuIdle - ch <- nc.cpuOther - ch <- nc.cpuTotal - ch <- nc.memAlloc - ch <- nc.memTotal -} - -func (nc *NodeCollector) Collect(ch chan<- prometheus.Metric) { - nodes := NodeGetMetrics() - for node := range nodes { - ch <- prometheus.MustNewConstMetric(nc.cpuAlloc, prometheus.GaugeValue, float64(nodes[node].cpuAlloc), node, nodes[node].nodeStatus) - ch <- prometheus.MustNewConstMetric(nc.cpuIdle, prometheus.GaugeValue, float64(nodes[node].cpuIdle), node, nodes[node].nodeStatus) - ch <- prometheus.MustNewConstMetric(nc.cpuOther, prometheus.GaugeValue, float64(nodes[node].cpuOther), node, nodes[node].nodeStatus) - ch <- prometheus.MustNewConstMetric(nc.cpuTotal, prometheus.GaugeValue, float64(nodes[node].cpuTotal), node, nodes[node].nodeStatus) - ch <- prometheus.MustNewConstMetric(nc.memAlloc, prometheus.GaugeValue, float64(nodes[node].memAlloc), node, nodes[node].nodeStatus) - ch <- prometheus.MustNewConstMetric(nc.memTotal, prometheus.GaugeValue, float64(nodes[node].memTotal), node, nodes[node].nodeStatus) - } -} diff --git a/node_test.go b/node_test.go deleted file mode 100644 index b554ddc..0000000 --- a/node_test.go +++ /dev/null @@ -1,57 +0,0 @@ -/* Copyright 2021 Chris Read - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . */ - -package main - -import ( - "io/ioutil" - "testing" - - "github.com/stretchr/testify/assert" -) - -/* -For this example data line: - -a048,79384,193000,3/13/0/16,mix - -We want output that looks like: - -slurm_node_cpus_allocated{name="a048",status="mix"} 3 -slurm_node_cpus_idle{name="a048",status="mix"} 3 -slurm_node_cpus_other{name="a048",status="mix"} 0 -slurm_node_cpus_total{name="a048",status="mix"} 16 -slurm_node_mem_allocated{name="a048",status="mix"} 179384 -slurm_node_mem_total{name="a048",status="mix"} 193000 - -*/ - -func TestNodeMetrics(t *testing.T) { - // Read the input data from a file - data, err := ioutil.ReadFile("test_data/sinfo_mem.txt") - if err != nil { - t.Fatalf("Can not open test data: %v", err) - } - metrics := ParseNodeMetrics(data) - t.Logf("%+v", metrics) - - assert.Contains(t, metrics, "b001") - assert.Equal(t, uint64(327680), metrics["b001"].memAlloc) - assert.Equal(t, uint64(386000), metrics["b001"].memTotal) - assert.Equal(t, uint64(32), metrics["b001"].cpuAlloc) - assert.Equal(t, uint64(0), metrics["b001"].cpuIdle) - assert.Equal(t, uint64(0), metrics["b001"].cpuOther) - assert.Equal(t, uint64(32), metrics["b001"].cpuTotal) -} diff --git a/nodes.go b/nodes.go deleted file mode 100644 index 0a88a9a..0000000 --- a/nodes.go +++ /dev/null @@ -1,189 +0,0 @@ -/* Copyright 2017 Victor Penso, Matteo Dessalvi - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . */ - -package main - -import ( - "github.com/prometheus/client_golang/prometheus" - "io/ioutil" - "log" - "os/exec" - "regexp" - "sort" - "strconv" - "strings" -) - -type NodesMetrics struct { - alloc float64 - comp float64 - down float64 - drain float64 - err float64 - fail float64 - idle float64 - maint float64 - mix float64 - resv float64 -} - -func NodesGetMetrics() *NodesMetrics { - return ParseNodesMetrics(NodesData()) -} - -func RemoveDuplicates(s []string) []string { - m := map[string]bool{} - t := []string{} - - // Walk through the slice 's' and for each value we haven't seen so far, append it to 't'. - for _, v := range s { - if _, seen := m[v]; !seen { - if len(v) > 0 { - t = append(t, v) - m[v] = true - } - } - } - - return t -} - -func ParseNodesMetrics(input []byte) *NodesMetrics { - var nm NodesMetrics - lines := strings.Split(string(input), "\n") - - // Sort and remove all the duplicates from the 'sinfo' output - sort.Strings(lines) - lines_uniq := RemoveDuplicates(lines) - - for _, line := range lines_uniq { - if strings.Contains(line, ",") { - split := strings.Split(line, ",") - count, _ := strconv.ParseFloat(strings.TrimSpace(split[0]), 64) - state := split[1] - alloc := regexp.MustCompile(`^alloc`) - comp := regexp.MustCompile(`^comp`) - down := regexp.MustCompile(`^down`) - drain := regexp.MustCompile(`^drain`) - fail := regexp.MustCompile(`^fail`) - err := regexp.MustCompile(`^err`) - idle := regexp.MustCompile(`^idle`) - maint := regexp.MustCompile(`^maint`) - mix := regexp.MustCompile(`^mix`) - resv := regexp.MustCompile(`^res`) - switch { - case alloc.MatchString(state) == true: - nm.alloc += count - case comp.MatchString(state) == true: - nm.comp += count - case down.MatchString(state) == true: - nm.down += count - case drain.MatchString(state) == true: - nm.drain += count - case fail.MatchString(state) == true: - nm.fail += count - case err.MatchString(state) == true: - nm.err += count - case idle.MatchString(state) == true: - nm.idle += count - case maint.MatchString(state) == true: - nm.maint += count - case mix.MatchString(state) == true: - nm.mix += count - case resv.MatchString(state) == true: - nm.resv += count - } - } - } - return &nm -} - -// Execute the sinfo command and return its output -func NodesData() []byte { - cmd := exec.Command("sinfo", "-h", "-o %D,%T") - stdout, err := cmd.StdoutPipe() - if err != nil { - log.Fatal(err) - } - if err := cmd.Start(); err != nil { - log.Fatal(err) - } - out, _ := ioutil.ReadAll(stdout) - if err := cmd.Wait(); err != nil { - log.Fatal(err) - } - return out -} - -/* - * Implement the Prometheus Collector interface and feed the - * Slurm scheduler metrics into it. - * https://godoc.org/github.com/prometheus/client_golang/prometheus#Collector - */ - -func NewNodesCollector() *NodesCollector { - return &NodesCollector{ - alloc: prometheus.NewDesc("slurm_nodes_alloc", "Allocated nodes", nil, nil), - comp: prometheus.NewDesc("slurm_nodes_comp", "Completing nodes", nil, nil), - down: prometheus.NewDesc("slurm_nodes_down", "Down nodes", nil, nil), - drain: prometheus.NewDesc("slurm_nodes_drain", "Drain nodes", nil, nil), - err: prometheus.NewDesc("slurm_nodes_err", "Error nodes", nil, nil), - fail: prometheus.NewDesc("slurm_nodes_fail", "Fail nodes", nil, nil), - idle: prometheus.NewDesc("slurm_nodes_idle", "Idle nodes", nil, nil), - maint: prometheus.NewDesc("slurm_nodes_maint", "Maint nodes", nil, nil), - mix: prometheus.NewDesc("slurm_nodes_mix", "Mix nodes", nil, nil), - resv: prometheus.NewDesc("slurm_nodes_resv", "Reserved nodes", nil, nil), - } -} - -type NodesCollector struct { - alloc *prometheus.Desc - comp *prometheus.Desc - down *prometheus.Desc - drain *prometheus.Desc - err *prometheus.Desc - fail *prometheus.Desc - idle *prometheus.Desc - maint *prometheus.Desc - mix *prometheus.Desc - resv *prometheus.Desc -} - -// Send all metric descriptions -func (nc *NodesCollector) Describe(ch chan<- *prometheus.Desc) { - ch <- nc.alloc - ch <- nc.comp - ch <- nc.down - ch <- nc.drain - ch <- nc.err - ch <- nc.fail - ch <- nc.idle - ch <- nc.maint - ch <- nc.mix - ch <- nc.resv -} -func (nc *NodesCollector) Collect(ch chan<- prometheus.Metric) { - nm := NodesGetMetrics() - ch <- prometheus.MustNewConstMetric(nc.alloc, prometheus.GaugeValue, nm.alloc) - ch <- prometheus.MustNewConstMetric(nc.comp, prometheus.GaugeValue, nm.comp) - ch <- prometheus.MustNewConstMetric(nc.down, prometheus.GaugeValue, nm.down) - ch <- prometheus.MustNewConstMetric(nc.drain, prometheus.GaugeValue, nm.drain) - ch <- prometheus.MustNewConstMetric(nc.err, prometheus.GaugeValue, nm.err) - ch <- prometheus.MustNewConstMetric(nc.fail, prometheus.GaugeValue, nm.fail) - ch <- prometheus.MustNewConstMetric(nc.idle, prometheus.GaugeValue, nm.idle) - ch <- prometheus.MustNewConstMetric(nc.maint, prometheus.GaugeValue, nm.maint) - ch <- prometheus.MustNewConstMetric(nc.mix, prometheus.GaugeValue, nm.mix) - ch <- prometheus.MustNewConstMetric(nc.resv, prometheus.GaugeValue, nm.resv) -} diff --git a/nodes_test.go b/nodes_test.go deleted file mode 100644 index f734ccf..0000000 --- a/nodes_test.go +++ /dev/null @@ -1,36 +0,0 @@ -/* Copyright 2017 Victor Penso, Matteo Dessalvi - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . */ - -package main - -import ( - "io/ioutil" - "os" - "testing" -) - -func TestNodesMetrics(t *testing.T) { - // Read the input data from a file - file, err := os.Open("test_data/sinfo.txt") - if err != nil { - t.Fatalf("Can not open test data: %v", err) - } - data, err := ioutil.ReadAll(file) - t.Logf("%+v", ParseNodesMetrics(data)) -} - -func TestNodesGetMetrics(t *testing.T) { - t.Logf("%+v", NodesGetMetrics()) -} diff --git a/partitions.go b/partitions.go deleted file mode 100644 index 16c6b36..0000000 --- a/partitions.go +++ /dev/null @@ -1,149 +0,0 @@ -/* Copyright 2020 Victor Penso - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . */ - -package main - -import ( - "io/ioutil" - "os/exec" - "log" - "strings" - "strconv" - "github.com/prometheus/client_golang/prometheus" -) - -func PartitionsData() []byte { - cmd := exec.Command("sinfo", "-h", "-o%R,%C") - stdout, err := cmd.StdoutPipe() - if err != nil { - log.Fatal(err) - } - if err := cmd.Start(); err != nil { - log.Fatal(err) - } - out, _ := ioutil.ReadAll(stdout) - if err := cmd.Wait(); err != nil { - log.Fatal(err) - } - return out -} - -func PartitionsPendingJobsData() []byte { - cmd := exec.Command("squeue","-a","-r","-h","-o%P","--states=PENDING") - stdout, err := cmd.StdoutPipe() - if err != nil { - log.Fatal(err) - } - if err := cmd.Start(); err != nil { - log.Fatal(err) - } - out, _ := ioutil.ReadAll(stdout) - if err := cmd.Wait(); err != nil { - log.Fatal(err) - } - return out -} - -type PartitionMetrics struct { - allocated float64 - idle float64 - other float64 - pending float64 - total float64 -} - -func ParsePartitionsMetrics() map[string]*PartitionMetrics { - partitions := make(map[string]*PartitionMetrics) - lines := strings.Split(string(PartitionsData()), "\n") - for _, line := range lines { - if strings.Contains(line,",") { - // name of a partition - partition := strings.Split(line,",")[0] - _,key := partitions[partition] - if !key { - partitions[partition] = &PartitionMetrics{0,0,0,0,0} - } - states := strings.Split(line,",")[1] - allocated,_ := strconv.ParseFloat(strings.Split(states,"/")[0],64) - idle,_ := strconv.ParseFloat(strings.Split(states,"/")[1],64) - other,_ := strconv.ParseFloat(strings.Split(states,"/")[2],64) - total,_ := strconv.ParseFloat(strings.Split(states,"/")[3],64) - partitions[partition].allocated = allocated - partitions[partition].idle = idle - partitions[partition].other = other - partitions[partition].total = total - } - } - // get list of pending jobs by partition name - list := strings.Split(string(PartitionsPendingJobsData()),"\n") - for _,partition := range list { - // accumulate the number of pending jobs - _,key := partitions[partition] - if key { - partitions[partition].pending += 1 - } - } - - - return partitions -} - -type PartitionsCollector struct { - allocated *prometheus.Desc - idle *prometheus.Desc - other *prometheus.Desc - pending *prometheus.Desc - total *prometheus.Desc -} - -func NewPartitionsCollector() *PartitionsCollector { - labels := []string{"partition"} - return &PartitionsCollector{ - allocated: prometheus.NewDesc("slurm_partition_cpus_allocated", "Allocated CPUs for partition", labels,nil), - idle: prometheus.NewDesc("slurm_partition_cpus_idle", "Idle CPUs for partition", labels,nil), - other: prometheus.NewDesc("slurm_partition_cpus_other", "Other CPUs for partition", labels,nil), - pending: prometheus.NewDesc("slurm_partition_jobs_pending", "Pending jobs for partition", labels,nil), - total: prometheus.NewDesc("slurm_partition_cpus_total", "Total CPUs for partition", labels,nil), - } -} - -func (pc *PartitionsCollector) Describe(ch chan<- *prometheus.Desc) { - ch <- pc.allocated - ch <- pc.idle - ch <- pc.other - ch <- pc.pending - ch <- pc.total -} - -func (pc *PartitionsCollector) Collect(ch chan<- prometheus.Metric) { - pm := ParsePartitionsMetrics() - for p := range pm { - if pm[p].allocated > 0 { - ch <- prometheus.MustNewConstMetric(pc.allocated, prometheus.GaugeValue, pm[p].allocated, p) - } - if pm[p].idle > 0 { - ch <- prometheus.MustNewConstMetric(pc.idle, prometheus.GaugeValue, pm[p].idle, p) - } - if pm[p].other > 0 { - ch <- prometheus.MustNewConstMetric(pc.other, prometheus.GaugeValue, pm[p].other, p) - } - if pm[p].pending > 0 { - ch <- prometheus.MustNewConstMetric(pc.pending, prometheus.GaugeValue, pm[p].pending, p) - } - if pm[p].total > 0 { - ch <- prometheus.MustNewConstMetric(pc.total, prometheus.GaugeValue, pm[p].total, p) - } - } -} diff --git a/pkg/ldapsearch/ldapsearch.go b/pkg/ldapsearch/ldapsearch.go new file mode 100644 index 0000000..14af79b --- /dev/null +++ b/pkg/ldapsearch/ldapsearch.go @@ -0,0 +1,83 @@ +package ldapsearch + +import ( + "bytes" + "context" + "fmt" + "io/ioutil" + "log" + "os/exec" + "strings" + "time" + + "github.com/go-ldap/ldif" +) + +const ( + searchQuery = "-LLL -E pr=1000/noprompt -h %s -b %s (&(objectClass=user)(uidNumber=*)(sAMAccountName=*))" + attributeKeyUID = "uidNumber" + attributeKeyUsername = "sAMAccountName" +) + +type Search struct { + uids map[string]string +} + +func Init(ldapServer, baseSearch, testFile string) (*Search, error) { + var output []byte + var err error + if testFile == "" { + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, "ldapsearch", strings.Split(fmt.Sprintf(searchQuery, ldapServer, baseSearch), " ")...) + stderr := "" + buf := bytes.NewBufferString(stderr) + cmd.Stderr = buf + output, err = cmd.Output() + if err != nil { + return nil, fmt.Errorf("error running ldapserarch %v: %v - stderr: %v", err, string(output), buf.String()) + } + } else { + output, err = ioutil.ReadFile(testFile) + if err != nil { + return nil, err + } + } + objects, err := ldif.Parse(string(output)) + if err != nil { + return nil, err + } + u := map[string]string{} + s := Search{ + uids: u, + } + log.Printf("Found %v entries! \n", len(objects.Entries)) + for _, entry := range objects.Entries { + obj := entry.Entry + uid := "" + username := "" + for _, attr := range obj.Attributes { + if attr.Name == attributeKeyUID { + uid = strings.Join(attr.Values, "-") + } + if attr.Name == attributeKeyUsername { + username = strings.Join(attr.Values, "-") + // log.Println("Found username: ", username) + } + } + if uid != "" { + s.uids[uid] = username + } + } + // log.Println(s.uids) + return &s, nil +} + +// GetUsername is used to very quickly retried a username from memory +func (s *Search) GetUsername(uid string) string { + user, ok := s.uids[uid] + if !ok { + return uid + } + return user +} diff --git a/pkg/ldapsearch/ldapsearch_test.go b/pkg/ldapsearch/ldapsearch_test.go new file mode 100644 index 0000000..b70eb5d --- /dev/null +++ b/pkg/ldapsearch/ldapsearch_test.go @@ -0,0 +1,22 @@ +package ldapsearch + +// import ( +// "testing" + +// "github.com/stretchr/testify/assert" +// ) + +// func TestParseLDIfFile(t *testing.T) { +// s, err := Init("./testdata/data.ldif") +// assert.NoError(t, err) +// username := s.GetUsername("234234234") +// assert.Equal(t, "test-user", username) +// } + +// This is a full on e2e test and must be run from someone's computer +// func TestRunCmd(t *testing.T) { +// s, err := Init("") +// assert.NoError(t, err) +// username := s.GetUsername("234234234") +// assert.Equal(t, "test-user", username) +// } diff --git a/pkg/slurm/accounts.go b/pkg/slurm/accounts.go new file mode 100644 index 0000000..6bf1a1e --- /dev/null +++ b/pkg/slurm/accounts.go @@ -0,0 +1,105 @@ +/* Copyright 2020 Victor Penso + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . */ + +package slurm + +import ( + "regexp" + "strconv" + "strings" + + "github.com/prometheus/client_golang/prometheus" +) + +type JobMetrics struct { + pending float64 + running float64 + running_cpus float64 + suspended float64 +} + +func ParseAccountsMetrics() map[string]*JobMetrics { + out := execCommand("squeue -a -r -h -o %A|%a|%T|%C") + + accounts := make(map[string]*JobMetrics) + lines := strings.Split(out, "\n") + for _, line := range lines { + if strings.Contains(line, "|") { + account := strings.Split(line, "|")[1] + _, key := accounts[account] + if !key { + accounts[account] = &JobMetrics{0, 0, 0, 0} + } + state := strings.Split(line, "|")[2] + state = strings.ToLower(state) + cpus, _ := strconv.ParseFloat(strings.Split(line, "|")[3], 64) + pending := regexp.MustCompile(`^pending`) + running := regexp.MustCompile(`^running`) + suspended := regexp.MustCompile(`^suspended`) + switch { + case pending.MatchString(state) == true: + accounts[account].pending++ + case running.MatchString(state) == true: + accounts[account].running++ + accounts[account].running_cpus += cpus + case suspended.MatchString(state) == true: + accounts[account].suspended++ + } + } + } + return accounts +} + +type AccountsCollector struct { + pending *prometheus.Desc + running *prometheus.Desc + running_cpus *prometheus.Desc + suspended *prometheus.Desc +} + +func NewAccountsCollector() *AccountsCollector { + labels := []string{"account"} + return &AccountsCollector{ + pending: prometheus.NewDesc("slurm_account_jobs_pending", "Pending jobs for account", labels, nil), + running: prometheus.NewDesc("slurm_account_jobs_running", "Running jobs for account", labels, nil), + running_cpus: prometheus.NewDesc("slurm_account_cpus_running", "Running cpus for account", labels, nil), + suspended: prometheus.NewDesc("slurm_account_jobs_suspended", "Suspended jobs for account", labels, nil), + } +} + +func (ac *AccountsCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- ac.pending + ch <- ac.running + ch <- ac.running_cpus + ch <- ac.suspended +} + +func (ac *AccountsCollector) Collect(ch chan<- prometheus.Metric) { + am := ParseAccountsMetrics() + for a := range am { + if am[a].pending > 0 { + ch <- prometheus.MustNewConstMetric(ac.pending, prometheus.GaugeValue, am[a].pending, a) + } + if am[a].running > 0 { + ch <- prometheus.MustNewConstMetric(ac.running, prometheus.GaugeValue, am[a].running, a) + } + if am[a].running_cpus > 0 { + ch <- prometheus.MustNewConstMetric(ac.running_cpus, prometheus.GaugeValue, am[a].running_cpus, a) + } + if am[a].suspended > 0 { + ch <- prometheus.MustNewConstMetric(ac.suspended, prometheus.GaugeValue, am[a].suspended, a) + } + } +} diff --git a/cpus.go b/pkg/slurm/cpus.go similarity index 63% rename from cpus.go rename to pkg/slurm/cpus.go index 384a034..948d679 100644 --- a/cpus.go +++ b/pkg/slurm/cpus.go @@ -13,15 +13,18 @@ GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . */ -package main +package slurm import ( - "github.com/prometheus/client_golang/prometheus" - "io/ioutil" - "log" - "os/exec" "strconv" "strings" + + "github.com/prometheus/client_golang/prometheus" +) + +const ( + CpuMetricsCommand = "sinfo -h -o %C" + CpuMetricsTestData = "test_data/sinfo_cpus.txt" ) type CPUsMetrics struct { @@ -31,14 +34,11 @@ type CPUsMetrics struct { total float64 } -func CPUsGetMetrics() *CPUsMetrics { - return ParseCPUsMetrics(CPUsData()) -} - -func ParseCPUsMetrics(input []byte) *CPUsMetrics { +func (cc *CPUsCollector) CPUsGetMetrics() *CPUsMetrics { var cm CPUsMetrics - if strings.Contains(string(input), "/") { - splitted := strings.Split(strings.TrimSpace(string(input)), "/") + out := getData(cc.isTest, CpuMetricsCommand, CpuMetricsTestData) + if strings.Contains(out, "/") { + splitted := strings.Split(strings.TrimSpace(out), "/") cm.alloc, _ = strconv.ParseFloat(splitted[0], 64) cm.idle, _ = strconv.ParseFloat(splitted[1], 64) cm.other, _ = strconv.ParseFloat(splitted[2], 64) @@ -47,43 +47,28 @@ func ParseCPUsMetrics(input []byte) *CPUsMetrics { return &cm } -// Execute the sinfo command and return its output -func CPUsData() []byte { - cmd := exec.Command("sinfo", "-h", "-o %C") - stdout, err := cmd.StdoutPipe() - if err != nil { - log.Fatal(err) - } - if err := cmd.Start(); err != nil { - log.Fatal(err) - } - out, _ := ioutil.ReadAll(stdout) - if err := cmd.Wait(); err != nil { - log.Fatal(err) - } - return out -} - /* * Implement the Prometheus Collector interface and feed the * Slurm scheduler metrics into it. * https://godoc.org/github.com/prometheus/client_golang/prometheus#Collector */ -func NewCPUsCollector() *CPUsCollector { +func NewCPUsCollector(isTest bool) *CPUsCollector { return &CPUsCollector{ - alloc: prometheus.NewDesc("slurm_cpus_alloc", "Allocated CPUs", nil, nil), - idle: prometheus.NewDesc("slurm_cpus_idle", "Idle CPUs", nil, nil), - other: prometheus.NewDesc("slurm_cpus_other", "Mix CPUs", nil, nil), - total: prometheus.NewDesc("slurm_cpus_total", "Total CPUs", nil, nil), + isTest: isTest, + alloc: prometheus.NewDesc("slurm_cpus_alloc", "Allocated CPUs", nil, nil), + idle: prometheus.NewDesc("slurm_cpus_idle", "Idle CPUs", nil, nil), + other: prometheus.NewDesc("slurm_cpus_other", "Mix CPUs", nil, nil), + total: prometheus.NewDesc("slurm_cpus_total", "Total CPUs", nil, nil), } } type CPUsCollector struct { - alloc *prometheus.Desc - idle *prometheus.Desc - other *prometheus.Desc - total *prometheus.Desc + isTest bool + alloc *prometheus.Desc + idle *prometheus.Desc + other *prometheus.Desc + total *prometheus.Desc } // Send all metric descriptions @@ -94,7 +79,7 @@ func (cc *CPUsCollector) Describe(ch chan<- *prometheus.Desc) { ch <- cc.total } func (cc *CPUsCollector) Collect(ch chan<- prometheus.Metric) { - cm := CPUsGetMetrics() + cm := cc.CPUsGetMetrics() ch <- prometheus.MustNewConstMetric(cc.alloc, prometheus.GaugeValue, cm.alloc) ch <- prometheus.MustNewConstMetric(cc.idle, prometheus.GaugeValue, cm.idle) ch <- prometheus.MustNewConstMetric(cc.other, prometheus.GaugeValue, cm.other) diff --git a/cpus_test.go b/pkg/slurm/cpus_test.go similarity index 71% rename from cpus_test.go rename to pkg/slurm/cpus_test.go index 61b0ea2..cc807be 100644 --- a/cpus_test.go +++ b/pkg/slurm/cpus_test.go @@ -13,24 +13,14 @@ GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . */ -package main +package slurm import ( - "io/ioutil" - "os" "testing" ) func TestCPUsMetrics(t *testing.T) { // Read the input data from a file - file, err := os.Open("test_data/sinfo_cpus.txt") - if err != nil { - t.Fatalf("Can not open test data: %v", err) - } - data, err := ioutil.ReadAll(file) - t.Logf("%+v", ParseCPUsMetrics(data)) -} - -func TestCPUssGetMetrics(t *testing.T) { - t.Logf("%+v", CPUsGetMetrics()) + coll := NewCPUsCollector(true) + t.Logf("%+v", coll.CPUsGetMetrics()) } diff --git a/gpus.go b/pkg/slurm/gpus.go similarity index 67% rename from gpus.go rename to pkg/slurm/gpus.go index ca3bcaf..c3890b9 100644 --- a/gpus.go +++ b/pkg/slurm/gpus.go @@ -13,15 +13,13 @@ GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . */ -package main +package slurm import ( - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/common/log" - "io/ioutil" - "os/exec" - "strings" "strconv" + "strings" + + "github.com/prometheus/client_golang/prometheus" ) type GPUsMetrics struct { @@ -38,15 +36,18 @@ func GPUsGetMetrics() *GPUsMetrics { func ParseAllocatedGPUs() float64 { var num_gpus = 0.0 - args := []string{"-a", "-X", "--format=Allocgres", "--state=RUNNING", "--noheader", "--parsable2"} - output := string(Execute("sacct", args)) + output := execCommand("sacct -a -X --format=AllocTRES --state=RUNNING --noheader --parsable2") if len(output) > 0 { for _, line := range strings.Split(output, "\n") { if len(line) > 0 { line = strings.Trim(line, "\"") - descriptor := strings.TrimPrefix(line, "gpu:") - job_gpus, _ := strconv.ParseFloat(descriptor, 64) - num_gpus += job_gpus + for _, resource := range strings.Split(line, ",") { + if strings.HasPrefix(resource, "gres/gpu=") { + descriptor := strings.TrimPrefix(resource, "gres/gpu=") + job_gpus, _ := strconv.ParseFloat(descriptor, 64) + num_gpus += job_gpus + } + } } } } @@ -56,18 +57,22 @@ func ParseAllocatedGPUs() float64 { func ParseTotalGPUs() float64 { var num_gpus = 0.0 - - args := []string{"-h", "-o \"%n %G\""} - output := string(Execute("sinfo", args)) - if len(output) > 0 { - for _, line := range strings.Split(output, "\n") { + out := execCommand("sinfo -h -o \"%n %G\"") + if len(out) > 0 { + for _, line := range strings.Split(out, "\n") { if len(line) > 0 { line = strings.Trim(line, "\"") - descriptor := strings.Fields(line)[1] - descriptor = strings.TrimPrefix(descriptor, "gpu:") - descriptor = strings.Split(descriptor, "(")[0] - node_gpus, _ := strconv.ParseFloat(descriptor, 64) - num_gpus += node_gpus + gres := strings.Fields(line)[1] + // gres column format: comma-delimited list of resources + for _, resource := range strings.Split(gres, ",") { + if strings.HasPrefix(resource, "gpu:") { + // format: gpu::N(S:), e.g. gpu:RTX2070:2(S:0) + descriptor := strings.Split(resource, ":")[2] + descriptor = strings.Split(descriptor, "(")[0] + node_gpus, _ := strconv.ParseFloat(descriptor, 64) + num_gpus += node_gpus + } + } } } } @@ -86,23 +91,6 @@ func ParseGPUsMetrics() *GPUsMetrics { return &gm } -// Execute the sinfo command and return its output -func Execute(command string, arguments []string) []byte { - cmd := exec.Command(command, arguments...) - stdout, err := cmd.StdoutPipe() - if err != nil { - log.Fatal(err) - } - if err := cmd.Start(); err != nil { - log.Fatal(err) - } - out, _ := ioutil.ReadAll(stdout) - if err := cmd.Wait(); err != nil { - log.Fatal(err) - } - return out -} - /* * Implement the Prometheus Collector interface and feed the * Slurm scheduler metrics into it. @@ -111,9 +99,9 @@ func Execute(command string, arguments []string) []byte { func NewGPUsCollector() *GPUsCollector { return &GPUsCollector{ - alloc: prometheus.NewDesc("slurm_gpus_alloc", "Allocated GPUs", nil, nil), - idle: prometheus.NewDesc("slurm_gpus_idle", "Idle GPUs", nil, nil), - total: prometheus.NewDesc("slurm_gpus_total", "Total GPUs", nil, nil), + alloc: prometheus.NewDesc("slurm_gpus_alloc", "Allocated GPUs", nil, nil), + idle: prometheus.NewDesc("slurm_gpus_idle", "Idle GPUs", nil, nil), + total: prometheus.NewDesc("slurm_gpus_total", "Total GPUs", nil, nil), utilization: prometheus.NewDesc("slurm_gpus_utilization", "Total GPU utilization", nil, nil), } } diff --git a/pkg/slurm/jobs.go b/pkg/slurm/jobs.go new file mode 100644 index 0000000..5209d58 --- /dev/null +++ b/pkg/slurm/jobs.go @@ -0,0 +1,308 @@ +/* Copyright 2017 Victor Penso, Matteo Dessalvi + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . */ + +package slurm + +import ( + "encoding/json" + "fmt" + "strconv" + + "github.com/prometheus/client_golang/prometheus" + "github.com/vpenso/prometheus-slurm-exporter/pkg/ldapsearch" +) + +const ( + showJobsCommand = "squeue -a --json" + showJobsTestDataInput = "./test_data/jobs.json" + showJobsTestDataProm = "./test_data/jobs.prom" + + minHistogramBucketRange = 1 // 1 second + maxHistogramBucketRange = 3600 * 24 * 14 //14 days + numberOfHistogramBuckets = 15 +) + +var ( + jobLabels = []string{"name", "job_id", "state", "state_reason", "partition", "user", "node"} + durationBuckets = prometheus.ExponentialBucketsRange(minHistogramBucketRange, maxHistogramBucketRange, numberOfHistogramBuckets) +) + +type jobsCollector struct { + jobsInfo *prometheus.GaugeVec + jobsReqCPU *prometheus.GaugeVec + jobsReqMemory *prometheus.GaugeVec + jobsReqBilling *prometheus.GaugeVec + jobsReqNodes *prometheus.GaugeVec + jobsRestartCount *prometheus.GaugeVec + jobExecDuration *prometheus.HistogramVec + jobSchedlingDuration *prometheus.HistogramVec + isTest bool + ldap *ldapsearch.Search +} + +func NewJobsCollector(isTest bool, ldap *ldapsearch.Search) *jobsCollector { + return &jobsCollector{ + isTest: isTest, + ldap: ldap, + jobsInfo: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Subsystem: "", + Name: "slurm_job_info", + Help: "General informations about slurm jobs.", + }, jobLabels), + jobExecDuration: prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Subsystem: "", + Name: "slurm_job_exec_duration", + Help: "Slurm job execution duration only for COMPLETED jobs.", + Buckets: durationBuckets, + }, jobLabels), + jobSchedlingDuration: prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Subsystem: "", + Name: "slurm_job_scheduling_duration", + Help: "Slurm job scheduling duration only for COMPLETED or RUNNING jobs.", + Buckets: durationBuckets, + }, jobLabels), + jobsReqCPU: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Subsystem: "", + Name: "slurm_job_req_cpu", + Help: "Requested CPU per job.", + }, jobLabels), + jobsReqMemory: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Subsystem: "", + Name: "slurm_job_req_memory_bytes", + Help: "Requested Memory per job.", + }, jobLabels), + jobsReqBilling: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Subsystem: "", + Name: "slurm_job_req_billing", + Help: "Requested billing per job.", + }, jobLabels), + jobsReqNodes: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Subsystem: "", + Name: "slurm_job_req_nodes", + Help: "Requested Nodes per job.", + }, jobLabels), + jobsRestartCount: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Subsystem: "", + Name: "slurm_job_restart_count", + Help: "Requested Restart count per job.", + }, jobLabels), + } +} + +func (s *jobsCollector) getJobsMetrics() { + // Get json data + data := getData(s.isTest, showJobsCommand, showJobsTestDataInput) + + // Parse json + squeueJson := &SqueuOutput{} + err := json.Unmarshal([]byte(data), squeueJson) + if err != nil { + ExporterErrors.WithLabelValues("json-encoding-sqeueu", err.Error()).Inc() + fmt.Println(err) + } + // create metrics from json object + for _, job := range squeueJson.Jobs { + user := job.UserName + if user == "" { + user = strconv.Itoa(job.UserID) + if s.ldap != nil { + user = s.ldap.GetUsername(user) + } + } + labelValues := []string{job.Name, strconv.Itoa(job.JobID), job.JobState, job.StateReason, job.Partition, user, job.Nodes} + s.jobsInfo.WithLabelValues(labelValues...).Set(1) + s.jobsRestartCount.WithLabelValues(labelValues...).Set(float64(job.RestartCnt)) + s.jobsReqCPU.WithLabelValues(labelValues...).Set(float64(job.Cpus)) + s.jobsReqMemory.WithLabelValues(labelValues...).Set(float64(job.MemoryPerCPU * job.Cpus)) + s.jobsReqNodes.WithLabelValues(labelValues...).Set(float64(job.NodeCount)) + s.jobsReqBilling.WithLabelValues(labelValues...).Set(job.BillableTres) + if job.StartTime != 0 { + schedulDuration := float64(job.StartTime - job.SubmitTime) + s.jobSchedlingDuration.WithLabelValues(labelValues...).Observe(schedulDuration) + } + if job.EndTime != 0 { + execDuration := float64(job.EndTime - job.StartTime) + s.jobExecDuration.WithLabelValues(labelValues...).Observe(execDuration) + } + } +} + +func (s *jobsCollector) Describe(ch chan<- *prometheus.Desc) { + s.jobsInfo.Describe(ch) + s.jobExecDuration.Describe(ch) + s.jobSchedlingDuration.Describe(ch) + s.jobsReqCPU.Describe(ch) + s.jobsReqMemory.Describe(ch) + s.jobsReqBilling.Describe(ch) + s.jobsReqNodes.Describe(ch) + s.jobsRestartCount.Describe(ch) +} + +func (s *jobsCollector) Collect(ch chan<- prometheus.Metric) { + s.jobsInfo.Reset() + s.jobExecDuration.Reset() + s.jobSchedlingDuration.Reset() + s.jobsReqCPU.Reset() + s.jobsReqMemory.Reset() + s.jobsReqBilling.Reset() + s.jobsReqNodes.Reset() + s.jobsRestartCount.Reset() + s.getJobsMetrics() + s.jobsInfo.Collect(ch) + s.jobExecDuration.Collect(ch) + s.jobSchedlingDuration.Collect(ch) + s.jobsReqCPU.Collect(ch) + s.jobsReqMemory.Collect(ch) + s.jobsReqBilling.Collect(ch) + s.jobsReqNodes.Collect(ch) + s.jobsRestartCount.Collect(ch) +} + +type SqueuOutput struct { + Meta struct { + Plugin struct { + Type string `json:"type"` + Name string `json:"name"` + } `json:"plugin"` + Slurm struct { + Version struct { + Major int `json:"major"` + Micro int `json:"micro"` + Minor int `json:"minor"` + } `json:"version"` + Release string `json:"release"` + } `json:"Slurm"` + } `json:"meta"` + Errors []interface{} `json:"errors"` + Jobs []struct { + Account string `json:"account"` + AccrueTime int `json:"accrue_time"` + AdminComment string `json:"admin_comment"` + ArrayJobID int `json:"array_job_id"` + ArrayTaskID interface{} `json:"array_task_id"` + ArrayMaxTasks int `json:"array_max_tasks"` + ArrayTaskString string `json:"array_task_string"` + AssociationID int `json:"association_id"` + BatchFeatures string `json:"batch_features"` + BatchFlag bool `json:"batch_flag"` + BatchHost string `json:"batch_host"` + Flags []string `json:"flags"` + BurstBuffer string `json:"burst_buffer"` + BurstBufferState string `json:"burst_buffer_state"` + Cluster string `json:"cluster"` + ClusterFeatures string `json:"cluster_features"` + Command string `json:"command"` + Comment string `json:"comment"` + Contiguous bool `json:"contiguous"` + CoreSpec interface{} `json:"core_spec"` + ThreadSpec interface{} `json:"thread_spec"` + CoresPerSocket interface{} `json:"cores_per_socket"` + BillableTres float64 `json:"billable_tres"` + CpusPerTask interface{} `json:"cpus_per_task"` + CPUFrequencyMinimum interface{} `json:"cpu_frequency_minimum"` + CPUFrequencyMaximum interface{} `json:"cpu_frequency_maximum"` + CPUFrequencyGovernor interface{} `json:"cpu_frequency_governor"` + CpusPerTres string `json:"cpus_per_tres"` + Deadline int `json:"deadline"` + DelayBoot int `json:"delay_boot"` + Dependency string `json:"dependency"` + DerivedExitCode int `json:"derived_exit_code"` + EligibleTime int `json:"eligible_time"` + EndTime int `json:"end_time"` + ExcludedNodes string `json:"excluded_nodes"` + ExitCode int `json:"exit_code"` + Features string `json:"features"` + FederationOrigin string `json:"federation_origin"` + FederationSiblingsActive string `json:"federation_siblings_active"` + FederationSiblingsViable string `json:"federation_siblings_viable"` + GresDetail []interface{} `json:"gres_detail"` + GroupID int `json:"group_id"` + JobID int `json:"job_id"` + JobResources struct { + } `json:"job_resources"` + JobState string `json:"job_state"` + LastSchedEvaluation int `json:"last_sched_evaluation"` + Licenses string `json:"licenses"` + MaxCpus int `json:"max_cpus"` + MaxNodes int `json:"max_nodes"` + McsLabel string `json:"mcs_label"` + MemoryPerTres string `json:"memory_per_tres"` + Name string `json:"name"` + Nodes string `json:"nodes"` + Nice int `json:"nice"` + TasksPerCore interface{} `json:"tasks_per_core"` + TasksPerNode int `json:"tasks_per_node"` + TasksPerSocket interface{} `json:"tasks_per_socket"` + TasksPerBoard int `json:"tasks_per_board"` + Cpus int `json:"cpus"` + NodeCount int `json:"node_count"` + Tasks int `json:"tasks"` + HetJobID int `json:"het_job_id"` + HetJobIDSet string `json:"het_job_id_set"` + HetJobOffset int `json:"het_job_offset"` + Partition string `json:"partition"` + MemoryPerNode interface{} `json:"memory_per_node"` + MemoryPerCPU int `json:"memory_per_cpu"` + MinimumCpusPerNode int `json:"minimum_cpus_per_node"` + MinimumTmpDiskPerNode int `json:"minimum_tmp_disk_per_node"` + PreemptTime int `json:"preempt_time"` + PreSusTime int `json:"pre_sus_time"` + Priority int `json:"priority"` + Profile interface{} `json:"profile"` + Qos string `json:"qos"` + Reboot bool `json:"reboot"` + RequiredNodes string `json:"required_nodes"` + Requeue bool `json:"requeue"` + ResizeTime int `json:"resize_time"` + RestartCnt int `json:"restart_cnt"` + ResvName string `json:"resv_name"` + Shared string `json:"shared"` + ShowFlags []string `json:"show_flags"` + SocketsPerBoard int `json:"sockets_per_board"` + SocketsPerNode interface{} `json:"sockets_per_node"` + StartTime int `json:"start_time"` + StateDescription string `json:"state_description"` + StateReason string `json:"state_reason"` + StandardError string `json:"standard_error"` + StandardInput string `json:"standard_input"` + StandardOutput string `json:"standard_output"` + SubmitTime int `json:"submit_time"` + SuspendTime int `json:"suspend_time"` + SystemComment string `json:"system_comment"` + TimeLimit int `json:"time_limit"` + TimeMinimum int `json:"time_minimum"` + ThreadsPerCore interface{} `json:"threads_per_core"` + TresBind string `json:"tres_bind"` + TresFreq string `json:"tres_freq"` + TresPerJob string `json:"tres_per_job"` + TresPerNode string `json:"tres_per_node"` + TresPerSocket string `json:"tres_per_socket"` + TresPerTask string `json:"tres_per_task"` + TresReqStr string `json:"tres_req_str"` + TresAllocStr string `json:"tres_alloc_str"` + UserID int `json:"user_id"` + UserName string `json:"user_name"` + Wckey string `json:"wckey"` + CurrentWorkingDirectory string `json:"current_working_directory"` + } `json:"jobs"` +} diff --git a/pkg/slurm/jobs_test.go b/pkg/slurm/jobs_test.go new file mode 100644 index 0000000..b720a94 --- /dev/null +++ b/pkg/slurm/jobs_test.go @@ -0,0 +1,44 @@ +/* Copyright 2017 Victor Penso, Matteo Dessalvi + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . */ + +package slurm + +// import ( +// "os" +// "testing" + +// "github.com/prometheus/client_golang/prometheus/testutil" +// "github.com/stretchr/testify/assert" +// ) + +// func TestJobsMetrics(t *testing.T) { +// collector := NewJobsCollector(true, nil) +// f, err := os.Open(showJobsTestDataProm) +// assert.NoError(t, err) +// defer f.Close() +// err = testutil.CollectAndCompare(collector, f) +// assert.NoError(t, err) +// } + +// TestGenerateJobsMetrics is only used to getnerate the .prom file for +// func TestGenerateJobsMetrics(t *testing.T) { +// reg := prometheus.NewRegistry() +// collector := NewJobsCollector(true, nil) +// err := reg.Register(collector) +// assert.NoError(t, err) +// gatherers := prometheus.Gatherers{reg} +// err = prometheus.WriteToTextfile(showJobsTestDataProm, gatherers) +// assert.NoError(t, err) +// } diff --git a/pkg/slurm/nodes.go b/pkg/slurm/nodes.go new file mode 100644 index 0000000..870ce9c --- /dev/null +++ b/pkg/slurm/nodes.go @@ -0,0 +1,511 @@ +/* Copyright 2017 Victor Penso, Matteo Dessalvi + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . */ + +package slurm + +import ( + "encoding/json" + "fmt" + "strconv" + "strings" + + "github.com/prometheus/client_golang/prometheus" +) + +const ( + showNodesDetailsCommand = "sinfo -R --json" + showNodesDetailsTestDataInput = "./test_data/sinfo-nodes.json" + showNodesDetailsTestDataProm = "./test_data/sinfo-nodes.prom" +) + +var ( + nodeResourcesLabels = []string{"name", "partition"} +) + +type nodesCollector struct { + scontrolNodesInfo *prometheus.GaugeVec + scontrolNodeCPULoad *prometheus.GaugeVec + scontrolNodeCPUTot *prometheus.GaugeVec + scontrolNodeCPUAllocated *prometheus.GaugeVec + scontrolNodeMemoryTot *prometheus.GaugeVec + scontrolNodeMemoryAllocated *prometheus.GaugeVec + scontrolNodeMemoryFree *prometheus.GaugeVec + scontrolNodeGPUTot *prometheus.GaugeVec + scontrolNodeGPUFree *prometheus.GaugeVec + isTest bool + nodeAddressSuffix string + + // Old metrics, keeping them for dashboards/alerts compatibility reasons + cpuAlloc *prometheus.GaugeVec + cpuIdle *prometheus.GaugeVec + cpuOther *prometheus.GaugeVec + cpuTotal *prometheus.GaugeVec + memAlloc *prometheus.GaugeVec + memTotal *prometheus.GaugeVec + alloc *prometheus.GaugeVec + comp *prometheus.GaugeVec + down *prometheus.GaugeVec + draining *prometheus.GaugeVec + drained *prometheus.GaugeVec + err *prometheus.GaugeVec + fail *prometheus.GaugeVec + idle *prometheus.GaugeVec + maint *prometheus.GaugeVec + mix *prometheus.GaugeVec + resv *prometheus.GaugeVec +} + +func NewNodesCollector(isTest bool, nodeAddressSuffix string) *nodesCollector { + labelsOldmetrics := []string{"node", "status"} + return &nodesCollector{ + isTest: isTest, + nodeAddressSuffix: nodeAddressSuffix, + scontrolNodesInfo: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Subsystem: "", + Name: "slurm_node_info", + Help: "Informations about nodes.", + }, + []string{"name", "arch", "partition", "feature", "address", "version", "os", "weight", "state", "reason"}), + scontrolNodeCPULoad: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Subsystem: "", + Name: "slurm_node_cpu_load", + Help: "CPU Load per node as reported by slurm CLI.", + }, + nodeResourcesLabels), + scontrolNodeCPUTot: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Subsystem: "", + Name: "slurm_node_cpu_tot", + Help: "CPU total available per node as reported by slurm CLI.", + }, + nodeResourcesLabels), + scontrolNodeCPUAllocated: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Subsystem: "", + Name: "slurm_node_cpu_allocated", + Help: "CPU Allocated per node as reported by slurm CLI.", + }, + nodeResourcesLabels), + scontrolNodeMemoryTot: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Subsystem: "", + Name: "slurm_node_memory_total_bytes", + Help: "Total memory per node as reported by slurm CLI.", + }, + nodeResourcesLabels), + scontrolNodeMemoryAllocated: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Subsystem: "", + Name: "slurm_node_memory_allocated_bytes", + Help: "Allocated memory per node as reported by slurm CLI.", + }, + nodeResourcesLabels), + scontrolNodeMemoryFree: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Subsystem: "", + Name: "slurm_node_memory_free_bytes", + Help: "Free memory per node as reported by slurm CLI.", + }, + nodeResourcesLabels), + scontrolNodeGPUTot: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Subsystem: "", + Name: "slurm_node_gpu_tot", + Help: "Number of total GPU on the node.", + }, + nodeResourcesLabels), + scontrolNodeGPUFree: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Subsystem: "", + Name: "slurm_node_gpu_free", + Help: "Number of free GPU on the node.", + }, + nodeResourcesLabels), + + // Old metrics, keeping them for dashboards/alerts compatibility reasons + + cpuAlloc: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "slurm_node_cpu_alloc", + Help: "Allocated CPUs per node", + }, labelsOldmetrics, + ), + cpuIdle: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "slurm_node_cpu_idle", + Help: "Idle CPUs per node", + }, labelsOldmetrics, + ), + cpuOther: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "slurm_node_cpu_other", + Help: "Other CPUs per node", + }, labelsOldmetrics, + ), + cpuTotal: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "slurm_node_cpu_total", + Help: "Total CPUs per node", + }, labelsOldmetrics, + ), + memAlloc: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "slurm_node_mem_alloc", + Help: "Allocated memory per node", + }, labelsOldmetrics, + ), + memTotal: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "slurm_node_mem_total", + Help: "Total memory per node", + }, labelsOldmetrics, + ), + alloc: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "slurm_nodes_alloc", + Help: "Allocated nodes", + }, []string{}, + ), + comp: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "slurm_nodes_comp", + Help: "Completing nodes", + }, []string{}, + ), + down: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "slurm_nodes_down", + Help: "Down nodes", + }, []string{}, + ), + draining: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "slurm_nodes_draining", + Help: "Draining nodes", + }, []string{}, + ), + drained: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "slurm_nodes_drained", + Help: "Draining nodes", + }, []string{}, + ), + err: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "slurm_nodes_err", + Help: "Error nodes", + }, []string{}, + ), + fail: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "slurm_nodes_fail", + Help: "Fail nodes", + }, []string{}, + ), + idle: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "slurm_nodes_idle", + Help: "Idle nodes", + }, []string{}, + ), + maint: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "slurm_nodes_maint", + Help: "Maint nodes", + }, []string{}, + ), + mix: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "slurm_nodes_mix", + Help: "Mix nodes", + }, []string{}, + ), + resv: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "slurm_nodes_resv", + Help: "Reserved nodes", + }, []string{}, + ), + } +} + +func (s *nodesCollector) getNodesMetrics() { + nodes := &NodeDetails{} + data := getData(s.isTest, showNodesDetailsCommand, showNodesDetailsTestDataInput) + err := json.Unmarshal([]byte(data), nodes) + if err != nil { + ExporterErrors.WithLabelValues("json-encoding-sinfo-nodes", err.Error()).Inc() + fmt.Println(err) + } + // create metrics from json object + for _, n := range nodes.Nodes { + // Prepare state and reason variables + state := evaluateState(n.State, n.StateFlags) + reason := "" + if n.Reason != "" { + reason = fmt.Sprintf("%s by %s", n.Reason, n.ReasonSetByUser) + } + // Iterating over partitions and active_features + for _, partition := range n.Partitions { + for _, feature := range strings.Split(n.ActiveFeatures, ",") { + fqdn := n.Address + s.nodeAddressSuffix + s.scontrolNodesInfo.WithLabelValues(n.Name, n.Architecture, partition, feature, fqdn, n.SlurmdVersion, n.OperatingSystem, strconv.Itoa(n.Weight), state, reason).Set(1) + } + // Now populating all other metrics + s.scontrolNodeCPUTot.WithLabelValues(n.Name, partition).Set(float64(n.Cpus)) + s.scontrolNodeCPUAllocated.WithLabelValues(n.Name, partition).Set(float64(n.AllocCpus)) + s.scontrolNodeCPULoad.WithLabelValues(n.Name, partition).Set(float64(n.CPULoad) / 100) + s.scontrolNodeMemoryTot.WithLabelValues(n.Name, partition).Set(float64(n.RealMemory)) + s.scontrolNodeMemoryFree.WithLabelValues(n.Name, partition).Set(float64(n.FreeMemory)) + s.scontrolNodeMemoryAllocated.WithLabelValues(n.Name, partition).Set(float64(n.AllocMemory)) + gpuTot := 0 + if n.Gres != "" { + // this is a bit obscure, but the standard gres configuration is `gpu:nvidia:3` and we need to get the 3 + gpuTot, err = strconv.Atoi(strings.Split(n.Gres, ":")[2]) + if err != nil { + ExporterErrors.WithLabelValues("atoi-gpu-tot", err.Error()).Inc() + fmt.Println(err) + } + + } + gpuUsed := 0 + if n.GresUsed != "gpu:0" { + // this is a bit obscure, but the standard gres configuration is `gpu:nvidia:0(IDX:N\/A)` and we need to get the 0 + gpuUsed, err = strconv.Atoi(strings.Split(strings.Split(n.GresUsed, "(")[0], ":")[2]) + if err != nil { + ExporterErrors.WithLabelValues("atoi-gpu-used", err.Error()).Inc() + fmt.Println(err) + } + } + s.scontrolNodeGPUTot.WithLabelValues(n.Name, partition).Set(float64(gpuTot)) + s.scontrolNodeGPUFree.WithLabelValues(n.Name, partition).Set(float64(gpuTot - gpuUsed)) + + } + + // Old metrics, keeping them for dashboards/alerts compatibility reasons + s.cpuAlloc.WithLabelValues(n.Name, state).Set(float64(n.AllocCpus)) + s.cpuIdle.WithLabelValues(n.Name, state).Set(float64(n.IdleCpus)) + s.cpuOther.WithLabelValues(n.Name, state).Set(float64(n.CPUBinding)) + s.cpuTotal.WithLabelValues(n.Name, state).Set(float64(n.Cpus)) + s.memAlloc.WithLabelValues(n.Name, state).Set(float64(n.AllocMemory)) + s.memTotal.WithLabelValues(n.Name, state).Set(float64(n.RealMemory)) + + s.aggregateNodeMetrics(state) + } +} + +// aggregateNodeMetrics aggregates metrics https://slurm.schedmd.com/sinfo.html +// This aggregation shoudl be done on prometheus level +// these are deprecated metrics +func (s *nodesCollector) aggregateNodeMetrics(state string) { + switch state { + case "ALLOCATED": + s.alloc.WithLabelValues().Inc() + case "COMPLETING": + s.comp.WithLabelValues().Inc() + case "DOWN": + s.down.WithLabelValues().Inc() + case "DRAINING": + s.draining.WithLabelValues().Inc() + case "DRAINED": + s.drained.WithLabelValues().Inc() + case "FAILING": + s.err.WithLabelValues().Inc() + case "FAIL": + s.fail.WithLabelValues().Inc() + case "IDLE": + s.idle.WithLabelValues().Inc() + case "MAINT": + s.maint.WithLabelValues().Inc() + case "MIXED": + s.mix.WithLabelValues().Inc() + case "RESERVED": + s.resv.WithLabelValues().Inc() + } +} + +func evaluateState(state string, stateFlags []string) string { + if len(stateFlags) == 0 { + return strings.ToUpper(state) + } + stateCombination := strings.ToUpper(state + "-" + strings.Join(stateFlags, "_")) + switch stateCombination { + case "IDLE-DRAIN": + return "DRAINED" + case "MIXED-DRAIN": + return "DRAINING" + case "ALLOCATED-DRAIN": + return "DRAINING" + case "DOWN-NOT_RESPONDING": + return "DOWN" + default: + return stateCombination + } +} + +func (s *nodesCollector) Describe(ch chan<- *prometheus.Desc) { + s.scontrolNodesInfo.Describe(ch) + s.scontrolNodeCPUAllocated.Describe(ch) + s.scontrolNodeCPULoad.Describe(ch) + s.scontrolNodeCPUTot.Describe(ch) + s.scontrolNodeMemoryAllocated.Describe(ch) + s.scontrolNodeMemoryFree.Describe(ch) + s.scontrolNodeMemoryTot.Describe(ch) + s.scontrolNodeGPUTot.Describe(ch) + s.scontrolNodeGPUFree.Describe(ch) + // Old metrics, keeping them for dashboards/alerts compatibility reasons + s.cpuAlloc.Describe(ch) + s.cpuIdle.Describe(ch) + s.cpuOther.Describe(ch) + s.cpuTotal.Describe(ch) + s.memAlloc.Describe(ch) + s.memTotal.Describe(ch) + s.alloc.Describe(ch) + s.comp.Describe(ch) + s.down.Describe(ch) + s.draining.Describe(ch) + s.drained.Describe(ch) + s.err.Describe(ch) + s.fail.Describe(ch) + s.idle.Describe(ch) + s.maint.Describe(ch) + s.mix.Describe(ch) + s.resv.Describe(ch) +} + +func (s *nodesCollector) Collect(ch chan<- prometheus.Metric) { + s.scontrolNodesInfo.Reset() + s.scontrolNodeCPUAllocated.Reset() + s.scontrolNodeCPULoad.Reset() + s.scontrolNodeCPUTot.Reset() + s.scontrolNodeMemoryAllocated.Reset() + s.scontrolNodeMemoryFree.Reset() + s.scontrolNodeMemoryTot.Reset() + s.scontrolNodeGPUFree.Reset() + s.scontrolNodeGPUTot.Reset() + // Old metrics, keeping them for dashboards/alerts compatibility reasons + s.cpuAlloc.Reset() + s.cpuIdle.Reset() + s.cpuOther.Reset() + s.cpuTotal.Reset() + s.memAlloc.Reset() + s.memTotal.Reset() + s.alloc.Reset() + s.comp.Reset() + s.down.Reset() + s.draining.Reset() + s.drained.Reset() + s.err.Reset() + s.fail.Reset() + s.idle.Reset() + s.maint.Reset() + s.mix.Reset() + s.resv.Reset() + s.getNodesMetrics() + s.scontrolNodesInfo.Collect(ch) + s.scontrolNodeCPUAllocated.Collect(ch) + s.scontrolNodeCPULoad.Collect(ch) + s.scontrolNodeCPUTot.Collect(ch) + s.scontrolNodeMemoryAllocated.Collect(ch) + s.scontrolNodeMemoryFree.Collect(ch) + s.scontrolNodeMemoryTot.Collect(ch) + s.scontrolNodeGPUFree.Collect(ch) + s.scontrolNodeGPUTot.Collect(ch) + // Old metrics, keeping them for dashboards/alerts compatibility reasons + s.cpuAlloc.Collect(ch) + s.cpuIdle.Collect(ch) + s.cpuOther.Collect(ch) + s.cpuTotal.Collect(ch) + s.memAlloc.Collect(ch) + s.memTotal.Collect(ch) + s.alloc.Collect(ch) + s.comp.Collect(ch) + s.down.Collect(ch) + s.draining.Collect(ch) + s.drained.Collect(ch) + s.err.Collect(ch) + s.fail.Collect(ch) + s.idle.Collect(ch) + s.maint.Collect(ch) + s.mix.Collect(ch) + s.resv.Collect(ch) +} + +type NodeDetails struct { + Meta struct { + Plugin struct { + Type string `json:"type"` + Name string `json:"name"` + } `json:"plugin"` + Slurm struct { + Version struct { + Major int `json:"major"` + Micro int `json:"micro"` + Minor int `json:"minor"` + } `json:"version"` + Release string `json:"release"` + } `json:"Slurm"` + } `json:"meta"` + Errors []interface{} `json:"errors"` + Nodes []struct { + Architecture string `json:"architecture"` + BurstbufferNetworkAddress string `json:"burstbuffer_network_address"` + Boards int `json:"boards"` + BootTime int `json:"boot_time"` + Comment string `json:"comment"` + Cores int `json:"cores"` + CPUBinding int `json:"cpu_binding"` + CPULoad int `json:"cpu_load"` + Extra string `json:"extra"` + FreeMemory int `json:"free_memory"` + Cpus int `json:"cpus"` + LastBusy int `json:"last_busy"` + Features string `json:"features"` + ActiveFeatures string `json:"active_features"` + Gres string `json:"gres"` + GresDrained string `json:"gres_drained"` + GresUsed string `json:"gres_used"` + McsLabel string `json:"mcs_label"` + Name string `json:"name"` + NextStateAfterReboot string `json:"next_state_after_reboot"` + Address string `json:"address"` + Hostname string `json:"hostname"` + State string `json:"state"` + StateFlags []string `json:"state_flags"` + NextStateAfterRebootFlags []string `json:"next_state_after_reboot_flags"` + OperatingSystem string `json:"operating_system"` + Owner interface{} `json:"owner"` + Partitions []string `json:"partitions"` + Port int `json:"port"` + RealMemory int `json:"real_memory"` + Reason string `json:"reason"` + ReasonChangedAt int `json:"reason_changed_at"` + ReasonSetByUser interface{} `json:"reason_set_by_user"` + SlurmdStartTime int `json:"slurmd_start_time"` + Sockets int `json:"sockets"` + Threads int `json:"threads"` + TemporaryDisk int `json:"temporary_disk"` + Weight int `json:"weight"` + Tres string `json:"tres"` + SlurmdVersion string `json:"slurmd_version"` + AllocMemory int `json:"alloc_memory"` + AllocCpus int `json:"alloc_cpus"` + IdleCpus int `json:"idle_cpus"` + TresUsed string `json:"tres_used"` + TresWeighted float64 `json:"tres_weighted"` + } `json:"nodes"` +} diff --git a/pkg/slurm/nodes_test.go b/pkg/slurm/nodes_test.go new file mode 100644 index 0000000..81d5893 --- /dev/null +++ b/pkg/slurm/nodes_test.go @@ -0,0 +1,48 @@ +/* Copyright 2017 Victor Penso, Matteo Dessalvi + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . */ + +package slurm + +// import ( +// "os" +// "testing" + +// "github.com/prometheus/client_golang/prometheus/testutil" +// "github.com/stretchr/testify/assert" +// ) + +// func TestNodesNodeMetrics(t *testing.T) { +// collector := NewNodesCollector(true, ".example.com") +// f, err := os.Open(showNodesDetailsTestDataProm) +// assert.NoError(t, err) +// defer f.Close() +// err = testutil.CollectAndCompare(collector, f) +// assert.NoError(t, err) +// } + +// TestGenerateNodesNodeMetrics is only used to getnerate the .prom file for +// func TestGenerateNodesNodeMetrics(t *testing.T) { +// reg := prometheus.NewRegistry() +// collector := NewNodesCollector(true, ".example.com") +// err := reg.Register(collector) +// assert.NoError(t, err) +// err = reg.Register(ExporterErrors) +// assert.NoError(t, err) +// _, err = reg.Gather() +// assert.NoError(t, err) +// gatherers := prometheus.Gatherers{reg} +// err = prometheus.WriteToTextfile(showNodesDetailsTestDataProm, gatherers) +// assert.NoError(t, err) +// } diff --git a/pkg/slurm/partitions.go b/pkg/slurm/partitions.go new file mode 100644 index 0000000..d024657 --- /dev/null +++ b/pkg/slurm/partitions.go @@ -0,0 +1,134 @@ +/* Copyright 2020 Victor Penso + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . */ + +package slurm + +import ( + "strconv" + "strings" + + "github.com/prometheus/client_golang/prometheus" +) + +type PartitionMetrics struct { + allocated float64 + idle float64 + other float64 + pending float64 + running float64 + total float64 +} + +func ParsePartitionsMetrics() map[string]*PartitionMetrics { + partitions := make(map[string]*PartitionMetrics) + out := execCommand("sinfo -h -o%R,%C") + lines := strings.Split(out, "\n") + for _, line := range lines { + if strings.Contains(line, ",") { + // name of a partition + partition := strings.Split(line, ",")[0] + _, key := partitions[partition] + if !key { + partitions[partition] = &PartitionMetrics{0, 0, 0, 0, 0, 0} + } + states := strings.Split(line, ",")[1] + allocated, _ := strconv.ParseFloat(strings.Split(states, "/")[0], 64) + idle, _ := strconv.ParseFloat(strings.Split(states, "/")[1], 64) + other, _ := strconv.ParseFloat(strings.Split(states, "/")[2], 64) + total, _ := strconv.ParseFloat(strings.Split(states, "/")[3], 64) + partitions[partition].allocated = allocated + partitions[partition].idle = idle + partitions[partition].other = other + partitions[partition].total = total + } + } + // get list of pending jobs by partition name + pendingOut := execCommand("squeue -a -r -h -o%P --states=PENDING") + list := strings.Split(pendingOut, "\n") + for _, partition := range list { + // accumulate the number of pending jobs + _, key := partitions[partition] + if key { + partitions[partition].pending += 1 + } + } + + // get list of running jobs by partition name + runningOut := execCommand("squeue -a -r -h -o%P --states=RUNNING") + list_r := strings.Split(runningOut, "\n") + for _, partition := range list_r { + // accumulate the number of running jobs + _, key := partitions[partition] + if key { + partitions[partition].running += 1 + } + } + + return partitions +} + +type PartitionsCollector struct { + allocated *prometheus.Desc + idle *prometheus.Desc + other *prometheus.Desc + pending *prometheus.Desc + running *prometheus.Desc + total *prometheus.Desc +} + +func NewPartitionsCollector() *PartitionsCollector { + labels := []string{"partition"} + return &PartitionsCollector{ + allocated: prometheus.NewDesc("slurm_partition_cpus_allocated", "Allocated CPUs for partition", labels, nil), + idle: prometheus.NewDesc("slurm_partition_cpus_idle", "Idle CPUs for partition", labels, nil), + other: prometheus.NewDesc("slurm_partition_cpus_other", "Other CPUs for partition", labels, nil), + pending: prometheus.NewDesc("slurm_partition_jobs_pending", "Pending jobs for partition", labels, nil), + running: prometheus.NewDesc("slurm_partition_jobs_running", "Running jobs for partition", labels, nil), + total: prometheus.NewDesc("slurm_partition_cpus_total", "Total CPUs for partition", labels, nil), + } +} + +func (pc *PartitionsCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- pc.allocated + ch <- pc.idle + ch <- pc.other + ch <- pc.pending + ch <- pc.running + ch <- pc.total +} + +func (pc *PartitionsCollector) Collect(ch chan<- prometheus.Metric) { + pm := ParsePartitionsMetrics() + for p := range pm { + if pm[p].allocated > 0 { + ch <- prometheus.MustNewConstMetric(pc.allocated, prometheus.GaugeValue, pm[p].allocated, p) + } + if pm[p].idle > 0 { + ch <- prometheus.MustNewConstMetric(pc.idle, prometheus.GaugeValue, pm[p].idle, p) + } + if pm[p].other > 0 { + ch <- prometheus.MustNewConstMetric(pc.other, prometheus.GaugeValue, pm[p].other, p) + } + if pm[p].pending > 0 { + ch <- prometheus.MustNewConstMetric(pc.pending, prometheus.GaugeValue, pm[p].pending, p) + } + if pm[p].running > 0 { + ch <- prometheus.MustNewConstMetric(pc.running, prometheus.GaugeValue, pm[p].running, p) + } + if pm[p].total > 0 { + ch <- prometheus.MustNewConstMetric(pc.total, prometheus.GaugeValue, pm[p].total, p) + } + } +} diff --git a/queue.go b/pkg/slurm/queue.go similarity index 54% rename from queue.go rename to pkg/slurm/queue.go index 1f701cf..3818e39 100644 --- a/queue.go +++ b/pkg/slurm/queue.go @@ -13,39 +13,40 @@ GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . */ -package main +package slurm import ( - "github.com/prometheus/client_golang/prometheus" - "io/ioutil" - "log" - "os/exec" "strings" + + "github.com/prometheus/client_golang/prometheus" +) + +const ( + queueCommand = "squeue -a -r -h -o %A,%T,%r --states=all" + queueTestData = "./test_data/squeue.txt" ) type QueueMetrics struct { - pending float64 - pending_dep float64 - running float64 - suspended float64 - cancelled float64 - completing float64 - completed float64 - configuring float64 - failed float64 - timeout float64 - preempted float64 - node_fail float64 + pending float64 + pending_dep float64 + running float64 + suspended float64 + cancelled float64 + completing float64 + completed float64 + configuring float64 + failed float64 + timeout float64 + preempted float64 + node_fail float64 + out_of_memory float64 } -// Returns the scheduler metrics -func QueueGetMetrics() *QueueMetrics { - return ParseQueueMetrics(QueueData()) -} +func (qc *QueueCollector) QueueGetMetrics() *QueueMetrics { + data := getData(qc.isTest, queueCommand, queueTestData) -func ParseQueueMetrics(input []byte) *QueueMetrics { var qm QueueMetrics - lines := strings.Split(string(input), "\n") + lines := strings.Split(data, "\n") for _, line := range lines { if strings.Contains(line, ",") { splitted := strings.Split(line, ",") @@ -76,65 +77,54 @@ func ParseQueueMetrics(input []byte) *QueueMetrics { qm.preempted++ case "NODE_FAIL": qm.node_fail++ + case "OUT_OF_MEMORY": + qm.out_of_memory++ } } } return &qm } -// Execute the squeue command and return its output -func QueueData() []byte { - cmd := exec.Command("squeue", "-a", "-r", "-h", "-o %A,%T,%r", "--states=all") - stdout, err := cmd.StdoutPipe() - if err != nil { - log.Fatal(err) - } - if err := cmd.Start(); err != nil { - log.Fatal(err) - } - out, _ := ioutil.ReadAll(stdout) - if err := cmd.Wait(); err != nil { - log.Fatal(err) - } - return out -} - /* * Implement the Prometheus Collector interface and feed the * Slurm queue metrics into it. * https://godoc.org/github.com/prometheus/client_golang/prometheus#Collector */ -func NewQueueCollector() *QueueCollector { +func NewQueueCollector(isTest bool) *QueueCollector { return &QueueCollector{ - pending: prometheus.NewDesc("slurm_queue_pending", "Pending jobs in queue", nil, nil), - pending_dep: prometheus.NewDesc("slurm_queue_pending_dependency", "Pending jobs because of dependency in queue", nil, nil), - running: prometheus.NewDesc("slurm_queue_running", "Running jobs in the cluster", nil, nil), - suspended: prometheus.NewDesc("slurm_queue_suspended", "Suspended jobs in the cluster", nil, nil), - cancelled: prometheus.NewDesc("slurm_queue_cancelled", "Cancelled jobs in the cluster", nil, nil), - completing: prometheus.NewDesc("slurm_queue_completing", "Completing jobs in the cluster", nil, nil), - completed: prometheus.NewDesc("slurm_queue_completed", "Completed jobs in the cluster", nil, nil), - configuring: prometheus.NewDesc("slurm_queue_configuring", "Configuring jobs in the cluster", nil, nil), - failed: prometheus.NewDesc("slurm_queue_failed", "Number of failed jobs", nil, nil), - timeout: prometheus.NewDesc("slurm_queue_timeout", "Jobs stopped by timeout", nil, nil), - preempted: prometheus.NewDesc("slurm_queue_preempted", "Number of preempted jobs", nil, nil), - node_fail: prometheus.NewDesc("slurm_queue_node_fail", "Number of jobs stopped due to node fail", nil, nil), + isTest: isTest, + pending: prometheus.NewDesc("slurm_queue_pending", "Pending jobs in queue", nil, nil), + pending_dep: prometheus.NewDesc("slurm_queue_pending_dependency", "Pending jobs because of dependency in queue", nil, nil), + running: prometheus.NewDesc("slurm_queue_running", "Running jobs in the cluster", nil, nil), + suspended: prometheus.NewDesc("slurm_queue_suspended", "Suspended jobs in the cluster", nil, nil), + cancelled: prometheus.NewDesc("slurm_queue_cancelled", "Cancelled jobs in the cluster", nil, nil), + completing: prometheus.NewDesc("slurm_queue_completing", "Completing jobs in the cluster", nil, nil), + completed: prometheus.NewDesc("slurm_queue_completed", "Completed jobs in the cluster", nil, nil), + configuring: prometheus.NewDesc("slurm_queue_configuring", "Configuring jobs in the cluster", nil, nil), + failed: prometheus.NewDesc("slurm_queue_failed", "Number of failed jobs", nil, nil), + timeout: prometheus.NewDesc("slurm_queue_timeout", "Jobs stopped by timeout", nil, nil), + preempted: prometheus.NewDesc("slurm_queue_preempted", "Number of preempted jobs", nil, nil), + node_fail: prometheus.NewDesc("slurm_queue_node_fail", "Number of jobs stopped due to node fail", nil, nil), + out_of_memory: prometheus.NewDesc("slurm_queue_out_of_memory", "Number of jobs stopped by oomkiller", nil, nil), } } type QueueCollector struct { - pending *prometheus.Desc - pending_dep *prometheus.Desc - running *prometheus.Desc - suspended *prometheus.Desc - cancelled *prometheus.Desc - completing *prometheus.Desc - completed *prometheus.Desc - configuring *prometheus.Desc - failed *prometheus.Desc - timeout *prometheus.Desc - preempted *prometheus.Desc - node_fail *prometheus.Desc + isTest bool + pending *prometheus.Desc + pending_dep *prometheus.Desc + running *prometheus.Desc + suspended *prometheus.Desc + cancelled *prometheus.Desc + completing *prometheus.Desc + completed *prometheus.Desc + configuring *prometheus.Desc + failed *prometheus.Desc + timeout *prometheus.Desc + preempted *prometheus.Desc + node_fail *prometheus.Desc + out_of_memory *prometheus.Desc } func (qc *QueueCollector) Describe(ch chan<- *prometheus.Desc) { @@ -150,10 +140,11 @@ func (qc *QueueCollector) Describe(ch chan<- *prometheus.Desc) { ch <- qc.timeout ch <- qc.preempted ch <- qc.node_fail + ch <- qc.out_of_memory } func (qc *QueueCollector) Collect(ch chan<- prometheus.Metric) { - qm := QueueGetMetrics() + qm := qc.QueueGetMetrics() ch <- prometheus.MustNewConstMetric(qc.pending, prometheus.GaugeValue, qm.pending) ch <- prometheus.MustNewConstMetric(qc.pending_dep, prometheus.GaugeValue, qm.pending_dep) ch <- prometheus.MustNewConstMetric(qc.running, prometheus.GaugeValue, qm.running) @@ -166,4 +157,5 @@ func (qc *QueueCollector) Collect(ch chan<- prometheus.Metric) { ch <- prometheus.MustNewConstMetric(qc.timeout, prometheus.GaugeValue, qm.timeout) ch <- prometheus.MustNewConstMetric(qc.preempted, prometheus.GaugeValue, qm.preempted) ch <- prometheus.MustNewConstMetric(qc.node_fail, prometheus.GaugeValue, qm.node_fail) + ch <- prometheus.MustNewConstMetric(qc.out_of_memory, prometheus.GaugeValue, qm.out_of_memory) } diff --git a/queue_test.go b/pkg/slurm/queue_test.go similarity index 68% rename from queue_test.go rename to pkg/slurm/queue_test.go index a9ab379..a15664b 100644 --- a/queue_test.go +++ b/pkg/slurm/queue_test.go @@ -13,24 +13,13 @@ GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . */ -package main +package slurm import ( - "io/ioutil" - "os" "testing" ) -func TestParseQueueMetrics(t *testing.T) { - // Read the input data from a file - file, err := os.Open("test_data/squeue.txt") - if err != nil { - t.Fatalf("Can not open test data: %v", err) - } - data, err := ioutil.ReadAll(file) - t.Logf("%+v", ParseQueueMetrics(data)) -} - func TestQueueGetMetrics(t *testing.T) { - t.Logf("%+v", QueueGetMetrics()) + collector := NewQueueCollector(true) + t.Logf("%+v", collector.QueueGetMetrics()) } diff --git a/scheduler.go b/pkg/slurm/scheduler.go similarity index 93% rename from scheduler.go rename to pkg/slurm/scheduler.go index aac9b30..9496790 100644 --- a/scheduler.go +++ b/pkg/slurm/scheduler.go @@ -13,16 +13,19 @@ GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . */ -package main +package slurm import ( - "github.com/prometheus/client_golang/prometheus" - "io/ioutil" - "log" - "os/exec" "regexp" "strconv" "strings" + + "github.com/prometheus/client_golang/prometheus" +) + +const ( + schedulerCommand = "sdiag" + schedulerTestData = "test_data/sdiag.txt" ) /* @@ -47,27 +50,11 @@ type SchedulerMetrics struct { total_backfilled_heterogeneous float64 } -// Execute the sdiag command and return its output -func SchedulerData() []byte { - cmd := exec.Command("sdiag") - stdout, err := cmd.StdoutPipe() - if err != nil { - log.Fatal(err) - } - if err := cmd.Start(); err != nil { - log.Fatal(err) - } - out, _ := ioutil.ReadAll(stdout) - if err := cmd.Wait(); err != nil { - log.Fatal(err) - } - return out -} - // Extract the relevant metrics from the sdiag output -func ParseSchedulerMetrics(input []byte) *SchedulerMetrics { +func (sc *SchedulerCollector) SchedulerGetMetrics() *SchedulerMetrics { var sm SchedulerMetrics - lines := strings.Split(string(input), "\n") + out := getData(sc.isTest, schedulerCommand, schedulerTestData) + lines := strings.Split(out, "\n") // Guard variables to check for string repetitions in the output of sdiag // (two occurencies of the following strings: 'Last cycle', 'Mean cycle') lc_count := 0 @@ -124,11 +111,6 @@ func ParseSchedulerMetrics(input []byte) *SchedulerMetrics { return &sm } -// Returns the scheduler metrics -func SchedulerGetMetrics() *SchedulerMetrics { - return ParseSchedulerMetrics(SchedulerData()) -} - /* * Implement the Prometheus Collector interface and feed the * Slurm scheduler metrics into it. @@ -137,6 +119,7 @@ func SchedulerGetMetrics() *SchedulerMetrics { // Collector strcture type SchedulerCollector struct { + isTest bool threads *prometheus.Desc queue_size *prometheus.Desc dbd_queue_size *prometheus.Desc @@ -169,7 +152,7 @@ func (c *SchedulerCollector) Describe(ch chan<- *prometheus.Desc) { // Send the values of all metrics func (sc *SchedulerCollector) Collect(ch chan<- prometheus.Metric) { - sm := SchedulerGetMetrics() + sm := sc.SchedulerGetMetrics() ch <- prometheus.MustNewConstMetric(sc.threads, prometheus.GaugeValue, sm.threads) ch <- prometheus.MustNewConstMetric(sc.queue_size, prometheus.GaugeValue, sm.queue_size) ch <- prometheus.MustNewConstMetric(sc.dbd_queue_size, prometheus.GaugeValue, sm.dbd_queue_size) @@ -185,8 +168,9 @@ func (sc *SchedulerCollector) Collect(ch chan<- prometheus.Metric) { } // Returns the Slurm scheduler collector, used to register with the prometheus client -func NewSchedulerCollector() *SchedulerCollector { +func NewSchedulerCollector(isTest bool) *SchedulerCollector { return &SchedulerCollector{ + isTest: isTest, threads: prometheus.NewDesc( "slurm_scheduler_threads", "Information provided by the Slurm sdiag command, number of scheduler threads ", diff --git a/scheduler_test.go b/pkg/slurm/scheduler_test.go similarity index 68% rename from scheduler_test.go rename to pkg/slurm/scheduler_test.go index 4a63c72..dd580a8 100644 --- a/scheduler_test.go +++ b/pkg/slurm/scheduler_test.go @@ -13,24 +13,13 @@ GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . */ -package main +package slurm import ( - "io/ioutil" - "os" "testing" ) -func TestSchedulerMetrics(t *testing.T) { - // Read the input data from a file - file, err := os.Open("test_data/sdiag.txt") - if err != nil { - t.Fatalf("Can not open test data: %v", err) - } - data, err := ioutil.ReadAll(file) - t.Logf("%+v", ParseSchedulerMetrics(data)) -} - func TestSchedulerGetMetrics(t *testing.T) { - t.Logf("%+v", SchedulerGetMetrics()) + coll := NewSchedulerCollector(true) + t.Logf("%+v", coll.SchedulerGetMetrics()) } diff --git a/pkg/slurm/slurm.go b/pkg/slurm/slurm.go new file mode 100644 index 0000000..1abc940 --- /dev/null +++ b/pkg/slurm/slurm.go @@ -0,0 +1,146 @@ +/* Copyright 2017 Victor Penso, Matteo Dessalvi + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . */ + +package slurm + +import ( + "context" + "fmt" + "io/ioutil" + "os/exec" + "strings" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/vpenso/prometheus-slurm-exporter/pkg/ldapsearch" +) + +var ( + ExporterErrors = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Subsystem: "", + Name: "slurm_exporter_errors_total", + Help: "Total number of Errors from the exporter.", + }, + []string{"command", "reason"}) + ExecDuration = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Subsystem: "", + Name: "slurm_exporter_exec_duration", + Help: "Duration of exec commands.", + }, + []string{"command"}) + execTimeoutSeconds = 10 +) + +func getData(isTest bool, command, file string) string { + if isTest { + return readFile(file) + } + return execCommand(command) +} + +func execCommand(command string) string { + cmdList := strings.Split(command, " ") + before := time.Now() + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(execTimeoutSeconds)*time.Second) + defer cancel() + out, err := exec.CommandContext(ctx, cmdList[0], cmdList[1:]...).Output() + elapsed := time.Since(before) + ExecDuration.WithLabelValues(command).Observe(elapsed.Seconds()) + if err != nil { + ExporterErrors.WithLabelValues(command, err.Error()).Inc() + return "" + } + return string(out) +} + +func readFile(filePath string) string { + rawData, err := ioutil.ReadFile(filePath) + if err != nil { + ExporterErrors.WithLabelValues("readFile", err.Error()).Inc() + fmt.Println(err) + } + return string(rawData) +} + +func NewRegistry(gpuCollectorEnabled bool, exectimeout int, nodeAddressSuffix, ldapServer, ldapBaseSearch string) (*prometheus.Registry, error) { + reg := prometheus.NewRegistry() + execTimeoutSeconds = exectimeout + err := reg.Register(NewAccountsCollector()) // from accounts.go + if err != nil { + return nil, err + } + err = reg.Register(NewCPUsCollector(false)) // from cpus.go + if err != nil { + return nil, err + } + err = reg.Register(NewPartitionsCollector()) // from partitions.go + if err != nil { + return nil, err + } + err = reg.Register(NewQueueCollector(false)) // from queue.go + if err != nil { + return nil, err + } + err = reg.Register(NewSchedulerCollector(false)) // from scheduler.go + if err != nil { + return nil, err + } + err = reg.Register(NewFairShareCollector()) // from sshare.go + if err != nil { + return nil, err + } + err = reg.Register(NewUsersCollector()) // from users.go + if err != nil { + return nil, err + } + err = reg.Register(ExporterErrors) // from this file + if err != nil { + return nil, err + } + err = reg.Register(ExecDuration) // from this file + if err != nil { + return nil, err + } + err = reg.Register(NewNodesCollector(false, nodeAddressSuffix)) // from scontrol.go + if err != nil { + return nil, err + } + if ldapServer != "" { + ldap, err := ldapsearch.Init("", ldapServer, ldapBaseSearch) + if err != nil { + ExporterErrors.WithLabelValues("ldapsearch", err.Error()).Inc() + fmt.Println(err) + } + err = reg.Register(NewJobsCollector(false, ldap)) // from jobs.go + if err != nil { + return nil, err + } + } else { + err = reg.Register(NewJobsCollector(false, nil)) // from jobs.go + if err != nil { + return nil, err + } + } + if gpuCollectorEnabled { + err = reg.Register(NewGPUsCollector()) // from gpus.go + if err != nil { + return nil, err + } + } + + return reg, nil +} diff --git a/pkg/slurm/sshare.go b/pkg/slurm/sshare.go new file mode 100644 index 0000000..2e5c6a9 --- /dev/null +++ b/pkg/slurm/sshare.go @@ -0,0 +1,69 @@ +/* Copyright 2021 Victor Penso + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . */ + +package slurm + +import ( + "strconv" + "strings" + + "github.com/prometheus/client_golang/prometheus" +) + +type FairShareMetrics struct { + fairshare float64 +} + +func ParseFairShareMetrics() map[string]*FairShareMetrics { + accounts := make(map[string]*FairShareMetrics) + out := execCommand("sshare -n -P -o account,fairshare") + lines := strings.Split(out, "\n") + for _, line := range lines { + if !strings.HasPrefix(line, " ") { + if strings.Contains(line, "|") { + account := strings.Trim(strings.Split(line, "|")[0], " ") + _, key := accounts[account] + if !key { + accounts[account] = &FairShareMetrics{0} + } + fairshare, _ := strconv.ParseFloat(strings.Split(line, "|")[1], 64) + accounts[account].fairshare = fairshare + } + } + } + return accounts +} + +type FairShareCollector struct { + fairshare *prometheus.Desc +} + +func NewFairShareCollector() *FairShareCollector { + labels := []string{"account"} + return &FairShareCollector{ + fairshare: prometheus.NewDesc("slurm_account_fairshare", "FairShare for account", labels, nil), + } +} + +func (fsc *FairShareCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- fsc.fairshare +} + +func (fsc *FairShareCollector) Collect(ch chan<- prometheus.Metric) { + fsm := ParseFairShareMetrics() + for f := range fsm { + ch <- prometheus.MustNewConstMetric(fsc.fairshare, prometheus.GaugeValue, fsm[f].fairshare, f) + } +} diff --git a/test_data/sdiag.txt b/pkg/slurm/test_data/sdiag.txt similarity index 100% rename from test_data/sdiag.txt rename to pkg/slurm/test_data/sdiag.txt diff --git a/pkg/slurm/test_data/sinfo-nodes.txt b/pkg/slurm/test_data/sinfo-nodes.txt new file mode 100644 index 0000000..b678e7c --- /dev/null +++ b/pkg/slurm/test_data/sinfo-nodes.txt @@ -0,0 +1,5 @@ +94,allocated +1,down* +2,draining +1,drained +54,mixed \ No newline at end of file diff --git a/test_data/sinfo.txt b/pkg/slurm/test_data/sinfo.txt similarity index 100% rename from test_data/sinfo.txt rename to pkg/slurm/test_data/sinfo.txt diff --git a/test_data/sinfo_cpus.txt b/pkg/slurm/test_data/sinfo_cpus.txt similarity index 100% rename from test_data/sinfo_cpus.txt rename to pkg/slurm/test_data/sinfo_cpus.txt diff --git a/test_data/sinfo_mem.txt b/pkg/slurm/test_data/sinfo_mem.txt similarity index 100% rename from test_data/sinfo_mem.txt rename to pkg/slurm/test_data/sinfo_mem.txt diff --git a/test_data/squeue.txt b/pkg/slurm/test_data/squeue.txt similarity index 100% rename from test_data/squeue.txt rename to pkg/slurm/test_data/squeue.txt diff --git a/pkg/slurm/users.go b/pkg/slurm/users.go new file mode 100644 index 0000000..99464f7 --- /dev/null +++ b/pkg/slurm/users.go @@ -0,0 +1,104 @@ +/* Copyright 2020 Victor Penso + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . */ + +package slurm + +import ( + "regexp" + "strconv" + "strings" + + "github.com/prometheus/client_golang/prometheus" +) + +type UserJobMetrics struct { + pending float64 + running float64 + running_cpus float64 + suspended float64 +} + +func ParseUsersMetrics() map[string]*UserJobMetrics { + users := make(map[string]*UserJobMetrics) + out := execCommand("squeue -a -r -h -o %A|%u|%T|%C") + lines := strings.Split(out, "\n") + for _, line := range lines { + if strings.Contains(line, "|") { + user := strings.Split(line, "|")[1] + _, key := users[user] + if !key { + users[user] = &UserJobMetrics{0, 0, 0, 0} + } + state := strings.Split(line, "|")[2] + state = strings.ToLower(state) + cpus, _ := strconv.ParseFloat(strings.Split(line, "|")[3], 64) + pending := regexp.MustCompile(`^pending`) + running := regexp.MustCompile(`^running`) + suspended := regexp.MustCompile(`^suspended`) + switch { + case pending.MatchString(state) == true: + users[user].pending++ + case running.MatchString(state) == true: + users[user].running++ + users[user].running_cpus += cpus + case suspended.MatchString(state) == true: + users[user].suspended++ + } + } + } + return users +} + +type UsersCollector struct { + pending *prometheus.Desc + running *prometheus.Desc + running_cpus *prometheus.Desc + suspended *prometheus.Desc +} + +func NewUsersCollector() *UsersCollector { + labels := []string{"user"} + return &UsersCollector{ + pending: prometheus.NewDesc("slurm_user_jobs_pending", "Pending jobs for user", labels, nil), + running: prometheus.NewDesc("slurm_user_jobs_running", "Running jobs for user", labels, nil), + running_cpus: prometheus.NewDesc("slurm_user_cpus_running", "Running cpus for user", labels, nil), + suspended: prometheus.NewDesc("slurm_user_jobs_suspended", "Suspended jobs for user", labels, nil), + } +} + +func (uc *UsersCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- uc.pending + ch <- uc.running + ch <- uc.running_cpus + ch <- uc.suspended +} + +func (uc *UsersCollector) Collect(ch chan<- prometheus.Metric) { + um := ParseUsersMetrics() + for u := range um { + if um[u].pending > 0 { + ch <- prometheus.MustNewConstMetric(uc.pending, prometheus.GaugeValue, um[u].pending, u) + } + if um[u].running > 0 { + ch <- prometheus.MustNewConstMetric(uc.running, prometheus.GaugeValue, um[u].running, u) + } + if um[u].running_cpus > 0 { + ch <- prometheus.MustNewConstMetric(uc.running_cpus, prometheus.GaugeValue, um[u].running_cpus, u) + } + if um[u].suspended > 0 { + ch <- prometheus.MustNewConstMetric(uc.suspended, prometheus.GaugeValue, um[u].suspended, u) + } + } +} diff --git a/sshare.go b/sshare.go deleted file mode 100644 index ecbaa69..0000000 --- a/sshare.go +++ /dev/null @@ -1,86 +0,0 @@ -/* Copyright 2021 Victor Penso - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . */ - -package main - -import ( - "io/ioutil" - "os/exec" - "log" - "strings" - "strconv" - "github.com/prometheus/client_golang/prometheus" -) - -func FairShareData() []byte { - cmd := exec.Command( "sshare", "-n", "-P", "-o", "account,fairshare" ) - stdout, err := cmd.StdoutPipe() - if err != nil { - log.Fatal(err) - } - if err := cmd.Start(); err != nil { - log.Fatal(err) - } - out, _ := ioutil.ReadAll(stdout) - if err := cmd.Wait(); err != nil { - log.Fatal(err) - } - return out -} - -type FairShareMetrics struct { - fairshare float64 -} - -func ParseFairShareMetrics() map[string]*FairShareMetrics { - accounts := make(map[string]*FairShareMetrics) - lines := strings.Split(string(FairShareData()), "\n") - for _, line := range lines { - if ! strings.HasPrefix(line," ") { - if strings.Contains(line,"|") { - account := strings.Trim(strings.Split(line,"|")[0]," ") - _,key := accounts[account] - if !key { - accounts[account] = &FairShareMetrics{0} - } - fairshare,_ := strconv.ParseFloat(strings.Split(line,"|")[1],64) - accounts[account].fairshare = fairshare - } - } - } - return accounts -} - -type FairShareCollector struct { - fairshare *prometheus.Desc -} - -func NewFairShareCollector() *FairShareCollector { - labels := []string{"account"} - return &FairShareCollector{ - fairshare: prometheus.NewDesc("slurm_account_fairshare","FairShare for account" , labels,nil), - } -} - -func (fsc *FairShareCollector) Describe(ch chan<- *prometheus.Desc) { - ch <- fsc.fairshare -} - -func (fsc *FairShareCollector) Collect(ch chan<- prometheus.Metric) { - fsm := ParseFairShareMetrics() - for f := range fsm { - ch <- prometheus.MustNewConstMetric(fsc.fairshare, prometheus.GaugeValue, fsm[f].fairshare, f) - } -} diff --git a/users.go b/users.go deleted file mode 100644 index 2b0e85e..0000000 --- a/users.go +++ /dev/null @@ -1,122 +0,0 @@ -/* Copyright 2020 Victor Penso - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . */ - -package main - -import ( - "io/ioutil" - "os/exec" - "log" - "strings" - "strconv" - "regexp" - "github.com/prometheus/client_golang/prometheus" -) - -func UsersData() []byte { - cmd := exec.Command("squeue","-a","-r","-h","-o %A|%u|%T|%C") - stdout, err := cmd.StdoutPipe() - if err != nil { - log.Fatal(err) - } - if err := cmd.Start(); err != nil { - log.Fatal(err) - } - out, _ := ioutil.ReadAll(stdout) - if err := cmd.Wait(); err != nil { - log.Fatal(err) - } - return out -} - -type UserJobMetrics struct { - pending float64 - running float64 - running_cpus float64 - suspended float64 -} - -func ParseUsersMetrics(input []byte) map[string]*UserJobMetrics { - users := make(map[string]*UserJobMetrics) - lines := strings.Split(string(input), "\n") - for _, line := range lines { - if strings.Contains(line,"|") { - user := strings.Split(line,"|")[1] - _,key := users[user] - if !key { - users[user] = &UserJobMetrics{0,0,0,0} - } - state := strings.Split(line,"|")[2] - state = strings.ToLower(state) - cpus,_ := strconv.ParseFloat(strings.Split(line,"|")[3],64) - pending := regexp.MustCompile(`^pending`) - running := regexp.MustCompile(`^running`) - suspended := regexp.MustCompile(`^suspended`) - switch { - case pending.MatchString(state) == true: - users[user].pending++ - case running.MatchString(state) == true: - users[user].running++ - users[user].running_cpus += cpus - case suspended.MatchString(state) == true: - users[user].suspended++ - } - } - } - return users -} - -type UsersCollector struct { - pending *prometheus.Desc - running *prometheus.Desc - running_cpus *prometheus.Desc - suspended *prometheus.Desc -} - -func NewUsersCollector() *UsersCollector { - labels := []string{"user"} - return &UsersCollector { - pending: prometheus.NewDesc("slurm_user_jobs_pending", "Pending jobs for user", labels, nil), - running: prometheus.NewDesc("slurm_user_jobs_running", "Running jobs for user", labels, nil), - running_cpus: prometheus.NewDesc("slurm_user_cpus_running", "Running cpus for user", labels, nil), - suspended: prometheus.NewDesc("slurm_user_jobs_suspended", "Suspended jobs for user", labels, nil), - } -} - -func (uc *UsersCollector) Describe(ch chan<- *prometheus.Desc) { - ch <- uc.pending - ch <- uc.running - ch <- uc.running_cpus - ch <- uc.suspended -} - -func (uc *UsersCollector) Collect(ch chan<- prometheus.Metric) { - um := ParseUsersMetrics(UsersData()) - for u := range um { - if um[u].pending > 0 { - ch <- prometheus.MustNewConstMetric(uc.pending, prometheus.GaugeValue, um[u].pending, u) - } - if um[u].running > 0 { - ch <- prometheus.MustNewConstMetric(uc.running, prometheus.GaugeValue, um[u].running, u) - } - if um[u].running_cpus > 0 { - ch <- prometheus.MustNewConstMetric(uc.running_cpus, prometheus.GaugeValue, um[u].running_cpus, u) - } - if um[u].suspended > 0 { - ch <- prometheus.MustNewConstMetric(uc.suspended, prometheus.GaugeValue, um[u].suspended, u) - } - } -} -