Merge pull request #4 from treydock/hostkey

Add known_hosts and host_key_algorithms options to configuration to support verifying SSH host keys

CHANGELOG.md

@@ -1,6 +1,8 @@
 ## 0.2.0 / TBD
 
 * Update to 1.15 and update Go module dependencies
+* Add `known_hosts` configuration option to allow verifying SSH hosts against known hosts
+* Add `host_key_algorithms` configuration option to specify host key algorithms to use when verifying SSH hosts
 
 ## 0.1.1 / 2020-04-01
 

README.md

@@ -33,6 +33,15 @@ modules:
   password:
     id: prometheus
     password: secret
+  verify:
+    user: prometheus
+    private_key: /home/prometheus/.ssh/id_rsa
+    known_hosts: /etc/ssh/ssh_known_hosts
+    host_key_algorithms:
+    - ssh-rsa
+    command: uptime
+    command_expect: "load average"
+    timeout: 5
 ```
 
 Example with curl would query host1 with the password module and host2 with the default module.
@@ -47,6 +56,9 @@ Configuration options for each module:
 * `user` - The username for the SSH connection
 * `password` - The password for the SSH connection, required if `private_key` is not specified
 * `private_key` - The SSH private key for the SSH connection, required if `password` is not specified
+* `known_hosts` - Optional SSH known hosts file to use to verify hosts
+* `host_key_algorithms` - Optional list of SSH host key algorithms to use
+  * See constants beginning with `KeyAlgo*` in [crypto/ssh](https://godoc.org/golang.org/x/crypto/ssh#pkg-constants)
 * `timeout` - Optional timeout of the SSH connection, session and optional command.
     * The default comes from the `--collector.ssh.default-timeout` flag.
 * `command` - Optional command to run.
@@ -110,15 +122,27 @@ The following example assumes this exporter is running on the Prometheus server
   metrics_path: /ssh
   static_configs:
   - targets:
-    - ssh1.example.com
-    - ssh2.example.com
+    - host1.example.com:22
+    - host2.example.com:22
+    labels:
+      module: default
+  - targets:
+    - host3.example.com:22
+    - host4.example.com:22
+    labels:
+      module: verify
   relabel_configs:
   - source_labels: [__address__]
     target_label: __param_target
   - source_labels: [__param_target]
     target_label: instance
   - target_label: __address__
     replacement: 127.0.0.1:9312
+  - source_labels: [module]
+    target_label: __param_module
+  metric_relabel_configs:
+  - regex: "^(module)$"
+    action: labeldrop
 - job_name: ssh-metrics
   metrics_path: /metrics
   static_configs:

collector/collector.go

@@ -15,7 +15,9 @@ package collector
 
 import (
 	"bytes"
+	"encoding/base64"
 	"io/ioutil"
+	"net"
 	"regexp"
 	"strings"
 	"time"
@@ -25,14 +27,15 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/treydock/ssh_exporter/config"
 	"golang.org/x/crypto/ssh"
+	"golang.org/x/crypto/ssh/knownhosts"
 )
 
 const (
 	namespace = "ssh"
 )
 
 type Metric struct {
-	Success       bool
+	Success       float64
 	FailureReason string
 }
 
@@ -70,7 +73,7 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) {
 
 	metric := c.collect()
 
-	ch <- prometheus.MustNewConstMetric(c.Success, prometheus.GaugeValue, boolToFloat64(metric.Success))
+	ch <- prometheus.MustNewConstMetric(c.Success, prometheus.GaugeValue, metric.Success)
 	for _, reason := range failureReasons {
 		var value float64
 		if reason == metric.FailureReason {
@@ -103,10 +106,11 @@ func (c *Collector) collect() Metric {
 	}
 
 	sshConfig := &ssh.ClientConfig{
-		User:            c.target.User,
-		Auth:            []ssh.AuthMethod{auth},
-		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
-		Timeout:         time.Duration(c.target.Timeout) * time.Second,
+		User:              c.target.User,
+		Auth:              []ssh.AuthMethod{auth},
+		HostKeyCallback:   hostKeyCallback(&metric, c.target, c.logger),
+		HostKeyAlgorithms: c.target.HostKeyAlgorithms,
+		Timeout:           time.Duration(c.target.Timeout) * time.Second,
 	}
 	connection, err := ssh.Dial("tcp", c.target.Host, sshConfig)
 	if err != nil {
@@ -170,7 +174,7 @@ func (c *Collector) collect() Metric {
 			return metric
 		}
 	}
-	metric.Success = true
+	metric.Success = 1
 	return metric
 }
 
@@ -186,10 +190,22 @@ func getPrivateKeyAuth(privatekey string) (ssh.AuthMethod, error) {
 	return ssh.PublicKeys(key), nil
 }
 
-func boolToFloat64(data bool) float64 {
-	if data {
-		return float64(1)
-	} else {
-		return float64(0)
+func hostKeyCallback(metric *Metric, target *config.Target, logger log.Logger) ssh.HostKeyCallback {
+	return func(hostname string, remote net.Addr, key ssh.PublicKey) error {
+		var hostKeyCallback ssh.HostKeyCallback
+		var err error
+		if target.KnownHosts != "" {
+			publicKey := base64.StdEncoding.EncodeToString(key.Marshal())
+			level.Debug(logger).Log("msg", "Verify SSH known hosts", "hostname", hostname, "remote", remote.String(), "key", publicKey)
+			hostKeyCallback, err = knownhosts.New(target.KnownHosts)
+			if err != nil {
+				metric.FailureReason = "error"
+				level.Error(logger).Log("msg", "Error creating hostkeycallback function", "err", err)
+				return err
+			}
+		} else {
+			hostKeyCallback = ssh.InsecureIgnoreHostKey()
+		}
+		return hostKeyCallback(hostname, remote, key)
 	}
 }

collector/collector_test.go

@@ -14,6 +14,8 @@
 package collector
 
 import (
+	"crypto/rand"
+	"crypto/rsa"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -26,12 +28,16 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/testutil"
 	"github.com/treydock/ssh_exporter/config"
+	gossh "golang.org/x/crypto/ssh"
+	"golang.org/x/crypto/ssh/knownhosts"
 )
 
 const (
 	listen = 60022
 )
 
+var knownHosts *os.File
+
 func publicKeyHandler(ctx ssh.Context, key ssh.PublicKey) bool {
 	buffer, err := ioutil.ReadFile("testdata/id_rsa_test1.pub")
 	if err != nil {
@@ -69,6 +75,27 @@ func TestMain(m *testing.M) {
 		PublicKeyHandler: publicKeyHandler,
 		PasswordHandler:  passwordHandler,
 	}
+	hostKey, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		fmt.Printf("ERROR generating RSA host key: %s", err)
+		os.Exit(1)
+	}
+	signer, err := gossh.NewSignerFromKey(hostKey)
+	if err != nil {
+		fmt.Printf("ERROR generating host key signer: %s", err)
+		os.Exit(1)
+	}
+	s.AddHostKey(signer)
+	knownHosts, err = ioutil.TempFile("", "knowm_hosts")
+	if err != nil {
+		fmt.Printf("ERROR creating known hosts: %s", err)
+		os.Exit(1)
+	}
+	defer os.Remove(knownHosts.Name())
+	knownHostsLine := knownhosts.Line([]string{fmt.Sprintf("localhost:%d", listen)}, s.HostSigners[0].PublicKey())
+	if _, err = knownHosts.Write([]byte(knownHostsLine)); err != nil {
+		fmt.Printf("ERROR writing known hosts: %s", err)
+	}
 	go func() {
 		if err := s.ListenAndServe(); err != nil {
 			fmt.Printf("ERROR starting SSH server: %s", err)
@@ -279,6 +306,108 @@ func TestCollectorPrivateKey(t *testing.T) {
 	}
 }
 
+func TestCollectorKnownHosts(t *testing.T) {
+	expected := `
+	# HELP ssh_failure Indicates a failure
+	# TYPE ssh_failure gauge
+	ssh_failure{reason="command-error"} 0
+	ssh_failure{reason="command-output"} 0
+	ssh_failure{reason="error"} 0
+	ssh_failure{reason="timeout"} 0
+	# HELP ssh_success SSH connection was successful
+	# TYPE ssh_success gauge
+	ssh_success 1
+	`
+	target := &config.Target{
+		Host:       fmt.Sprintf("localhost:%d", listen),
+		User:       "test",
+		PrivateKey: "testdata/id_rsa_test1",
+		KnownHosts: knownHosts.Name(),
+		Timeout:    2,
+	}
+	w := log.NewSyncWriter(os.Stderr)
+	logger := log.NewLogfmtLogger(w)
+	collector := NewCollector(target, logger)
+	gatherers := setupGatherer(collector)
+	if val, err := testutil.GatherAndCount(gatherers); err != nil {
+		t.Errorf("Unexpected error: %v", err)
+	} else if val != 6 {
+		t.Errorf("Unexpected collection count %d, expected 6", val)
+	}
+	if err := testutil.GatherAndCompare(gatherers, strings.NewReader(expected),
+		"ssh_success", "ssh_failure"); err != nil {
+		t.Errorf("unexpected collecting result:\n%s", err)
+	}
+}
+
+func TestCollectorKnownHostsError(t *testing.T) {
+	expected := `
+	# HELP ssh_failure Indicates a failure
+	# TYPE ssh_failure gauge
+	ssh_failure{reason="command-error"} 0
+	ssh_failure{reason="command-output"} 0
+	ssh_failure{reason="error"} 1
+	ssh_failure{reason="timeout"} 0
+	# HELP ssh_success SSH connection was successful
+	# TYPE ssh_success gauge
+	ssh_success 0
+	`
+	target := &config.Target{
+		Host:       fmt.Sprintf("127.0.0.1:%d", listen),
+		User:       "test",
+		PrivateKey: "testdata/id_rsa_test1",
+		KnownHosts: knownHosts.Name(),
+		Timeout:    2,
+	}
+	w := log.NewSyncWriter(os.Stderr)
+	logger := log.NewLogfmtLogger(w)
+	collector := NewCollector(target, logger)
+	gatherers := setupGatherer(collector)
+	if val, err := testutil.GatherAndCount(gatherers); err != nil {
+		t.Errorf("Unexpected error: %v", err)
+	} else if val != 6 {
+		t.Errorf("Unexpected collection count %d, expected 6", val)
+	}
+	if err := testutil.GatherAndCompare(gatherers, strings.NewReader(expected),
+		"ssh_success", "ssh_failure"); err != nil {
+		t.Errorf("unexpected collecting result:\n%s", err)
+	}
+}
+
+func TestCollectorKnownHostsDNE(t *testing.T) {
+	expected := `
+	# HELP ssh_failure Indicates a failure
+	# TYPE ssh_failure gauge
+	ssh_failure{reason="command-error"} 0
+	ssh_failure{reason="command-output"} 0
+	ssh_failure{reason="error"} 1
+	ssh_failure{reason="timeout"} 0
+	# HELP ssh_success SSH connection was successful
+	# TYPE ssh_success gauge
+	ssh_success 0
+	`
+	target := &config.Target{
+		Host:       fmt.Sprintf("localhost:%d", listen),
+		User:       "test",
+		PrivateKey: "testdata/id_rsa_test1",
+		KnownHosts: "/dne",
+		Timeout:    2,
+	}
+	w := log.NewSyncWriter(os.Stderr)
+	logger := log.NewLogfmtLogger(w)
+	collector := NewCollector(target, logger)
+	gatherers := setupGatherer(collector)
+	if val, err := testutil.GatherAndCount(gatherers); err != nil {
+		t.Errorf("Unexpected error: %v", err)
+	} else if val != 6 {
+		t.Errorf("Unexpected collection count %d, expected 6", val)
+	}
+	if err := testutil.GatherAndCompare(gatherers, strings.NewReader(expected),
+		"ssh_success", "ssh_failure"); err != nil {
+		t.Errorf("unexpected collecting result:\n%s", err)
+	}
+}
+
 func TestCollectDNEKey(t *testing.T) {
 	target := &config.Target{
 		Host:       fmt.Sprintf("localhost:%d", listen),

config/config.go

@@ -31,23 +31,27 @@ type SafeConfig struct {
 }
 
 type Module struct {
-	ModuleName    string
-	User          string `yaml:"user"`
-	Password      string `yaml:"password"`
-	PrivateKey    string `yaml:"private_key"`
-	Timeout       int    `yaml:"timeout"`
-	Command       string `yaml:"command"`
-	CommandExpect string `yaml:"command_expect"`
+	ModuleName        string
+	User              string   `yaml:"user"`
+	Password          string   `yaml:"password"`
+	PrivateKey        string   `yaml:"private_key"`
+	KnownHosts        string   `yaml:"known_hosts"`
+	HostKeyAlgorithms []string `yaml:"host_key_algorithms"`
+	Timeout           int      `yaml:"timeout"`
+	Command           string   `yaml:"command"`
+	CommandExpect     string   `yaml:"command_expect"`
 }
 
 type Target struct {
-	Host          string
-	User          string
-	Password      string
-	PrivateKey    string
-	Timeout       int
-	Command       string
-	CommandExpect string
+	Host              string
+	User              string
+	Password          string
+	PrivateKey        string
+	KnownHosts        string
+	HostKeyAlgorithms []string
+	Timeout           int
+	Command           string
+	CommandExpect     string
 }
 
 func (sc *SafeConfig) ReloadConfig(configFile string) error {