Initial consul support

This commit adds initial support for HashiCorp consul as a source of scrapable targets for the scraper. It is confiugred using the typical environment variables one would use for a consul client. This commit also includes an integration test against a running consul instance. It functions pretty much the same as the nomad source in terms of logic. Signed-off-by: David Bond <[email protected]>
davidsbond · Nov 25, 2024 · 2d7b205 · 2d7b205
1 parent d7983f0
commit 2d7b205
Show file tree

Hide file tree

Showing 11 changed files with 504 additions and 41 deletions.
diff --git a/README.md b/README.md
@@ -37,7 +37,7 @@ autopgo scrape
 #### Configuration
 
 The `scrape` command accepts a single argument that is contextual depending on the mode specified via the `--mode` flag.
-The `mode` flag accepts `file`, `kube` & `nomad` as values, defaulting to `file`.
+The `mode` flag accepts `file`, `kube`, `consul` & `nomad` as values, defaulting to `file`.
 
 The `scrape` command also accepts some command-line flags that may also be set via environment variables. They are
 described in the table below:
@@ -51,7 +51,7 @@ described in the table below:
 |     `--app`, `-a`     |     `AUTOPGO_APP`     |          None           | Specifies the the application name that profiles will be uploaded for                    |
 |  `--frequency`, `-f`  |  `AUTOPGO_FREQUENCY`  |          `60s`          | Specifies the interval between profiling runs                                            |
 |  `--duration`, `-d`   |  `AUTOPGO_DURATION`   |          `30s`          | Specifies the amount of time a target will be profiled for                               |
-|    `--mode`, `-m`     |    `AUTOPGO_MODE`     |         `file`          | What mode to run the scraper in (file, kube, nomad)                                      |
+|    `--mode`, `-m`     |    `AUTOPGO_MODE`     |         `file`          | What mode to run the scraper in (file, kube, nomad, consul)                              |
 
 ##### File Mode
 
@@ -142,7 +142,7 @@ As in all other operating modes, a single scraper instance is required per appli
 example of a Nomad job specification that contains a service with all usable tags:
 
 ```hcl
-job "example-app   {
+job "example-app    {
   type = "service"
   group "example-app" {
     count = 1
@@ -174,6 +174,26 @@ job "example-app   {
 }
 ```
 
+##### Consul Mode
+
+When running the scraper in `consul` mode the first argument usually reserved for a configuration file is no longer
+required. If running the scraper against the Consul service catalogue you instead need to set the typical environment
+variables for any consul client, such as `CONSUL_ADDR` etc. One key difference between this mode and
+[kube mode](#kube-mode) is that the `autopgo.port` tag is not required as it can be obtained from the service itself.
+
+The scraper will then source targets from the Consul service catalogue, searching for any services with appropriate tags
+added to their entry. The table below describes these tags and provides examples:
+
+|           Key           |                Example                 | Required | Description                                                                             |
+|:-----------------------:|:--------------------------------------:|:--------:|:----------------------------------------------------------------------------------------|
+|    `autopgo.scrape`     |        `autopgo.scrape: "true"`        |   Yes    | Informs the scraper that this is a scrape target.                                       |
+|  `autopgo.scrape.app`   |      `autopgo.app: "hello-world"`      |   Yes    | Informs the scraper which application the profile belongs to.                           |
+|  `autopgo.scrape.path`  | `autopgo.path: "/debug/pprof/profile"` |    No    | Allows for specifying the path to the pprof endpoint, defaults to /debug/pprof/profile. |
+| `autopgo.scrape.scheme` |        `autopgo.scheme: "http"`        |    No    | Informs the scraper whether the endpoint uses HTTP or HTTPS, defaults to HTTP.          |
+
+As in all other operating modes, a single scraper instance is required per application you wish to scrape. Below is an
+example of a Nomad job specification that contains a service with all usable tags:
+
 #### Sampling
 
 The sampling behaviour of the scraper is fairly simple. At the interval defined by the `--frequency` flag, a number

diff --git a/cmd/scrape/scrape.go b/cmd/scrape/scrape.go
@@ -4,7 +4,8 @@ package scrape
 import (
 	"time"
 
-	"github.com/hashicorp/nomad/api"
+	consul "github.com/hashicorp/consul/api"
+	nomad "github.com/hashicorp/nomad/api"
 	"github.com/spf13/cobra"
 	"golang.org/x/sync/errgroup"
 	"k8s.io/client-go/kubernetes"
@@ -19,9 +20,10 @@ import (
 )
 
 const (
-	modeFile  = "file"
-	modeKube  = "kube"
-	modeNomad = "nomad"
+	modeFile   = "file"
+	modeKube   = "kube"
+	modeNomad  = "nomad"
+	modeConsul = "consul"
 )
 
 // Command returns a cobra.Command instance used to run the scraper.
@@ -58,6 +60,8 @@ func Command() *cobra.Command {
 				source, err = target.NewFileSource(ctx, args[0])
 			case modeNomad:
 				source, err = nomadTargetSource(app)
+			case modeConsul:
+				source, err = consulTargetSource(app)
 			case modeKube:
 				var configLocation string
 				if len(args) != 0 {
@@ -103,7 +107,7 @@ func Command() *cobra.Command {
 	flags.UintVarP(&sampleSize, "sample-size", "s", 0, "The maximum number of targets to scrape concurrently")
 	flags.DurationVarP(&duration, "duration", "d", time.Second*30, "How long to profile targets for")
 	flags.DurationVarP(&frequency, "frequency", "f", time.Minute, "Interval between scraping targets")
-	flags.StringVarP(&mode, "mode", "m", modeFile, "Mode to use for obtaining targets (file, kube, nomad)")
+	flags.StringVarP(&mode, "mode", "m", modeFile, "Mode to use for obtaining targets (file, kube, nomad, consul)")
 
 	cmd.MarkFlagRequired("app")
 	cmd.MarkFlagRequired("sample-size")
@@ -135,10 +139,19 @@ func kubeTargetSource(configLocation, app string) (*target.KubernetesSource, err
 }
 
 func nomadTargetSource(app string) (*target.NomadSource, error) {
-	nomad, err := api.NewClient(api.DefaultConfig())
+	cl, err := nomad.NewClient(nomad.DefaultConfig())
 	if err != nil {
 		return nil, err
 	}
 
-	return target.NewNomadSource(nomad, app), nil
+	return target.NewNomadSource(cl, app), nil
+}
+
+func consulTargetSource(app string) (*target.ConsulSource, error) {
+	cl, err := consul.NewClient(consul.DefaultConfig())
+	if err != nil {
+		return nil, err
+	}
+
+	return target.NewConsulSource(cl, app), nil
 }
diff --git a/go.mod b/go.mod
@@ -9,11 +9,13 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/sns v1.33.6
 	github.com/google/pprof v0.0.0-20241023014458-598669927662
 	github.com/google/uuid v1.6.0
+	github.com/hashicorp/consul/api v1.28.2
 	github.com/hashicorp/nomad/api v0.0.0-20241121182148-997da25cdb49
 	github.com/spf13/cobra v1.8.1
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.19.0
 	github.com/stretchr/testify v1.10.0
+	github.com/testcontainers/testcontainers-go/modules/consul v0.34.0
 	github.com/testcontainers/testcontainers-go/modules/k3s v0.34.0
 	gocloud.dev v0.40.0
 	gocloud.dev/pubsub/kafkapubsub v0.40.0
@@ -48,6 +50,7 @@ require (
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.3 // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.3 // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
+	github.com/armon/go-metrics v0.4.1 // indirect
 	github.com/aws/aws-sdk-go v1.55.5 // indirect
 	github.com/aws/aws-sdk-go-v2 v1.32.5 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 // indirect
@@ -88,6 +91,7 @@ require (
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
 	github.com/envoyproxy/go-control-plane v0.13.1 // indirect
 	github.com/envoyproxy/protoc-gen-validate v1.1.0 // indirect
+	github.com/fatih/color v1.14.1 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
@@ -113,10 +117,14 @@ require (
 	github.com/hashicorp/cronexpr v1.1.2 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/hashicorp/go-hclog v1.5.0 // indirect
+	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
 	github.com/hashicorp/go-uuid v1.0.3 // indirect
+	github.com/hashicorp/golang-lru v0.5.4 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/hashicorp/serf v0.10.1 // indirect
 	github.com/imdario/mergo v0.3.13 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jcmturner/aescts/v2 v2.0.0 // indirect
@@ -132,6 +140,8 @@ require (
 	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.17 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/moby/docker-image-spec v1.3.1 // indirect