Add glooctl snapshot cmd (#10239)

Signed-off-by: Benjamin Leggett <[email protected]>
Co-authored-by: soloio-bulldozer[bot] <48420018+soloio-bulldozer[bot]@users.noreply.github.com>

changelog/v1.18.0-beta34/add-glooctl-snapshot-cmd.yaml

@@ -0,0 +1,7 @@
+changelog:
+  - type: NEW_FEATURE
+    issueLink: https://github.com/solo-io/solo-projects/issues/7131
+    resolvesIssue: true
+    description: >-
+      Add `glooctl proxy snapshot` command, which can be pointed at a Gloo Gateway instance and will produce
+      a zip archive containing all Envoy state, for the purposes of simplified issue reporting and triage.

docs/content/reference/cli/glooctl_proxy.md

@@ -43,6 +43,7 @@ these commands can be used to interact directly with the Proxies Gloo is managin
 * [glooctl proxy dump](../glooctl_proxy_dump)	 - dump Envoy config from one of the proxy instances
 * [glooctl proxy logs](../glooctl_proxy_logs)	 - dump Envoy logs from one of the proxy instancesNote: this will enable verbose logging on Envoy
 * [glooctl proxy served-config](../glooctl_proxy_served-config)	 - dump Envoy config being served by the Gloo xDS server
+* [glooctl proxy snapshot](../glooctl_proxy_snapshot)	 - snapshot complete proxy state for the given instance to an archive
 * [glooctl proxy stats](../glooctl_proxy_stats)	 - stats for one of the proxy instances
 * [glooctl proxy url](../glooctl_proxy_url)	 - print the http endpoint for a proxy
 

docs/content/reference/cli/glooctl_proxy_snapshot.md

@@ -0,0 +1,43 @@
+---
+title: "glooctl proxy snapshot"
+description: "Reference for the 'glooctl proxy snapshot' command."
+weight: 5
+---
+## glooctl proxy snapshot
+
+snapshot complete proxy state for the given instance to an archive
+
+```
+glooctl proxy snapshot [flags]
+```
+
+### Options
+
+```
+  -h, --help          help for snapshot
+      --include-eds   include EDS in the config snapshot (default true)
+```
+
+### Options inherited from parent commands
+
+```
+  -c, --config string              set the path to the glooctl config file (default "<home_directory>/.gloo/glooctl-config.yaml")
+      --consul-address string      address of the Consul server. Use with --use-consul (default "127.0.0.1:8500")
+      --consul-allow-stale-reads   Allows reading using Consul's stale consistency mode.
+      --consul-datacenter string   Datacenter to use. If not provided, the default agent datacenter is used. Use with --use-consul
+      --consul-root-key string     key prefix for the Consul key-value storage. (default "gloo")
+      --consul-scheme string       URI scheme for the Consul server. Use with --use-consul (default "http")
+      --consul-token string        Token is used to provide a per-request ACL token which overrides the agent's default token. Use with --use-consul
+  -i, --interactive                use interactive mode
+      --kube-context string        kube context to use when interacting with kubernetes
+      --kubeconfig string          kubeconfig to use, if not standard one
+      --name string                the name of the proxy service/deployment to use (default "gateway-proxy")
+  -n, --namespace string           namespace for reading or writing resources (default "gloo-system")
+      --port string                the name of the service port to connect to (default "http")
+      --use-consul                 use Consul Key-Value storage as the backend for reading and writing config (VirtualServices, Upstreams, and Proxies)
+```
+
+### SEE ALSO
+
+* [glooctl proxy](../glooctl_proxy)	 - interact with proxy instances managed by Gloo
+

pkg/utils/envoyutils/admincli/client.go

@@ -1,6 +1,7 @@
 package admincli
 
 import (
+	"archive/zip"
 	"context"
 	"fmt"
 	"io"
@@ -13,6 +14,10 @@ import (
 	"github.com/solo-io/gloo/pkg/utils/protoutils"
 	"github.com/solo-io/gloo/pkg/utils/requestutils/curl"
 	"github.com/solo-io/go-utils/threadsafe"
+
+	"github.com/solo-io/gloo/pkg/utils/kubeutils/kubectl"
+	"github.com/solo-io/gloo/pkg/utils/kubeutils/portforward"
+	"github.com/solo-io/gloo/projects/gloo/pkg/defaults"
 )
 
 const (
@@ -25,10 +30,14 @@ const (
 	HealthCheckPath    = "healthcheck"
 	LoggingPath        = "logging"
 	ServerInfoPath     = "server_info"
-
-	DefaultAdminPort = 19000
 )
 
+// DumpOptions should have flags for any kind of underlying optional
+// filtering or inclusion of Envoy dump data, such as including EDS, filters, etc.
+type DumpOptions struct {
+	ConfigIncludeEDS bool
+}
+
 // Client is a utility for executing requests against the Envoy Admin API
 // The Admin API handlers can be found here:
 // https://github.com/envoyproxy/envoy/blob/63bc9b564b1a76a22a0d029bcac35abeffff2a61/source/server/admin/admin.cc#L127
@@ -47,13 +56,45 @@ func NewClient() *Client {
 		curlOptions: []curl.Option{
 			curl.WithScheme("http"),
 			curl.WithHost("127.0.0.1"),
-			curl.WithPort(DefaultAdminPort),
+			curl.WithPort(int(defaults.EnvoyAdminPort)),
 			// 3 retries, exponential back-off, 10 second max
 			curl.WithRetries(3, 0, 10),
 		},
 	}
 }
 
+// NewPortForwardedClient takes a pod selector like <podname> or `deployment/<podname`,
+// and returns a port-forwarded Envoy admin client pointing at that pod,
+// as well as a deferrable shutdown function.
+//
+// Designed to be used by tests and CLI from outside of a cluster where `kubectl` is present.
+// In all other cases, `NewClient` is preferred
+func NewPortForwardedClient(ctx context.Context, proxySelector, namespace string) (*Client, func(), error) {
+	selector := portforward.WithResourceSelector(proxySelector, namespace)
+
+	// 1. Open a port-forward to the Kubernetes Deployment, so that we can query the Envoy Admin API directly
+	portForwarder, err := kubectl.NewCli().StartPortForward(ctx,
+		selector,
+		portforward.WithRemotePort(int(defaults.EnvoyAdminPort)))
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// 2. Close the port-forward when we're done accessing data
+	deferFunc := func() {
+		portForwarder.Close()
+		portForwarder.WaitForStop()
+	}
+
+	// 3. Create a CLI that connects to the Envoy Admin API
+	adminCli := NewClient().
+		WithCurlOptions(
+			curl.WithHostPort(portForwarder.Address()),
+		)
+
+	return adminCli, deferFunc, err
+}
+
 // WithReceiver sets the io.Writer that will be used by default for the stdout and stderr
 // of cmdutils.Cmd created by the Client
 func (c *Client) WithReceiver(receiver io.Writer) *Client {
@@ -110,6 +151,11 @@ func (c *Client) GetStats(ctx context.Context) (string, error) {
 	return outLocation.String(), nil
 }
 
+// ServerInfoCmd returns the cmdutils.Cmd that can be run to request data from the server_info endpoint
+func (c *Client) ServerInfoCmd(ctx context.Context) cmdutils.Cmd {
+	return c.RequestPathCmd(ctx, ServerInfoPath)
+}
+
 // ClustersCmd returns the cmdutils.Cmd that can be run to request data from the clusters endpoint
 func (c *Client) ClustersCmd(ctx context.Context) cmdutils.Cmd {
 	return c.RequestPathCmd(ctx, ClustersPath)
@@ -208,10 +254,7 @@ func (c *Client) GetServerInfo(ctx context.Context) (*adminv3.ServerInfo, error)
 		outLocation threadsafe.Buffer
 	)
 
-	err := c.RequestPathCmd(ctx, ServerInfoPath).
-		WithStdout(&outLocation).
-		Run().
-		Cause()
+	err := c.ServerInfoCmd(ctx).WithStdout(&outLocation).Run().Cause()
 	if err != nil {
 		return nil, err
 	}
@@ -261,3 +304,42 @@ func (c *Client) GetSingleListenerFromDynamicListeners(
 	}
 	return &listener, nil
 }
+
+// WriteEnvoyDumpToZip will dump config, stats, clusters and listeners to zipfile in the current directory.
+// Useful for diagnostics or testing
+func (c *Client) WriteEnvoyDumpToZip(ctx context.Context, options DumpOptions, zip *zip.Writer) error {
+	configParams := make(map[string]string)
+	if options.ConfigIncludeEDS {
+		configParams["include_eds"] = "on"
+	}
+
+	// zip writer has the benefit of not requiring tmpdirs or file ops (all in mem)
+	// - but it can't support async writes, so do these sequentally
+	// Also don't join errors, we want to fast-fail
+	if err := c.ServerInfoCmd(ctx).WithStdout(fileInArchive(zip, "server_info.json")).Run().Cause(); err != nil {
+		return err
+	}
+	if err := c.ConfigDumpCmd(ctx, configParams).WithStdout(fileInArchive(zip, "config.json")).Run().Cause(); err != nil {
+		return err
+	}
+	if err := c.StatsCmd(ctx).WithStdout(fileInArchive(zip, "stats.txt")).Run().Cause(); err != nil {
+		return err
+	}
+	if err := c.ClustersCmd(ctx).WithStdout(fileInArchive(zip, "clusters.txt")).Run().Cause(); err != nil {
+		return err
+	}
+	if err := c.ListenersCmd(ctx).WithStdout(fileInArchive(zip, "listeners.txt")).Run().Cause(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// fileInArchive creates a file at the given path within the archive, and returns the file object for writing.
+func fileInArchive(w *zip.Writer, path string) io.Writer {
+	f, err := w.Create(path)
+	if err != nil {
+		fmt.Printf("unable to create file: %f\n", err)
+	}
+	return f
+}

pkg/utils/glooadminutils/admincli/client.go

@@ -13,8 +13,7 @@ import (
 
 const (
 	InputSnapshotPath = "/snapshots/input"
-
-	DefaultAdminPort = 9091
+	DefaultAdminPort  = 9091
 )
 
 // Client is a utility for executing requests against the Gloo Admin API

pkg/utils/kubeutils/portforward/cli_portforwarder.go

@@ -1,11 +1,13 @@
 package portforward
 
 import (
+	"bufio"
 	"context"
 	"fmt"
 	"net"
 	"os/exec"
 	"strconv"
+	"strings"
 
 	errors "github.com/rotisserie/eris"
 
@@ -64,13 +66,43 @@ func (c *cliPortForwarder) startOnce(ctx context.Context) error {
 		fmt.Sprintf("%s/%s", c.properties.resourceType, c.properties.resourceName),
 		fmt.Sprintf("%d:%d", c.properties.localPort, c.properties.remotePort),
 	)
-	c.cmd.Stdout = c.properties.stdout
-	c.cmd.Stderr = c.properties.stderr
+
+	// Errors should not happen here unless some other thing has futzed
+	// with this cmd's stdout/err.
+	fwdOut, err := c.cmd.StdoutPipe()
+	if err != nil {
+		return err
+	}
+	fwdErr, err := c.cmd.StderrPipe()
+	if err != nil {
+		return err
+	}
+
 	c.cmdCancel = cmdCancel
 
 	c.errCh = make(chan error, 1)
 
-	return c.cmd.Start()
+	// short circuit error return if we can't even start
+	if err := c.cmd.Start(); err != nil {
+		return err
+	}
+
+	// TODO Because we are not using a real Go-only kube client but are spawning a long-running
+	// subprocess, wait until the subprocess actually writes a success msg to stdout before
+	// trying to query the endpoint or we will get spurious failures because this func
+	// will return even though the port-forward hasn't happened yet.
+	outScan := bufio.NewScanner(fwdOut)
+	for outScan.Scan() {
+		if strings.Contains(outScan.Text(), "Forwarding from") {
+			// We are good, port-forward is ready.
+			return nil
+		}
+	}
+
+	// If we get here, we didn't get any stdout, so grab stderr and return it as error
+	stdErr := bufio.NewScanner(fwdErr)
+	stdErr.Scan()
+	return errors.Errorf("failed to start port-forward: %s", stdErr.Text())
 }
 
 func (c *cliPortForwarder) Address() string {

pkg/utils/kubeutils/portforward/options.go

@@ -4,6 +4,8 @@ import (
 	"fmt"
 	"io"
 	"os"
+
+	"strings"
 )
 
 type Option func(*properties)
@@ -31,6 +33,21 @@ func WithKubeContext(kubeContext string) Option {
 	}
 }
 
+// WithResourceSelector takes a kubectl-style selector like `deployment/<name>`
+// or `pod/<name>` and tries to construct the correct Option for it.
+//
+// If no `<resource>/<name>` style selector supplied, assumes a raw pod name has been provided.
+func WithResourceSelector(resourceSelector, namespace string) Option {
+	if sel := strings.Split(resourceSelector, "/"); len(sel) == 2 {
+		if strings.HasPrefix(sel[0], "deploy") {
+			return WithDeployment(sel[1], namespace)
+		} else if strings.HasPrefix(sel[0], "po") {
+			return WithPod(sel[1], namespace)
+		}
+	}
+	return WithPod(resourceSelector, namespace)
+}
+
 func WithDeployment(name, namespace string) Option {
 	return WithResource(name, namespace, "deployment")
 }
@@ -39,6 +56,10 @@ func WithService(name, namespace string) Option {
 	return WithResource(name, namespace, "service")
 }
 
+func WithPod(name, namespace string) Option {
+	return WithResource(name, namespace, "pod")
+}
+
 func WithResource(name, namespace, resourceType string) Option {
 	return func(config *properties) {
 		config.resourceName = name

projects/gateway2/deployer/deployer_test.go

@@ -30,6 +30,15 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 	api "sigs.k8s.io/gateway-api/apis/v1"
+
+	// TODO BML tests in this suite fail if this no-op import is not imported first.
+	//
+	// I know, I know, you're reading this, and you're skeptical. I can feel it.
+	// Don't take my word for it.
+	//
+	// There is some import within this package that this suite relies on. Chasing that down is
+	// *hard* tho due to the import tree, and best done in a followup.
+	_ "github.com/solo-io/gloo/projects/gloo/pkg/translator"
 )
 
 // testBootstrap implements resources.Resource in order to use protoutils.UnmarshalYAML