backport/v1.0/: bugtool gops pprofheap

.github/workflows/packages-e2e-tests.yaml

@@ -56,6 +56,11 @@ jobs:
           sudo mkdir -p /etc/tetragon/tetragon.tp.d/
           sudo cp examples/tracingpolicy/bpf.yaml /etc/tetragon/tetragon.tp.d/bpf.yaml
 
+      - name: Configure gops
+        run: |
+          sudo mkdir -p /etc/tetragon/tetragon.conf.d/
+          echo "localhost:8118" | sudo tee /etc/tetragon/tetragon.conf.d/gops-address
+
       - name: Install Tetragon Tarball
         run: |
           tar zxvf tetragon-${{ steps.tag.outputs.tag }}-${{ matrix.arch }}.tar.gz
@@ -85,6 +90,7 @@ jobs:
           sudo tetra tracingpolicy list | grep bpf -
           sudo tetra bugtool
           test $(stat -c %a /var/run/tetragon/tetragon.sock) -eq "660"
+          sudo tetra bugtool 2>&1 | grep "Successfully dumped gops pprof-heap" -
 
       - name: Uninstall Tetragon Tarball
         run: |

Dockerfile.tarball

@@ -21,6 +21,7 @@ COPY --from=tetragon-release /usr/bin/tetra /usr/local/bin/
 
 # Helper binaries
 COPY --from=tetragon-release /usr/bin/bpftool /usr/local/lib/tetragon/
+COPY --from=tetragon-release /usr/bin/gops /usr/local/lib/tetragon/
 
 # BPF files
 COPY --from=tetragon-release /var/lib/tetragon /usr/local/lib/tetragon/bpf

cmd/tetra/bugtool/bugtool.go

@@ -12,19 +12,21 @@ import (
 var (
 	outFile string
 	bpfTool string
+	gops    string
 )
 
 func New() *cobra.Command {
 	bugtoolCmd := &cobra.Command{
 		Use:   "bugtool",
 		Short: "Produce a tar archive with debug information",
 		Run: func(cmd *cobra.Command, args []string) {
-			bugtool.Bugtool(outFile, bpfTool)
+			bugtool.Bugtool(outFile, bpfTool, gops)
 		},
 	}
 
 	flags := bugtoolCmd.Flags()
 	flags.StringVarP(&outFile, "out", "o", "tetragon-bugtool.tar.gz", "Output filename")
 	flags.StringVar(&bpfTool, "bpftool", "", "Path to bpftool binary")
+	flags.StringVar(&gops, "gops", "", "Path to gops binary")
 	return bugtoolCmd
 }

pkg/bugtool/bugtool.go

@@ -14,6 +14,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"net"
 	"net/http"
 	"os"
 	"os/exec"
@@ -26,6 +27,7 @@ import (
 	"github.com/cilium/tetragon/pkg/defaults"
 	"github.com/cilium/tetragon/pkg/logger"
 	"github.com/cilium/tetragon/pkg/policyfilter"
+	gopssignal "github.com/google/gops/signal"
 	"go.uber.org/multierr"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
@@ -44,6 +46,7 @@ type InitInfo struct {
 	GopsAddr    string `json:"gops_address"`
 	MapDir      string `json:"map_dir"`
 	BpfToolPath string `json:"bpftool_path"`
+	GopsPath    string `json:"gops_path"`
 }
 
 // LoadInitInfo returns the InitInfo by reading the info file from its default location
@@ -83,6 +86,14 @@ func doSaveInitInfo(fname string, info *InitInfo) error {
 		logger.GetLogger().WithField("bpftool", info.BpfToolPath).Info("Successfully detected bpftool path")
 	}
 
+	gops, err := exec.LookPath("gops")
+	if err != nil {
+		logger.GetLogger().Warn("failed to locate gops binary, on bugtool debugging ensure you have gops installed")
+	} else {
+		info.GopsPath = gops
+		logger.GetLogger().WithField("gops", info.GopsPath).Info("Successfully detected gops path")
+	}
+
 	// Create DefaultRunDir if it does not already exist
 	if err := os.MkdirAll(defaults.DefaultRunDir, 0755); err != nil {
 		logger.GetLogger().WithField("infoFile", fname).Warn("failed to directory exists")
@@ -182,7 +193,7 @@ func (s *bugtoolInfo) tarAddFile(tarWriter *tar.Writer, fnameSrc string, fnameDs
 }
 
 // Bugtool gathers information and writes it as a tar archive in the given filename
-func Bugtool(outFname string, bpftool string) error {
+func Bugtool(outFname string, bpftool string, gops string) error {
 	info, err := LoadInitInfo()
 	if err != nil {
 		return err
@@ -192,6 +203,10 @@ func Bugtool(outFname string, bpftool string) error {
 		info.BpfToolPath = bpftool
 	}
 
+	if gops != "" {
+		info.GopsPath = gops
+	}
+
 	return doBugtool(info, outFname)
 }
 
@@ -488,21 +503,69 @@ func (s *bugtoolInfo) addBpftoolInfo(tarWriter *tar.Writer) {
 
 	_, err := os.Stat(s.info.BpfToolPath)
 	if err != nil {
-		s.multiLog.WithError(err).Warn("Failed to locate bpftool, please install it.")
+		s.multiLog.WithError(err).Warn("Failed to locate bpftool. Please install it or specify its path, see 'bugtool --help'")
 		return
 	}
 	s.execCmd(tarWriter, "bpftool-maps.json", s.info.BpfToolPath, "map", "show", "-j")
 	s.execCmd(tarWriter, "bpftool-progs.json", s.info.BpfToolPath, "prog", "show", "-j")
 	s.execCmd(tarWriter, "bpftool-cgroups.json", s.info.BpfToolPath, "cgroup", "tree", "-j")
 }
 
+func (s *bugtoolInfo) getPProf(tarWriter *tar.Writer, file string) error {
+	if s.info.GopsAddr == "" {
+		s.multiLog.Info("Skipping gops dump info as daemon is running without gops, use --gops-address to enable gops")
+		return nil
+	}
+
+	s.multiLog.WithField("gops-address", s.info.GopsAddr).Info("Contacting gops server for pprof dump")
+
+	conn, err := net.Dial("tcp", s.info.GopsAddr)
+	if err != nil {
+		s.multiLog.WithField("gops-address", s.info.GopsAddr).WithError(err).Warn("Failed to contact gops server")
+		return err
+	}
+
+	buf := []byte{gopssignal.HeapProfile}
+	if _, err := conn.Write(buf); err != nil {
+		s.multiLog.WithField("gops-address", s.info.GopsAddr).WithError(err).Warn("Failed to send gops pprof-heap command")
+		return err
+	}
+
+	buff := new(bytes.Buffer)
+	if _, err = buff.ReadFrom(conn); err != nil {
+		s.multiLog.WithField("gops-address", s.info.GopsAddr).WithError(err).Warn("Failed reading gops pprof-heap response")
+	}
+	return s.tarAddBuff(tarWriter, file, buff)
+}
+
 func (s *bugtoolInfo) addGopsInfo(tarWriter *tar.Writer) {
 	if s.info.GopsAddr == "" {
+		s.multiLog.Info("Skipping gops dump info as daemon is running without gops, use --gops-address to enable gops")
 		return
 	}
-	s.execCmd(tarWriter, "gops.stack", "gops", "stack", s.info.GopsAddr)
-	s.execCmd(tarWriter, "gpos.stats", "gops", "stats", s.info.GopsAddr)
-	s.execCmd(tarWriter, "gops.memstats", "gops", "memstats", s.info.GopsAddr)
+
+	if s.info.GopsPath == "" {
+		s.multiLog.WithField("gops-address", s.info.GopsAddr).Warn("Failed to locate gops. Please install it or specify its path, see 'bugtool --help'")
+		return
+	}
+
+	_, err := os.Stat(s.info.GopsPath)
+	if err != nil {
+		s.multiLog.WithField("gops-address", s.info.GopsAddr).WithError(err).Warn("Failed to locate gops, please install it")
+		return
+	}
+
+	s.multiLog.WithField("gops-address", s.info.GopsAddr).WithField("gops-path", s.info.GopsPath).Info("Dumping gops information")
+
+	s.execCmd(tarWriter, "gops.stack", s.info.GopsPath, "stack", s.info.GopsAddr)
+	s.execCmd(tarWriter, "gops.stats", s.info.GopsPath, "stats", s.info.GopsAddr)
+	s.execCmd(tarWriter, "gops.memstats", s.info.GopsPath, "memstats", s.info.GopsAddr)
+	err = s.getPProf(tarWriter, "gops.pprof-heap")
+	if err != nil {
+		s.multiLog.WithField("gops-address", s.info.GopsAddr).WithField("gops-path", s.info.GopsPath).WithError(err).Warn("Failed to dump gops pprof-heap")
+	} else {
+		s.multiLog.WithField("gops-address", s.info.GopsAddr).WithField("gops-path", s.info.GopsPath).Info("Successfully dumped gops pprof-heap")
+	}
 }
 
 func (s *bugtoolInfo) dumpPolicyFilterMap(tarWriter *tar.Writer) error {

pkg/observer/observertesthelper/observer_test_helper.go

@@ -125,7 +125,7 @@ func WithLib(lib string) TestOption {
 func testDone(tb testing.TB, obs *observer.Observer) {
 	if tb.Failed() {
 		bugtoolFname := "/tmp/tetragon-bugtool.tar.gz"
-		if err := bugtool.Bugtool(bugtoolFname, ""); err == nil {
+		if err := bugtool.Bugtool(bugtoolFname, "", ""); err == nil {
 			logger.GetLogger().WithField("test", tb.Name()).
 				WithField("file", bugtoolFname).Info("Dumped bugtool info")
 		} else {