Skip to content

Commit

Permalink
daemon: always dump stack to file
Browse files Browse the repository at this point in the history
Dumping to log is unusable in 90% of cases and inspecting file is much
more convenient.

Signed-off-by: Alexander Morozov <[email protected]>
  • Loading branch information
LK4D4 committed Nov 2, 2016
1 parent c600cfa commit e5d3658
Show file tree
Hide file tree
Showing 3 changed files with 33 additions and 33 deletions.
13 changes: 10 additions & 3 deletions daemon/debugtrap_unix.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,15 +7,22 @@ import (
"os/signal"
"syscall"

psignal "github.com/docker/docker/pkg/signal"
stackdump "github.com/docker/docker/pkg/signal"

"github.com/Sirupsen/logrus"
)

func setupDumpStackTrap(_ string) {
func setupDumpStackTrap(root string) {
c := make(chan os.Signal, 1)
signal.Notify(c, syscall.SIGUSR1)
go func() {
for range c {
psignal.DumpStacks("")
path, err := stackdump.DumpStacks(root)
if err != nil {
logrus.WithError(err).Error("failed to write goroutines dump")
continue
}
logrus.Infof("goroutine stacks written to %s", path)
}
}()
}
7 changes: 6 additions & 1 deletion daemon/debugtrap_windows.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,12 @@ func setupDumpStackTrap(root string) {
logrus.Debugf("Stackdump - waiting signal at %s", ev)
for {
syscall.WaitForSingleObject(h, syscall.INFINITE)
signal.DumpStacks(root)
path, err := signal.DumpStacks(root)
if err != nil {
logrus.WithError(err).Error("failed to write goroutines dump")
continue
}
logrus.Infof("goroutine stacks written to %s", path)
}
}()
}
46 changes: 17 additions & 29 deletions pkg/signal/trap.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package signal

import (
"fmt"
"os"
gosignal "os/signal"
"path/filepath"
Expand All @@ -10,6 +11,7 @@ import (
"time"

"github.com/Sirupsen/logrus"
"github.com/pkg/errors"
)

// Trap sets up a simplified signal "trap", appropriate for common
Expand Down Expand Up @@ -64,8 +66,11 @@ func Trap(cleanup func()) {
}()
}

// DumpStacks dumps the runtime stack.
func DumpStacks(root string) {
const stacksLogNameTemplate = "goroutine-stacks-%s.log"

// DumpStacks appends the runtime stack into file in dir and returns full path
// to that file.
func DumpStacks(dir string) (string, error) {
var (
buf []byte
stackSize int
Expand All @@ -77,32 +82,15 @@ func DumpStacks(root string) {
bufferLen *= 2
}
buf = buf[:stackSize]
// Note that if the daemon is started with a less-verbose log-level than "info" (the default), the goroutine
// traces won't show up in the log.
if root == "" {
logrus.Infof("=== BEGIN goroutine stack dump ===\n%s\n=== END goroutine stack dump ===", buf)
} else {
// Dumps the stacks to a file in the root directory of the daemon
// On Windows, this overcomes two issues - one being that if the stack is too big, it doesn't
// get written to the event log when the Windows daemon is running as a service.
// Second, using logrus, the tabs and new-lines end up getting written as literal
// \t and \n's, meaning you need to use something like notepad++ to convert the
// output into something readable using 'type' from a command line or notepad/notepad++ etc.
path := filepath.Join(root, "goroutine-stacks.log")
f, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0666)
if err != nil {
logrus.Warnf("Could not open %s to write the goroutine stacks: %v", path, err)
return
}
defer f.Close()
f.WriteString("=== BEGIN goroutine stack dump ===\n")
f.WriteString(time.Now().String() + "\n")
if _, err := f.Write(buf); err != nil {
logrus.Warnf("Could not write goroutine stacks to %s: %v", path, err)
return
}
f.WriteString("=== END goroutine stack dump ===\n")
f.Sync()
logrus.Infof("goroutine stacks written to %s", path)
path := filepath.Join(dir, fmt.Sprintf(stacksLogNameTemplate, time.Now().Format(time.RFC3339)))
f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0666)
if err != nil {
return "", errors.Wrap(err, "failed to open file to write the goroutine stacks")
}
defer f.Close()
if _, err := f.Write(buf); err != nil {
return "", errors.Wrap(err, "failed to write goroutine stacks")
}
f.Sync()
return path, nil
}

0 comments on commit e5d3658

Please sign in to comment.