Merge pull request #277 from orbs-network/bugfix/memory-leak-test

Add memory leak test

test.manual-memory-leaks.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+rm -rf /tmp/*.prof
+
+go test ./test/_manual -count 1 > test.out
+
+go tool pprof --inuse_space -nodecount 10 -weblist orbs-network-go -hide /orbs-network-go/test/ --base /tmp/mem-tx-before.prof /tmp/mem-tx-after.prof
+go tool pprof --inuse_space -nodecount 20 -weblist orbs-network-go -hide /orbs-network-go/test/ --base /tmp/mem-shutdown-before.prof /tmp/mem-shutdown-after.prof
+
+echo ""
+echo ""
+echo "TestMemoryLeaks_AfterSomeTransactions:"
+echo ""
+
+go tool pprof --inuse_space -nodecount 10 -top -show orbs-network-go -hide /orbs-network-go/test/ --base /tmp/mem-tx-before.prof /tmp/mem-tx-after.prof
+
+echo ""
+echo ""
+echo "TestMemoryLeaks_OnSystemShutdown:"
+echo ""
+
+go tool pprof --inuse_space -nodecount 20 -top --base /tmp/mem-shutdown-before.prof /tmp/mem-shutdown-after.prof

test.memory-leaks.sh

@@ -1,22 +1,17 @@
 #!/bin/sh
 
-rm -rf /tmp/*.prof
+go test ./test/acceptance -tags memoryleak -run TestMemoryLeaks -count 1 > test.out
 
-go test ./test/_manual -count 1 > test.out
+export EXIT_CODE=$?
 
-go tool pprof --inuse_space -nodecount 10 -weblist orbs-network-go -hide /orbs-network-go/test/ --base /tmp/mem-tx-before.prof /tmp/mem-tx-after.prof
-go tool pprof --inuse_space -nodecount 20 -weblist orbs-network-go -hide /orbs-network-go/test/ --base /tmp/mem-shutdown-before.prof /tmp/mem-shutdown-after.prof
+if [ $EXIT_CODE != 0 ]; then
+  echo "Test failed! Found leaking memory"
 
-echo ""
-echo ""
-echo "TestMemoryLeaks_AfterSomeTransactions:"
-echo ""
+  echo ""
+  echo ""
+  echo "****** Memory delta:"
+  echo ""
+  go tool pprof --inuse_space -nodecount 10 -top --base /tmp/mem-shutdown-before.prof /tmp/mem-shutdown-after.prof
 
-go tool pprof --inuse_space -nodecount 10 -top -show orbs-network-go -hide /orbs-network-go/test/ --base /tmp/mem-tx-before.prof /tmp/mem-tx-after.prof
-
-echo ""
-echo ""
-echo "TestMemoryLeaks_OnSystemShutdown:"
-echo ""
-
-go tool pprof --inuse_space -nodecount 20 -top --base /tmp/mem-shutdown-before.prof /tmp/mem-shutdown-after.prof
+  exit $EXIT_CODE
+fi

test.sh

@@ -8,5 +8,8 @@ check_exit_code_and_report
 # this test must run separately since zero parallel package tests are allowed concurrently
 source ./test.goroutine-leaks.sh
 
+# this test must run separately since zero parallel package tests are allowed concurrently
+source ./test.memory-leaks.sh
+
 # uncomment to run component tests
 # ./test.components.sh

test/_manual/README.md

@@ -8,7 +8,7 @@ It is possible that as we gather more evidence, some of these tests will become
 
 This test attempts to show the top memory consumers of a running acceptance node. The test shows a report of all memory consumers and the amount of memory they currently have in-use.
 
-* Run with `PROJECT-ROOT/test.memory-leaks.sh`
+* Run with `PROJECT-ROOT/test.manual-memory-leaks.sh`
 
 * Test scenarios:
 

test/acceptance/memory_leak_test.go

@@ -0,0 +1,61 @@
+//+build memoryleak
+
+package acceptance
+
+import (
+	"github.com/stretchr/testify/require"
+	"os"
+	"runtime"
+	"runtime/pprof"
+	"testing"
+	"time"
+)
+
+// this test should not run in parallel with any other test (even package parallel) since it's examining shared global system state (num goroutines)
+// if another test is running, the other test may create goroutines which we may mistake as leaks because the numbers won't add up
+// therefore, this file is marked on top with a build flag ("memoryleak") meaning without this flag it won't build or run
+// to run this test, add to the go command "-tags memoryleak", this is done in test.sh while making sure it's the only test running
+func TestMemoryLeaks_OnSystemShutdown(t *testing.T) {
+
+	before, _ := os.Create("/tmp/mem-shutdown-before.prof")
+	defer before.Close()
+	after, _ := os.Create("/tmp/mem-shutdown-after.prof")
+	defer after.Close()
+
+	t.Run("TestCreateGazillionTransactionsWhileTransportIsDuplicatingRandomMessages", TestCreateGazillionTransactionsWhileTransportIsDuplicatingRandomMessages)
+	t.Run("TestCreateGazillionTransactionsWhileTransportIsDroppingRandomMessages", TestCreateGazillionTransactionsWhileTransportIsDroppingRandomMessages)
+	t.Run("TestCreateGazillionTransactionsWhileTransportIsDelayingRandomMessages", TestCreateGazillionTransactionsWhileTransportIsDelayingRandomMessages)
+
+	runtime.GC()
+	runtime.GC()
+	runtime.GC()
+	runtime.GC()
+	time.Sleep(50 * time.Millisecond)
+
+	memUsageBefore := getMemUsage()
+	pprof.WriteHeapProfile(before)
+
+	for i := 0; i < 10; i++ {
+		t.Run("TestCreateGazillionTransactionsWhileTransportIsDuplicatingRandomMessages", TestCreateGazillionTransactionsWhileTransportIsDuplicatingRandomMessages)
+		t.Run("TestCreateGazillionTransactionsWhileTransportIsDroppingRandomMessages", TestCreateGazillionTransactionsWhileTransportIsDroppingRandomMessages)
+		t.Run("TestCreateGazillionTransactionsWhileTransportIsDelayingRandomMessages", TestCreateGazillionTransactionsWhileTransportIsDelayingRandomMessages)
+	}
+
+	runtime.GC()
+	runtime.GC()
+	runtime.GC()
+	runtime.GC()
+	time.Sleep(50 * time.Millisecond)
+
+	memUsageAfter := getMemUsage()
+	pprof.WriteHeapProfile(after)
+
+	require.InDelta(t, memUsageAfter, memUsageBefore, 0.1*float64(memUsageBefore), "added memory should be around than 10%, compare /tmp/mem-shutdown-before.prof and /tmp/mem-shutdown-after.prof to see memory consumers")
+}
+
+func getMemUsage() uint64 {
+	var m runtime.MemStats
+	runtime.ReadMemStats(&m)
+
+	return m.Alloc
+}