Add integration test

Signed-off-by: Chun-Hung Tseng <[email protected]>

.github/workflows/integration_tests.yml

@@ -0,0 +1,26 @@
+---
+name: integration_tests
+on:
+  push:
+    tags:
+      - v*
+    branches:
+      - master
+      - main
+  pull_request:
+permissions:
+  contents: read
+  # Optional: allow read access to pull request. Use with `only-new-issues` option.
+  # pull-requests: read
+jobs:
+  test:
+    name: lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-go@v5
+        with:
+          go-version: 1.21.10
+      - uses: actions/checkout@v4
+      - name: tests
+        run: |
+          make integration-test

Makefile

@@ -18,6 +18,10 @@ verify-gofmt:
 test:
 	go test -v --race -cpu=1,2,4 ./code/ ./runtime/
 
+.PHONY: integration-test
+integration-test: gofail
+	./integration/sleep/execute.sh
+
 fix: fix-gofmt
 
 .PHONY: fix-gofmt

integration/README.md

@@ -0,0 +1,5 @@
+# Integration Tests
+
+Each directory contains a scenario
+
+- sleep: the enabling and disabling of a failpoint won't be delayed due to an ongoing sleep() action

integration/sleep/execute.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+GOFAIL_BINARY="$(pwd)/gofail"
+
+cd $(dirname $0)
+
+pushd failpoints
+$GOFAIL_BINARY enable
+popd
+
+go build -o integration_test_sleep . 
+
+pushd failpoints
+$GOFAIL_BINARY disable
+popd
+
+./integration_test_sleep
+rm integration_test_sleep

integration/sleep/failpoints/failpoints.go

@@ -0,0 +1,40 @@
+package failpoints
+
+import (
+	"log"
+	"sync"
+	"time"
+)
+
+func Worker1(wg *sync.WaitGroup) {
+	defer wg.Done()
+
+	log.Println("worker1 in")
+	defer log.Println("worker1 out")
+
+	// gofail: var worker1Failpoint struct{}
+
+	time.Sleep(3 * time.Second)
+}
+
+func Worker2(wg *sync.WaitGroup) {
+	defer wg.Done()
+
+	log.Println("worker2 in")
+	defer log.Println("worker2 out")
+
+	// gofail: var worker2Failpoint struct{}
+
+	time.Sleep(3 * time.Second)
+}
+
+func Worker3(wg *sync.WaitGroup) {
+	defer wg.Done()
+
+	log.Println("worker3 in")
+	defer log.Println("worker3 out")
+
+	// gofail: var worker3Failpoint struct{}
+
+	time.Sleep(3 * time.Second)
+}

integration/sleep/go.mod

@@ -0,0 +1,9 @@
+module go.etcd.io/gofail/integration/sleep
+
+go 1.21
+
+toolchain go1.21.10
+
+replace go.etcd.io/gofail => ./../../
+
+require go.etcd.io/gofail v0.1.1-0.20240328162059-93c579a86c46

integration/sleep/go.sum

@@ -0,0 +1,8 @@
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

integration/sleep/main.go

@@ -0,0 +1,81 @@
+package main
+
+import (
+	"log"
+	"sync"
+	"time"
+
+	"go.etcd.io/gofail/integration/sleep/failpoints"
+	gofail "go.etcd.io/gofail/runtime"
+)
+
+func main() {
+	{
+		// expectation: this part of the code will take about 3s to execute, because all go routines will be executing concurrently
+		start := time.Now()
+
+		log.Println("Stage 1: Run 3 workers under normal logic")
+		var wg sync.WaitGroup
+		wg.Add(1)
+		go failpoints.Worker1(&wg)
+
+		wg.Add(1)
+		go failpoints.Worker2(&wg)
+
+		wg.Add(1)
+		go failpoints.Worker3(&wg)
+
+		wg.Wait()
+
+		elapsed := time.Since(start)
+		if elapsed > (3*time.Second + 100*time.Millisecond) {
+			log.Fatalln("invalid execution time", elapsed)
+		}
+
+		log.Println("Stage 1: Done")
+	}
+
+	{
+		// expectation: this part of the code will take about 6s to execute only,
+		// because all go routines will be executing concurrently, with both the sleep
+		// from failpoint and the original sleep actions
+		//
+		// The gofail implementation up till commit 93c579a86c46 is executing the
+		// program sequentially, due to the failpoint action execution and enable/disable
+		// flows are under the same locking mechanism, only one of the actions can make
+		// progress at a given moment
+		log.Println("Stage 2: Run 3 workers under failpoint logic")
+
+		start := time.Now()
+
+		var wg sync.WaitGroup
+		gofail.Enable("worker1Failpoint", `sleep("3s")`)
+		wg.Add(1)
+		go failpoints.Worker1(&wg)
+		time.Sleep(10 * time.Millisecond)
+
+		gofail.Enable("worker2Failpoint", `sleep("3s")`)
+		wg.Add(1)
+		go failpoints.Worker2(&wg)
+		time.Sleep(10 * time.Millisecond)
+
+		gofail.Enable("worker3Failpoint", `sleep("3s")`)
+		wg.Add(1)
+		go failpoints.Worker3(&wg)
+		time.Sleep(10 * time.Millisecond)
+
+		// the failpoint can be disabled during failpoint execution
+		gofail.Disable("worker1Failpoint")
+		gofail.Disable("worker2Failpoint")
+		gofail.Disable("worker3Failpoint")
+
+		wg.Wait()
+
+		elapsed := time.Since(start)
+		if elapsed > (6*time.Second + 100*time.Millisecond) {
+			log.Fatalln("invalid execution time", elapsed)
+		}
+
+		log.Println("Stage 2: Done")
+	}
+}