[ENG-430] Addresses intermittent Redis timeouts (#17)

* [ENG-430] Adding configurable Redis timeout (-timeout)

* [ENG-430] Added max-retries option and enable retries in Redis GET

Makefile

@@ -18,8 +18,8 @@ compile:
                --go-grpc_opt=paths=source_relative \
                protos/*.proto
 
-all: compile cmd/server.go
-	go build -o $(out) cmd/server.go
+all: compile cmd/main.go
+	go build -o $(out) cmd/main.go
 	@chmod +x $(out)
 
 services:

api/statemachine_test.go

@@ -29,7 +29,10 @@ import (
 )
 
 var _ = Describe("FSM Protocol Buffers", func() {
-    BeforeEach(func() { Logger = log.NullLog })
+    BeforeEach(func() {
+        Logger = log.NewLog("statemachine-test")
+        Logger.Level = log.NONE
+    })
     Context("if well defined", func() {
         It("can be initialized", func() {
             var spaceship = Configuration{

cmd/server.go → cmd/main.go

@@ -85,6 +85,10 @@ func main() {
         "The name of the Dead-Letter Queue ("+"DLQ) in SQS to post errors to; if not "+
             "specified, the DLQ will not be used")
     var grpcPort = flag.Int("grpc-port", 7398, "The port for the gRPC server")
+    var maxRetries = flag.Int("max-retries", storage.DefaultMaxRetries,
+        "Max number of attempts for a recoverable error to be retried against the Redis cluster")
+    var timeout = flag.Duration("timeout", storage.DefaultTimeout,
+        "Timeout for Redis (as a Duration string, e.g. 1s, 20ms, etc.)")
     flag.Parse()
 
     if *localOnly {
@@ -100,7 +104,8 @@ func main() {
         store = storage.NewInMemoryStore()
     } else {
         logger.Info("Connecting to Redis server at %s", *redisUrl)
-        store = storage.NewRedisStore(*redisUrl, 1)
+        logger.Info("with timeout: %s, max-retries: %d", *timeout, *maxRetries)
+        store = storage.NewRedisStore(*redisUrl, 1, *timeout, *maxRetries)
     }
     server.SetStore(store)
 

docker/Dockerfile

@@ -5,16 +5,15 @@
 # NOTE: Use the Ubuntu image, others will encounter errors,
 #       including the "`GLIBC_2.32' not found" error.
 FROM  ubuntu:22.04
-WORKDIR /app
-
-COPY build/bin/sm-server docker/entrypoint.sh ./
 
 RUN groupadd -r sm-bot && useradd -r -g sm-bot sm-bot
-RUN apt-get update
-RUN apt-get install ca-certificates -y
+RUN apt-get update && apt-get install ca-certificates -y
+
+WORKDIR /app
+
 ADD docker/aws-credentials /home/sm-bot/.aws/credentials
-RUN chmod +x entrypoint.sh
-RUN chown sm-bot:sm-bot entrypoint.sh
+COPY build/bin/sm-server docker/entrypoint.sh ./
+RUN chmod a+x entrypoint.sh
 
 USER sm-bot
 

docker/entrypoint.sh

@@ -25,12 +25,12 @@ then
   endpoint="--endpoint-url ${AWS_ENDPOINT}"
 fi
 
-echo "./sm-server -port ${SERVER_PORT}  ${endpoint:-} ${DEBUG} \
-    -redis ${REDIS}:${REDIS_PORT} \
-    -events ${EVENTS_Q} -errors ${ERRORS_Q}"
-echo "$@"
+echo -e "./sm-server -port ${SERVER_PORT}  ${endpoint:-} ${DEBUG}\n\
+    -redis ${REDIS}:${REDIS_PORT} -timeout ${TIMEOUT:-25ms} -max-retries ${RETRIES:-3}\n\
+    -events ${EVENTS_Q} -errors ${ERRORS_Q}\n\
+    $@"
 
 ./sm-server -http-port "${SERVER_PORT}"  ${endpoint:-} ${DEBUG} \
-    -redis ${REDIS}:${REDIS_PORT} \
+    -redis ${REDIS}:${REDIS_PORT} -timeout ${TIMEOUT:-25ms} -max-retries ${RETRIES:-3} \
     -events "${EVENTS_Q}" -errors "${ERRORS_Q}" \
     "$@"

go.mod

@@ -9,7 +9,7 @@ require (
 	github.com/golang/protobuf v1.5.2
 	github.com/google/uuid v1.3.0
 	github.com/gorilla/mux v1.8.0
-	github.com/massenz/slf4go v0.1.1-gd1a5998
+	github.com/massenz/slf4go v0.3.1-gb35df61
 	github.com/onsi/ginkgo v1.16.5
 	github.com/onsi/gomega v1.18.1
 	google.golang.org/grpc v1.32.0
@@ -28,4 +28,5 @@ require (
 	google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -61,6 +61,8 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/massenz/slf4go v0.1.1-gd1a5998 h1:sXODxQ9HxPkbRvS0tsssrGtvV0rfNxRUrQnS7WkG5sk=
 github.com/massenz/slf4go v0.1.1-gd1a5998/go.mod h1:LYuqnSMv0FrjFqD7uWEd/oZjuYf0MH9ukqt91bjBxhI=
+github.com/massenz/slf4go v0.3.1-gb35df61 h1:X/rcmd918F2nkkPbahMcQE0Qb8wc6xg37rpsfunjGMM=
+github.com/massenz/slf4go v0.3.1-gb35df61/go.mod h1:ZJjthXAnZMJGwXUz3Z3v5uyban00uAFFoDYODOoLFpw=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
 github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
@@ -182,5 +184,7 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=