chore: add backdoor to test for quick demo

The README has all the details on how to use it.

In the longer term we will of course want to do this in a shell without
involving go test, but this is convenient enough with what we have at
the moment.

README.md

@@ -42,7 +42,7 @@ Begin running the updated simapp commands from the `gcosmos` directory.
 sh ./scripts/run_gcosmos.sh
 ```
 
-# Interact
+### Interact
 ```bash
 # Install the grpcurl binary in your relative directory to interact with the GRPC server.
 # GOBIN="$PWD" go install github.com/fullstorydev/grpcurl/cmd/grpcurl@v1
@@ -54,7 +54,7 @@ sh ./scripts/run_gcosmos.sh
 ./grpcurl -plaintext -d '{"address":"cosmos1r5v5srda7xfth3hn2s26txvrcrntldjumt8mhl","denom":"stake"}' localhost:9092 gordian.server.v1.GordianGRPC/QueryAccountBalance
 ```
 
-# Transaction Testing
+### Transaction Testing
 ```bash
 ./gcosmos tx bank send val cosmos10r39fueph9fq7a6lgswu4zdsg8t3gxlqvvvyvn 1stake --chain-id=TODO:TEMPORARY_CHAIN_ID --generate-only > example-tx.json
 
@@ -65,3 +65,70 @@ sh ./scripts/run_gcosmos.sh
 
 ./grpcurl -plaintext -emit-defaults -d '{"tx":"'$(cat example-tx-signed.json | base64 | tr -d '\n')'"}' localhost:9092 gordian.server.v1.GordianGRPC/SubmitTransaction
 ```
+
+## Hacking on a demo with four validators
+
+We have a current hack that uses a test setup with four validators,
+and then keeps them alive so that you can temporarily interact with the network.
+
+From the root of this repository, run:
+`HACK_TEST_DEMO=1 go test . -v -run=TestTx_multiple_simpleSend -timeout=5m`
+
+Which will run the test and then print out details like:
+```
+main_test.go:1195: >>>>>>>>>>>>> Test will run until 2024-10-25T13:36:32-04:00 (38.95884875s)
+main_test.go:1196: >>>>>>>>>>>>> (pass flag -timeout=0 to run forever)
+main_test.go:1201: VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV
+main_test.go:1202: >            CONNECTION INFO           <
+main_test.go:1204: Validator 0:
+main_test.go:1205: 	HTTP address: http://127.0.0.1:51758
+main_test.go:1206: 	Home directory: /tmp/TestTx_multiple_simpleSend2865145828/001
+main_test.go:1204: Validator 1:
+main_test.go:1205: 	HTTP address: http://127.0.0.1:51760
+main_test.go:1206: 	Home directory: /tmp/TestTx_multiple_simpleSend2865145828/002
+main_test.go:1204: Validator 2:
+main_test.go:1205: 	HTTP address: http://127.0.0.1:51759
+main_test.go:1206: 	Home directory: /tmp/TestTx_multiple_simpleSend2865145828/003
+main_test.go:1204: Validator 3:
+main_test.go:1205: 	HTTP address: http://127.0.0.1:51761
+main_test.go:1206: 	Home directory: /tmp/TestTx_multiple_simpleSend2865145828/004
+main_test.go:1208: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+```
+
+From there, you can interact with the HTTP API of each validator, or inspect their home directories as desired.
+
+Useful HTTP interactions include:
+
+Check the validator's view of the voting and committing heights:
+```shell
+$ curl "$ADDR/blocks/watermark"
+{"VotingHeight":4,"VotingRound":4,"CommittingHeight":3,"CommittingRound":0}
+```
+
+See the current validator set:
+```shell
+$ curl -s http://127.0.0.1:53779/validators | jq
+{
+  "FinalizationHeight": 3,
+  "Validators": [
+    {
+      "PubKey": "ZWQyNTUxOQACP0CD6CdN4xH9mUgwIR5ntLO0DgUkT5iASs8vHTpA8A==",
+      "Power": 100000003
+    },
+    {
+      "PubKey": "ZWQyNTUxOQCQjWqgssuCmzDdkK15aPb3xQ2iMtTNvC3u1F9pu3Wctw==",
+      "Power": 100000002
+    },
+    ...
+```
+
+Refer to the `http*.go` files in [the gserver/internal/gsi directory](gserver/internal/gsi/) for more details on available HTTP paths.
+
+### Known issues
+
+There was (and there may still be) a bug where validator counts were not reported correctly if the validator count was between 2 and 10, inclusive.
+So, the tests in `main_test.go` routinely start 11 validators, but only give meaningful voting power to the first 4.
+Due to the way the proposer selection is currently implemented,
+it is normal to see heights 1, 2, and 3 pass successfully; then heights 4 through 11 go through several timeouts with no proposed blocks;
+then 4 more heights pass on the first round, and the cycle continues.
+We still need to get to the bottom of that bug.

main_test.go

@@ -1183,6 +1183,40 @@ func TestTx_multiple_simpleSend(t *testing.T) {
 		resp.Body.Close()
 		require.Equal(t, "10100", newBalance.Balance.Amount, "validator reported wrong receiver balance") // Was at 10k, added 100.
 	})
+
+	// See the "hacking on a demo" section of the README for details on what we are doing here.
+	defer func() {
+		if os.Getenv("HACK_TEST_DEMO") == "" {
+			t.Log("VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV")
+			t.Log("End of test. Set environment variable HACK_TEST_DEMO=1 to sleep and print out connection details of validators.")
+			t.Log("^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^")
+			return
+		}
+
+		deadline, ok := t.Deadline()
+		if ok {
+			t.Logf(">>>>>>>>>>>>> Test will run until %s (%s)", deadline.Add(-time.Second).Format(time.RFC3339), time.Until(deadline))
+			t.Log(">>>>>>>>>>>>> (pass flag -timeout=0 to run forever)")
+		} else {
+			t.Log(">>>>>>>>>>> Test sleeping forever due to -timeout=0 flag; press ^C to stop")
+		}
+
+		t.Log("VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV")
+		t.Log(">                   CONNECTION INFO                   < ")
+		for i, a := range httpAddrs {
+			t.Logf("Validator %d:", i)
+			t.Logf("\tHTTP address: http://%s", a)
+			t.Logf("\tHome directory: %s", c.RootCmds[i].homeDir)
+		}
+		t.Log("^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^")
+
+		if ok {
+			time.Sleep(time.Until(deadline) - time.Second) // Allow test to still pass.
+			t.Logf(">>>>>>>>>> Sleep timeout elapsed")
+		} else {
+			select {}
+		}
+	}()
 }
 
 type balance struct {