Fix flaky TestMultipleClientsWithMixedLabelsAndExpectFailure

This failed in a bunch of runs in #598

When it fails, it has 3 updates, but they're all from the same member. I believe what we want to check is that the cluster never gets joined by members with labels

.github/workflows/find-flaky-tests.yml

@@ -8,8 +8,8 @@ jobs:
   find-flaky-tests:
     # Only run this workflow when the comment contains '/find-flaky-tests' or it's manually triggered
     if: contains(github.event.comment.body, '/find-flaky-tests') || github.event_name == 'workflow_dispatch'
-    fail-fast: false
     strategy:
+      fail-fast: false
       matrix:
         runs: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20] # Run the build workflow 20 times
     uses: ./.github/workflows/test-build.yml

kv/memberlist/memberlist_client_test.go

@@ -610,7 +610,7 @@ func TestMultipleClientsWithMixedLabelsAndExpectFailure(t *testing.T) {
 
 	err := testMultipleClientsWithConfigGenerator(t, len(membersLabel), configGen)
 	require.Error(t, err)
-	require.Contains(t, err.Error(), fmt.Sprintf("expected to see at least %d updates, got", len(membersLabel)))
+	require.Contains(t, err.Error(), fmt.Sprintf("expected to see %d members, got", len(membersLabel)))
 }
 
 func TestMultipleClientsWithMixedLabelsAndClusterLabelVerificationDisabled(t *testing.T) {
@@ -721,6 +721,7 @@ func testMultipleClientsWithConfigGenerator(t *testing.T, members int, configGen
 	firstKv := clients[0]
 	ctx, cancel := context.WithTimeout(context.Background(), casInterval*3) // Watch for 3x cas intervals.
 	updates := 0
+	gotMembers := 0
 	firstKv.WatchKey(ctx, key, func(in interface{}) bool {
 		updates++
 
@@ -733,12 +734,18 @@ func testMultipleClientsWithConfigGenerator(t *testing.T, members int, configGen
 			"tokens, oldest timestamp:", now.Sub(time.Unix(minTimestamp, 0)).String(),
 			"avg timestamp:", now.Sub(time.Unix(avgTimestamp, 0)).String(),
 			"youngest timestamp:", now.Sub(time.Unix(maxTimestamp, 0)).String())
+		gotMembers = len(r.Members)
 		return true // yes, keep watching
 	})
 	cancel() // make linter happy
 
 	t.Logf("Ring updates observed: %d", updates)
 
+	// We expect that all members are in the ring
+	if gotMembers != members {
+		return fmt.Errorf("expected to see %d members, got %d", members, gotMembers)
+	}
+
 	if updates < members {
 		// in general, at least one update from each node. (although that's not necessarily true...
 		// but typically we get more updates than that anyway)