From dc5418818174fd8fc38ebab07f55dcfb326e75f1 Mon Sep 17 00:00:00 2001 From: Ashesh Vidyut Date: Tue, 6 Jun 2023 09:51:24 +0000 Subject: [PATCH 1/6] backport of commit 2f94024b356f774d0032ad069babb1a3fce6cfbd --- api/operator_raft.go | 17 +++++ api/operator_raft_test.go | 19 +++++ .../raft/listpeers/operator_raft_list.go | 71 +++++++++++++++++-- .../raft/listpeers/operator_raft_list_test.go | 27 +++++++ 4 files changed, 129 insertions(+), 5 deletions(-) diff --git a/api/operator_raft.go b/api/operator_raft.go index 1da20e899ff5..b59721ee98b6 100644 --- a/api/operator_raft.go +++ b/api/operator_raft.go @@ -119,3 +119,20 @@ func (op *Operator) RaftRemovePeerByID(id string, q *WriteOptions) error { } return nil } + +// GetAutoPilotHealth is used to query the autopilot health. +func (op *Operator) GetAutoPilotHealth(q *QueryOptions) (*OperatorHealthReply, error) { + r := op.c.newRequest("GET", "/v1/operator/autopilot/health") + r.setQueryOptions(q) + _, resp, err := op.c.doRequest(r) + if err != nil { + return nil, err + } + defer closeResponseBody(resp) + + var out OperatorHealthReply + if err := decodeBody(resp, &out); err != nil { + return nil, err + } + return &out, nil +} diff --git a/api/operator_raft_test.go b/api/operator_raft_test.go index ecefaa971920..dc2012c0ff16 100644 --- a/api/operator_raft_test.go +++ b/api/operator_raft_test.go @@ -54,3 +54,22 @@ func TestAPI_OperatorRaftLeaderTransfer(t *testing.T) { t.Fatalf("err:%v", transfer) } } + +func TestAPI_GetAutoPilotHealth(t *testing.T) { + t.Parallel() + c, s := makeClient(t) + defer s.Stop() + + operator := c.Operator() + out, err := operator.GetAutoPilotHealth(nil) + if err != nil { + t.Fatalf("err: %v", err) + } + + if len(out.Servers) != 1 || + !out.Servers[0].Leader || + !out.Servers[0].Voter || + out.Servers[0].LastIndex <= 0 { + t.Fatalf("bad: %v", out) + } +} diff --git a/command/operator/raft/listpeers/operator_raft_list.go b/command/operator/raft/listpeers/operator_raft_list.go index 98934d8d0eb2..17f92b30033f 100644 --- a/command/operator/raft/listpeers/operator_raft_list.go +++ b/command/operator/raft/listpeers/operator_raft_list.go @@ -21,10 +21,16 @@ type cmd struct { flags *flag.FlagSet http *flags.HTTPFlags help string + + // flags + detailed bool } func (c *cmd) init() { c.flags = flag.NewFlagSet("", flag.ContinueOnError) + c.flags.BoolVar(&c.detailed, "detailed", false, + "Outputs additional information 'commit_index' which is "+ + "the index of the server's last committed Raft log entry.") c.http = &flags.HTTPFlags{} flags.Merge(c.flags, c.http.ClientFlags()) flags.Merge(c.flags, c.http.ServerFlags()) @@ -48,13 +54,22 @@ func (c *cmd) Run(args []string) int { } // Fetch the current configuration. - result, err := raftListPeers(client, c.http.Stale()) - if err != nil { - c.UI.Error(fmt.Sprintf("Error getting peers: %v", err)) - return 1 + if c.detailed { + result, err := raftListPeersDetailed(client, c.http.Stale()) + if err != nil { + c.UI.Error(fmt.Sprintf("Error getting peers: %v", err)) + return 1 + } + c.UI.Output(result) + } else { + result, err := raftListPeers(client, c.http.Stale()) + if err != nil { + c.UI.Error(fmt.Sprintf("Error getting peers: %v", err)) + return 1 + } + c.UI.Output(result) } - c.UI.Output(result) return 0 } @@ -86,6 +101,52 @@ func raftListPeers(client *api.Client, stale bool) (string, error) { return columnize.Format(result, &columnize.Config{Delim: string([]byte{0x1f})}), nil } +func raftListPeersDetailed(client *api.Client, stale bool) (string, error) { + q := &api.QueryOptions{ + AllowStale: stale, + } + reply, err := client.Operator().RaftGetConfiguration(q) + if err != nil { + return "", fmt.Errorf("Failed to retrieve raft configuration: %v", err) + } + + autoPilotReply, err := client.Operator().GetAutoPilotHealth(q) + if err != nil { + return "", fmt.Errorf("Failed to retrieve autopilot health: %v", err) + } + + serverHealthDataMap := make(map[string]api.ServerHealth) + + for _, serverHealthData := range autoPilotReply.Servers { + serverHealthDataMap[serverHealthData.ID] = serverHealthData + } + + // Format it as a nice table. + result := []string{"Node\x1fID\x1fAddress\x1fState\x1fVoter\x1fRaftProtocol\x1fCommitIndex"} + for _, s := range reply.Servers { + raftProtocol := s.ProtocolVersion + + if raftProtocol == "" { + raftProtocol = "<=1" + } + state := "follower" + if s.Leader { + state = "leader" + } + + serverHealthData, ok := serverHealthDataMap[s.ID] + if ok { + result = append(result, fmt.Sprintf("%s\x1f%s\x1f%s\x1f%s\x1f%v\x1f%s\x1f%v", + s.Node, s.ID, s.Address, state, s.Voter, raftProtocol, serverHealthData.LastIndex)) + } else { + result = append(result, fmt.Sprintf("%s\x1f%s\x1f%s\x1f%s\x1f%v\x1f%s\x1f%v", + s.Node, s.ID, s.Address, state, s.Voter, raftProtocol, "")) + } + } + + return columnize.Format(result, &columnize.Config{Delim: string([]byte{0x1f})}), nil +} + func (c *cmd) Synopsis() string { return synopsis } diff --git a/command/operator/raft/listpeers/operator_raft_list_test.go b/command/operator/raft/listpeers/operator_raft_list_test.go index 8d53e4945398..18ec43a1b109 100644 --- a/command/operator/raft/listpeers/operator_raft_list_test.go +++ b/command/operator/raft/listpeers/operator_raft_list_test.go @@ -43,6 +43,33 @@ func TestOperatorRaftListPeersCommand(t *testing.T) { } } +func TestOperatorRaftListPeersCommandDetailed(t *testing.T) { + if testing.Short() { + t.Skip("too slow for testing.Short") + } + + t.Parallel() + a := agent.NewTestAgent(t, ``) + defer a.Shutdown() + + expected := fmt.Sprintf("%s %s 127.0.0.1:%d leader true 3", + a.Config.NodeName, a.Config.NodeID, a.Config.ServerPort) + + // Test the list-peers subcommand directly + ui := cli.NewMockUi() + c := New(ui) + args := []string{"-http-addr=" + a.HTTPAddr(), "-detailed"} + + code := c.Run(args) + if code != 0 { + t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String()) + } + output := strings.TrimSpace(ui.OutputWriter.String()) + if !strings.Contains(output, expected) { + t.Fatalf("bad: %q, %q", output, expected) + } +} + func TestOperatorRaftListPeersCommand_verticalBar(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") From e85e8fccc788bcae9fbd1b897bd91d8e2692dae1 Mon Sep 17 00:00:00 2001 From: Ashesh Vidyut Date: Tue, 6 Jun 2023 09:56:50 +0000 Subject: [PATCH 2/6] backport of commit 7626d0992dd59f424d893dd462bbde3a9cb0646c --- command/operator/raft/listpeers/operator_raft_list_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/command/operator/raft/listpeers/operator_raft_list_test.go b/command/operator/raft/listpeers/operator_raft_list_test.go index 18ec43a1b109..a6f314143a3f 100644 --- a/command/operator/raft/listpeers/operator_raft_list_test.go +++ b/command/operator/raft/listpeers/operator_raft_list_test.go @@ -52,7 +52,7 @@ func TestOperatorRaftListPeersCommandDetailed(t *testing.T) { a := agent.NewTestAgent(t, ``) defer a.Shutdown() - expected := fmt.Sprintf("%s %s 127.0.0.1:%d leader true 3", + expected := fmt.Sprintf("%s %s 127.0.0.1:%d leader true 3 1", a.Config.NodeName, a.Config.NodeID, a.Config.ServerPort) // Test the list-peers subcommand directly From 4068cc65b9e5e7d0e2344244edaa24635a6ea388 Mon Sep 17 00:00:00 2001 From: Ashesh Vidyut Date: Tue, 6 Jun 2023 11:01:54 +0000 Subject: [PATCH 3/6] backport of commit 79aabc958aed60b2bd97c836529d53a627262124 --- website/content/commands/operator/raft.mdx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/website/content/commands/operator/raft.mdx b/website/content/commands/operator/raft.mdx index 857a9ee1ac12..7c948c1e14e3 100644 --- a/website/content/commands/operator/raft.mdx +++ b/website/content/commands/operator/raft.mdx @@ -73,6 +73,10 @@ configuration. we recommend setting this option to `true. Default is `false`. +- `-detailed` - Outputs additional information 'commit_index' which is +the index of the server's last committed Raft log entry. +Default is `false`. + ## remove-peer Corresponding HTTP API Endpoint: [\[DELETE\] /v1/operator/raft/peer](/consul/api-docs/operator/raft#delete-raft-peer) From 44996a3d619f48eb56cf4ca70c4b28a62434b510 Mon Sep 17 00:00:00 2001 From: Ashesh Vidyut Date: Tue, 6 Jun 2023 11:14:01 +0000 Subject: [PATCH 4/6] backport of commit 44eee4168b5a0d405636dbcb5b14ee7959334550 --- .changelog/17582.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .changelog/17582.txt diff --git a/.changelog/17582.txt b/.changelog/17582.txt new file mode 100644 index 000000000000..3beae9d726a0 --- /dev/null +++ b/.changelog/17582.txt @@ -0,0 +1,3 @@ +```release-note:feature +improved consul operator raft list-peers command added -detailed flag which will print CommitIndex in the table in response +``` From cf0e5ef8c9892b697527601da7bd90bed32a932d Mon Sep 17 00:00:00 2001 From: absolutelightning Date: Wed, 21 Jun 2023 11:44:41 +0530 Subject: [PATCH 5/6] fix conflicts --- api/operator_raft.go | 17 ----- api/operator_raft_test.go | 19 ----- .../raft/listpeers/operator_raft_list.go | 71 ++----------------- .../raft/listpeers/operator_raft_list_test.go | 27 ------- website/content/commands/operator/raft.mdx | 6 +- 5 files changed, 6 insertions(+), 134 deletions(-) diff --git a/api/operator_raft.go b/api/operator_raft.go index 7147714b600a..f0ffa1522872 100644 --- a/api/operator_raft.go +++ b/api/operator_raft.go @@ -122,20 +122,3 @@ func (op *Operator) RaftRemovePeerByID(id string, q *WriteOptions) error { } return nil } - -// GetAutoPilotHealth is used to query the autopilot health. -func (op *Operator) GetAutoPilotHealth(q *QueryOptions) (*OperatorHealthReply, error) { - r := op.c.newRequest("GET", "/v1/operator/autopilot/health") - r.setQueryOptions(q) - _, resp, err := op.c.doRequest(r) - if err != nil { - return nil, err - } - defer closeResponseBody(resp) - - var out OperatorHealthReply - if err := decodeBody(resp, &out); err != nil { - return nil, err - } - return &out, nil -} diff --git a/api/operator_raft_test.go b/api/operator_raft_test.go index dc2012c0ff16..ecefaa971920 100644 --- a/api/operator_raft_test.go +++ b/api/operator_raft_test.go @@ -54,22 +54,3 @@ func TestAPI_OperatorRaftLeaderTransfer(t *testing.T) { t.Fatalf("err:%v", transfer) } } - -func TestAPI_GetAutoPilotHealth(t *testing.T) { - t.Parallel() - c, s := makeClient(t) - defer s.Stop() - - operator := c.Operator() - out, err := operator.GetAutoPilotHealth(nil) - if err != nil { - t.Fatalf("err: %v", err) - } - - if len(out.Servers) != 1 || - !out.Servers[0].Leader || - !out.Servers[0].Voter || - out.Servers[0].LastIndex <= 0 { - t.Fatalf("bad: %v", out) - } -} diff --git a/command/operator/raft/listpeers/operator_raft_list.go b/command/operator/raft/listpeers/operator_raft_list.go index 43b45d0efdf3..a54ccb9d3ea2 100644 --- a/command/operator/raft/listpeers/operator_raft_list.go +++ b/command/operator/raft/listpeers/operator_raft_list.go @@ -21,16 +21,10 @@ type cmd struct { flags *flag.FlagSet http *flags.HTTPFlags help string - - // flags - detailed bool } func (c *cmd) init() { c.flags = flag.NewFlagSet("", flag.ContinueOnError) - c.flags.BoolVar(&c.detailed, "detailed", false, - "Outputs additional information 'commit_index' which is "+ - "the index of the server's last committed Raft log entry.") c.http = &flags.HTTPFlags{} flags.Merge(c.flags, c.http.ClientFlags()) flags.Merge(c.flags, c.http.ServerFlags()) @@ -54,22 +48,13 @@ func (c *cmd) Run(args []string) int { } // Fetch the current configuration. - if c.detailed { - result, err := raftListPeersDetailed(client, c.http.Stale()) - if err != nil { - c.UI.Error(fmt.Sprintf("Error getting peers: %v", err)) - return 1 - } - c.UI.Output(result) - } else { - result, err := raftListPeers(client, c.http.Stale()) - if err != nil { - c.UI.Error(fmt.Sprintf("Error getting peers: %v", err)) - return 1 - } - c.UI.Output(result) + result, err := raftListPeers(client, c.http.Stale()) + if err != nil { + c.UI.Error(fmt.Sprintf("Error getting peers: %v", err)) + return 1 } + c.UI.Output(result) return 0 } @@ -129,52 +114,6 @@ func raftListPeers(client *api.Client, stale bool) (string, error) { return columnize.Format(result, &columnize.Config{Delim: string([]byte{0x1f})}), nil } -func raftListPeersDetailed(client *api.Client, stale bool) (string, error) { - q := &api.QueryOptions{ - AllowStale: stale, - } - reply, err := client.Operator().RaftGetConfiguration(q) - if err != nil { - return "", fmt.Errorf("Failed to retrieve raft configuration: %v", err) - } - - autoPilotReply, err := client.Operator().GetAutoPilotHealth(q) - if err != nil { - return "", fmt.Errorf("Failed to retrieve autopilot health: %v", err) - } - - serverHealthDataMap := make(map[string]api.ServerHealth) - - for _, serverHealthData := range autoPilotReply.Servers { - serverHealthDataMap[serverHealthData.ID] = serverHealthData - } - - // Format it as a nice table. - result := []string{"Node\x1fID\x1fAddress\x1fState\x1fVoter\x1fRaftProtocol\x1fCommitIndex"} - for _, s := range reply.Servers { - raftProtocol := s.ProtocolVersion - - if raftProtocol == "" { - raftProtocol = "<=1" - } - state := "follower" - if s.Leader { - state = "leader" - } - - serverHealthData, ok := serverHealthDataMap[s.ID] - if ok { - result = append(result, fmt.Sprintf("%s\x1f%s\x1f%s\x1f%s\x1f%v\x1f%s\x1f%v", - s.Node, s.ID, s.Address, state, s.Voter, raftProtocol, serverHealthData.LastIndex)) - } else { - result = append(result, fmt.Sprintf("%s\x1f%s\x1f%s\x1f%s\x1f%v\x1f%s\x1f%v", - s.Node, s.ID, s.Address, state, s.Voter, raftProtocol, "")) - } - } - - return columnize.Format(result, &columnize.Config{Delim: string([]byte{0x1f})}), nil -} - func (c *cmd) Synopsis() string { return synopsis } diff --git a/command/operator/raft/listpeers/operator_raft_list_test.go b/command/operator/raft/listpeers/operator_raft_list_test.go index 3cb9597b13d5..c40ae8c48d7b 100644 --- a/command/operator/raft/listpeers/operator_raft_list_test.go +++ b/command/operator/raft/listpeers/operator_raft_list_test.go @@ -43,33 +43,6 @@ func TestOperatorRaftListPeersCommand(t *testing.T) { } } -func TestOperatorRaftListPeersCommandDetailed(t *testing.T) { - if testing.Short() { - t.Skip("too slow for testing.Short") - } - - t.Parallel() - a := agent.NewTestAgent(t, ``) - defer a.Shutdown() - - expected := fmt.Sprintf("%s %s 127.0.0.1:%d leader true 3 1", - a.Config.NodeName, a.Config.NodeID, a.Config.ServerPort) - - // Test the list-peers subcommand directly - ui := cli.NewMockUi() - c := New(ui) - args := []string{"-http-addr=" + a.HTTPAddr(), "-detailed"} - - code := c.Run(args) - if code != 0 { - t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String()) - } - output := strings.TrimSpace(ui.OutputWriter.String()) - if !strings.Contains(output, expected) { - t.Fatalf("bad: %q, %q", output, expected) - } -} - func TestOperatorRaftListPeersCommand_verticalBar(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") diff --git a/website/content/commands/operator/raft.mdx b/website/content/commands/operator/raft.mdx index 7c888e0b46cf..eaa0081a60ad 100644 --- a/website/content/commands/operator/raft.mdx +++ b/website/content/commands/operator/raft.mdx @@ -73,10 +73,6 @@ configuration. we recommend setting this option to `true`. Default is `false`. -- `-detailed` - Outputs additional information 'commit_index' which is -the index of the server's last committed Raft log entry. -Default is `false`. - ## remove-peer Corresponding HTTP API Endpoint: [\[DELETE\] /v1/operator/raft/peer](/consul/api-docs/operator/raft#delete-raft-peer) @@ -113,7 +109,7 @@ The return code will indicate success or failure. Corresponding HTTP API Endpoint: [\[POST\] /v1/operator/raft/transfer-leader](/consul/api-docs/operator/raft#transfer-raft-leadership) -This command transfers Raft leadership to another server agent. If an `id` is provided, Consul transfers leadership to the server with that id. +This command transfers Raft leadership to another server agent. If an `id` is provided, Consul transfers leadership to the server with that id. Use this command to change leadership without restarting the leader node, which maintains quorum and workload capacity. From c4e3902b47f395249acad4816863e69fc8c4d4b3 Mon Sep 17 00:00:00 2001 From: Ashesh Vidyut <134911583+absolutelightning@users.noreply.github.com> Date: Thu, 15 Jun 2023 20:11:04 +0530 Subject: [PATCH 6/6] Updated docs added explanation. (#17751) * init * fix tests * added -detailed in docs * added change log * fix doc * checking for entry in map * fix tests * removed detailed flag * removed detailed flag * revert unwanted changes * removed unwanted changes * updated change log * pr review comment changes * pr comment changes single API instead of two * fix change log * fix tests * fix tests * fix test operator raft endpoint test * Update .changelog/17582.txt Co-authored-by: Semir Patel * nits * updated docs * explanation added --------- Co-authored-by: Semir Patel --- website/content/commands/operator/raft.mdx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/website/content/commands/operator/raft.mdx b/website/content/commands/operator/raft.mdx index eaa0081a60ad..b89c0385ac88 100644 --- a/website/content/commands/operator/raft.mdx +++ b/website/content/commands/operator/raft.mdx @@ -66,6 +66,10 @@ Raft configuration. `Voter` is "true" or "false", indicating if the server has a vote in the Raft configuration. +`Commit Index` is the last log index the server has a record of in its Raft log. + +`Trails Leader By` is the difference in commit index the server has from the leader. + #### Command Options - `-stale` - Enables non-leader servers to provide cluster state information.