From 7a9dc3458f442ec33d220abafc2e5b1c8b767830 Mon Sep 17 00:00:00 2001 From: Abhishek K <32607604+abhishek9686@users.noreply.github.com> Date: Tue, 31 Oct 2023 14:34:28 +0400 Subject: [PATCH] NET-667: force delete daemon node (#2645) * force delete zombie node * return correct resp * fix zombie hosts processing * add nil check rather checking error * pr comments --- controllers/hosts.go | 17 ++++++++++++++ controllers/node.go | 30 ++---------------------- logic/hosts.go | 2 +- logic/nodes.go | 56 +++++++++++++++++++++++++++++++++++++------- logic/zombie.go | 8 +++++-- 5 files changed, 73 insertions(+), 40 deletions(-) diff --git a/controllers/hosts.go b/controllers/hosts.go index 6c6bbeccb..3274c3598 100644 --- a/controllers/hosts.go +++ b/controllers/hosts.go @@ -321,6 +321,23 @@ func deleteHostFromNetwork(w http.ResponseWriter, r *http.Request) { node, err := logic.UpdateHostNetwork(currHost, network, false) if err != nil { + if node == nil && forceDelete { + // force cleanup the node + node, err := logic.GetNodeByHostRef(hostid, network) + if err != nil { + slog.Error("couldn't get node for host", "hostid", hostid, "network", network, "error", err) + logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest")) + return + } + if err = logic.DeleteNodeByID(&node); err != nil { + slog.Error("failed to force delete daemon node", + "nodeid", node.ID.String(), "hostid", hostid, "network", network, "error", err) + logic.ReturnErrorResponse(w, r, logic.FormatError(fmt.Errorf("failed to force delete daemon node: "+err.Error()), "internal")) + return + } + logic.ReturnSuccessResponse(w, r, "force deleted daemon node successfully") + return + } logger.Log(0, r.Header.Get("user"), "failed to remove host from network:", hostid, network, err.Error()) logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal")) return diff --git a/controllers/node.go b/controllers/node.go index 4b66b158d..c75666fa1 100644 --- a/controllers/node.go +++ b/controllers/node.go @@ -725,34 +725,6 @@ func deleteNode(w http.ResponseWriter, r *http.Request) { } forceDelete := r.URL.Query().Get("force") == "true" fromNode := r.Header.Get("requestfrom") == "node" - if node.IsRelayed { - // cleanup node from relayednodes on relay node - relayNode, err := logic.GetNodeByID(node.RelayedBy) - if err == nil { - relayedNodes := []string{} - for _, relayedNodeID := range relayNode.RelayedNodes { - if relayedNodeID == node.ID.String() { - continue - } - relayedNodes = append(relayedNodes, relayedNodeID) - } - relayNode.RelayedNodes = relayedNodes - logic.UpsertNode(&relayNode) - } - } - if node.IsRelay { - // unset all the relayed nodes - logic.SetRelayedNodes(false, node.ID.String(), node.RelayedNodes) - } - if node.IsIngressGateway { - // delete ext clients belonging to ingress gatewa - go func(node models.Node) { - if err = logic.DeleteGatewayExtClients(node.ID.String(), node.Network); err != nil { - slog.Error("failed to delete extclients", "gatewayid", node.ID.String(), "network", node.Network, "error", err.Error()) - } - }(node) - - } purge := forceDelete || fromNode if err := logic.DeleteNode(&node, purge); err != nil { @@ -764,6 +736,8 @@ func deleteNode(w http.ResponseWriter, r *http.Request) { logger.Log(1, r.Header.Get("user"), "Deleted node", nodeid, "from network", params["network"]) go func() { // notify of peer change if !fromNode { + node.PendingDelete = true + node.Action = models.NODE_DELETE if err := mq.NodeUpdate(&node); err != nil { slog.Error("error publishing node update to node", "node", node.ID, "error", err) } diff --git a/logic/hosts.go b/logic/hosts.go index c8911a5e1..9778ddf81 100644 --- a/logic/hosts.go +++ b/logic/hosts.go @@ -392,7 +392,7 @@ func DissasociateNodeFromHost(n *models.Node, h *models.Host) error { } } }() - if err := deleteNodeByID(n); err != nil { + if err := DeleteNodeByID(n); err != nil { return err } diff --git a/logic/nodes.go b/logic/nodes.go index c9cf8da01..9a643e9d3 100644 --- a/logic/nodes.go +++ b/logic/nodes.go @@ -183,6 +183,35 @@ func UpdateNode(currentNode *models.Node, newNode *models.Node) error { func DeleteNode(node *models.Node, purge bool) error { alreadyDeleted := node.PendingDelete || node.Action == models.NODE_DELETE node.Action = models.NODE_DELETE + + if !alreadyDeleted { + //delete ext clients if node is ingress gw + if node.IsIngressGateway { + if err := DeleteGatewayExtClients(node.ID.String(), node.Network); err != nil { + slog.Error("failed to delete ext clients", "nodeid", node.ID.String(), "error", err.Error()) + } + } + if node.IsRelayed { + // cleanup node from relayednodes on relay node + relayNode, err := GetNodeByID(node.RelayedBy) + if err == nil { + relayedNodes := []string{} + for _, relayedNodeID := range relayNode.RelayedNodes { + if relayedNodeID == node.ID.String() { + continue + } + relayedNodes = append(relayedNodes, relayedNodeID) + } + relayNode.RelayedNodes = relayedNodes + UpsertNode(&relayNode) + } + } + if node.IsRelay { + // unset all the relayed nodes + SetRelayedNodes(false, node.ID.String(), node.RelayedNodes) + } + } + if !purge && !alreadyDeleted { newnode := *node newnode.PendingDelete = true @@ -198,7 +227,7 @@ func DeleteNode(node *models.Node, purge bool) error { host, err := GetHost(node.HostID.String()) if err != nil { logger.Log(1, "no host found for node", node.ID.String(), "deleting..") - if delErr := deleteNodeByID(node); delErr != nil { + if delErr := DeleteNodeByID(node); delErr != nil { logger.Log(0, "failed to delete node", node.ID.String(), delErr.Error()) } return err @@ -215,16 +244,25 @@ func DeleteNode(node *models.Node, purge bool) error { return nil } -// deleteNodeByID - deletes a node from database -func deleteNodeByID(node *models.Node) error { - var err error - var key = node.ID.String() - //delete any ext clients as required - if node.IsIngressGateway { - if err := DeleteGatewayExtClients(node.ID.String(), node.Network); err != nil { - logger.Log(0, "failed to deleted ext clients", err.Error()) +// GetNodeByHostRef - gets the node by host id and network +func GetNodeByHostRef(hostid, network string) (node models.Node, err error) { + nodes, err := GetNetworkNodes(network) + if err != nil { + return models.Node{}, err + } + for _, node := range nodes { + if node.HostID.String() == hostid && node.Network == network { + return node, nil } } + return models.Node{}, errors.New("node not found") +} + +// DeleteNodeByID - deletes a node from database +func DeleteNodeByID(node *models.Node) error { + var err error + var key = node.ID.String() + if err = database.DeleteRecord(database.NODES_TABLE_NAME, key); err != nil { if !database.IsEmptyRecord(err) { return err diff --git a/logic/zombie.go b/logic/zombie.go index d191aaa3d..2e1692edd 100644 --- a/logic/zombie.go +++ b/logic/zombie.go @@ -107,6 +107,7 @@ func ManageZombies(ctx context.Context, peerUpdate chan *models.Node) { logger.Log(1, "error deleting zombie node", zombies[i].String(), err.Error()) continue } + node.PendingDelete = true node.Action = models.NODE_DELETE peerUpdate <- &node logger.Log(1, "deleting zombie node", node.ID.String()) @@ -120,14 +121,17 @@ func ManageZombies(ctx context.Context, peerUpdate chan *models.Node) { host, err := GetHost(hostZombies[i].String()) if err != nil { logger.Log(1, "error retrieving zombie host", err.Error()) - logger.Log(1, "deleting ", host.ID.String(), " from zombie list") - zombies = append(zombies[:i], zombies[i+1:]...) + if host != nil { + logger.Log(1, "deleting ", host.ID.String(), " from zombie list") + } + hostZombies = append(hostZombies[:i], hostZombies[i+1:]...) continue } if len(host.Nodes) == 0 { if err := RemoveHost(host, true); err != nil { logger.Log(0, "error deleting zombie host", host.ID.String(), err.Error()) } + hostZombies = append(hostZombies[:i], hostZombies[i+1:]...) } } }