CLIENT-3117 Replace an existing node in the cluster when a new peer h…

…as the same node name, but a different IP address. The existing node must also have recent cluster tend failures and not be a localhost IP address (127.0.0.1) for the node replacement to proceed.

client/src/com/aerospike/client/cluster/Cluster.java

@@ -569,11 +569,11 @@ private final void tend(boolean failIfNotConnected, boolean isInit) {
 				}
 
 				// Handle nodes changes determined from refreshes.
-				ArrayList<Node> removeList = findNodesToRemove(peers.refreshCount);
+				findNodesToRemove(peers);
 
 				// Remove nodes in a batch.
-				if (removeList.size() > 0) {
-					removeNodes(removeList);
+				if (peers.removeList.size() > 0) {
+					removeNodes(peers.removeList);
 				}
 			}
 
@@ -762,8 +762,9 @@ protected Node createNode(NodeValidator nv) {
 		return node;
 	}
 
-	private final ArrayList<Node> findNodesToRemove(int refreshCount) {
-		ArrayList<Node> removeList = new ArrayList<Node>();
+	private final void findNodesToRemove(Peers peers) {
+		int refreshCount = peers.refreshCount;	
+		ArrayList<Node> removeList = peers.removeList;
 
 		for (Node node : nodes) {
 			if (! node.isActive()) {
@@ -797,7 +798,6 @@ private final ArrayList<Node> findNodesToRemove(int refreshCount) {
 				}
 			}
 		}
-		return removeList;
 	}
 
 	private final boolean findNodeInPartitionMap(Node filter) {

client/src/com/aerospike/client/cluster/Node.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2012-2023 Aerospike, Inc.
+ * Copyright 2012-2024 Aerospike, Inc.
  *
  * Portions may be licensed to Aerospike, Inc. under one or more contributor
  * license agreements WHICH ARE COMPATIBLE WITH THE APACHE LICENSE, VERSION 2.0.
@@ -426,7 +426,7 @@ protected final void refreshPeers(Peers peers) {
 			boolean peersValidated = true;
 
 			for (Peer peer : peers.peers) {
-				if (findPeerNode(cluster, peers, peer.nodeName)) {
+				if (findPeerNode(cluster, peers, peer)) {
 					// Node already exists. Do not even try to connect to hosts.
 					continue;
 				}
@@ -450,19 +450,16 @@ protected final void refreshPeers(Peers peers) {
 							if (Log.warnEnabled()) {
 								Log.warn("Peer node " + peer.nodeName + " is different than actual node " + nv.name + " for host " + host);
 							}
-
-							if (findPeerNode(cluster, peers, nv.name)) {
-								// Node already exists. Do not even try to connect to hosts.
-								nv.primaryConn.close();
-								nodeValidated = true;
-								break;
-							}
 						}
 
 						// Create new node.
 						Node node = cluster.createNode(nv);
 						peers.nodes.put(nv.name, node);
-						nodeValidated = true;
+						nodeValidated = true;							
+
+						if (peer.replaceNode != null) {
+							peers.removeList.add(peer.replaceNode);
+						}
 						break;
 					}
 					catch (Throwable e) {
@@ -490,20 +487,37 @@ protected final void refreshPeers(Peers peers) {
 		}
 	}
 
-	private static boolean findPeerNode(Cluster cluster, Peers peers, String nodeName) {
+	private static boolean findPeerNode(Cluster cluster, Peers peers, Peer peer) {
 		// Check global node map for existing cluster.
-		Node node = cluster.nodesMap.get(nodeName);
+		Node node = cluster.nodesMap.get(peer.nodeName);
 
 		if (node != null) {
-			node.referenceCount++;
-			return true;
+			// Node name found.
+			if (node.failures <= 0 || node.address.getAddress().isLoopbackAddress()) {
+				// If the node does not have cluster tend errors or is localhost,
+				// reject new peer as the IP address does not need to change.
+				node.referenceCount++;
+				return true;
+			}
+
+			// Match peer hosts with the node host.
+			for (Host h : peer.hosts) {
+				if (h.equals(node.host)) {
+					// Main node host is also the same as one of the peer hosts.
+					// Peer should not be added.
+					node.referenceCount++;
+					return true;
+				}
+			}
+			peer.replaceNode = node;
 		}
 
 		// Check local node map for this tend iteration.
-		node = peers.nodes.get(nodeName);
+		node = peers.nodes.get(peer.nodeName);
 
 		if (node != null) {
 			node.referenceCount++;
+			peer.replaceNode = null;
 			return true;
 		}
 		return false;

client/src/com/aerospike/client/cluster/Peer.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2012-2021 Aerospike, Inc.
+ * Copyright 2012-2024 Aerospike, Inc.
  *
  * Portions may be licensed to Aerospike, Inc. under one or more contributor
  * license agreements WHICH ARE COMPATIBLE WITH THE APACHE LICENSE, VERSION 2.0.
@@ -24,4 +24,5 @@ public final class Peer {
 	String nodeName;
 	String tlsName;
 	List<Host> hosts;
+	Node replaceNode;
 }

client/src/com/aerospike/client/cluster/Peers.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2012-2022 Aerospike, Inc.
+ * Copyright 2012-2024 Aerospike, Inc.
  *
  * Portions may be licensed to Aerospike, Inc. under one or more contributor
  * license agreements WHICH ARE COMPATIBLE WITH THE APACHE LICENSE, VERSION 2.0.
@@ -26,13 +26,15 @@
 public final class Peers {
 	public final ArrayList<Peer> peers;
 	public final HashMap<String,Node> nodes;
+	public final ArrayList<Node> removeList;
 	private final HashSet<Host> invalidHosts;
 	public int refreshCount;
 	public boolean genChanged;
 
 	public Peers(int peerCapacity) {
 		peers = new ArrayList<Peer>(peerCapacity);
 		nodes = new HashMap<String,Node>(16);
+		removeList = new ArrayList<Node>();
 		invalidHosts = new HashSet<Host>(8);
 	}