node1 [node_2409:8086:8110:3f0::2d]
node2 [node_2409:8086:8110:3f0::2e]
node3 [node_2409:8086:8110:3f0::2f]
here is node1 log
[2020-12-23T02:38:57,563][INFO ][o.e.c.c.JoinHelper ] [node_2409:8086:8110:3f0::2d] failed to join {node_2409:8086:8110:3f0::2e}{2H9c-7LTT-eha7ICDCCqcA}{BlX_bI3bSx2uJpouoyjf2A}{2409:8086:8110:3f0:0:0:0:2e}{[2409:8086:8110:3f0::2e]:9300}{dim} with JoinRequest{sourceNode={node_2409:8086:8110:3f0::2d}{44C3jmp9RNis66QaYeJj5A}{HdogZVSKQtu2V61zBCdNeQ}{2409:8086:8110:3f0:0:0:0:2d}{[2409:8086:8110:3f0::2d]:9300}{dim}, optionalJoin=Optional[Join{term=19, lastAcceptedTerm=13, lastAcceptedVersion=46097, sourceNode={node_2409:8086:8110:3f0::2d}{44C3jmp9RNis66QaYeJj5A}{HdogZVSKQtu2V61zBCdNeQ}{2409:8086:8110:3f0:0:0:0:2d}{[2409:8086:8110:3f0::2d]:9300}{dim}, targetNode={node_2409:8086:8110:3f0::2e}{2H9c-7LTT-eha7ICDCCqcA}{BlX_bI3bSx2uJpouoyjf2A}{2409:8086:8110:3f0:0:0:0:2e}{[2409:8086:8110:3f0::2e]:9300}{dim}}]}
org.elasticsearch.transport.RemoteTransportException: [node_2409:8086:8110:3f0::2e][[2409:8086:8110:3f0::2e]:9300][internal:cluster/coordination/join]
Caused by: java.lang.IllegalStateException: failure when sending a validation request to node
at org.elasticsearch.cluster.coordination.Coordinator$2.onFailure(Coordinator.java:506) ~[elasticsearch-7.3.1.jar:7.3.1]
at org.elasticsearch.cluster.coordination.JoinHelper$5.handleException(JoinHelper.java:360) ~[elasticsearch-7.3.1.jar:7.3.1]
at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1111) ~[elasticsearch-7.3.1.jar:7.3.1]
at org.elasticsearch.transport.InboundHandler.lambda$handleException$2(InboundHandler.java:246) ~[elasticsearch-7.3.1.jar:7.3.1]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:688) ~[elasticsearch-7.3.1.jar:7.3.1]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_242]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_242]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_242]
Caused by: org.elasticsearch.transport.RemoteTransportException: [node_2409:8086:8110:3f0::2d][[2409:8086:8110:3f0::2d]:9300][internal:cluster/coordination/join/validate]
Caused by: org.elasticsearch.cluster.coordination.CoordinationStateRejectedException: join validation on cluster state with a different cluster uuid _4xxif5aQ_qY2qjy4VzOKw than local cluster uuid x2h6qG3gQG-FAgdHGS_aqA, rejecting
at org.elasticsearch.cluster.coordination.JoinHelper.lambda$new$4(JoinHelper.java:148) ~[elasticsearch-7.3.1.jar:7.3.1]
at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) ~[elasticsearch-7.3.1.jar:7.3.1]
node2：
[2020-12-23T02:38:57,372][INFO ][o.e.c.s.ClusterApplierService] [node_2409:8086:8110:3f0::2e] master node changed {previous , current [{node_2409:8086:8110:3f0::2e}{2H9c-7LTT-eha7ICDCCqcA}{BlX_bI3bSx2uJpouoyjf2A}{2409:8086:8110:3f0:0:0:0:2e}{[2409:8086:8110:3f0::2e]:9300}{dim}]}, removed {{node_2409:8086:8110:3f0::2f}{2dFgJYXoTyWJ12es9CuiCQ}{tAO43Q-sRjGypNdmlUcNQQ}{2409:8086:8110:3f0:0:0:0:2f}{[2409:8086:8110:3f0::2f]:9300}{dim},}, added {{node_2409:8086:8110:3f0::2f}{2dFgJYXoTyWJ12es9CuiCQ}{p6QGBla8QkuMi-6-428Zgw}{2409:8086:8110:3f0:0:0:0:2f}{[2409:8086:8110:3f0::2f]:9300}{dim},}, term: 19, version: 58, reason: Publication{term=19, version=58}
[2020-12-23T02:38:57,393][INFO ][o.e.c.r.DelayedAllocationService] [node_2409:8086:8110:3f0::2e] scheduling reroute for delayed shards in [59.3s] (1 delayed shards)
[2020-12-23T02:38:57,450][INFO ][o.e.i.s.IndexShard ] [node_2409:8086:8110:3f0::2e] [i2000.mim][0] primary-replica resync completed with 0 operations
[2020-12-23T02:38:57,561][WARN ][o.e.c.c.Coordinator ] [node_2409:8086:8110:3f0::2e] failed to validate incoming join request from node [{node_2409:8086:8110:3f0::2d}{44C3jmp9RNis66QaYeJj5A}{HdogZVSKQtu2V61zBCdNeQ}{2409:8086:8110:3f0:0:0:0:2d}{[2409:8086:8110:3f0::2d]:9300}{dim}]
org.elasticsearch.transport.RemoteTransportException: [node_2409:8086:8110:3f0::2d][[2409:8086:8110:3f0::2d]:9300][internal:cluster/coordination/join/validate]
Caused by: org.elasticsearch.cluster.coordination.CoordinationStateRejectedException: join validation on cluster state with a different cluster uuid _4xxif5aQ_qY2qjy4VzOKw than local cluster uuid x2h6qG3gQG-FAgdHGS_aqA, rejecting
at org.elasticsearch.cluster.coordination.JoinHelper.lambda$new$4(JoinHelper.java:148) ~[elasticsearch-7.3.1.jar:7.3.1]
at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) ~[elasticsearch-7.3.1.jar:7.3.1]
[2020-12-23T02:38:58,547][WARN ][o.e.c.c.Coordinator ] [node_2409:8086:8110:3f0::2e] failed to validate incoming join request from node [{node_2409:8086:8110:3f0::2d}{44C3jmp9RNis66QaYeJj5A}{HdogZVSKQtu2V61zBCdNeQ}{2409:8086:8110:3f0:0:0:0:2d}{[2409:8086:8110:3f0::2d]:9300}{dim}]
org.elasticsearch.transport.RemoteTransportException: [node_2409:8086:8110:3f0::2d][[2409:8086:8110:3f0::2d]:9300][internal:cluster/coordination/join/validate]
Caused by: org.elasticsearch.cluster.coordination.CoordinationStateRejectedException: join validation on cluster state with a different cluster uuid _4xxif5aQ_qY2qjy4VzOKw than local cluster uuid x2h6qG3gQG-FAgdHGS_aqA, rejecting
at org.elasticsearch.cluster.coordination.JoinHelper.lambda$new$4(JoinHelper.java:148) ~[elasticsearch-7.3.1.jar:7.3.1]
at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) ~[elasticsearch-7.3.1.jar:7.3.1]
at org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:267) ~[elasticsearch-7.3.1.jar:7.3.1]
node3
[2020-12-23T02:38:56,761][INFO ][o.e.c.c.JoinHelper ] [node_2409:8086:8110:3f0::2f] failed to join {node_2409:8086:8110:3f0::2f}{2dFgJYXoTyWJ12es9CuiCQ}{p6QGBla8QkuMi-6-428Zgw}{2409:8086:8110:3f0:0:0:0:2f}{[2409:8086:8110:3f0::2f]:9300}{dim} with JoinRequest{sourceNode={node_2409:8086:8110:3f0::2f}{2dFgJYXoTyWJ12es9CuiCQ}{p6QGBla8QkuMi-6-428Zgw}{2409:8086:8110:3f0:0:0:0:2f}{[2409:8086:8110:3f0::2f]:9300}{dim}, optionalJoin=Optional[Join{term=18, lastAcceptedTerm=1, lastAcceptedVersion=57, sourceNode={node_2409:8086:8110:3f0::2f}{2dFgJYXoTyWJ12es9CuiCQ}{p6QGBla8QkuMi-6-428Zgw}{2409:8086:8110:3f0:0:0:0:2f}{[2409:8086:8110:3f0::2f]:9300}{dim}, targetNode={node_2409:8086:8110:3f0::2f}{2dFgJYXoTyWJ12es9CuiCQ}{p6QGBla8QkuMi-6-428Zgw}{2409:8086:8110:3f0:0:0:0:2f}{[2409:8086:8110:3f0::2f]:9300}{dim}}]}
org.elasticsearch.transport.RemoteTransportException: [node_2409:8086:8110:3f0::2f][[2409:8086:8110:3f0::2f]:9300][internal:cluster/coordination/join]
Caused by: org.elasticsearch.cluster.coordination.CoordinationStateRejectedException: became follower
at org.elasticsearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.lambda$close$3(JoinHelper.java:477) ~[elasticsearch-7.3.1.jar:7.3.1]
at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_242]
at org.elasticsearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.close(JoinHelper.java:477) ~[elasticsearch-7.3.1.jar:7.3.1]
at org.elasticsearch.cluster.coordination.Coordinator.becomeFollower(Coordinator.java:606) ~[elasticsearch-7.3.1.jar:7.3.1]
at org.elasticsearch.cluster.coordination.Coordinator.onFollowerCheckRequest(Coordinator.java:243) ~[elasticsearch-7.3.1.jar:7.3.1]
at org.elasticsearch.cluster.coordination.FollowersChecker$2.doRun(FollowersChecker.java:187) [elasticsearch-7.3.1.jar:7.3.1]
[2020-12-23T02:38:57,131][INFO ][o.e.c.s.ClusterApplierService] [node_2409:8086:8110:3f0::2f] master node changed {previous , current [{node_2409:8086:8110:3f0::2e}{2H9c-7LTT-eha7ICDCCqcA}{BlX_bI3bSx2uJpouoyjf2A}{2409:8086:8110:3f0:0:0:0:2e}{[2409:8086:8110:3f0::2e]:9300}{dim}]}, added {{node_2409:8086:8110:3f0::2e}{2H9c-7LTT-eha7ICDCCqcA}{BlX_bI3bSx2uJpouoyjf2A}{2409:8086:8110:3f0:0:0:0:2e}{[2409:8086:8110:3f0::2e]:9300}{dim},}, term: 19, version: 58, reason: ApplyCommitRequest{term=19, version=58, sourceNode={node_2409:8086:8110:3f0::2e}{2H9c-7LTT-eha7ICDCCqcA}{BlX_bI3bSx2uJpouoyjf2A}{2409:8086:8110:3f0:0:0:0:2e}{[2409:8086:8110:3f0::2e]:9300}{dim}}
[2020-12-23T02:38:57,390][INFO ][o.e.h.AbstractHttpServerTransport] [node_2409:8086:8110:3f0::2f] publish_address {[2409:8086:8110:3f0::2f]:9200}, bound_addresses {[2409:8086:8110:3f0::2f]:9200}
[2020-12-23T02:38:57,394][INFO ][o.e.n.Node ] [node_2409:8086:8110:3f0::2f] started
[2020-12-23T02:48:13,025][INFO ][o.e.c.c.Coordinator ] [node_2409:8086:8110:3f0::2f] master node [{node_2409:8086:8110:3f0::2e}{2H9c-7LTT-eha7ICDCCqcA}{BlX_bI3bSx2uJpouoyjf2A}{2409:8086:8110:3f0:0:0:0:2e}{[2409:8086:8110:3f0::2e]:9300}{dim}] failed, restarting discovery
org.elasticsearch.transport.NodeDisconnectedException: [node_2409:8086:8110:3f0::2e][[2409:8086:8110:3f0::2e]:9300][disconnected] disconnected
[2020-12-23T02:48:13,030][INFO ][o.e.c.s.ClusterApplierService] [node_2409:8086:8110:3f0::2f] master node changed {previous [{node_2409:8086:8110:3f0::2e}{2H9c-7LTT-eha7ICDCCqcA}{BlX_bI3bSx2uJpouoyjf2A}{2409:8086:8110:3f0:0:0:0:2e}{[2409:8086:8110:3f0::2e]:9300}{dim}], current }, term: 19, version: 112, reason: becoming candidate: onLeaderFailure
[2020-12-23T02:48:13,163][WARN ][o.e.c.NodeConnectionsService] [node_2409:8086:8110:3f0::2f] failed to connect to {node_2409:8086:8110:3f0::2e}{2H9c-7LTT-eha7ICDCCqcA}{BlX_bI3bSx2uJpouoyjf2A}{2409:8086:8110:3f0:0:0:0:2e}{[2409:8086:8110:3f0::2e]:9300}{dim} (tried [1] times)
org.elasticsearch.transport.ConnectTransportException: [node_2409:8086:8110:3f0::2e][[2409:8086:8110:3f0::2e]:9300] connect_exception
at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:957) ~[elasticsearch-7.3.1.jar:7.3.1]
at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$3(ActionListener.java:161) ~[elasticsearch-7.3.1.jar:7.3.1]
at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.3.1.jar:7.3.1]
at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:774) ~[?:1.8.0_242]
at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:750) ~[?:1.8.0_242]
at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:488) ~[?:1.8.0_242]
at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:1990) ~[?:1.8.0_242]
at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.3.1.jar:7.3.1]
at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]
[2020-12-23T02:48:23,028][WARN ][o.e.c.c.ClusterFormationFailureHelper] [node_2409:8086:8110:3f0::2f] master not discovered or elected yet, an election requires 2 nodes with ids [2dFgJYXoTyWJ12es9CuiCQ, 2H9c-7LTT-eha7ICDCCqcA], have discovered [{node_2409:8086:8110:3f0::2f}{2dFgJYXoTyWJ12es9CuiCQ}{p6QGBla8QkuMi-6-428Zgw}{2409:8086:8110:3f0:0:0:0:2f}{[2409:8086:8110:3f0::2f]:9300}{dim}, {node_2409:8086:8110:3f0::2d}{44C3jmp9RNis66QaYeJj5A}{HdogZVSKQtu2V61zBCdNeQ}{2409:8086:8110:3f0:0:0:0:2d}{[2409:8086:8110:3f0::2d]:9300}{dim}] which is not a quorum; discovery will continue using [[2409:8086:8110:3f0::2d]:9300, [2409:8086:8110:3f0::2e]:9300] from hosts providers and [{node_2409:8086:8110:3f0::2e}{2H9c-7LTT-eha7ICDCCqcA}{BlX_bI3bSx2uJpouoyjf2A}{2409:8086:8110:3f0:0:0:0:2e}{[2409:8086:8110:3f0::2e]:9300}{dim}, {node_2409:8086:8110:3f0::2f}{2dFgJYXoTyWJ12es9CuiCQ}{p6QGBla8QkuMi-6-428Zgw}{2409:8086:8110:3f0:0:0:0:2f}{[2409:8086:8110:3f0::2f]:9300}{dim}] from last-known cluster state; node term 19, last-accepted version 112 in term 19