I have a cluster with 4 nodes , 2 master and 2 data nodes, currently only one master is responding, all the other nodes connect for 10 mins and shard allocation start, then again it becomes NA , not sure what is the issue?
[2023-07-23T22:22:11,855][TRACE][o.e.d.HandshakingTransportAddressConnector] [ES-Master-2] [connectToRemoteMasterNode[10.182.5.175:9840]] opened probe connection
[2023-07-23T22:22:11,855][TRACE][o.e.d.HandshakingTransportAddressConnector] [ES-Master-2] [connectToRemoteMasterNode[10.182.5.175:9840]] handshake successful: {ES-Aggr-2}{sfViGaurRaecVCv2RfDTig}{SoRVzQJkQX6yx4JsMMzyig}{ES-Aggr-2}{10.182.5.175:9840}{cdfhilrstw}{ml.machine_memory=16495927296, ml.max_open_jobs=512, xpack.installed=true, ml.max_jvm_size=2147483648, transform.node=true}
[2023-07-23T22:22:11,855][DEBUG][o.e.d.PeerFinder ] [ES-Master-2] address [10.182.5.175:9840], node [null], requesting [false] connection failed
org.elasticsearch.transport.ConnectTransportException: [ES-Aggr-2][10.182.5.175:9840] non-master-eligible node found
at org.elasticsearch.discovery.HandshakingTransportAddressConnector$1$1.innerOnResponse(HandshakingTransportAddressConnector.java:115) [elasticsearch-7.16.2.jar:7.16.2]
at org.elasticsearch.discovery.HandshakingTransportAddressConnector$1$1.innerOnResponse(HandshakingTransportAddressConnector.java:103) [elasticsearch-7.16.2.jar:7.16.2]
at org.elasticsearch.action.NotifyOnceListener.onResponse(NotifyOnceListener.java:29) [elasticsearch-7.16.2.jar:7.16.2]
at org.elasticsearch.action.ActionListener$MappedActionListener.onResponse(ActionListener.java:101) [elasticsearch-7.16.2.jar:7.16.2]
at org.elasticsearch.transport.TransportService.lambda$handshake$9(TransportService.java:577) [elasticsearch-7.16.2.jar:7.16.2]
at org.elasticsearch.action.ActionListener$DelegatingFailureActionListener.onResponse(ActionListener.java:219) [elasticsearch-7.16.2.jar:7.16.2]
at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:43) [elasticsearch-7.16.2.jar:7.16.2]
at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1471) [elasticsearch-7.16.2.jar:7.16.2]
at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1471) [elasticsearch-7.16.2.jar:7.16.2]
at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:340) [elasticsearch-7.16.2.jar:7.16.2]
at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:328) [elasticsearch-7.16.2.jar:7.16.2]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:718) [elasticsearch-7.16.2.jar:7.16.2]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?]
at java.lang.Thread.run(Thread.java:833) [?:?]
[2023-07-23T22:22:13,464][TRACE][o.e.d.PeerFinder ] [ES-Master-2] startProbe(10.182.5.183:9840) not probing local node
[2023-07-23T22:22:13,636][TRACE][o.e.d.PeerFinder ] [ES-Master-2] address [10.182.4.184:9840], node [{ES-Master-1}{GW9UYH3tSR2LiMUTedmPuw}{o_TwPom5THemHV1O4YbmzA}{ES-Master-1}{10.182.4.184:9840}{cdfhilmrstw}{ml.machine_memory=16495927296, ml.max_open_jobs=512, xpack.installed=true, ml.max_jvm_size=2147483648, transform.node=true}], requesting [false] requesting peers
[2023-07-23T22:22:13,636][TRACE][o.e.d.PeerFinder ] [ES-Master-2] probing master nodes from cluster state: nodes:
{ES-Master-1}{GW9UYH3tSR2LiMUTedmPuw}{o_TwPom5THemHV1O4YbmzA}{ES-Master-1}{10.182.4.184:9840}{cdfhilmrstw}{ml.machine_memory=16495927296, ml.max_open_jobs=512, xpack.installed=true, ml.max_jvm_size=2147483648, transform.node=true}, master
{ES-Master-2}{nfO_NdY6QdK5PWI1-Qj6Ew}{9ajB76fiTUiXnmp0RzZNug}{ES-Master-2}{10.182.5.183:9840}{cdfhilmrstw}{ml.machine_memory=16495927296, xpack.installed=true, transform.node=true, ml.max_open_jobs=512, ml.max_jvm_size=2147483648}, local
{ES-Aggr-2}{sfViGaurRaecVCv2RfDTig}{SoRVzQJkQX6yx4JsMMzyig}{ES-Aggr-2}{10.182.5.175:9840}{cdfhilrstw}{ml.machine_memory=16495927296, ml.max_open_jobs=512, xpack.installed=true, ml.max_jvm_size=2147483648, transform.node=true}
{ES-Aggr-1}{beiqGC2KTeuwlx4ZOvkulQ}{0py7SdmPS6iXVIdFVF3oiQ}{ES-Aggr-1}{10.182.4.170:9840}{cdfhilrstw}{ml.machine_memory=16495927296, ml.max_open_jobs=512, xpack.installed=true, ml.max_jvm_size=2147483648, transform.node=true}
[2023-07-23T22:22:13,636][TRACE][o.e.d.PeerFinder ] [ES-Master-2] startProbe(10.182.5.183:9840) not probing local node
[2023-07-23T22:22:13,636][TRACE][o.e.d.SeedHostsResolver ] [ES-Master-2] resolved host [ES-Master-1] to [10.182.4.184:9840]
[2023-07-23T22:22:13,637][TRACE][o.e.d.SeedHostsResolver ] [ES-Master-2] resolved host [ES-Master-2] to [10.182.5.183:9840]
[2023-07-23T22:22:13,637][TRACE][o.e.d.SeedHostsResolver ] [ES-Master-2] resolved host [ES-Aggr-1] to [10.182.4.170:9840]
[2023-07-23T22:22:13,637][TRACE][o.e.d.SeedHostsResolver ] [ES-Master-2] resolved host [ES-Aggr-2] to [10.182.5.175:9840]
[2023-07-23T22:22:13,637][TRACE][o.e.d.PeerFinder ] [ES-Master-2] probing resolved transport addresses [10.182.4.184:9840, 10.182.4.170:9840, 10.182.5.175:9840]
[2023-07-23T22:22:13,637][TRACE][o.e.d.PeerFinder ] [ES-Master-2] address [10.182.4.170:9840], node [null], requesting [false] attempting connection
[2023-07-23T22:22:13,637][TRACE][o.e.d.PeerFinder ] [ES-Master-2] address [10.182.5.175:9840], node [null], requesting [false] attempting connection
[2023-07-23T22:22:13,637][TRACE][o.e.d.HandshakingTransportAddressConnector] [ES-Master-2] [connectToRemoteMasterNode[10.182.4.170:9840]] opening probe connection
[2023-07-23T22:22:13,637][TRACE][o.e.d.HandshakingTransportAddressConnector] [ES-Master-2] [connectToRemoteMasterNode[10.182.5.175:9840]] opening probe connection
[2023-07-23T22:22:13,638][TRACE][o.e.d.PeerFinder ] [ES-Master-2] address [10.182.4.184:9840], node [{ES-Master-1}{GW9UYH3tSR2LiMUTedmPuw}{o_TwPom5THemHV1O4YbmzA}{ES-Master-1}{10.182.4.184:9840}{cdfhilmrstw}{ml.machine_memory=16495927296, ml.max_open_jobs=512, xpack.installed=true, ml.max_jvm_size=2147483648, transform.node=true}], requesting [true] received PeersResponse{masterNode=Optional[{ES-Master-1}{GW9UYH3tSR2LiMUTedmPuw}{o_TwPom5THemHV1O4YbmzA}{ES-Master-1}{10.182.4.184:9840}{cdfhilmrstw}{ml.machine_memory=16495927296, ml.max_open_jobs=512, xpack.installed=true, ml.max_jvm_size=2147483648, transform.node=true}], knownPeers=[], term=2357}
[2023-07-23T22:22:13,651][TRACE][o.e.d.HandshakingTransportAddressConnector] [ES-Master-2] [connectToRemoteMasterNode[10.182.4.170:9840]] opened probe connection
[2023-07-23T22:22:13,651][TRACE][o.e.d.HandshakingTransportAddressConnector] [ES-Master-2] [connectToRemoteMasterNode[10.182.5.175:9840]] opened probe connection
[2023-07-23T22:22:13,728][TRACE][o.e.d.HandshakingTransportAddressConnector] [ES-Master-2] [connectToRemoteMasterNode[10.182.5.175:9840]] handshake successful: {ES-Aggr-2}{sfViGaurRaecVCv2RfDTig}{SoRVzQJkQX6yx4JsMMzyig}{ES-Aggr-2}{10.182.5.175:9840}{cdfhilrstw}{ml.machine_memory=16495927296, ml.max_open_jobs=512, xpack.installed=true, ml.max_jvm_size=2147483648, transform.node=true}
[2023-07-23T22:22:13,729][DEBUG][o.e.d.PeerFinder ] [ES-Master-2] address [10.182.5.175:9840], node [null], requesting [false] connection failed
org.elasticsearch.transport.ConnectTransportException: [ES-Aggr-2][10.182.5.175:9840] non-master-eligible node found
at org.elasticsearch.discovery.HandshakingTransportAddressConnector$1$1.innerOnResponse(HandshakingTransportAddressConnector.java:115) [elasticsearch-7.16.2.jar:7.16.2]
at org.elasticsearch.discovery.HandshakingTransportAddressConnector$1$1.innerOnResponse(HandshakingTransportAddressConnector.java:103) [elasticsearch-7.16.2.jar:7.16.2]
at org.elasticsearch.action.NotifyOnceListener.onResponse(NotifyOnceListener.java:29) [elasticsearch-7.16.2.jar:7.16.2]
at org.elasticsearch.action.ActionListener$MappedActionListener.onResponse(ActionListener.java:101) [elasticsearch-7.16.2.jar:7.16.2]
at org.elasticsearch.transport.TransportService.lambda$handshake$9(TransportService.java:577) [elasticsearch-7.16.2.jar:7.16.2]
at org.elasticsearch.action.ActionListener$DelegatingFailureActionListener.onResponse(ActionListener.java:219) [elasticsearch-7.16.2.jar:7.16.2]
at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:43) [elasticsearch-7.16.2.jar:7.16.2]
at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1471) [elasticsearch-7.16.2.jar:7.16.2]
at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1471) [elasticsearch-7.16.2.jar:7.16.2]
at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:340) [elasticsearch-7.16.2.jar:7.16.2]
at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:328) [elasticsearch-7.16.2.jar:7.16.2]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:718) [elasticsearch-7.16.2.jar:7.16.2]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?]
at java.lang.Thread.run(Thread.java:833) [?:?]