Configured the Cross Cluster Search in elasticsearch.yml with two remote clusters. Everything is fine for the first time it starts.
But when the CCS node is restarted, it does not connect to the remote cluster and throws the below exception,
[2018-09-12T07:43:52,505][WARN ][o.e.t.RemoteClusterService] [node-2] failed to update seed list for cluster: one_six_four
org.elasticsearch.transport.ConnectTransportException: [][[2620:108:70:9303:5054:ff:fe29:4e72]:9300] handshake_timeout[30s]
at org.elasticsearch.transport.TcpTransport.executeHandshake(TcpTransport.java:1676) ~[elasticsearch-6.4.0.jar:6.4.0]
at org.elasticsearch.transport.TcpTransport.openConnection(TcpTransport.java:654) ~[elasticsearch-6.4.0.jar:6.4.0]
at org.elasticsearch.transport.TcpTransport.openConnection(TcpTransport.java:124) ~[elasticsearch-6.4.0.jar:6.4.0]
at org.elasticsearch.transport.TransportService.openConnection(TransportService.java:348) ~[elasticsearch-6.4.0.jar:6.4.0]
at org.elasticsearch.transport.RemoteClusterConnection$ConnectHandler.lambda$collectRemoteNodes$2(RemoteClusterConnection.java:458) ~[elasticsearch-6.4.0.jar:6.4.0]
at org.elasticsearch.common.util.CancellableThreads.executeIO(CancellableThreads.java:105) ~[elasticsearch-6.4.0.jar:6.4.0]
at org.elasticsearch.transport.RemoteClusterConnection$ConnectHandler.collectRemoteNodes(RemoteClusterConnection.java:455) [elasticsearch-6.4.0.jar:6.4.0]
at org.elasticsearch.transport.RemoteClusterConnection$ConnectHandler$1.doRun(RemoteClusterConnection.java:443) [elasticsearch-6.4.0.jar:6.4.0]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-6.4.0.jar:6.4.0]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [?:1.8.0_91]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) [?:1.8.0_91]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:624) [elasticsearch-6.4.0.jar:6.4.0]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [?:1.8.0_91]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [?:1.8.0_91]
at java.lang.Thread.run(Thread.java:745) [?:1.8.0_91]
[2018-09-12T07:43:52,520][WARN ][o.e.b.ElasticsearchUncaughtExceptionHandler] [node-2] uncaught exception in thread [main]
org.elasticsearch.bootstrap.StartupException: java.lang.IllegalStateException: failed to connect to remote clusters
at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:140) ~[elasticsearch-6.4.0.jar:6.4.0]
at org.elasticsearch.bootstrap.Elasticsearch.execute(Elasticsearch.java:127) ~[elasticsearch-6.4.0.jar:6.4.0]
at org.elasticsearch.cli.EnvironmentAwareCommand.execute(EnvironmentAwareCommand.java:86) ~[elasticsearch-6.4.0.jar:6.4.0]
at org.elasticsearch.cli.Command.mainWithoutErrorHandling(Command.java:124) ~[elasticsearch-cli-6.4.0.jar:6.4.0]
at org.elasticsearch.cli.Command.main(Command.java:90) ~[elasticsearch-cli-6.4.0.jar:6.4.0]
at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:93) ~[elasticsearch-6.4.0.jar:6.4.0]
at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:86) ~[elasticsearch-6.4.0.jar:6.4.0]
Caused by: java.lang.IllegalStateException: failed to connect to remote clusters
at org.elasticsearch.transport.RemoteClusterService.initializeRemoteClusters(RemoteClusterService.java:347) ~[elasticsearch-6.4.0.jar:6.4.0]
at org.elasticsearch.transport.TransportService.doStart(TransportService.java:227) ~[elasticsearch-6.4.0.jar:6.4.0]
at org.elasticsearch.common.component.AbstractLifecycleComponent.start(AbstractLifecycleComponent.java:66) ~[elasticsearch-6.4.0.jar:6.4.0]
at org.elasticsearch.node.Node.start(Node.java:667) ~[elasticsearch-6.4.0.jar:6.4.0]
at org.elasticsearch.bootstrap.Bootstrap.start(Bootstrap.java:265) ~[elasticsearch-6.4.0.jar:6.4.0]
at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:335) ~[elasticsearch-6.4.0.jar:6.4.0]
at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:136) ~[elasticsearch-6.4.0.jar:6.4.0]
... 6 more
Caused by: java.util.concurrent.ExecutionException: ConnectTransportException[[][[2620:108:70:9303:5054:ff:fe29:4e72]:9300] handshake_timeout[30s]]
at org.elasticsearch.common.util.concurrent.BaseFuture$Sync.getValue(BaseFuture.java:265) ~[elasticsearch-6.4.0.jar:6.4.0]
at org.elasticsearch.common.util.concurrent.BaseFuture$Sync.get(BaseFuture.java:238) ~[elasticsearch-6.4.0.jar:6.4.0]
at org.elasticsearch.common.util.concurrent.BaseFuture.get(BaseFuture.java:69) ~[elasticsearch-6.4.0.jar:6.4.0]
at org.elasticsearch.transport.RemoteClusterService.initializeRemoteClusters(RemoteClusterService.java:341) ~[elasticsearch-6.4.0.jar:6.4.0]
at org.elasticsearch.transport.TransportService.doStart(TransportService.java:227) ~[elasticsearch-6.4.0.jar:6.4.0]
at org.elasticsearch.common.component.AbstractLifecycleComponent.start(AbstractLifecycleComponent.java:66) ~[elasticsearch-6.4.0.jar:6.4.0]
at org.elasticsearch.node.Node.start(Node.java:667) ~[elasticsearch-6.4.0.jar:6.4.0]
at org.elasticsearch.bootstrap.Bootstrap.start(Bootstrap.java:265) ~[elasticsearch-6.4.0.jar:6.4.0]
at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:335) ~[elasticsearch-6.4.0.jar:6.4.0]
at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:136) ~[elasticsearch-6.4.0.jar:6.4.0]
... 6 more
Caused by: org.elasticsearch.transport.ConnectTransportException: [][[2620:108:70:9303:5054:ff:fe29:4e72]:9300] handshake_timeout[30s]
at org.elasticsearch.transport.TcpTransport.executeHandshake(TcpTransport.java:1676) ~[elasticsearch-6.4.0.jar:6.4.0]
at org.elasticsearch.transport.TcpTransport.openConnection(TcpTransport.java:654) ~[elasticsearch-6.4.0.jar:6.4.0]
at org.elasticsearch.transport.TcpTransport.openConnection(TcpTransport.java:124) ~[elasticsearch-6.4.0.jar:6.4.0]
at org.elasticsearch.transport.TransportService.openConnection(TransportService.java:348) ~[elasticsearch-6.4.0.jar:6.4.0]
at org.elasticsearch.transport.RemoteClusterConnection$ConnectHandler.lambda$collectRemoteNodes$2(RemoteClusterConnection.java:458) ~[elasticsearch-6.4.0.jar:6.4.0]
at org.elasticsearch.common.util.CancellableThreads.executeIO(CancellableThreads.java:105) ~[elasticsearch-6.4.0.jar:6.4.0]
Any idea?
Thanks in advance!