Hello Team,
We are using Elastic Enterprise edition license. We have created on-premise ECK (Kubenetes) clusters (2 clusters). We are trying to create remote cluster from Kibana(Cluster1) to test Cross Cluster Replication. We are using name and namespace from YAML files like below to configure seed nodes (cluster 2 nodes)
name.namespace.svc:9300
But we are getting below errors on data node logs and we are unable to create remote cluster. Please share your thoughts
=================================================================================
{"type": "server", "timestamp": "2022-03-10T05:59:51,774Z", "level": "WARN", "component": "o.e.t.SniffConnectionStrategy", "cluster.name": "osl-onprem", "node.name": "osl-onprem-es-data-2", "message": "fetching nodes from external cluster [ccr-dev2test] failed", "cluster.uuid": “xxxxxxx”, "node.id": “xxxxxxxx” ,
"stacktrace": ["org.Elasticsearch.transport.ConnectTransportException: [xx.xx.xx.xx:9300] general node connection failure",
"at org.Elasticsearch.transport.TcpTransport$ChannelsConnectedListener.lambda$onResponse$2(TcpTransport.java:1035) ~[Elasticsearch-7.16.2.jar:7.16.2]",
"at org.Elasticsearch.action.ActionListener$1.onFailure(ActionListener.java:144) ~[Elasticsearch-7.16.2.jar:7.16.2]",
"at org.Elasticsearch.transport.TransportHandshaker$HandshakeResponseHandler.handleLocalException(TransportHandshaker.java:155) ~[Elasticsearch-7.16.2.jar:7.16.2]",
"at org.Elasticsearch.transport.TransportHandshaker.lambda$sendHandshake$0(TransportHandshaker.java:52) ~[Elasticsearch-7.16.2.jar:7.16.2]",
"at org.Elasticsearch.action.ActionListener$2.onResponse(ActionListener.java:241) ~[Elasticsearch-7.16.2.jar:7.16.2]",
"at org.Elasticsearch.action.ActionListener.lambda$toBiConsumer$0(ActionListener.java:277) ~[Elasticsearch-7.16.2.jar:7.16.2]",
"at org.Elasticsearch.core.CompletableContext.lambda$addListener$0(CompletableContext.java:28) ~[Elasticsearch-core-7.16.2.jar:7.16.2]",
"at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?]",
"at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?]",
"at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]",
"at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?]",
"at org.Elasticsearch.core.CompletableContext.complete(CompletableContext.java:50) ~[Elasticsearch-core-7.16.2.jar:7.16.2]",
"at org.Elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:51) ~[?:?]",
"at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:578) ~[?:?]",
"at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:571) ~[?:?]",
"at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:550) ~[?:?]",
"at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:491) ~[?:?]",
"at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:616) ~[?:?]",
"at io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:605) ~[?:?]",
"at io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:104) ~[?:?]",
"at io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[?:?]",
"at io.netty.channel.AbstractChannel$CloseFuture.setClosed(AbstractChannel.java:1182) ~[?:?]",
"at io.netty.channel.AbstractChannel$AbstractUnsafe.doClose0(AbstractChannel.java:773) ~[?:?]",
"at io.netty.channel.AbstractChannel$AbstractUnsafe.close(AbstractChannel.java:749) ~[?:?]",
"at io.netty.channel.AbstractChannel$AbstractUnsafe.close(AbstractChannel.java:620) ~[?:?]",
"at io.netty.channel.DefaultChannelPipeline$HeadContext.close(DefaultChannelPipeline.java:1352) ~[?:?]",
"at io.netty.channel.AbstractChannelHandlerContext.invokeClose(AbstractChannelHandlerContext.java:622) ~[?:?]",
"at io.netty.channel.AbstractChannelHandlerContext.close(AbstractChannelHandlerContext.java:606) ~[?:?]",
"at io.netty.channel.AbstractChannelHandlerContext.close(AbstractChannelHandlerContext.java:472) ~[?:?]",
"at io.netty.handler.ssl.SslUtils.handleHandshakeFailure(SslUtils.java:440) ~[?:?]",
"at io.netty.handler.ssl.SslHandler.setHandshakeFailure(SslHandler.java:1883) ~[?:?]",
"at io.netty.handler.ssl.SslHandler.handleUnwrapThrowable(SslHandler.java:1275) ~[?:?]",
"at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1242) ~[?:?]",
"at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1286) ~[?:?]",
"at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) ~[?:?]",
"at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) ~[?:?]",
"at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) ~[?:?]",
"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) ~[?:?]",
"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) ~[?:?]",
"at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) ~[?:?]",
"at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[?:?]",
"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) ~[?:?]",
"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) ~[?:?]",
"at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[?:?]",
"at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[?:?]",
"at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:719) ~[?:?]",
"at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:620) ~[?:?]",
"at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:583) ~[?:?]",
"at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]",
"at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) ~[?:?]",
"at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]",
"at java.lang.Thread.run(Thread.java:833) [?:?]",
"Caused by: org.Elasticsearch.transport.TransportException: handshake failed because connection reset",
"... 49 more"] }
{"type": "server", "timestamp": "2022-03-10T05:59:51,773Z", "level": "WARN", "component": "o.e.t.TcpTransport", "cluster.name": "osl-onprem", "node.name": "osl-onprem-es-data-2", "message": "exception caught on transport layer [Netty4TcpChannel{localAddress=/xx.xx.xx.xx:xxxxx, remoteAddress=osl-onprem-es-transport.apl-cms-data-search-l3-dev2.svc/xx.xx.xx.xx:xxxx, profile=default}], closing connection", "cluster.uuid": “xxxxxx”, "node.id": “xxxxxxx” ,
"stacktrace": ["io.netty.handler.codec.DecoderException: javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed",
"at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:477) ~[netty-codec-4.1.66.Final.jar:4.1.66.Final]",
"at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) ~[netty-codec-4.1.66.Final.jar:4.1.66.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.66.Final.jar:4.1.66.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.66.Final.jar:4.1.66.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.66.Final.jar:4.1.66.Final]",
"at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) [netty-transport-4.1.66.Final.jar:4.1.66.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.66.Final.jar:4.1.66.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.66.Final.jar:4.1.66.Final]",
"at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) [netty-transport-4.1.66.Final.jar:4.1.66.Final]",
"at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) [netty-transport-4.1.66.Final.jar:4.1.66.Final]",
"at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:719) [netty-transport-4.1.66.Final.jar:4.1.66.Final]",
"at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:620) [netty-transport-4.1.66.Final.jar:4.1.66.Final]",
"at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:583) [netty-transport-4.1.66.Final.jar:4.1.66.Final]",