Unable to add Remote Cluster for Cross Cluster Replication

Hello Team,

We are using Elastic Enterprise edition license. We have created on-premise ECK (Kubenetes) clusters (2 clusters). We are trying to create remote cluster from Kibana(Cluster1) to test Cross Cluster Replication. We are using name and namespace from YAML files like below to configure seed nodes (cluster 2 nodes)

name.namespace.svc:9300

But we are getting below errors on data node logs and we are unable to create remote cluster. Please share your thoughts

=================================================================================

{"type": "server", "timestamp": "2022-03-10T05:59:51,774Z", "level": "WARN", "component": "o.e.t.SniffConnectionStrategy", "cluster.name": "osl-onprem", "node.name": "osl-onprem-es-data-2", "message": "fetching nodes from external cluster [ccr-dev2test] failed", "cluster.uuid": “xxxxxxx”, "node.id": “xxxxxxxx” ,

"stacktrace": ["org.Elasticsearch.transport.ConnectTransportException: [xx.xx.xx.xx:9300] general node connection failure",

"at org.Elasticsearch.transport.TcpTransport$ChannelsConnectedListener.lambda$onResponse$2(TcpTransport.java:1035) ~[Elasticsearch-7.16.2.jar:7.16.2]",

"at org.Elasticsearch.action.ActionListener$1.onFailure(ActionListener.java:144) ~[Elasticsearch-7.16.2.jar:7.16.2]",

"at org.Elasticsearch.transport.TransportHandshaker$HandshakeResponseHandler.handleLocalException(TransportHandshaker.java:155) ~[Elasticsearch-7.16.2.jar:7.16.2]",

"at org.Elasticsearch.transport.TransportHandshaker.lambda$sendHandshake$0(TransportHandshaker.java:52) ~[Elasticsearch-7.16.2.jar:7.16.2]",

"at org.Elasticsearch.action.ActionListener$2.onResponse(ActionListener.java:241) ~[Elasticsearch-7.16.2.jar:7.16.2]",

"at org.Elasticsearch.action.ActionListener.lambda$toBiConsumer$0(ActionListener.java:277) ~[Elasticsearch-7.16.2.jar:7.16.2]",

"at org.Elasticsearch.core.CompletableContext.lambda$addListener$0(CompletableContext.java:28) ~[Elasticsearch-core-7.16.2.jar:7.16.2]",

"at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?]",

"at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?]",

"at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]",

"at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?]",

"at org.Elasticsearch.core.CompletableContext.complete(CompletableContext.java:50) ~[Elasticsearch-core-7.16.2.jar:7.16.2]",

"at org.Elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:51) ~[?:?]",

"at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:578) ~[?:?]",

"at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:571) ~[?:?]",

"at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:550) ~[?:?]",

"at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:491) ~[?:?]",

"at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:616) ~[?:?]",

"at io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:605) ~[?:?]",

"at io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:104) ~[?:?]",

"at io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[?:?]",

"at io.netty.channel.AbstractChannel$CloseFuture.setClosed(AbstractChannel.java:1182) ~[?:?]",

"at io.netty.channel.AbstractChannel$AbstractUnsafe.doClose0(AbstractChannel.java:773) ~[?:?]",

"at io.netty.channel.AbstractChannel$AbstractUnsafe.close(AbstractChannel.java:749) ~[?:?]",

"at io.netty.channel.AbstractChannel$AbstractUnsafe.close(AbstractChannel.java:620) ~[?:?]",

"at io.netty.channel.DefaultChannelPipeline$HeadContext.close(DefaultChannelPipeline.java:1352) ~[?:?]",

"at io.netty.channel.AbstractChannelHandlerContext.invokeClose(AbstractChannelHandlerContext.java:622) ~[?:?]",

"at io.netty.channel.AbstractChannelHandlerContext.close(AbstractChannelHandlerContext.java:606) ~[?:?]",

"at io.netty.channel.AbstractChannelHandlerContext.close(AbstractChannelHandlerContext.java:472) ~[?:?]",

"at io.netty.handler.ssl.SslUtils.handleHandshakeFailure(SslUtils.java:440) ~[?:?]",

"at io.netty.handler.ssl.SslHandler.setHandshakeFailure(SslHandler.java:1883) ~[?:?]",

"at io.netty.handler.ssl.SslHandler.handleUnwrapThrowable(SslHandler.java:1275) ~[?:?]",

"at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1242) ~[?:?]",

"at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1286) ~[?:?]",

"at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) ~[?:?]",

"at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) ~[?:?]",

"at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) ~[?:?]",

"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) ~[?:?]",

"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) ~[?:?]",

"at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) ~[?:?]",

"at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[?:?]",

"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) ~[?:?]",

"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) ~[?:?]",

"at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[?:?]",

"at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[?:?]",

"at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:719) ~[?:?]",

"at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:620) ~[?:?]",

"at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:583) ~[?:?]",

"at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]",

"at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) ~[?:?]",

"at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]",

"at java.lang.Thread.run(Thread.java:833) [?:?]",

"Caused by: org.Elasticsearch.transport.TransportException: handshake failed because connection reset",

"... 49 more"] }

{"type": "server", "timestamp": "2022-03-10T05:59:51,773Z", "level": "WARN", "component": "o.e.t.TcpTransport", "cluster.name": "osl-onprem", "node.name": "osl-onprem-es-data-2", "message": "exception caught on transport layer [Netty4TcpChannel{localAddress=/xx.xx.xx.xx:xxxxx, remoteAddress=osl-onprem-es-transport.apl-cms-data-search-l3-dev2.svc/xx.xx.xx.xx:xxxx, profile=default}], closing connection", "cluster.uuid": “xxxxxx”, "node.id": “xxxxxxx” ,

"stacktrace": ["io.netty.handler.codec.DecoderException: javax.net.ssl.SSLHandshakeException: PKIX path validation failed: java.security.cert.CertPathValidatorException: signature check failed",

"at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:477) ~[netty-codec-4.1.66.Final.jar:4.1.66.Final]",

"at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) ~[netty-codec-4.1.66.Final.jar:4.1.66.Final]",

"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.66.Final.jar:4.1.66.Final]",

"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.66.Final.jar:4.1.66.Final]",

"at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.66.Final.jar:4.1.66.Final]",

"at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) [netty-transport-4.1.66.Final.jar:4.1.66.Final]",

"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.66.Final.jar:4.1.66.Final]",

"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.66.Final.jar:4.1.66.Final]",

"at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) [netty-transport-4.1.66.Final.jar:4.1.66.Final]",

"at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) [netty-transport-4.1.66.Final.jar:4.1.66.Final]",

"at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:719) [netty-transport-4.1.66.Final.jar:4.1.66.Final]",

"at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:620) [netty-transport-4.1.66.Final.jar:4.1.66.Final]",

"at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:583) [netty-transport-4.1.66.Final.jar:4.1.66.Final]",

Any idea?

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.