2019-11-19 15:48:29.829 WARN 15682 --- [orker-epoll-2-1] c.l.a.c.HttpClientPipelineConfigurator : [id: 0x40b6a66f, L:/172.17.2.154:60238 - R:172.17.2.154/172.17.2.154:9200] TLS handshake failed:
javax.net.ssl.SSLHandshakeException: error:10000410:SSL routines:OPENSSL_internal:SSLV3_ALERT_HANDSHAKE_FAILURE
at io.netty.handler.ssl.ReferenceCountedOpenSslEngine.shutdownWithError(ReferenceCountedOpenSslEngine.java:1002) ~[netty-handler-4.1.43.Final.jar!/:4.1.43.Final]
at io.netty.handler.ssl.ReferenceCountedOpenSslEngine.sslReadErrorResult(ReferenceCountedOpenSslEngine.java:1268) ~[netty-handler-4.1.43.Final.jar!/:4.1.43.Final]
at io.netty.handler.ssl.ReferenceCountedOpenSslEngine.unwrap(ReferenceCountedOpenSslEngine.java:1222) ~[netty-handler-4.1.43.Final.jar!/:4.1.43.Final]
at io.netty.handler.ssl.ReferenceCountedOpenSslEngine.unwrap(ReferenceCountedOpenSslEngine.java:1293) ~[netty-handler-4.1.43.Final.jar!/:4.1.43.Final]
at io.netty.handler.ssl.ReferenceCountedOpenSslEngine.unwrap(ReferenceCountedOpenSslEngine.java:1336) ~[netty-handler-4.1.43.Final.jar!/:4.1.43.Final]
at io.netty.handler.ssl.SslHandler$SslEngineType$1.unwrap(SslHandler.java:204) ~[netty-handler-4.1.43.Final.jar!/:4.1.43.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1332) [netty-handler-4.1.43.Final.jar!/:4.1.43.Final]
at io.netty.handler.ssl.SslHandler.decodeNonJdkCompatible(SslHandler.java:1239) [netty-handler-4.1.43.Final.jar!/:4.1.43.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1276) [netty-handler-4.1.43.Final.jar!/:4.1.43.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:503) [netty-codec-4.1.43.Final.jar!/:4.1.43.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:442) [netty-codec-4.1.43.Final.jar!/:4.1.43.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:281) [netty-codec-4.1.43.Final.jar!/:4.1.43.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) [netty-transport-4.1.43.Final.jar!/:4.1.43.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) [netty-transport-4.1.43.Final.jar!/:4.1.43.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) [netty-transport-4.1.43.Final.jar!/:4.1.43.Final]
at io.netty.handler.flush.FlushConsolidationHandler.channelRead(FlushConsolidationHandler.java:154) [netty-handler-4.1.43.Final.jar!/:4.1.43.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) [netty-transport-4.1.43.Final.jar!/:4.1.43.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) [netty-transport-4.1.43.Final.jar!/:4.1.43.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) [netty-transport-4.1.43.Final.jar!/:4.1.43.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1422) [netty-transport-4.1.43.Final.jar!/:4.1.43.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) [netty-transport-4.1.43.Final.jar!/:4.1.43.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) [netty-transport-4.1.43.Final.jar!/:4.1.43.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:931) [netty-transport-4.1.43.Final.jar!/:4.1.43.Final]
at io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:792) [netty-transport-native-epoll-4.1.43.Final-linux-x86_64.jar!/:4.1.43.Final]
at io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:502) [netty-transport-native-epoll-4.1.43.Final-linux-x86_64.jar!/:4.1.43.Final]
at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:407) [netty-transport-native-epoll-4.1.43.Final-linux-x86_64.jar!/:4.1.43.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$6.run(SingleThreadEventExecutor.java:1050) [netty-common-4.1.43.Final.jar!/:4.1.43.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.43.Final.jar!/:4.1.43.Final]
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) [netty-common-4.1.43.Final.jar!/:4.1.43.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_212]
In addition, these log keep printing ,may be need do some optimization
The logs you have posted are from zipkin, not from Elasticsearch, so I'm not sure if many people in these forums can help you with these. How about you send us the logs from elasticsearch instead ?
Also , you mention No cipher suits in common in the topic but I don't see this anywhere in the logs you sent us. Maybe some part of your post didn't make it through ?
@ikakavas, Thanks again for taking time to answer my question
Elasticsearch version : 6.8.0
Zipkin version 2.19.1
Elasticsearch log
[2019-11-25T10:01:04,749][WARN ][o.e.h.n.Netty4HttpServerTransport] [lFAgPu1] caught exception while handling client http traffic, closing connection [id: 0x2359f2a9, L:0.0.0.0/0.0.0.0:9200 ! R:/172.17.2.154:43979]
io.netty.handler.codec.DecoderException: javax.net.ssl.SSLHandshakeException: no cipher suites in common
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:472) ~[netty-codec-4.1.32.Final.jar:4.1.32.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:278) ~[netty-codec-4.1.32.Final.jar:4.1.32.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) [netty-transport-4.1.32.Final.jar:4.1.32.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) [netty-transport-4.1.32.Final.jar:4.1.32.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.32.Final.jar:4.1.32.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) [netty-transport-4.1.32.Final.jar:4.1.32.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) [netty-transport-4.1.32.Final.jar:4.1.32.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) [netty-transport-4.1.32.Final.jar:4.1.32.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) [netty-transport-4.1.32.Final.jar:4.1.32.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) [netty-common-4.1.32.Final.jar:4.1.32.Final]
at java.lang.Thread.run(Thread.java:745) [?:1.8.0_121]
Caused by: javax.net.ssl.SSLHandshakeException: no cipher suites in common
at sun.security.ssl.Handshaker.checkThrown(Handshaker.java:1478) ~[?:?]
at sun.security.ssl.SSLEngineImpl.checkTaskThrown(SSLEngineImpl.java:535) ~[?:?]
at sun.security.ssl.SSLEngineImpl.readNetRecord(SSLEngineImpl.java:813) ~[?:?]
at sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:781) ~[?:?]
at javax.net.ssl.SSLEngine.unwrap(SSLEngine.java:624) ~[?:1.8.0_121]
at io.netty.handler.ssl.SslHandler$SslEngineType$3.unwrap(SslHandler.java:295) ~[netty-handler-4.1.32.Final.jar:4.1.32.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1301) ~[netty-handler-4.1.32.Final.jar:4.1.32.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1203) ~[netty-handler-4.1.32.Final.jar:4.1.32.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1247) ~[netty-handler-4.1.32.Final.jar:4.1.32.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:502) ~[netty-codec-4.1.32.Final.jar:4.1.32.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:441) ~[netty-codec-4.1.32.Final.jar:4.1.32.Final]
... 15 more
Caused by: javax.net.ssl.SSLHandshakeException: no cipher suites in common
at sun.security.ssl.Alerts.getSSLException(Alerts.java:192) ~[?:?]
at sun.security.ssl.SSLEngineImpl.fatal(SSLEngineImpl.java:1666) ~[?:?]
at sun.security.ssl.Handshaker.fatalSE(Handshaker.java:304) ~[?:?]
at sun.security.ssl.Handshaker.fatalSE(Handshaker.java:292) ~[?:?]
at sun.security.ssl.ServerHandshaker.chooseCipherSuite(ServerHandshaker.java:1045) ~[?:?]
at sun.security.ssl.ServerHandshaker.clientHello(ServerHandshaker.java:741) ~[?:?]
at sun.security.ssl.ServerHandshaker.processMessage(ServerHandshaker.java:224) ~[?:?]
at sun.security.ssl.Handshaker.processLoop(Handshaker.java:1026) ~[?:?]
at sun.security.ssl.Handshaker$1.run(Handshaker.java:966) ~[?:?]
at sun.security.ssl.Handshaker$1.run(Handshaker.java:963) ~[?:?]
Generate certificate steps
1.bin/elasticsearch-certutil ca
2.bin/elasticsearch-certutil cert --ca elastic-stack-ca.p12
It's not clear what TLS version Zipking tries to use to talk to Elasticsearch, but you can probably force it to use TLSv1.2 with -Djdk.tls.client.protocols=TLSv1.2
I am remove the part and add -Djdk.tls.client.protocols=TLSv1.2,but still No cipher suits in common
then
I add -Djavax.net.debug=all check the detail
I think these logs from zipkin may help.
System property jdk.tls.client.cipherSuites is set to 'null'
System property jdk.tls.server.cipherSuites is set to 'null'
Ignoring disabled cipher suite: TLS_DH_anon_WITH_AES_256_CBC_SHA
Ignoring disabled cipher suite: TLS_DH_anon_WITH_AES_256_CBC_SHA256
Ignoring disabled cipher suite: TLS_ECDHE_RSA_WITH_NULL_SHA
Ignoring disabled cipher suite: SSL_RSA_WITH_DES_CBC_SHA
Ignoring disabled cipher suite: SSL_DHE_DSS_EXPORT_WITH_DES40_CBC_SHA
...
Ignoring disabled cipher suite: SSL_DHE_DSS_WITH_3DES_EDE_CBC_SHA
Ignoring disabled cipher suite: TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA
Ignoring disabled cipher suite: SSL_RSA_WITH_3DES_EDE_CBC_SHA
Ignoring disabled cipher suite: TLS_ECDH_RSA_WITH_3DES_EDE_CBC_SHA
Ignoring disabled cipher suite: TLS_ECDH_ECDSA_WITH_3DES_EDE_CBC_SHA
Ignoring disabled cipher suite: SSL_DHE_RSA_WITH_3DES_EDE_CBC_SHA
what's more , zipkin still prints log like javax.net.ssl.SSLHandshakeException: error:10000410:SSL routines:OPENSSL_internal:SSLV3_ALERT_HANDSHAKE_FAILURE
io.netty.handler.codec.DecoderException: javax.net.ssl.SSLHandshakeException: no cipher suites in common
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:472) ~[netty-codec-4.1.32.Final.jar:4.1.32.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:278) ~[netty-codec-4.1.32.Final.jar:4.1.32.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) [netty-transport-4.1.32.Final.jar:4.1.32.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) [netty-transport-4.1.32.Final.jar:4.1.32.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.32.Final.jar:4.1.32.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) [netty-transport-4.1.32.Final.jar:4.1.32.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) [netty-transport-4.1.32.Final.jar:4.1.32.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) [netty-transport-4.1.32.Final.jar:4.1.32.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) [netty-transport-4.1.32.Final.jar:4.1.32.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) [netty-common-4.1.32.Final.jar:4.1.32.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_211]
Caused by: javax.net.ssl.SSLHandshakeException: no cipher suites in common
at sun.security.ssl.Handshaker.checkThrown(Handshaker.java:1521) ~[?:?]
at sun.security.ssl.SSLEngineImpl.checkTaskThrown(SSLEngineImpl.java:528) ~[?:?]
at sun.security.ssl.SSLEngineImpl.readNetRecord(SSLEngineImpl.java:802) ~[?:?]
at sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:766) ~[?:?]
at javax.net.ssl.SSLEngine.unwrap(SSLEngine.java:624) ~[?:1.8.0_211]
at io.netty.handler.ssl.SslHandler$SslEngineType$3.unwrap(SslHandler.java:295) ~[netty-handler-4.1.32.Final.jar:4.1.32.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1301) ~[netty-handler-4.1.32.Final.jar:4.1.32.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1203) ~[netty-handler-4.1.32.Final.jar:4.1.32.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1247) ~[netty-handler-4.1.32.Final.jar:4.1.32.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:502) ~[netty-codec-4.1.32.Final.jar:4.1.32.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:441) ~[netty-codec-4.1.32.Final.jar:4.1.32.Final]
... 15 more
Caused by: javax.net.ssl.SSLHandshakeException: no cipher suites in common
at sun.security.ssl.Alerts.getSSLException(Alerts.java:192) ~[?:?]
at sun.security.ssl.SSLEngineImpl.fatal(SSLEngineImpl.java:1647) ~[?:?]
at sun.security.ssl.Handshaker.fatalSE(Handshaker.java:318) ~[?:?]
at sun.security.ssl.Handshaker.fatalSE(Handshaker.java:306) ~[?:?]
at sun.security.ssl.ServerHandshaker.chooseCipherSuite(ServerHandshaker.java:1127) ~[?:?]
at sun.security.ssl.ServerHandshaker.clientHello(ServerHandshaker.java:814) ~[?:?]
at sun.security.ssl.ServerHandshaker.processMessage(ServerHandshaker.java:221) ~[?:?]
at sun.security.ssl.Handshaker.processLoop(Handshaker.java:1037) ~[?:?]
at sun.security.ssl.Handshaker$1.run(Handshaker.java:970) ~[?:?]
at sun.security.ssl.Handshaker$1.run(Handshaker.java:967) ~[?:?]
at java.security.AccessController.doPrivileged(Native Method) ~[?:1.8.0_211]
at sun.security.ssl.Handshaker$DelegatedTask.run(Handshaker.java:1459) ~[?:?]
at io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1464) ~[netty-handler-4.1.32.Final.jar:4.1.32.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1369) ~[netty-handler-4.1.32.Final.jar:4.1.32.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1203) ~[netty-handler-4.1.32.Final.jar:4.1.32.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1247) ~[netty-handler-4.1.32.Final.jar:4.1.32.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:502) ~[netty-codec-4.1.32.Final.jar:4.1.32.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:441) ~[netty-codec-4.1.32.Final.jar:4.1.32.Final]
... 15 more
Are you running elasticsearch and zipkin on the same host ?
Yes.
Are you using the same java installation for both ? If not what Java versions are you using ?
Apache, Apache Lucene, Apache Hadoop, Hadoop, HDFS and the yellow elephant
logo are trademarks of the
Apache Software Foundation
in the United States and/or other countries.