(RecoveryFileChunkRequest.java:35) ~[elasticsearch-7.14.0.jar:7.14.0]
at org.elasticsearch.transport.RequestHandlerRegistry.newRequest(RequestHandlerRegistry.java:48) ~[elasticsearch-7.14.0.jar:7.14.0]
at org.elasticsearch.transport.InboundHandler.handleRequest(InboundHandler.java:188) ~[elasticsearch-7.14.0.jar:7.14.0]
at org.elasticsearch.transport.InboundHandler.messageReceived(InboundHandler.java:100) ~[elasticsearch-7.14.0.jar:7.14.0]
at org.elasticsearch.transport.InboundHandler.inboundMessage(InboundHandler.java:82) ~[elasticsearch-7.14.0.jar:7.14.0]
at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:710) ~[elasticsearch-7.14.0.jar:7.14.0]
at org.elasticsearch.transport.InboundPipeline.forwardFragments(InboundPipeline.java:129) ~[elasticsearch-7.14.0.jar:7.14.0]
at org.elasticsearch.transport.InboundPipeline.doHandleBytes(InboundPipeline.java:104) ~[elasticsearch-7.14.0.jar:7.14.0]
at org.elasticsearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:69) ~[elasticsearch-7.14.0.jar:7.14.0]
at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:63) ~[?:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) ~[?:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) ~[?:?]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) ~[?:?]
at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:271) ~[?:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) ~[?:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) ~[?:?]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) ~[?:?]
at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) ~[?:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) ~[?:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) ~[?:?]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) ~[?:?]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[?:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) ~[?:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) ~[?:?]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[?:?]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) ~[?:?]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:714) ~[?:?]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]
at java.lang.Thread.run(Thread.java:831) ~[?:?]
```
It's not always that character specifically. I also see characters 254, 152, and 149. Is this a helpful data point, or is it completely arbitrary?
---
### Author:  [@CWS-Dan](https://discuss.elastic.co/u/CWS-Dan)
#### Post date: [March 28, 2024, 4:03pm UTC](https://discuss.elastic.co/t/file-corruption-occurred-on-recovery-but-checksums-are-ok/356341/4 "2024-03-28T16:03:09Z")
Sorry, just one more line in the log that I think might be relevant to the problem.
`Caused by: org.elasticsearch.common.io.stream.NotSerializableExceptionWrapper: execution_exception: org.apache.lucene.index.CorruptIndexException: checksum failed (hardware problem?)`
These log excerpts are getting big and ugly, so I'll spare you the rest of the stack trace, unless you think it's important. 🙂
---
### Author:  [@DavidTurner](https://discuss.elastic.co/u/DavidTurner)
#### Post date: [March 28, 2024, 4:28pm UTC](https://discuss.elastic.co/t/file-corruption-occurred-on-recovery-but-checksums-are-ok/356341/5 "2024-03-28T16:28:07Z")
> [@CWS-Dan](#):
>
> it might help in identifying what data is being meddled with
It doesn't, sorry, but this exception is also an indication that the data sent over the network by `CWS-CWHELP` is different from the data that `CWS-CWHELP2` is receiving. Pretty sure there's something badly wrong with your network, but if you can't pin down the problem yourself you'll need to investigate that further with your network or infra folks, there's not a lot more we can do on the Elasticsearch side.
---
### Author:  [@CWS-Dan](https://discuss.elastic.co/u/CWS-Dan)
#### Post date: [March 28, 2024, 5:48pm UTC](https://discuss.elastic.co/t/file-corruption-occurred-on-recovery-but-checksums-are-ok/356341/6 "2024-03-28T17:48:39Z")
Understood, I will definitely do that. Just one more question, which might be related to Elasticsearch's config, so you might be able to help. For each of our nodes, I have them communicating on ports 4357/4358. Here are the relevant settings in elasticsearch.yml, from CWS-CWHELP2:
```auto
http.port: 4357
transport.tcp.port: 4358
discovery.seed_hosts: ["cws-cwhelp:4358","cws-cwhelp3:4358"]
```
However, when I look in the logs, I can see it attempting to communicate on a port (62538) that isn't in the config:
`[2024-03-28T11:41:36,847][WARN][o.e.t.TcpTransport] [CWS-CWHELP2] exception caught on transport layer [Netty4TcpChannel{localAddress=/10.50.2.5:62538, remoteAddress=CWS-CWHELP/10.50.2.27:4358, profile=default}], closing connection`
Do you know why it would be doing that? Is there another setting I must change? Or do you think this incidental and not related to Elasticsearch?
---
### Author:  [@DavidTurner](https://discuss.elastic.co/u/DavidTurner)
#### Post date: [March 28, 2024, 11:26pm UTC](https://discuss.elastic.co/t/file-corruption-occurred-on-recovery-but-checksums-are-ok/356341/7 "2024-03-28T23:26:51Z")
That's normal for TCP, the outbound port of every connection is different and isn't configurable.
---
### Author:  [@system](https://discuss.elastic.co/u/system)
#### Post date: [April 25, 2024, 11:27pm UTC](https://discuss.elastic.co/t/file-corruption-occurred-on-recovery-but-checksums-are-ok/356341/8 "2024-04-25T23:27:21Z")
This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.