continue to get multi_get and node is evicted
Whether multi_get could cause node eviction
It shouldn't, but you may be abusing it. Can you provide more information?
Attaching the logs
[2015-12-30 14:25:15,581][WARN ][netty.channel.socket.nio.AbstractNioSelector] Unexpected exception in the selector loop.
java.lang.OutOfMemoryError: Java heap space
[2015-12-30 14:25:15,063][WARN ][netty.channel.socket.nio.AbstractNioSelector] Unexpected exception in the selector loop.
java.lang.OutOfMemoryError: Java heap space
[2015-12-30 14:25:09,699][DEBUG][action.get ] [es-v2-24] [apptivodb2][1] failed to execute multi_get for [6]/[178901]
java.lang.OutOfMemoryError: Java heap space
[2015-12-30 14:26:04,999][WARN ][netty.channel.socket.nio.AbstractNioSelector] Unexpected exception in the selector loop.
java.lang.OutOfMemoryError: Java heap space
[2015-12-30 14:25:58,481][WARN ][netty.channel.socket.nio.AbstractNioSelector] Unexpected exception in the selector loop.
java.lang.OutOfMemoryError: Java heap space
[2015-12-30 14:25:57,213][WARN ][netty.channel.socket.nio.AbstractNioSelector] Unexpected exception in the selector loop.
java.lang.OutOfMemoryError: Java heap space
[2015-12-30 14:25:56,222][WARN ][transport.netty ] [es-v2-24] Failed to send error message back to client for action [count/s]
java.lang.OutOfMemoryError: Java heap space
[2015-12-30 14:25:26,203][DEBUG][action.get ] [es-v2-24] [apptivodb2][1] failed to execute multi_get for [6]/[178901]
[2015-12-30 14:27:56,785][WARN ][transport.netty ] [es-v2-24] exception caught on transport layer [[id: 0x0aab6b54, /10.10.2.131:36943 :> /10.10.2.124:31224]], closing connection
java.io.StreamCorruptedException: invalid internal transport message format
at org.elasticsearch.transport.netty.SizeHeaderFrameDecoder.decode(SizeHeaderFrameDecoder.java:46)
at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:425)
at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:482)
at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365)
at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102)
at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)
at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
at org.elasticsearch.common.netty.OpenChannelsHandler.handleUpstream(OpenChannelsHandler.java:74)
at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)
at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559)
at org.elasticsearch.common.netty.channel.Channels.fireChannelDisconnected(Channels.java:396)
at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.close(AbstractNioWorker.java:360)
at org.elasticsearch.common.netty.channel.socket.nio.NioServerSocketPipelineSink.handleAcceptedSocket(NioServerSocketPipelineSink.java:81)
at org.elasticsearch.common.netty.channel.socket.nio.NioServerSocketPipelineSink.eventSunk(NioServerSocketPipelineSink.java:36)
at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendDownstream(DefaultChannelPipeline.java:574)
at org.elasticsearch.common.netty.channel.Channels.close(Channels.java:812)
at org.elasticsearch.common.netty.channel.AbstractChannel.close(AbstractChannel.java:197)
at org.elasticsearch.transport.netty.NettyTransport.exceptionCaught(NettyTransport.java:539)
at org.elasticsearch.transport.netty.MessageChannelHandler.exceptionCaught(MessageChannelHandler.java:234)
at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:112)
at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)
at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.exceptionCaught(FrameDecoder.java:377)
at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:112)
at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)
at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
at org.elasticsearch.common.netty.OpenChannelsHandler.handleUpstream(OpenChannelsHandler.java:74)
Yeah, that's why.
How much heap does the node have? What is the mget query you are running.
"from":0,"size":10,"post_filter":{"and":{"filters":[{"term":{"firmId":XXXXXX}},{"term":{"activityType":"Task"}},{"term":{"objectStatus":0}},{"or":{"filters":[{"range":{"startDate":{"from":"2015-12-30
00:00:00","to":"2015-12-30
23:59:59","include_lower":true,"include_upper":true}}},{"and":{"filters":[{"range":{"startDate":{"from":null,"to":"2015-12-30
00:00:00","include_lower":true,"include_upper":true}}},{"range":{"endDate":{"from":"2015-12-30
00:00:00","to":null,"include_lower":true,"include_upper":true}}}]}}]}},{"not":{"filter":{"term":{"statusCode":"4"}}}},{"not":{"filter":{"term":{"statusCode":"5"}}}},{"nested":{"filter":{"term":{"assigneeDetails.objectRefId":37230}},"path":"assigneeDetails"}},{"nested":{"filter":{"and":{"filters":[{"term":{"assigneeDetails.objectId":8}},{"or":{"filters":[{"term":{"assigneeDetails.objectRefId":51235}},{"term":{"assigneeDetails.objectRefId":37230}}]}}]}},"path":"assigneeDetails"}},{"and":{"filters":[{"not":{"filter":{"term":{"activityId":XXXXXX}}}},{"not":{"filter":{"term":{"activityId":XXXXX}}}},{"not":{"filter":{"term":{"activityId":178848}}}},{"not":{"filter":{"term":{"activityId":XXXXX}},{"not":{"filter":{"term":{"activityId":xxxxxx}}}},{"not":{"filter":{"term":{"activityId":xxxxxxx}}}},{"not":{"filter":{"term":{"activityId":xxxxxx}}}},{"not":{"filter":{"term":{"activityId":xxxxxx}}}},{"not":{"filter":{"term":{"activityId":xxxxxx}}}},{"not":{"filter":{"term":{"activityId":xxxxxx}}}},{"not":{"filter":{"term":{"activityId":xxxxxx}}}},{"not":{"filter":{"term":{"activityId":xxxxxx}}}},{"not":{"filter":{"term":{"activityId":xxxxxx}}}},{"not":{"filter":{"term":{"activityId":xxxxxx}}}},{"not":{"filter":{"term":{"activityId":xxxxxx}}}},{"not":{"filter":{"term":{"activityId":xxxxxx}}}},{"not":{"filter":{"term":{"activityId":xxxxxx}}}},{"not":{"filter":{"term":{"activityId":xxxxxx}}}},{"not":{"filter":{"term":{"activityId":xxxxxx}}}},{"not":{"filter":{"term":{"activityId":xxxxxx}}}},{"not":{"filter":{"term":{"activityId":xxxxxxx}}}},{"not":{"filter":{"term":{"activityId":xxxxxx}}}},{"not":{"filter":{"term":{"activityId":xxxxxx}}},{"not":{"filter":{"term":{"activityId":xxxxxxx}}}},{"not":{"filter":{"term":{"activityId":xxxx}}}},{"not":{"filter":{"term":{"activityId":xxxxxx}}},{"not":{"filter":{"term":{"activityId":xxxxxx}}}},{"not":{"filter":{"term":{"activityId":xxxxxx}}}},{"not":{"filter":{"term":{"activityId":xxxxxx}}}},{"not":{"filter":{"term":{"activityId":xxxxxx}}}},{"not":{"filter":{"term":{"activityId":xxxxxx}}}},{"not":{"filter":{"term":{"activityId":xxxxxx}}}},{"not":{"filter":{"term":{"activityId":xxxxxx}}}},{"not":{"filter":{"term":{"activityId":xxxxxx}}}},{"not":{"filter":{"term":{"activityId":xxxxxx}}}},{"not":{"filter":{"term":{"activityId":xxxxxx}}}},{"not":{"filter":{"term":{"activityId":xxxxxx}}}},{"not":{"filter":{"term":{"activityId":xxxxxx}}}},{"not":{"filter":{"term":{"activityId":xxxxxx}}}},{"not":{"filter":{"term":{"activityId":xxxxxx}}}},{"not":{"filter":{"term":{"activityId":xxxxxx}}}},{"not":{"filter":{"term":{"activityId":xxxxxx}}}},{"not":{"filter":{"term":{"activityId":xxxxx}}}},{"not":{"filter":{"term":{"activityId":xxxxxx}}}},{"not":{"filter":{"term":{"activityId":xxxxxx}}}},{"not":{"filter":{"term":{"activityId":xxxxxx}}}},{"not":{"filter":{"term":{"activityId":xxxxxx}}}},{"not":{"filter":{"term":{"activityId":xxxxxx}}}},{"not":{"filter":{"term":{"activityId":xxxxxx}}}},{"not":{"filter":{"term":{"activityId":xxxxxx}}}},{"not":{"filter":{"term":{"activityId":xxxxxx}}}},{"not":{"filter":{"term":{"activityId":xxxxxx}}}},{"not":{"filter":{"term":{"activityId":xxxxxx}}}},{"not":{"filter":{"term":{"activityId":xxxxxx}}}},{"not":{"filter":{"term":{"activityId":xxxxxx}}}}]}}]}},"explain":false,"sort":[{"endDate":{"order":"asc"}}]}],
extra_source[]
Java heap size is 5.8gb
[jalbertemmanuel@yamuna21 ~]$ curl 'http://************/_cat/nodes?v&h=n,d,hp,hm,rm,fm,fe'
n d hp hm rm fm fe
es-v1-XX 1.1tb 60 5.8gb 252.2gb 171.3mb 0
es-v1-XX 4.4tb 43 5.8gb 251.3gb 147.7mb 0
es-v1-XX 4.4tb 59 5.8gb 251.3gb 168.1mb 0
es-v1-XX 4.2tb 58 5.8gb 251.3gb 114.2mb 0
es-v1-XX 4.2tb 14 5.8gb 252.1gb 112mb 0
Hi please let me know whether mget can cause OOM or default mapping will cause OOM
Can you post your query correctly formatted rather than the above, it's really hard to read.
2015-12-30 14:15:39,462]
[{"explain":false,"facets":{"f":{"terms":{"field":"caseTypeId","size":10000000},"facet_filter":{"and":{"filters":[{"term":{"firmId":52461}},{"term":{"deleted":"0"}},{"or":{"filters":[{"term":{"isActive":"Y"}},{"missing":{"field":"isActive","null_value":true,"existence":true}}]}},{"or":{"filters":[]}}]}}}}}], extra_source[],
2015-12-30 14:15:39,463
"size":10000,"query":{"bool":{"must":[{"term":{"firmId":123456}},{"term":{"objectStatus":0}},{"term":{"activityType":"Task"}},{"terms":{"parentActivityIds":[123121,11111111,128893,138893,148893,128894,128894,138894,158894,138894,138894,138894,138894,138895,138895,138895,138895,138895,138895,138895,138895,138895,138895,138895,138895,138895,138895,138895,138895,138895,138895,138895,138896,138896,138896,138896,138896,138896,138896,138896,138896,138896,138896,138896,138896,138896,138896,138896,138896,138896,138896,138896,138896,138896,138896,138896,138896,138896,138896,138896,138896,138896,138896,138896,138896,138896,138896,138896,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138900,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901,138901]}}]}},"fields":"id"}], extra_source[]