Elasticsearch version
bin/elasticsearch --version
7.1.1
Logstash version
7.1.1
Plugins installed
bin/elasticsearch-plugin list
analysis-ik
JVM version
java -version
java version "1.8.0_91"
Java(TM) SE Runtime Environment (build 1.8.0_91-b14)
Java HotSpot(TM) 64-Bit Server VM (build 25.91-b14, mixed mode)
OS version
uname -a
Linux log-es05.com 2.6.32-642.6.2.el6.x86_64 #1 SMP Wed Oct 26 06:52:09 UTC 2016 x86_64 x86_64 x86_64 GNU/Linux
Related config
node.master: false
node.data: true
node.ingest: true
node.ml: false
xpack.ml.enabled: true
issue
i am doing log analysis using Filebeat (7.1.1) -> logstash(7.1.1) -> Elasticsearch (7.1.1)
, The cluster have been worked well for more than three nounth , but there is a sudden crash of one of my es node.
i jump to the server and find some msg like this in the log file. It seems that the es met some error and exit it self.
[2019-09-28T10:08:35,123][ERROR][o.e.i.e.Engine ] [ESV06] [mm-xxxx-2019.09.28][7] already closed by tragic event on the index writer
java.lang.IncompatibleClassChangeError: vtable stub
at org.apache.lucene.util.RamUsageEstimator.sizeOf(RamUsageEstimator.java:366) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]
at org.apache.lucene.util.packed.PackedLongValues$Builder.build(PackedLongValues.java:196) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]
at org.apache.lucene.index.SortedSetDocValuesWriter.flush(SortedSetDocValuesWriter.java:200) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]
at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:262) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]
at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:137) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]
at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:469) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]
at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:554) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]
at org.apache.lucene.index.DocumentsWriter.preUpdate(DocumentsWriter.java:401) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]
at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:479) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]
at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1594) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]
at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1213) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]
at org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1092) ~[elasticsearch-7.1.1.jar:7.1.1]
at org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1037) ~[elasticsearch-7.1.1.jar:7.1.1]
at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:864) ~[elasticsearch-7.1.1.jar:7.1.1]
at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:789) ~[elasticsearch-7.1.1.jar:7.1.1]
at org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:762) ~[elasticsearch-7.1.1.jar:7.1.1]
at org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:719) ~[elasticsearch-7.1.1.jar:7.1.1]
at org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$executeIndexRequestOnPrimary$3(TransportShardBulkAction.java:452) ~[elasticsearch-7.1.1.jar:7.1.1]
at org.elasticsearch.action.bulk.TransportShardBulkAction.executeOnPrimaryWhileHandlingMappingUpdates(TransportShardBulkAction.java:475) ~[elasticsearch-7.1.1.jar:7.1.1]
at org.elasticsearch.action.bulk.TransportShardBulkAction.executeIndexRequestOnPrimary(TransportShardBulkAction.java:450) ~[elasticsearch-7.1.1.jar:7.1.1]
at org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:218) ~[elasticsearch-7.1.1.jar:7.1.1]
at org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:161) ~[elasticsearch-7.1.1.jar:7.1.1]
at org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:153) ~[elasticsearch-7.1.1.jar:7.1.1]
at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:141) ~[elasticsearch-7.1.1.jar:7.1.1]
at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:79) ~[elasticsearch-7.1.1.jar:7.1.1]
at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryShardReference.perform(TransportReplicationAction.java:1042) ~[elasticsearch-7.1.1.jar:7.1.1]
at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryShardReference.perform(TransportReplicationAction.java:1020) ~[elasticsearch-7.1.1.jar:7.1.1]
at org.elasticsearch.action.support.replication.ReplicationOperation.execute(ReplicationOperation.java:104) ~[elasticsearch-7.1.1.jar:7.1.1]
at org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.runWithPrimaryShardReference(TransportReplicationAction.java:422) ~[elasticsearch-7.1.1.jar:7.1.1]
at org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.lambda$doRun$0(TransportReplicationAction.java:363) ~[elasticsearch-7.1.1.jar:7.1.1]
at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:61) ~[elasticsearch-7.1.1.jar:7.1.1]
....
maybe this is because of some unexpected system error?