We have a pretty new Elasticsearch set up. We seem to be hitting a memory error that we can not overcome. I've increased the values to one half of the installed memory.
Any times on adjusting the JVM?
[2017-09-20T09:47:44,005][INFO ][o.e.m.j.JvmGcMonitorService] [elastic1.xxx.university.edu] [gc][old][353][80] duration [2.4m], collections [21]/[1.2m], total [2.4m]/[9.2m], memory [7.8gb]->[7.8gb]/[7.8gb], all_pools {[young] [998.5mb]->[998.5mb]/[998.5mb]}{[survivor] [120.1mb]->[124.7mb]/[124.7mb]}{[old] [6.7gb]->[6.7gb]/[6.7gb]}
[2017-09-20T09:47:44,005][WARN ][o.e.m.j.JvmGcMonitorService] [elastic1.xxx.university.edu] [gc][353] overhead, spent [2.4m] collecting in the last [1.2m]
[2017-09-20T09:45:07,033][DEBUG][o.e.a.b.TransportShardBulkAction] [elastic1.xxx.university.edu] [bro-2017.09.19.21][4] failed to execute bulk item (index) BulkShardRequest [[bro-2017.09.19.21][4]] containing [74] requests
org.elasticsearch.index.mapper.MapperParsingException: failed to parse [version]
at org.elasticsearch.index.mapper.FieldMapper.parse(FieldMapper.java:298) ~[elasticsearch-5.5.0.jar:5.5.0]
at org.elasticsearch.index.mapper.DocumentParser.parseObjectOrField(DocumentParser.java:468) ~[elasticsearch-5.5.0.jar:5.5.0]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-5.5.0.jar:5.5.0]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [?:1.8.0_91]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [?:1.8.0_91]
at java.lang.Thread.run(Thread.java:745) [?:1.8.0_91]
Caused by: java.lang.NumberFormatException: For input string: "1.1"
at java.lang.NumberFormatException.forInputString(NumberFormatException.java:65) ~[?:1.8.0_91]
at java.lang.Long.parseLong(Long.java:589) ~[?:1.8.0_91]
at java.lang.Long.parseLong(Long.java:631) ~[?:1.8.0_91]
at org.elasticsearch.common.xcontent.support.AbstractXContentParser.longValue(AbstractXContentParser.java:172) ~[elasticsearch-5.5.0.jar:5.5.0]
at org.elasticsearch.index.mapper.NumberFieldMapper$NumberType$7.parse(NumberFieldMapper.java:740) ~[elasticsearch-5.5.0.jar:5.5.0]
at org.elasticsearch.index.mapper.NumberFieldMapper$NumberType$7.parse(NumberFieldMapper.java:719) ~[elasticsearch-5.5.0.jar:5.5.0]
at org.elasticsearch.index.mapper.NumberFieldMapper.parseCreateField(NumberFieldMapper.java:1058) ~[elasticsearch-5.5.0.jar:5.5.0]
at org.elasticsearch.index.mapper.FieldMapper.parse(FieldMapper.java:287) ~[elasticsearch-5.5.0.jar:5.5.0]
... 36 more
[2017-09-20T09:47:44,022][ERROR][o.e.b.ElasticsearchUncaughtExceptionHandler] [elastic1.xxx.university.edu] fatal error in thread [elasticsearch[elastic1.xxx.university.edu][bulk][T#19]], exiting
java.lang.OutOfMemoryError: Java heap space
at org.apache.lucene.util.packed.PackedLongValues$Builder.(PackedLongValues.java:185) ~[lucene-core-6.6.0.jar:6.6.0 5c7a7b65d2aa7ce5ec96458315c661a18b320241 - ishan - 2017-05-30 07:29:46]
at org.apache.lucene.util.packed.DeltaPackedLongValues$Builder.(DeltaPackedLongValues.java:59) ~[lucene-core-6.6.0.jar:6.6.0 5c7a7b65d2aa7ce5ec96458315c661a18b320241 - ishan - 2017-05-30 07:29:46]
at org.apache.lucene.util.packed.PackedLongValues.deltaPackedBuilder(PackedLongValues.java:55) ~[lucene-core-6.6.0.jar:6.6.0 5c7a7b65d2aa7ce5ec96458315c661a18b320241 - ishan - 2017-05-30 07:29:46]
at org.apache.lucene.util.packed.PackedLongValues.deltaPackedBuilder(PackedLongValues.java:60) ~[lucene-core-6.6.0.jar:6.6.0 5c7a7b65d2aa7ce5ec96458315c661a18b320241 - ishan - 2017-05-30 07:29:46]
at org.apache.lucene.index.NormValuesWriter.(NormValuesWriter.java:41) ~[lucene-core-6.6.0.jar:6.6.0 5c7a7b65d2aa7ce5ec96458315c661a18b320241 - ishan - 2017-05-30 07:29:46]
at org.apache.lucene.index.DefaultIndexingChain$PerField.setInvertState(DefaultIndexingChain.java:692) ~[lucene-core-6.6.0.jar:6.6.0 5c7a7b65d2aa7ce5ec96458315c661a18b320241 - ishan - 2017-05-30 07:29:46]
at org.apache.lucene.index.DefaultIndexingChain$PerField.(DefaultIndexingChain.java:682) ~[lucene-core-6.6.0.jar:6.6.0 5c7a7b65d2aa7ce5ec96458315c661a18b320241 - ishan - 2017-05-30 07:29:46]
at org.apache.lucene.index.DefaultIndexingChain.getOrAddField(DefaultIndexingChain.java:622) ~[lucene-core-6.6.0.jar:6.6.0 5c7a7b65d2aa7ce5ec96458315c661a18b320241 - ishan - 2017-05-30 07:29:46]
at org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:445) ~[lucene-core-6.6.0.jar:6.6.0 5c7a7b65d2aa7ce5ec96458315c661a18b320241 - ishan - 2017-05-30 07:29:46]
at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:403) ~[lucene-core-6.6.0.jar:6.6.0 5c7a7b65d2aa7ce5ec96458315c661a18b320241 - ishan - 2017-05-30 07:29:46]
at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:232) ~[lucene-core-6.6.0.jar:6.6.0 5c7a7b65d2aa7ce5ec96458315c661a18b320241 - ishan - 2017-05-30 07:29:46]
at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:478) ~[lucene-core-6.6.0.jar:6.6.0 5c7a7b65d2aa7ce5ec96458315c661a18b320241 - ishan - 2017-05-30 07:29:46]
at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1571) ~[lucene-core-6.6.0.jar:6.6.0 5c7a7b65d2aa7ce5ec96458315c661a18b320241 - ishan - 2017-05-30 07:29:46]
at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1316) ~[lucene-core-6.6.0.jar:6.6.0 5c7a7b65d2aa7ce5ec96458315c661a18b320241 - ishan - 2017-05-30 07:29:46]
at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:661) ~[elasticsearch-5.5.0.jar:5.5.0]
at org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:605) ~[elasticsearch-5.5.0.jar:5.5.0]
at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:505) ~[elasticsearch-5.5.0.jar:5.5.0]
at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:556) ~[elasticsearch-5.5.0.jar:5.5.0]
at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:545) ~[elasticsearch-5.5.0.jar:5.5.0]
at org.elasticsearch.action.bulk.TransportShardBulkAction.executeIndexRequestOnPrimary(TransportShardBulkAction.java:484) ~[elasticsearch-5.5.0.jar:5.5.0]
at org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:143) ~[elasticsearch-5.5.0.jar:5.5.0]
at