Hi all,
I'm running an Elasticsearch cluster with just one node
JVM configuration uses the default values with
-Xms 2g
-Xmx 2g
but when I checked the memory usage I saw that the working set memory value is up to 25gb.
This could be really critical, so why seems that Elastic doesn't respect the configuration?
thanks in advance
[2024-02-07T08:55:54,816][WARN ][o.e.i.c.IndicesClusterStateService] [erp_master_ny1sv-vmutil02] [[rawsenttraf][0]] marking and sending shard failed due to [failed recovery]
org.elasticsearch.indices.recovery.RecoveryFailedException: [rawsenttraf][0]: Recovery failed on {erp_master_ny1sv-vmutil02}{g5-Z4JNHSS-VPiDzDZlLrg}{4p1YULUTS6aJBuX4UDxouw}{10.32.49.55}{10.32.49.55:9301}{hostname=ny1sv-vmutil02, erp=true, location=F:\ManageEngine\Log360\Exchange Reporter Plus\bin\..\elasticsearch}
at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$6(IndexShard.java:2139) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:624) [elasticsearch-6.5.4.jar:6.5.4]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_282]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_282]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_282]
Caused by: org.elasticsearch.index.shard.IndexShardRecoveryException: failed to recover from gateway
at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:430) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:302) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1645) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$6(IndexShard.java:2135) ~[elasticsearch-6.5.4.jar:6.5.4]
... 4 more
Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: failed to open reader on writer
at org.elasticsearch.index.engine.InternalEngine.createSearcherManager(InternalEngine.java:600) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:218) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:168) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:2266) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:2248) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.IndexShard.innerOpenEngineAndTranslog(IndexShard.java:1413) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.IndexShard.openEngineAndRecoverFromTranslog(IndexShard.java:1370) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:425) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:302) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1645) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$6(IndexShard.java:2135) ~[elasticsearch-6.5.4.jar:6.5.4]
... 4 more
Caused by: java.io.IOException: The paging file is too small for this operation to complete: MMapIndexInput(path="F:\ManageEngine\Log360\Exchange Reporter Plus\elasticsearch\data\nodes\0\indices\IqvR_kGsT8Sl52V6yZp-Fw\0\index\_6mj.fdt") [this may be caused by lack of enough unfragmented virtual address space or too restrictive virtual memory limits enforced by the operating system, preventing us to map a chunk of 1073741824 bytes. Windows is unfortunately very limited on virtual address space. If your index size is several hundred Gigabytes, consider changing to Linux. More information: http://blog.thetaphi.de/2012/07/use-lucenes-mmapdirectory-on-64bit.html]
at sun.nio.ch.FileChannelImpl.map0(Native Method) ~[?:1.8.0_282]
at sun.nio.ch.FileChannelImpl.map(FileChannelImpl.java:924) ~[?:1.8.0_282]
at org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:267) ~[lucene-core-7.5.0.jar:7.5.0 b5bf70b7e32d7ddd9742cc821d471c5fabd4e3df - jimczi - 2018-09-18 13:01:13]
at org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) ~[lucene-core-7.5.0.jar:7.5.0 b5bf70b7e32d7ddd9742cc821d471c5fabd4e3df - jimczi - 2018-09-18 13:01:13]
at org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) ~[lucene-core-7.5.0.jar:7.5.0 b5bf70b7e32d7ddd9742cc821d471c5fabd4e3df - jimczi - 2018-09-18 13:01:13]
at org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) ~[lucene-core-7.5.0.jar:7.5.0 b5bf70b7e32d7ddd9742cc821d471c5fabd4e3df - jimczi - 2018-09-18 13:01:13]
at org.apache.lucene.codecs.compressing.CompressingStoredFieldsReader.<init>(CompressingStoredFieldsReader.java:150) ~[lucene-core-7.5.0.jar:7.5.0 b5bf70b7e32d7ddd9742cc821d471c5fabd4e3df - jimczi - 2018-09-18 13:01:13]
at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsReader(CompressingStoredFieldsFormat.java:121) ~[lucene-core-7.5.0.jar:7.5.0 b5bf70b7e32d7ddd9742cc821d471c5fabd4e3df - jimczi - 2018-09-18 13:01:13]
at org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsReader(Lucene50StoredFieldsFormat.java:173) ~[lucene-core-7.5.0.jar:7.5.0 b5bf70b7e32d7ddd9742cc821d471c5fabd4e3df - jimczi - 2018-09-18 13:01:13]
at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:126) ~[lucene-core-7.5.0.jar:7.5.0 b5bf70b7e32d7ddd9742cc821d471c5fabd4e3df - jimczi - 2018-09-18 13:01:13]
at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:83) ~[lucene-core-7.5.0.jar:7.5.0 b5bf70b7e32d7ddd9742cc821d471c5fabd4e3df - jimczi - 2018-09-18 13:01:13]
at org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:172) ~[lucene-core-7.5.0.jar:7.5.0 b5bf70b7e32d7ddd9742cc821d471c5fabd4e3df - jimczi - 2018-09-18 13:01:13]
at org.apache.lucene.index.ReadersAndUpdates.getReadOnlyClone(ReadersAndUpdates.java:214) ~[lucene-core-7.5.0.jar:7.5.0 b5bf70b7e32d7ddd9742cc821d471c5fabd4e3df - jimczi - 2018-09-18 13:01:13]
at org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:106) ~[lucene-core-7.5.0.jar:7.5.0 b5bf70b7e32d7ddd9742cc821d471c5fabd4e3df - jimczi - 2018-09-18 13:01:13]
at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:524) ~[lucene-core-7.5.0.jar:7.5.0 b5bf70b7e32d7ddd9742cc821d471c5fabd4e3df - jimczi - 2018-09-18 13:01:13]
at org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:103) ~[lucene-core-7.5.0.jar:7.5.0 b5bf70b7e32d7ddd9742cc821d471c5fabd4e3df - jimczi - 2018-09-18 13:01:13]
at org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:79) ~[lucene-core-7.5.0.jar:7.5.0 b5bf70b7e32d7ddd9742cc821d471c5fabd4e3df - jimczi - 2018-09-18 13:01:13]
at org.elasticsearch.index.engine.InternalEngine.createSearcherManager(InternalEngine.java:585) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:218) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:168) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:2266) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:2248) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.IndexShard.innerOpenEngineAndTranslog(IndexShard.java:1413) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.IndexShard.openEngineAndRecoverFromTranslog(IndexShard.java:1370) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:425) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:302) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1645) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$6(IndexShard.java:2135) ~[elasticsearch-6.5.4.jar:6.5.4]
... 4 more
[2024-02-07T08:55:54,847][WARN ][o.e.c.r.a.AllocationService] [erp_master_ny1sv-vmutil02] failing shard [failed shard, shard [rawrcvdtraf][0], node[g5-Z4JNHSS-VPiDzDZlLrg], [P], recovery_source[existing store recovery; bootstrap_history_uuid=false], s[INITIALIZING], a[id=4GJxm7MFRVKOiadW5EBTyw], unassigned_info[[reason=ALLOCATION_FAILED], at[2024-02-07T13:55:52.829Z], failed_attempts[1], delayed=false, details[failed shard on node [g5-Z4JNHSS-VPiDzDZlLrg]: failed recovery, failure RecoveryFailedException[[rawrcvdtraf][0]: Recovery failed on {erp_master_ny1sv-vmutil02}{g5-Z4JNHSS-VPiDzDZlLrg}{4p1YULUTS6aJBuX4UDxouw}{10.32.49.55}{10.32.49.55:9301}{hostname=ny1sv-vmutil02, erp=true, location=F:\ManageEngine\Log360\Exchange Reporter Plus\bin\..\elasticsearch}]; nested: IndexShardRecoveryException[failed to recover from gateway]; nested: EngineCreationFailureException[failed to open reader on writer]; nested: IOException[The paging file is too small for this operation to complete: MMapIndexInput(path="F:\ManageEngine\Log360\Exchange Reporter Plus\elasticsearch\data\nodes\0\indices\446jrqMVQi2FVSdooV_ZVg\0\index\_46g.fdt") [this may be caused by lack of enough unfragmented virtual address space or too restrictive virtual memory limits enforced by the operating system, preventing us to map a chunk of 1073741824 bytes. Windows is unfortunately very limited on virtual address space. If your index size is several hundred Gigabytes, consider changing to Linux. More information: http://blog.thetaphi.de/2012/07/use-lucenes-mmapdirectory-on-64bit.html]]; ], allocation_status[no_valid_shard_copy]], message [failed recovery], failure [RecoveryFailedException[[rawrcvdtraf][0]: Recovery failed on {erp_master_ny1sv-vmutil02}{g5-Z4JNHSS-VPiDzDZlLrg}{4p1YULUTS6aJBuX4UDxouw}{10.32.49.55}{10.32.49.55:9301}{hostname=ny1sv-vmutil02, erp=true, location=F:\ManageEngine\Log360\Exchange Reporter Plus\bin\..\elasticsearch}]; nested: IndexShardRecoveryException[failed to recover from gateway]; nested: EngineCreationFailureException[failed to open reader on writer]; nested: IOException[The paging file is too small for this operation to complete: MMapIndexInput(path="F:\ManageEngine\Log360\Exchange Reporter Plus\elasticsearch\data\nodes\0\indices\446jrqMVQi2FVSdooV_ZVg\0\index\_46g.fdt") [this may be caused by lack of enough unfragmented virtual address space or too restrictive virtual memory limits enforced by the operating system, preventing us to map a chunk of 1073741824 bytes. Windows is unfortunately very limited on virtual address space. If your index size is several hundred Gigabytes, consider changing to Linux. More information: http://blog.thetaphi.de/2012/07/use-lucenes-mmapdirectory-on-64bit.html]]; ], markAsStale [true]]
org.elasticsearch.indices.recovery.RecoveryFailedException: [rawrcvdtraf][0]: Recovery failed on {erp_master_ny1sv-vmutil02}{g5-Z4JNHSS-VPiDzDZlLrg}{4p1YULUTS6aJBuX4UDxouw}{10.32.49.55}{10.32.49.55:9301}{hostname=ny1sv-vmutil02, erp=true, location=F:\ManageEngine\Log360\Exchange Reporter Plus\bin\..\elasticsearch}
at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$6(IndexShard.java:2139) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:624) ~[elasticsearch-6.5.4.jar:6.5.4]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_282]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_282]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_282]
Caused by: org.elasticsearch.index.shard.IndexShardRecoveryException: failed to recover from gateway
at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:430) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:302) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1645) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$6(IndexShard.java:2135) ~[elasticsearch-6.5.4.jar:6.5.4]
... 4 more
Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: failed to open reader on writer
at org.elasticsearch.index.engine.InternalEngine.createSearcherManager(InternalEngine.java:600) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:218) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:168) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:2266) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:2248) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.IndexShard.innerOpenEngineAndTranslog(IndexShard.java:1413) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.IndexShard.openEngineAndRecoverFromTranslog(IndexShard.java:1370) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:425) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:302) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1645) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$6(IndexShard.java:2135) ~[elasticsearch-6.5.4.jar:6.5.4]
... 4 more
Caused by: java.io.IOException: The paging file is too small for this operation to complete: MMapIndexInput(path="F:\ManageEngine\Log360\Exchange Reporter Plus\elasticsearch\data\nodes\0\indices\446jrqMVQi2FVSdooV_ZVg\0\index\_46g.fdt") [this may be caused by lack of enough unfragmented virtual address space or too restrictive virtual memory limits enforced by the operating system, preventing us to map a chunk of 1073741824 bytes. Windows is unfortunately very limited on virtual address space. If your index size is several hundred Gigabytes, consider changing to Linux. More information: http://blog.thetaphi.de/2012/07/use-lucenes-mmapdirectory-on-64bit.html]
at sun.nio.ch.FileChannelImpl.map0(Native Method) ~[?:1.8.0_282]
at sun.nio.ch.FileChannelImpl.map(FileChannelImpl.java:924) ~[?:1.8.0_282]
at org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:267) ~[lucene-core-7.5.0.jar:7.5.0 b5bf70b7e32d7ddd9742cc821d471c5fabd4e3df - jimczi - 2018-09-18 13:01:13]
at org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) ~[lucene-core-7.5.0.jar:7.5.0 b5bf70b7e32d7ddd9742cc821d471c5fabd4e3df - jimczi - 2018-09-18 13:01:13]
at org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) ~[lucene-core-7.5.0.jar:7.5.0 b5bf70b7e32d7ddd9742cc821d471c5fabd4e3df - jimczi - 2018-09-18 13:01:13]
at org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) ~[lucene-core-7.5.0.jar:7.5.0 b5bf70b7e32d7ddd9742cc821d471c5fabd4e3df - jimczi - 2018-09-18 13:01:13]
at org.apache.lucene.codecs.compressing.CompressingStoredFieldsReader.<init>(CompressingStoredFieldsReader.java:150) ~[lucene-core-7.5.0.jar:7.5.0 b5bf70b7e32d7ddd9742cc821d471c5fabd4e3df - jimczi - 2018-09-18 13:01:13]
at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsReader(CompressingStoredFieldsFormat.java:121) ~[lucene-core-7.5.0.jar:7.5.0 b5bf70b7e32d7ddd9742cc821d471c5fabd4e3df - jimczi - 2018-09-18 13:01:13]
at org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsReader(Lucene50StoredFieldsFormat.java:173) ~[lucene-core-7.5.0.jar:7.5.0 b5bf70b7e32d7ddd9742cc821d471c5fabd4e3df - jimczi - 2018-09-18 13:01:13]
at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:126) ~[lucene-core-7.5.0.jar:7.5.0 b5bf70b7e32d7ddd9742cc821d471c5fabd4e3df - jimczi - 2018-09-18 13:01:13]
at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:83) ~[lucene-core-7.5.0.jar:7.5.0 b5bf70b7e32d7ddd9742cc821d471c5fabd4e3df - jimczi - 2018-09-18 13:01:13]
at org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:172) ~[lucene-core-7.5.0.jar:7.5.0 b5bf70b7e32d7ddd9742cc821d471c5fabd4e3df - jimczi - 2018-09-18 13:01:13]
at org.apache.lucene.index.ReadersAndUpdates.getReadOnlyClone(ReadersAndUpdates.java:214) ~[lucene-core-7.5.0.jar:7.5.0 b5bf70b7e32d7ddd9742cc821d471c5fabd4e3df - jimczi - 2018-09-18 13:01:13]
at org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:106) ~[lucene-core-7.5.0.jar:7.5.0 b5bf70b7e32d7ddd9742cc821d471c5fabd4e3df - jimczi - 2018-09-18 13:01:13]
at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:524) ~[lucene-core-7.5.0.jar:7.5.0 b5bf70b7e32d7ddd9742cc821d471c5fabd4e3df - jimczi - 2018-09-18 13:01:13]
at org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:103) ~[lucene-core-7.5.0.jar:7.5.0 b5bf70b7e32d7ddd9742cc821d471c5fabd4e3df - jimczi - 2018-09-18 13:01:13]
at org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:79) ~[lucene-core-7.5.0.jar:7.5.0 b5bf70b7e32d7ddd9742cc821d471c5fabd4e3df - jimczi - 2018-09-18 13:01:13]
at org.elasticsearch.index.engine.InternalEngine.createSearcherManager(InternalEngine.java:585) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:218) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:168) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:2266) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:2248) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.IndexShard.innerOpenEngineAndTranslog(IndexShard.java:1413) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.IndexShard.openEngineAndRecoverFromTranslog(IndexShard.java:1370) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:425) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:302) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1645) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$6(IndexShard.java:2135) ~[elasticsearch-6.5.4.jar:6.5.4]
... 4 more
[2024-02-07T08:55:54,941][WARN ][o.e.c.r.a.AllocationService] [erp_master_ny1sv-vmutil02] failing shard [failed shard, shard [rawsenttraf][0], node[g5-Z4JNHSS-VPiDzDZlLrg], [P], recovery_source[existing store recovery; bootstrap_history_uuid=false], s[INITIALIZING], a[id=xuVK7QwoREqHfxBrD87UNw], unassigned_info[[reason=ALLOCATION_FAILED], at[2024-02-07T13:55:52.720Z], failed_attempts[1], delayed=false, details[failed shard on node [g5-Z4JNHSS-VPiDzDZlLrg]: failed recovery, failure RecoveryFailedException[[rawsenttraf][0]: Recovery failed on {erp_master_ny1sv-vmutil02}{g5-Z4JNHSS-VPiDzDZlLrg}{4p1YULUTS6aJBuX4UDxouw}{10.32.49.55}{10.32.49.55:9301}{hostname=ny1sv-vmutil02, erp=true, location=F:\ManageEngine\Log360\Exchange Reporter Plus\bin\..\elasticsearch}]; nested: IndexShardRecoveryException[failed to recover from gateway]; nested: EngineCreationFailureException[failed to open reader on writer]; nested: IOException[The paging file is too small for this operation to complete: MMapIndexInput(path="F:\ManageEngine\Log360\Exchange Reporter Plus\elasticsearch\data\nodes\0\indices\IqvR_kGsT8Sl52V6yZp-Fw\0\index\_i4j.fdt") [this may be caused by lack of enough unfragmented virtual address space or too restrictive virtual memory limits enforced by the operating system, preventing us to map a chunk of 1073741824 bytes. Windows is unfortunately very limited on virtual address space. If your index size is several hundred Gigabytes, consider changing to Linux. More information: http://blog.thetaphi.de/2012/07/use-lucenes-mmapdirectory-on-64bit.html]]; ], allocation_status[no_valid_shard_copy]], message [failed recovery], failure [RecoveryFailedException[[rawsenttraf][0]: Recovery failed on {erp_master_ny1sv-vmutil02}{g5-Z4JNHSS-VPiDzDZlLrg}{4p1YULUTS6aJBuX4UDxouw}{10.32.49.55}{10.32.49.55:9301}{hostname=ny1sv-vmutil02, erp=true, location=F:\ManageEngine\Log360\Exchange Reporter Plus\bin\..\elasticsearch}]; nested: IndexShardRecoveryException[failed to recover from gateway]; nested: EngineCreationFailureException[failed to open reader on writer]; nested: IOException[The paging file is too small for this operation to complete: MMapIndexInput(path="F:\ManageEngine\Log360\Exchange Reporter Plus\elasticsearch\data\nodes\0\indices\IqvR_kGsT8Sl52V6yZp-Fw\0\index\_6mj.fdt") [this may be caused by lack of enough unfragmented virtual address space or too restrictive virtual memory limits enforced by the operating system, preventing us to map a chunk of 1073741824 bytes. Windows is unfortunately very limited on virtual address space. If your index size is several hundred Gigabytes, consider changing to Linux. More information: http://blog.thetaphi.de/2012/07/use-lucenes-mmapdirectory-on-64bit.html]]; ], markAsStale [true]]
org.elasticsearch.indices.recovery.RecoveryFailedException: [rawsenttraf][0]: Recovery failed on {erp_master_ny1sv-vmutil02}{g5-Z4JNHSS-VPiDzDZlLrg}{4p1YULUTS6aJBuX4UDxouw}{10.32.49.55}{10.32.49.55:9301}{hostname=ny1sv-vmutil02, erp=true, location=F:\ManageEngine\Log360\Exchange Reporter Plus\bin\..\elasticsearch}
at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$6(IndexShard.java:2139) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:624) ~[elasticsearch-6.5.4.jar:6.5.4]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_282]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_282]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_282]
Caused by: org.elasticsearch.index.shard.IndexShardRecoveryException: failed to recover from gateway
at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:430) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:302) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1645) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$6(IndexShard.java:2135) ~[elasticsearch-6.5.4.jar:6.5.4]
... 4 more
Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: failed to open reader on writer
at org.elasticsearch.index.engine.InternalEngine.createSearcherManager(InternalEngine.java:600) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:218) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:168) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:2266) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:2248) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.IndexShard.innerOpenEngineAndTranslog(IndexShard.java:1413) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.IndexShard.openEngineAndRecoverFromTranslog(IndexShard.java:1370) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:425) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:302) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1645) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$6(IndexShard.java:2135) ~[elasticsearch-6.5.4.jar:6.5.4]
... 4 more
Caused by: java.io.IOException: The paging file is too small for this operation to complete: MMapIndexInput(path="F:\ManageEngine\Log360\Exchange Reporter Plus\elasticsearch\data\nodes\0\indices\IqvR_kGsT8Sl52V6yZp-Fw\0\index\_6mj.fdt") [this may be caused by lack of enough unfragmented virtual address space or too restrictive virtual memory limits enforced by the operating system, preventing us to map a chunk of 1073741824 bytes. Windows is unfortunately very limited on virtual address space. If your index size is several hundred Gigabytes, consider changing to Linux. More information: http://blog.thetaphi.de/2012/07/use-lucenes-mmapdirectory-on-64bit.html]
at sun.nio.ch.FileChannelImpl.map0(Native Method) ~[?:1.8.0_282]
at sun.nio.ch.FileChannelImpl.map(FileChannelImpl.java:924) ~[?:1.8.0_282]
at org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:267) ~[lucene-core-7.5.0.jar:7.5.0 b5bf70b7e32d7ddd9742cc821d471c5fabd4e3df - jimczi - 2018-09-18 13:01:13]
at org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) ~[lucene-core-7.5.0.jar:7.5.0 b5bf70b7e32d7ddd9742cc821d471c5fabd4e3df - jimczi - 2018-09-18 13:01:13]
at org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) ~[lucene-core-7.5.0.jar:7.5.0 b5bf70b7e32d7ddd9742cc821d471c5fabd4e3df - jimczi - 2018-09-18 13:01:13]
at org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) ~[lucene-core-7.5.0.jar:7.5.0 b5bf70b7e32d7ddd9742cc821d471c5fabd4e3df - jimczi - 2018-09-18 13:01:13]
at org.apache.lucene.codecs.compressing.CompressingStoredFieldsReader.<init>(CompressingStoredFieldsReader.java:150) ~[lucene-core-7.5.0.jar:7.5.0 b5bf70b7e32d7ddd9742cc821d471c5fabd4e3df - jimczi - 2018-09-18 13:01:13]
at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsReader(CompressingStoredFieldsFormat.java:121) ~[lucene-core-7.5.0.jar:7.5.0 b5bf70b7e32d7ddd9742cc821d471c5fabd4e3df - jimczi - 2018-09-18 13:01:13]
at org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsReader(Lucene50StoredFieldsFormat.java:173) ~[lucene-core-7.5.0.jar:7.5.0 b5bf70b7e32d7ddd9742cc821d471c5fabd4e3df - jimczi - 2018-09-18 13:01:13]
at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:126) ~[lucene-core-7.5.0.jar:7.5.0 b5bf70b7e32d7ddd9742cc821d471c5fabd4e3df - jimczi - 2018-09-18 13:01:13]
at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:83) ~[lucene-core-7.5.0.jar:7.5.0 b5bf70b7e32d7ddd9742cc821d471c5fabd4e3df - jimczi - 2018-09-18 13:01:13]
at org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:172) ~[lucene-core-7.5.0.jar:7.5.0 b5bf70b7e32d7ddd9742cc821d471c5fabd4e3df - jimczi - 2018-09-18 13:01:13]
at org.apache.lucene.index.ReadersAndUpdates.getReadOnlyClone(ReadersAndUpdates.java:214) ~[lucene-core-7.5.0.jar:7.5.0 b5bf70b7e32d7ddd9742cc821d471c5fabd4e3df - jimczi - 2018-09-18 13:01:13]
at org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:106) ~[lucene-core-7.5.0.jar:7.5.0 b5bf70b7e32d7ddd9742cc821d471c5fabd4e3df - jimczi - 2018-09-18 13:01:13]
at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:524) ~[lucene-core-7.5.0.jar:7.5.0 b5bf70b7e32d7ddd9742cc821d471c5fabd4e3df - jimczi - 2018-09-18 13:01:13]
at org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:103) ~[lucene-core-7.5.0.jar:7.5.0 b5bf70b7e32d7ddd9742cc821d471c5fabd4e3df - jimczi - 2018-09-18 13:01:13]
at org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:79) ~[lucene-core-7.5.0.jar:7.5.0 b5bf70b7e32d7ddd9742cc821d471c5fabd4e3df - jimczi - 2018-09-18 13:01:13]
at org.elasticsearch.index.engine.InternalEngine.createSearcherManager(InternalEngine.java:585) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:218) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:168) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:2266) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:2248) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.IndexShard.innerOpenEngineAndTranslog(IndexShard.java:1413) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.IndexShard.openEngineAndRecoverFromTranslog(IndexShard.java:1370) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:425) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:302) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1645) ~[elasticsearch-6.5.4.jar:6.5.4]
at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$6(IndexShard.java:2135) ~[elasticsearch-6.5.4.jar:6.5.4]
... 4 more
