We faced this error for at least 24 hours. We think we had no resource issues but we are 100 % sure. I checked out these metrics;
root@-0:/# curl https://vpc-es-eu-central-1-prd-xxx.eu-central-1.es.amazonaws.com/_nodes/stats/jvm?pretty
{
"_nodes" : {
"total" : 7,
"successful" : 7,
"failed" : 0
},
"cluster_name" : "889422606013:es-eu-central-1-prd",
"nodes" : {
"kFHzi8QKQnytZpVk-EzjtA" : {
"timestamp" : 1605256063183,
"name" : "kFHzi8Q",
"roles" : [ "data", "ingest" ],
"jvm" : {
"timestamp" : 1605256063183,
"uptime_in_millis" : 41116855,
"mem" : {
"heap_used_in_bytes" : 12507346208,
"heap_used_percent" : 72,
"heap_committed_in_bytes" : 17145004032,
"heap_max_in_bytes" : 17145004032,
"non_heap_used_in_bytes" : 272200560,
"non_heap_committed_in_bytes" : 316952576,
"pools" : {
"young" : {
"used_in_bytes" : 270172528,
"max_in_bytes" : 279183360,
"peak_used_in_bytes" : 279183360,
"peak_max_in_bytes" : 279183360
},
"survivor" : {
"used_in_bytes" : 23504832,
"max_in_bytes" : 34865152,
"peak_used_in_bytes" : 34865152,
"peak_max_in_bytes" : 34865152
},
"old" : {
"used_in_bytes" : 12213685344,
"max_in_bytes" : 16830955520,
"peak_used_in_bytes" : 12823036352,
"peak_max_in_bytes" : 16830955520
}
}
},
"threads" : {
"count" : 259,
"peak_count" : 262
},
"gc" : {
"collectors" : {
"young" : {
"collection_count" : 32426,
"collection_time_in_millis" : 1999027
},
"old" : {
"collection_count" : 27,
"collection_time_in_millis" : 2561
}
}
},
"buffer_pools" : {
"mapped" : {
"count" : 195396,
"used_in_bytes" : 133912888074,
"total_capacity_in_bytes" : 133912888074
},
"direct" : {
"count" : 196,
"used_in_bytes" : 220739814,
"total_capacity_in_bytes" : 220739812
}
},
"classes" : {
"current_loaded_count" : 31893,
"total_loaded_count" : 32143,
"total_unloaded_count" : 250
}
}
},
"CFqG_oEOQo-NQyj41640jQ" : {
"timestamp" : 1605256063183,
"name" : "CFqG_oE",
"roles" : [ "master", "data", "ingest" ],
"jvm" : {
"timestamp" : 1605256063183,
"uptime_in_millis" : 64406818,
"mem" : {
"heap_used_in_bytes" : 4194806472,
"heap_used_percent" : 48,
"heap_committed_in_bytes" : 8572502016,
"heap_max_in_bytes" : 8572502016,
"non_heap_used_in_bytes" : 274528520,
"non_heap_committed_in_bytes" : 325447680,
"pools" : {
"young" : {
"used_in_bytes" : 72265024,
"max_in_bytes" : 139591680,
"peak_used_in_bytes" : 139591680,
"peak_max_in_bytes" : 139591680
},
"survivor" : {
"used_in_bytes" : 7304592,
"max_in_bytes" : 17432576,
"peak_used_in_bytes" : 17432576,
"peak_max_in_bytes" : 17432576
},
"old" : {
"used_in_bytes" : 4115252744,
"max_in_bytes" : 8415477760,
"peak_used_in_bytes" : 8141765696,
"peak_max_in_bytes" : 8415477760
}
}
},
"threads" : {
"count" : 161,
"peak_count" : 161
},
"gc" : {
"collectors" : {
"young" : {
"collection_count" : 60105,
"collection_time_in_millis" : 5529158
},
"old" : {
"collection_count" : 1259,
"collection_time_in_millis" : 148285
}
}
},
"buffer_pools" : {
"mapped" : {
"count" : 0,
"used_in_bytes" : 0,
"total_capacity_in_bytes" : 0
},
"direct" : {
"count" : 101,
"used_in_bytes" : 102263475,
"total_capacity_in_bytes" : 102263473
}
},
"classes" : {
"current_loaded_count" : 32948,
"total_loaded_count" : 33136,
"total_unloaded_count" : 188
}
}
},
"8M7lqE4MR7Sqt-H7Tz-UEA" : {
"timestamp" : 1605256063183,
"name" : "8M7lqE4",
"roles" : [ "master", "data", "ingest" ],
"jvm" : {
"timestamp" : 1605256063183,
"uptime_in_millis" : 64459023,
"mem" : {
"heap_used_in_bytes" : 3501605816,
"heap_used_percent" : 40,
"heap_committed_in_bytes" : 8572502016,
"heap_max_in_bytes" : 8572502016,
"non_heap_used_in_bytes" : 279774752,
"non_heap_committed_in_bytes" : 334479360,
"pools" : {
"young" : {
"used_in_bytes" : 112279784,
"max_in_bytes" : 139591680,
"peak_used_in_bytes" : 139591680,
"peak_max_in_bytes" : 139591680
},
"survivor" : {
"used_in_bytes" : 17432576,
"max_in_bytes" : 17432576,
"peak_used_in_bytes" : 17432576,
"peak_max_in_bytes" : 17432576
},
"old" : {
"used_in_bytes" : 3371906304,
"max_in_bytes" : 8415477760,
"peak_used_in_bytes" : 8365638240,
"peak_max_in_bytes" : 8415477760
}
}
},
"threads" : {
"count" : 262,
"peak_count" : 262
},
"gc" : {
"collectors" : {
"young" : {
"collection_count" : 74900,
"collection_time_in_millis" : 8060362
},
"old" : {
"collection_count" : 1241,
"collection_time_in_millis" : 288708
}
}
},
"buffer_pools" : {
"mapped" : {
"count" : 10877,
"used_in_bytes" : 2633605902,
"total_capacity_in_bytes" : 2633605902
},
"direct" : {
"count" : 197,
"used_in_bytes" : 120509207,
"total_capacity_in_bytes" : 120509205
}
},
"classes" : {
"current_loaded_count" : 33331,
"total_loaded_count" : 33549,
"total_unloaded_count" : 218
}
}
},
"DFtqtNV8TiGDW07ZVfncgg" : {
"timestamp" : 1605256063182,
"name" : "DFtqtNV",
"roles" : [ "data", "ingest" ],
"jvm" : {
"timestamp" : 1605256063183,
"uptime_in_millis" : 41158585,
"mem" : {
"heap_used_in_bytes" : 8489169840,
"heap_used_percent" : 49,
"heap_committed_in_bytes" : 17145004032,
"heap_max_in_bytes" : 17145004032,
"non_heap_used_in_bytes" : 271273008,
"non_heap_committed_in_bytes" : 317698048,
"pools" : {
"young" : {
"used_in_bytes" : 75343832,
"max_in_bytes" : 279183360,
"peak_used_in_bytes" : 279183360,
"peak_max_in_bytes" : 279183360
},
"survivor" : {
"used_in_bytes" : 26611344,
"max_in_bytes" : 34865152,
"peak_used_in_bytes" : 34865152,
"peak_max_in_bytes" : 34865152
},
"old" : {
"used_in_bytes" : 8387227944,
"max_in_bytes" : 16830955520,
"peak_used_in_bytes" : 12816554688,
"peak_max_in_bytes" : 16830955520
}
}
},
"threads" : {
"count" : 265,
"peak_count" : 267
},
"gc" : {
"collectors" : {
"young" : {
"collection_count" : 33416,
"collection_time_in_millis" : 2149306
},
"old" : {
"collection_count" : 30,
"collection_time_in_millis" : 2846
}
}
},
"buffer_pools" : {
"mapped" : {
"count" : 206163,
"used_in_bytes" : 136551075054,
"total_capacity_in_bytes" : 136551075054
},
"direct" : {
"count" : 193,
"used_in_bytes" : 220495886,
"total_capacity_in_bytes" : 220495884
}
},
"classes" : {
"current_loaded_count" : 32171,
"total_loaded_count" : 32400,
"total_unloaded_count" : 229
}
}
}
}
What do you guys think?
