I had a Elastic search stack crash due to disk almost full, after enabling extra winlogbeat, which took me a bit with surprise, so I added some policies to reduce them a bit.
But after doing that and expanding the disk, and bringing it back up, it now shows three shards as unassigned:
{
"index": ".ds-ilm-history-5-2022.11.01-000014",
"shard": "0",
"prirep": "r",
"state": "UNASSIGNED",
"docs": null,
"store": null,
"ip": null,
"id": null,
"node": null,
"sync_id": null,
"unassigned.reason": "CLUSTER_RECOVERED",
"unassigned.at": "2022-11-01T12:20:58.547Z",
"unassigned.for": "50.2m",
"unassigned.details": null,
"recoverysource.type": "peer",
"completion.size": null,
"fielddata.memory_size": null,
"fielddata.evictions": null,
"query_cache.memory_size": null,
"query_cache.evictions": null,
"flush.total": null,
"flush.total_time": null,
"get.current": null,
"get.time": null,
"get.total": null,
"get.exists_time": null,
"get.exists_total": null,
"get.missing_time": null,
"get.missing_total": null,
"indexing.delete_current": null,
"indexing.delete_time": null,
"indexing.delete_total": null,
"indexing.index_current": null,
"indexing.index_time": null,
"indexing.index_total": null,
"indexing.index_failed": null,
"merges.current": null,
"merges.current_docs": null,
"merges.current_size": null,
"merges.total": null,
"merges.total_docs": null,
"merges.total_size": null,
"merges.total_time": null,
"refresh.total": null,
"refresh.time": null,
"refresh.external_total": null,
"refresh.external_time": null,
"refresh.listeners": null,
"search.fetch_current": null,
"search.fetch_time": null,
"search.fetch_total": null,
"search.open_contexts": null,
"search.query_current": null,
"search.query_time": null,
"search.query_total": null,
"search.scroll_current": null,
"search.scroll_time": null,
"search.scroll_total": null,
"segments.count": null,
"segments.memory": null,
"segments.index_writer_memory": null,
"segments.version_map_memory": null,
"segments.fixed_bitset_memory": null,
"seq_no.max": null,
"seq_no.local_checkpoint": null,
"seq_no.global_checkpoint": null,
"warmer.current": null,
"warmer.total": null,
"warmer.total_time": null,
"path.data": null,
"path.state": null
},
{
"index": ".ds-.slm-history-5-2022.11.01-000010",
"shard": "0",
"prirep": "r",
"state": "UNASSIGNED",
"docs": null,
"store": null,
"ip": null,
"id": null,
"node": null,
"sync_id": null,
"unassigned.reason": "CLUSTER_RECOVERED",
"unassigned.at": "2022-11-01T12:20:58.547Z",
"unassigned.for": "50.2m",
"unassigned.details": null,
"recoverysource.type": "peer",
"completion.size": null,
"fielddata.memory_size": null,
"fielddata.evictions": null,
"query_cache.memory_size": null,
"query_cache.evictions": null,
"flush.total": null,
"flush.total_time": null,
"get.current": null,
"get.time": null,
"get.total": null,
"get.exists_time": null,
"get.exists_total": null,
"get.missing_time": null,
"get.missing_total": null,
"indexing.delete_current": null,
"indexing.delete_time": null,
"indexing.delete_total": null,
"indexing.index_current": null,
"indexing.index_time": null,
"indexing.index_total": null,
"indexing.index_failed": null,
"merges.current": null,
"merges.current_docs": null,
"merges.current_size": null,
"merges.total": null,
"merges.total_docs": null,
"merges.total_size": null,
"merges.total_time": null,
"refresh.total": null,
"refresh.time": null,
"refresh.external_total": null,
"refresh.external_time": null,
"refresh.listeners": null,
"search.fetch_current": null,
"search.fetch_time": null,
"search.fetch_total": null,
"search.open_contexts": null,
"search.query_current": null,
"search.query_time": null,
"search.query_total": null,
"search.scroll_current": null,
"search.scroll_time": null,
"search.scroll_total": null,
"segments.count": null,
"segments.memory": null,
"segments.index_writer_memory": null,
"segments.version_map_memory": null,
"segments.fixed_bitset_memory": null,
"seq_no.max": null,
"seq_no.local_checkpoint": null,
"seq_no.global_checkpoint": null,
"warmer.current": null,
"warmer.total": null,
"warmer.total_time": null,
"path.data": null,
"path.state": null
},
{
"index": "winlogbeat-7.17.4-2022.08",
"shard": "0",
"prirep": "r",
"state": "UNASSIGNED",
"docs": null,
"store": null,
"ip": null,
"id": null,
"node": null,
"sync_id": null,
"unassigned.reason": "CLUSTER_RECOVERED",
"unassigned.at": "2022-11-01T12:20:58.558Z",
"unassigned.for": "50.2m",
"unassigned.details": null,
"recoverysource.type": "peer",
"completion.size": null,
"fielddata.memory_size": null,
"fielddata.evictions": null,
"query_cache.memory_size": null,
"query_cache.evictions": null,
"flush.total": null,
"flush.total_time": null,
"get.current": null,
"get.time": null,
"get.total": null,
"get.exists_time": null,
"get.exists_total": null,
"get.missing_time": null,
"get.missing_total": null,
"indexing.delete_current": null,
"indexing.delete_time": null,
"indexing.delete_total": null,
"indexing.index_current": null,
"indexing.index_time": null,
"indexing.index_total": null,
"indexing.index_failed": null,
"merges.current": null,
"merges.current_docs": null,
"merges.current_size": null,
"merges.total": null,
"merges.total_docs": null,
"merges.total_size": null,
"merges.total_time": null,
"refresh.total": null,
"refresh.time": null,
"refresh.external_total": null,
"refresh.external_time": null,
"refresh.listeners": null,
"search.fetch_current": null,
"search.fetch_time": null,
"search.fetch_total": null,
"search.open_contexts": null,
"search.query_current": null,
"search.query_time": null,
"search.query_total": null,
"search.scroll_current": null,
"search.scroll_time": null,
"search.scroll_total": null,
"segments.count": null,
"segments.memory": null,
"segments.index_writer_memory": null,
"segments.version_map_memory": null,
"segments.fixed_bitset_memory": null,
"seq_no.max": null,
"seq_no.local_checkpoint": null,
"seq_no.global_checkpoint": null,
"warmer.current": null,
"warmer.total": null,
"warmer.total_time": null,
"path.data": null,
"path.state": null
},
I also have one that is cluster recovered and is initializing
{
"index": "winlogbeat-7.17.4-2022.09",
"shard": "0",
"prirep": "r",
"state": "INITIALIZING",
"docs": null,
"store": null,
"ip": "172.18.0.3",
"id": "SNlOo98FSFKue0_eGa9B1w",
"node": "es01",
"sync_id": null,
"unassigned.reason": "CLUSTER_RECOVERED",
"unassigned.at": "2022-11-01T12:20:58.553Z",
"unassigned.for": "50.2m",
"unassigned.details": null,
"recoverysource.type": "peer",
"completion.size": null,
"fielddata.memory_size": null,
"fielddata.evictions": null,
"query_cache.memory_size": null,
"query_cache.evictions": null,
"flush.total": null,
"flush.total_time": null,
"get.current": null,
"get.time": null,
"get.total": null,
"get.exists_time": null,
"get.exists_total": null,
"get.missing_time": null,
"get.missing_total": null,
"indexing.delete_current": null,
"indexing.delete_time": null,
"indexing.delete_total": null,
"indexing.index_current": null,
"indexing.index_time": null,
"indexing.index_total": null,
"indexing.index_failed": null,
"merges.current": null,
"merges.current_docs": null,
"merges.current_size": null,
"merges.total": null,
"merges.total_docs": null,
"merges.total_size": null,
"merges.total_time": null,
"refresh.total": null,
"refresh.time": null,
"refresh.external_total": null,
"refresh.external_time": null,
"refresh.listeners": null,
"search.fetch_current": null,
"search.fetch_time": null,
"search.fetch_total": null,
"search.open_contexts": null,
"search.query_current": null,
"search.query_time": null,
"search.query_total": null,
"search.scroll_current": null,
"search.scroll_time": null,
"search.scroll_total": null,
"segments.count": null,
"segments.memory": null,
"segments.index_writer_memory": null,
"segments.version_map_memory": null,
"segments.fixed_bitset_memory": null,
"seq_no.max": null,
"seq_no.local_checkpoint": null,
"seq_no.global_checkpoint": null,
"warmer.current": null,
"warmer.total": null,
"warmer.total_time": null,
"path.data": null,
"path.state": null
},
Looking in elastichq, it shows:
So, do I just need to be patient, or can I nudge the elasticsearch somehow?
I can just loose the 'bad' shards if need be, I just want to be sure how to do that precisely and not mess things up further 