Hi,
Output from _cluster/stats?human&pretty is:
{
"_nodes" : {
"total" : 1,
"successful" : 1,
"failed" : 0
},
"cluster_name" : "elasticsearch",
"cluster_uuid" : "rqh_pT6GT-KD3CMXnBeBfQ",
"timestamp" : 1609849049336,
"status" : "yellow",
"indices" : {
"count" : 304,
"shards" : {
"total" : 304,
"primaries" : 304,
"replication" : 0.0,
"index" : {
"shards" : {
"min" : 1,
"max" : 1,
"avg" : 1.0
},
"primaries" : {
"min" : 1,
"max" : 1,
"avg" : 1.0
},
"replication" : {
"min" : 0.0,
"max" : 0.0,
"avg" : 0.0
}
}
},
"docs" : {
"count" : 281041112,
"deleted" : 4256731
},
"store" : {
"size" : "138.4gb",
"size_in_bytes" : 148653071123
},
"fielddata" : {
"memory_size" : "41kb",
"memory_size_in_bytes" : 42032,
"evictions" : 0
},
"query_cache" : {
"memory_size" : "28.3mb",
"memory_size_in_bytes" : 29725903,
"total_count" : 336809,
"hit_count" : 35675,
"miss_count" : 301134,
"cache_size" : 31897,
"cache_count" : 33613,
"evictions" : 1716
},
"completion" : {
"size" : "0b",
"size_in_bytes" : 0
},
"segments" : {
"count" : 3158,
"memory" : "184.9mb",
"memory_in_bytes" : 193915270,
"terms_memory" : "94.4mb",
"terms_memory_in_bytes" : 99046155,
"stored_fields_memory" : "68.9mb",
"stored_fields_memory_in_bytes" : 72312120,
"term_vectors_memory" : "0b",
"term_vectors_memory_in_bytes" : 0,
"norms_memory" : "4.6mb",
"norms_memory_in_bytes" : 4860736,
"points_memory" : "11.3mb",
"points_memory_in_bytes" : 11933003,
"doc_values_memory" : "5.4mb",
"doc_values_memory_in_bytes" : 5763256,
"index_writer_memory" : "412.8mb",
"index_writer_memory_in_bytes" : 432866840,
"version_map_memory" : "1.7mb",
"version_map_memory_in_bytes" : 1816675,
"fixed_bit_set" : "3.6mb",
"fixed_bit_set_memory_in_bytes" : 3823824,
"max_unsafe_auto_id_timestamp" : 1609836489917,
"file_sizes" : { }
}
},
"nodes" : {
"count" : {
"total" : 1,
"coordinating_only" : 0,
"data" : 1,
"ingest" : 1,
"master" : 1,
"voting_only" : 0
},
"versions" : [
"7.3.0"
],
"os" : {
"available_processors" : 8,
"allocated_processors" : 8,
"names" : [
{
"name" : "Linux",
"count" : 1
}
],
"pretty_names" : [
{
"pretty_name" : "CentOS Linux 7 (Core)",
"count" : 1
}
],
"mem" : {
"total" : "13.5gb",
"total_in_bytes" : 14542573568,
"free" : "185mb",
"free_in_bytes" : 194035712,
"used" : "13.3gb",
"used_in_bytes" : 14348537856,
"free_percent" : 1,
"used_percent" : 99
}
},
"process" : {
"cpu" : {
"percent" : 4
},
"open_file_descriptors" : {
"min" : 6514,
"max" : 6514,
"avg" : 6514
}
},
"jvm" : {
"max_uptime" : "4.8d",
"max_uptime_in_millis" : 418784197,
"versions" : [
{
"version" : "12.0.1",
"vm_name" : "OpenJDK 64-Bit Server VM",
"vm_version" : "12.0.1+12",
"vm_vendor" : "Oracle Corporation",
"bundled_jdk" : true,
"using_bundled_jdk" : true,
"count" : 1
}
],
"mem" : {
"heap_used" : "3.8gb",
"heap_used_in_bytes" : 4176793776,
"heap_max" : "5.9gb",
"heap_max_in_bytes" : 6372720640
},
"threads" : 101
},
"fs" : {
"total" : "199.8gb",
"total_in_bytes" : 214639316992,
"free" : "51.9gb",
"free_in_bytes" : 55795838976,
"available" : "51.9gb",
"available_in_bytes" : 55795838976
},
"plugins" : [ ],
"network_types" : {
"transport_types" : {
"security4" : 1
},
"http_types" : {
"security4" : 1
}
},
"discovery_types" : {
"zen" : 1
},
"packaging_types" : [
{
"flavor" : "default",
"type" : "rpm",
"count" : 1
}
]
}
}
At time of elasticsearch crush i see in elasticsearch.log:
[2020-12-27T13:42:35,964][ERROR][o.e.x.s.a.e.ReservedRealm] [certuy-elk.cert.uy] failed to retrieve password hash for reserved user [kibana]
org.elasticsearch.action.UnavailableShardsException: at least one primary shard for the index [.security-7] is unavailable
at org.elasticsearch.xpack.security.support.SecurityIndexManager.getUnavailableReason(SecurityIndexManager.java:181) ~[x-pack-security-7.3.0.jar:7.3.0]
at org.elasticsearch.xpack.security.authc.esnative.NativeUsersStore.getReservedUserInfo(NativeUsersStore.java:525) [x-pack-security-7.3.0.jar:7.3.0]
at org.elasticsearch.xpack.security.authc.esnative.ReservedRealm.getUserInfo(ReservedRealm.java:212) [x-pack-security-7.3.0.jar:7.3.0]
at org.elasticsearch.xpack.security.authc.esnative.ReservedRealm.doAuthenticate(ReservedRealm.java:93) [x-pack-security-7.3.0.jar:7.3.0]
at org.elasticsearch.xpack.security.authc.support.CachingUsernamePasswordRealm.authenticateWithCache(CachingUsernamePasswordRealm.java:166) [x-pack-security-7.3.0.jar:7.3.0]
at org.elasticsearch.xpack.security.authc.support.CachingUsernamePasswordRealm.authenticate(CachingUsernamePasswordRealm.java:103) [x-pack-security-7.3.0.jar:7.3.0]
at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$consumeToken$15(AuthenticationService.java:364) [x-pack-security-7.3.0.jar:7.3.0]
at org.elasticsearch.xpack.core.common.IteratingActionListener.run(IteratingActionListener.java:102) [x-pack-core-7.3.0.jar:7.3.0]
at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.consumeToken(AuthenticationService.java:407) [x-pack-security-7.3.0.jar:7.3.0]
at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$extractToken$11(AuthenticationService.java:335) [x-pack-security-7.3.0.jar:7.3.0]
at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.extractToken(AuthenticationService.java:345) [x-pack-security-7.3.0.jar:7.3.0]
at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$checkForApiKey$3(AuthenticationService.java:288) [x-pack-security-7.3.0.jar:7.3.0]
at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:62) [elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.xpack.security.authc.ApiKeyService.authenticateWithApiKeyIfPresent(ApiKeyService.java:357) [x-pack-security-7.3.0.jar:7.3.0]
at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.checkForApiKey(AuthenticationService.java:269) [x-pack-security-7.3.0.jar:7.3.0]
at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$0(AuthenticationService.java:252) [x-pack-security-7.3.0.jar:7.3.0]
at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:62) [elasticsearch-7.3.0.jar:7.3.0]
However after restarting index is recovered.