Hello everyone,
I have a problem with my .siem-signals-default index. The shard goes to red status and I can't reallocate the shard to another node. Because the shard seems to not be in my cluster and I can't get it back. See the message below:
"index" : ".siem-signals-default-000003",
"shard" : 0,
"primary" : true,
"current_state" : "unassigned",
"unassigned_info" : {
"reason" : "CLUSTER_RECOVERED",
"at" : "2021-12-15T10:17:49.049Z",
"last_allocation_status" : "no_valid_shard_copy"
},
"can_allocate" : "no_valid_shard_copy",
"allocate_explanation" : "cannot allocate because a previous copy of the primary shard existed but can no longer be found on the nodes in the cluster",
Has anyone encountered this problem? Or can someone tell me how to solve it?
Thanks a lot !
@ warkolm if you see this topic. Sorry for the previous one but I was in vacation and didn't notice that you replied 
This is my output of _cluster/stats?pretty&human:
{
"_nodes" : {
"total" : 5,
"successful" : 5,
"failed" : 0
},
"cluster_name" : "ELK-prod",
"cluster_uuid" : "u_lFGhvMTj2abtw3EWhydg",
"timestamp" : 1644315336181,
"status" : "red",
"indices" : {
"count" : 117,
"shards" : {
"total" : 202,
"primaries" : 117,
"replication" : 0.7264957264957265,
"index" : {
"shards" : {
"min" : 1,
"max" : 2,
"avg" : 1.7264957264957266
},
"primaries" : {
"min" : 1,
"max" : 1,
"avg" : 1.0
},
"replication" : {
"min" : 0.0,
"max" : 1.0,
"avg" : 0.7264957264957265
}
}
},
"docs" : {
"count" : 386105873,
"deleted" : 2498489
},
"store" : {
"size" : "198.6gb",
"size_in_bytes" : 213272419082,
"reserved" : "0b",
"reserved_in_bytes" : 0
},
"fielddata" : {
"memory_size" : "105.8kb",
"memory_size_in_bytes" : 108344,
"evictions" : 0
},
"query_cache" : {
"memory_size" : "1.1mb",
"memory_size_in_bytes" : 1187557,
"total_count" : 13321420,
"hit_count" : 323259,
"miss_count" : 12998161,
"cache_size" : 255,
"cache_count" : 34317,
"evictions" : 34062
},
"completion" : {
"size" : "0b",
"size_in_bytes" : 0
},
"segments" : {
"count" : 1823,
"memory" : "31.5mb",
"memory_in_bytes" : 33105672,
"terms_memory" : "17.3mb",
"terms_memory_in_bytes" : 18216664,
"stored_fields_memory" : "2.7mb",
"stored_fields_memory_in_bytes" : 2876648,
"term_vectors_memory" : "0b",
"term_vectors_memory_in_bytes" : 0,
"norms_memory" : "187kb",
"norms_memory_in_bytes" : 191488,
"points_memory" : "0b",
"points_memory_in_bytes" : 0,
"doc_values_memory" : "11.2mb",
"doc_values_memory_in_bytes" : 11820872,
"index_writer_memory" : "125.3mb",
"index_writer_memory_in_bytes" : 131447044,
"version_map_memory" : "732.8kb",
"version_map_memory_in_bytes" : 750446,
"fixed_bit_set" : "8.1mb",
"fixed_bit_set_memory_in_bytes" : 8559424,
"max_unsafe_auto_id_timestamp" : 1644278409399,
"file_sizes" : { }
},
"mappings" : {
"field_types" : [
{
"name" : "alias",
"count" : 321,
"index_count" : 14
},
{
"name" : "binary",
"count" : 14,
"index_count" : 3
},
{
"name" : "boolean",
"count" : 1642,
"index_count" : 79
},
{
"name" : "byte",
"count" : 11,
"index_count" : 11
},
{
"name" : "date",
"count" : 1841,
"index_count" : 115
},
{
"name" : "date_nanos",
"count" : 1,
"index_count" : 1
},
{
"name" : "date_range",
"count" : 1,
"index_count" : 1
},
{
"name" : "double",
"count" : 821,
"index_count" : 29
},
{
"name" : "double_range",
"count" : 1,
"index_count" : 1
},
{
"name" : "flattened",
"count" : 38,
"index_count" : 9
},
{
"name" : "float",
"count" : 1037,
"index_count" : 36
},
{
"name" : "float_range",
"count" : 1,
"index_count" : 1
},
{
"name" : "geo_point",
"count" : 213,
"index_count" : 48
},
{
"name" : "geo_shape",
"count" : 2,
"index_count" : 2
},
{
"name" : "half_float",
"count" : 155,
"index_count" : 50
},
{
"name" : "integer",
"count" : 390,
"index_count" : 39
},
{
"name" : "integer_range",
"count" : 1,
"index_count" : 1
},
{
"name" : "ip",
"count" : 1336,
"index_count" : 48
},
{
"name" : "ip_range",
"count" : 1,
"index_count" : 1
},
{
"name" : "keyword",
"count" : 40769,
"index_count" : 114
},
{
"name" : "long",
"count" : 24249,
"index_count" : 104
},
{
"name" : "long_range",
"count" : 1,
"index_count" : 1
},
{
"name" : "nested",
"count" : 108,
"index_count" : 38
},
{
"name" : "object",
"count" : 21904,
"index_count" : 115
},
{
"name" : "scaled_float",
"count" : 561,
"index_count" : 6
},
{
"name" : "shape",
"count" : 1,
"index_count" : 1
},
{
"name" : "short",
"count" : 942,
"index_count" : 23
},
{
"name" : "text",
"count" : 2843,
"index_count" : 91
}
]
},
"analysis" : {
"char_filter_types" : [ ],
"tokenizer_types" : [ ],
"filter_types" : [
{
"name" : "pattern_capture",
"count" : 1,
"index_count" : 1
}
],
"analyzer_types" : [
{
"name" : "custom",
"count" : 1,
"index_count" : 1
}
],
"built_in_char_filters" : [ ],
"built_in_tokenizers" : [
{
"name" : "uax_url_email",
"count" : 1,
"index_count" : 1
}
],
"built_in_filters" : [
{
"name" : "lowercase",
"count" : 1,
"index_count" : 1
},
{
"name" : "unique",
"count" : 1,
"index_count" : 1
}
],
"built_in_analyzers" : [ ]
}
},
"nodes" : {
"count" : {
"total" : 5,
"coordinating_only" : 0,
"data" : 5,
"ingest" : 1,
"master" : 3,
"ml" : 0,
"remote_cluster_client" : 5,
"transform" : 5,
"voting_only" : 0
},
"versions" : [
"7.9.0"
],
"os" : {
"available_processors" : 6,
"allocated_processors" : 6,
"names" : [
{
"name" : "Linux",
"count" : 5
}
],
"pretty_names" : [
{
"pretty_name" : "Debian GNU/Linux 9 (stretch)",
"count" : 5
}
],
"mem" : {
"total" : "31.1gb",
"total_in_bytes" : 33424068608,
"free" : "2.9gb",
"free_in_bytes" : 3123941376,
"used" : "28.2gb",
"used_in_bytes" : 30300127232,
"free_percent" : 9,
"used_percent" : 91
}
},
"process" : {
"cpu" : {
"percent" : 39
},
"open_file_descriptors" : {
"min" : 597,
"max" : 697,
"avg" : 642
}
},
"jvm" : {
"max_uptime" : "54.6d",
"max_uptime_in_millis" : 4724648677,
"versions" : [
{
"version" : "14.0.1",
"vm_name" : "OpenJDK 64-Bit Server VM",
"vm_version" : "14.0.1+7",
"vm_vendor" : "AdoptOpenJDK",
"bundled_jdk" : true,
"using_bundled_jdk" : true,
"count" : 5
}
],
"mem" : {
"heap_used" : "7.6gb",
"heap_used_in_bytes" : 8224373256,
"heap_max" : "14gb",
"heap_max_in_bytes" : 15032385536
},
"threads" : 366
},
"fs" : {
"total" : "343gb",
"total_in_bytes" : 368301158400,
"free" : "133.2gb",
"free_in_bytes" : 143107043328,
"available" : "115.7gb",
"available_in_bytes" : 124232675328
},
"plugins" : [ ],
"network_types" : {
"transport_types" : {
"security4" : 5
},
"http_types" : {
"security4" : 5
}
},
"discovery_types" : {
"zen" : 5
},
"packaging_types" : [
{
"flavor" : "default",
"type" : "deb",
"count" : 5
}
],
"ingest" : {
"number_of_pipelines" : 64,
"processor_stats" : {
"append" : {
"count" : 0,
"failed" : 0,
"current" : 0,
"time" : "0s",
"time_in_millis" : 0
},
"conditional" : {
"count" : 6700754,
"failed" : 0,
"current" : 0,
"time" : "14.1s",
"time_in_millis" : 14123
},
"convert" : {
"count" : 0,
"failed" : 0,
"current" : 0,
"time" : "0s",
"time_in_millis" : 0
},
"date" : {
"count" : 2529896,
"failed" : 0,
"current" : 0,
"time" : "1.3s",
"time_in_millis" : 1312
},
"dot_expander" : {
"count" : 58264,
"failed" : 0,
"current" : 0,
"time" : "8ms",
"time_in_millis" : 8
},
"geoip" : {
"count" : 1408696,
"failed" : 625597,
"current" : 0,
"time" : "706ms",
"time_in_millis" : 706
},
"grok" : {
"count" : 11595730,
"failed" : 2522609,
"current" : 0,
"time" : "11.2s",
"time_in_millis" : 11263
},
"gsub" : {
"count" : 0,
"failed" : 0,
"current" : 0,
"time" : "0s",
"time_in_millis" : 0
},
"json" : {
"count" : 7283,
"failed" : 0,
"current" : 0,
"time" : "12ms",
"time_in_millis" : 12
},
"pipeline" : {
"count" : 0,
"failed" : 0,
"current" : 0,
"time" : "0s",
"time_in_millis" : 0
},
"remove" : {
"count" : 10578416,
"failed" : 0,
"current" : 0,
"time" : "458ms",
"time_in_millis" : 458
},
"rename" : {
"count" : 10554704,
"failed" : 0,
"current" : 0,
"time" : "693ms",
"time_in_millis" : 693
},
"script" : {
"count" : 3255974,
"failed" : 3148205,
"current" : 0,
"time" : "5.5s",
"time_in_millis" : 5523
},
"set" : {
"count" : 12329294,
"failed" : 0,
"current" : 0,
"time" : "598ms",
"time_in_millis" : 598
},
"split" : {
"count" : 2522613,
"failed" : 0,
"current" : 0,
"time" : "55ms",
"time_in_millis" : 55
},
"user_agent" : {
"count" : 0,
"failed" : 0,
"current" : 0,
"time" : "0s",
"time_in_millis" : 0
}
}
}
}
}