Getting Error "all shards failed" when I go for re-indexing

{
"error" : {
"root_cause" : ,
"type" : "search_phase_execution_exception",
"reason" : "all shards failed",
"phase" : "query",
"grouped" : true,
"failed_shards" :
},
"status" : 503
}

I think you will need to provide a lot more detail about how you got that error if you want someone to help.

I have issue with two shards , it went in INITIALIZING state because of that my cluster become red.
Now I want to re-index , without loosing data .

Please provide more details.

Which version are you running?

What is the size and configuration of your cluster?

What is the use case?

How much data do you have in the cluster?

How did you end up where you are now?

Version: 7.6.0
ClusterSize: 1 node
Data: 1Tb

Config:
cluster.name: es-cluster-mo4-rs
node.master: true
node.data: true

I have ELK Stack , using it for logs management . I have Life Cycle policy which is rotating index on daily basis. Suddenly one index became in Red state and life policy failed to create new index.

What is the full output of the cluster stats API?

curl -X GET "x.x.x.x:9200/_cluster/health?pretty"
{
"cluster_name" : "xxxxxxxxxxx",
"status" : "red",
"timed_out" : false,
"number_of_nodes" : 1,
"number_of_data_nodes" : 1,
"active_primary_shards" : 29,
"active_shards" : 29,
"relocating_shards" : 0,
"initializing_shards" : 2,
"unassigned_shards" : 0,
"delayed_unassigned_shards" : 0,
"number_of_pending_tasks" : 0,
"number_of_in_flight_fetch" : 0,
"task_max_waiting_in_queue_millis" : 0,
"active_shards_percent_as_number" : 93.54838709677419
}

curl -X GET "x.x.x.x:9200/_cluster/stats?human&pretty&pretty"

{
"_nodes" : {
"total" : 1,
"successful" : 1,
"failed" : 0
},
"cluster_name" : "XXXXXXXXXXXXXXXXXXX",
"cluster_uuid" : "5qcyUKYKRSmFhvQCzYzqvQ",
"timestamp" : 1597214529651,
"status" : "red",
"indices" : {
"count" : 29,
"shards" : {
"total" : 29,
"primaries" : 29,
"replication" : 0.0,
"index" : {
"shards" : {
"min" : 1,
"max" : 1,
"avg" : 1.0
},
"primaries" : {
"min" : 1,
"max" : 1,
"avg" : 1.0
},
"replication" : {
"min" : 0.0,
"max" : 0.0,
"avg" : 0.0
}
}
},
"docs" : {
"count" : 183973730,
"deleted" : 38
},
"store" : {
"size" : "56.6gb",
"size_in_bytes" : 60866644362
},
"fielddata" : {
"memory_size" : "0b",
"memory_size_in_bytes" : 0,
"evictions" : 0
},
"query_cache" : {
"memory_size" : "0b",
"memory_size_in_bytes" : 0,
"total_count" : 0,
"hit_count" : 0,
"miss_count" : 0,
"cache_size" : 0,
"cache_count" : 0,
"evictions" : 0
},
"completion" : {
"size" : "0b",
"size_in_bytes" : 0
},
"segments" : {
"count" : 515,
"memory" : "96.7mb",
"memory_in_bytes" : 101432595,
"terms_memory" : "62.1mb",
"terms_memory_in_bytes" : 65146095,
"stored_fields_memory" : "34.3mb",
"stored_fields_memory_in_bytes" : 36015320,
"term_vectors_memory" : "0b",
"term_vectors_memory_in_bytes" : 0,
"norms_memory" : "8.6kb",
"norms_memory_in_bytes" : 8896,
"points_memory" : "0b",
"points_memory_in_bytes" : 0,
"doc_values_memory" : "256.1kb",
"doc_values_memory_in_bytes" : 262284,
"index_writer_memory" : "0b",
"index_writer_memory_in_bytes" : 0,
"version_map_memory" : "0b",
"version_map_memory_in_bytes" : 0,
"fixed_bit_set" : "568b",
"fixed_bit_set_memory_in_bytes" : 568,
"max_unsafe_auto_id_timestamp" : 1588912675144,
"file_sizes" : { }
}
},
"nodes" : {
"count" : {
"total" : 1,
"coordinating_only" : 0,
"data" : 1,
"ingest" : 1,
"master" : 1,
"ml" : 1,
"voting_only" : 0
},
"versions" : [
"7.6.0"
],
"os" : {
"available_processors" : 6,
"allocated_processors" : 6,
"names" : [
{
"name" : "Linux",
"count" : 1
}
],
"pretty_names" : [
{
"pretty_name" : "CentOS Linux 7 (Core)",
"count" : 1
}
],
"mem" : {
"total" : "15.5gb",
"total_in_bytes" : 16646995968,
"free" : "166.1mb",
"free_in_bytes" : 174231552,
"used" : "15.3gb",
"used_in_bytes" : 16472764416,
"free_percent" : 1,
"used_percent" : 99
}
},
"process" : {
"cpu" : {
"percent" : 0
},
"open_file_descriptors" : {
"min" : 1295,
"max" : 1295,
"avg" : 1295
}
},
"jvm" : {
"max_uptime" : "17.4h",
"max_uptime_in_millis" : 62843071,
"versions" : [
{
"version" : "13.0.2",
"vm_name" : "OpenJDK 64-Bit Server VM",
"vm_version" : "13.0.2+8",
"vm_vendor" : "AdoptOpenJDK",
"bundled_jdk" : true,
"using_bundled_jdk" : true,
"count" : 1
}
],
"mem" : {
"heap_used" : "1.7gb",
"heap_used_in_bytes" : 1861839712,
"heap_max" : "7.9gb",
"heap_max_in_bytes" : 8537636864
},
"threads" : 80
},
"fs" : {
"total" : "1.4tb",
"total_in_bytes" : 1585199681536,
"free" : "600.1gb",
"free_in_bytes" : 644449427456,
"available" : "532.6gb",
"available_in_bytes" : 571923664896
},
"plugins" : ,
"network_types" : {
"transport_types" : {
"security4" : 1
},
"http_types" : {
"security4" : 1
}
},
"discovery_types" : {
"zen" : 1
},
"packaging_types" : [
{
"flavor" : "default",
"type" : "rpm",
"count" : 1
}
],
"ingest" : {
"number_of_pipelines" : 1,
"processor_stats" : {
"gsub" : {
"count" : 0,
"failed" : 0,
"current" : 0,
"time" : "0s",
"time_in_millis" : 0
},
"script" : {
"count" : 0,
"failed" : 0,
"current" : 0,
"time" : "0s",
"time_in_millis" : 0
}
}
}
}
}

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.