Readiness Probe Failing - clusterHealthCheckParams: "wait_for_status=yellow&timeout=1s"

Elastic Stack Elasticsearch
1

We have a deployed a elastic master with single replica in kubernetes and have added the parameter
clusterHealthCheckParams: wait_for_status=yellow&timeout=1s as suggested in post.

The elastic master was working fine even after restarts. Now after almost 6months, we restarted the elastic master pod and the readiness probe is getting failed. The error shown is as below

Readiness probe failed: Waiting for Elasticsearch cluster to become cluster to be ready (request params: "wait_for_status=yellow&timeout=1s" ) Cluster is not yet ready (request params: "wait_for_status=yellow&timeout=1s" )

There is no error observed in the elastic master pod. Any inputs as to what may be the reason the readiness probe is failing would be helpful.

Regards,
Aniesh

2

What is the full output of the cluster stats API? What is in the Elasticsearch logs?

3

The output is as below:

curl -XGET 'localhost:9200/_cluster/st 
{
  "_nodes" : {
    "total" : 1,
    "successful" : 1,
    "failed" : 0
  },
  "cluster_name" : "elasticsearch",
  "cluster_uuid" : "ZjXHTVc4T_2cYcJGuniJUQ",
  "timestamp" : 1643104229456,
  "status" : "red",
  "indices" : {
    "count" : 2,
    "shards" : {
      "total" : 2,
      "primaries" : 2,
      "replication" : 0.0,
      "index" : {
        "shards" : {
          "min" : 1,
          "max" : 1,
          "avg" : 1.0
        },
        "primaries" : {
          "min" : 1,
          "max" : 1,
          "avg" : 1.0
        },
        "replication" : {
          "min" : 0.0,
          "max" : 0.0,
          "avg" : 0.0
        }
      }
    },
    "docs" : {
      "count" : 9,
      "deleted" : 1
    },
    "store" : {
      "size" : "158.5kb",
      "size_in_bytes" : 162406
    },
    "fielddata" : {
      "memory_size" : "0b",
      "memory_size_in_bytes" : 0,
      "evictions" : 0
    },
    "query_cache" : {
      "memory_size" : "0b",
      "memory_size_in_bytes" : 0,
      "total_count" : 0,
      "hit_count" : 0,
      "miss_count" : 0,
      "cache_size" : 0,
      "cache_count" : 0,
      "evictions" : 0
    },
    "completion" : {
      "size" : "0b",
      "size_in_bytes" : 0
    },
    "segments" : {
      "count" : 9,
      "memory" : "18kb",
      "memory_in_bytes" : 18439,
      "terms_memory" : "13.2kb",
      "terms_memory_in_bytes" : 13569,
      "stored_fields_memory" : "2.7kb",
      "stored_fields_memory_in_bytes" : 2808,
      "term_vectors_memory" : "0b",
      "term_vectors_memory_in_bytes" : 0,
      "norms_memory" : "1kb",
      "norms_memory_in_bytes" : 1088,
      "points_memory" : "66b",
      "points_memory_in_bytes" : 66,
      "doc_values_memory" : "908b",
      "doc_values_memory_in_bytes" : 908,
      "index_writer_memory" : "0b",
      "index_writer_memory_in_bytes" : 0,
      "version_map_memory" : "0b",
      "version_map_memory_in_bytes" : 0,
      "fixed_bit_set" : "336b",
      "fixed_bit_set_memory_in_bytes" : 336,
      "max_unsafe_auto_id_timestamp" : -1,
      "file_sizes" : { }
    }
  },
  "nodes" : {
    "count" : {
      "total" : 1,
      "coordinating_only" : 0,
      "data" : 1,
      "ingest" : 1,
      "master" : 1,
      "voting_only" : 0
    },
    "versions" : [
      "7.3.0"
    ],
    "os" : {
      "available_processors" : 1,
      "allocated_processors" : 1,
      "names" : [
        {
          "name" : "Linux",
          "count" : 1
        }
      ],
      "pretty_names" : [
        {
          "pretty_name" : "CentOS Linux 7 (Core)",
          "count" : 1
        }
      ],
      "mem" : {
        "total" : "503.1gb",
        "total_in_bytes" : 540296163328,
        "free" : "251.5gb",
        "free_in_bytes" : 270106763264,
        "used" : "251.6gb",
        "used_in_bytes" : 270189400064,
        "free_percent" : 50,
        "used_percent" : 50
      }
    },
    "process" : {
      "cpu" : {
        "percent" : 0
      },
      "open_file_descriptors" : {
        "min" : 226,
        "max" : 226,
        "avg" : 226
      }
    },
    "jvm" : {
      "max_uptime" : "1.3h",
      "max_uptime_in_millis" : 4760870,
      "versions" : [
        {
          "version" : "12.0.1",
          "vm_name" : "OpenJDK 64-Bit Server VM",
          "vm_version" : "12.0.1+12",
          "vm_vendor" : "Oracle Corporation",
          "bundled_jdk" : true,
          "using_bundled_jdk" : true,
          "count" : 1
        }
      ],
      "mem" : {
        "heap_used" : "118.1mb",
        "heap_used_in_bytes" : 123915720,
        "heap_max" : "1015.6mb",
        "heap_max_in_bytes" : 1065025536
      },
      "threads" : 32
    },
    "fs" : {
      "total" : "99.9gb",
      "total_in_bytes" : 107321622528,
      "free" : "99.7gb",
      "free_in_bytes" : 107078459392,
      "available" : "99.7gb",
      "available_in_bytes" : 107078459392
    },
    "plugins" : [ ],
    "network_types" : {
      "transport_types" : {
        "security4" : 1
      },
      "http_types" : {
        "security4" : 1
      }
    },
    "discovery_types" : {
      "zen" : 1
    },
    "packaging_types" : [
      {
        "flavor" : "default",
        "type" : "docker",
        "count" : 1
      }
    ]
  }
}
4

I have attached the cluster status and shards details.

curl -XGET 'localhost:9200/_cluster/health?pretty'
{
  "cluster_name" : "elasticsearch",
  "status" : "red",
  "timed_out" : false,
  "number_of_nodes" : 1,
  "number_of_data_nodes" : 1,
  "active_primary_shards" : 2,
  "active_shards" : 2,
  "relocating_shards" : 0,
  "initializing_shards" : 0,
  "unassigned_shards" : 4,
  "delayed_unassigned_shards" : 0,
  "number_of_pending_tasks" : 0,
  "number_of_in_flight_fetch" : 0,
  "task_max_waiting_in_queue_millis" : 0,
  "active_shards_percent_as_number" : 33.33333333333333
}

curl -XGET localhost:9200/_cat/shards
.kibana_task_manager 0 p STARTED    2  12.8kb 10.20.164.98 elasticsearch-master-0
missingperson        0 p UNASSIGNED                        
missingperson        0 r UNASSIGNED                        
.kibana_1            0 p STARTED    7 145.7kb 10.20.164.98 elasticsearch-master-0
fir                  0 p UNASSIGNED                        
fir                  0 r UNASSIGNED
5

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.