Unable to access kibana shows HTTP 503 error in kibana logs

Hi,
I have been using Kibana and Elastic Search, whereas elastic search service is up and running and kibana was down due to below error because of this unable to access the kibana and health URL shows red
"tags":["warning","stats-collection"],"pid":16426,"message":"Unable to fetch data from rollups collector"}
{"type":"error","@timestamp":"2019-05-09T08:44:56Z","tags":["warning","stats-collection"],"pid":16426,"level":"error","error":{"message":"[no_shard_available_action_exception] No shard available for [get [.kibana][doc][config:6.6.2]: routing [null]]","name":"Error","stack":"[no_shard_available_action_exception] No shard available for [get [.kibana][doc][config:6.6.2]: routing [null]] :: {"path":"/.kibana/doc/config%3A6.6.2","query":{},"statusCode":503,"response":"{\"error\":{\"root_cause\":[{\"type\":\"no_shard_available_action_exception\",\"reason\":\"No shard available for [get [.kibana][doc][config:6.6.2]: routing [null]]\"}],\"type\":\"no_shard_available_action_exception\",\"reason\":\"No shard available for [get [.kibana][doc][config:6.6.2]: routing [null]]\"},\"status\":503}"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector. (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[no_shard_available_action_exception] No shard available for [get [.kibana][doc][config:6.6.2]: routing [null]]"}
{"type":"log","@timestamp":"2019-05-09T08:44:56Z","tags":["warning","stats-collection"],"pid":16426,"message":"Unable to fetch data from kibana_settings collector"}
{"type":"error","@timestamp":"2019-05-09T08:44:56Z","tags":["warning","stats-collection"],"pid":16426,"level":"error","error":{"message":"[search_phase_execution_exception] all shards failed","name":"Error","stack":"[search_phase_execution_exception] all shards failed :: {"path":"/.kibana/_search","query":{"ignore_unavailable":true,"filter_path":"aggregations.types.buckets"},"body":"{\"size\":0,\"query\":{\"terms\":{\"type\":[\"dashboard\",\"visualization\",\"search\",\"index-pattern\",\"graph-workspace\",\"timelion-sheet\"]}},\"aggs\":{\"types\":{\"terms\":{\"field\":\"type\",\"size\":6}}}}","statusCode":503,"response":"{\"error\":{\"root_cause\":,\"type\":\"search_phase_execution_exception\",\"reason\":\"all shards failed\",\"phase\":\"query\",\"grouped\":true,\"failed_shards\":},\"status\":503}"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector. (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[search_phase_execution_exception] all shards failed"}
{"type":"log","@timestamp":"2019-05-09T08:44:56Z","tags":["warning","stats-collection"],"pid":16426,"message":"Unable to fetch data from kibana collector"}
{"type":"log","@timestamp":"2019-05-09T08:44:57Z","tags":["status","plugin:spaces@6.6.2","error"],"pid":16426,"state":"red","message":"Status changed from red to red - all shards failed: [search_phase_execution_exception] all shards failed","prevState":"red","prevMsg":"[data] Elasticsearch cluster did not respond with license information."}
{"type":"log","@timestamp":"2019-05-09T08:44:57Z","tags":["fatal","root"],"pid":16426,"message":"{ [search_phase_execution_exception] all shards failed :: {"path":"/.kibana/doc/_count","query":{},"body":"{\"query\":{\"bool\":{\"should\":[{\"bool\":{\"must\":[{\"exists\":{\"field\":\"index-pattern\"}},{\"bool\":{\"must_not\":{\"term\":{\"migrationVersion.index-pattern\":\"6.5.0\"}}}}]}}]}}}","statusCode":503,"response":"{\"error\":{\"root_cause\":,\"type\":\"search_phase_execution_exception\",\"reason\":\"all shards failed\",\"phase\":\"query\",\"grouped\":true,\"failed_shards\":},\"status\":503}"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector. (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)\n status: 503,\n displayName: 'ServiceUnavailable',\n message:\n 'all shards failed: [search_phase_execution_exception] all shards failed',\n path: '/.kibana/doc/_count',\n query: {},\n body:\n { error:\n { root_cause: ,\n type: 'search_phase_execution_exception',\n reason: 'all shards failed',\n phase: 'query',\n grouped: true,\n failed_shards: },\n status: 503 },\n statusCode: 503,\n response:\n '{"error":{"root_cause":,"type":"search_phase_execution_exception","reason":"all shards failed","phase":"query","grouped":true,"failed_shards":},"status":503}',\n toString: [Function],\n toJSON: [Function],\n isBoom: true,\n isServer: true,\n data: null,\n output:\n { statusCode: 503,\n payload:\n { message:\n 'all shards failed: [search_phase_execution_exception] all shards failed',\n statusCode: 503,\n error: 'Service Unavailable' },\n headers: {} },\n reformat: [Function],\n [Symbol(SavedObjectsClientErrorCode)]: 'SavedObjectsClient/esUnavailable' }"}

curl -X GET http://X.X.X.X:9200/_cluster/health
{"cluster_name":"XXX-apm-cluster","status":"red","timed_out":false,"number_of_nodes":2,"number_of_data_nodes":2,"active_primary_shards":0,"active_shards":0,"relocating_shards":0,"initializing_shards":0,"unassigned_shards":0,"delayed_unassigned_shards":0,"number_of_pending_tasks":0,"number_of_in_flight_fetch":0,"task_max_waiting_in_queue_millis":0,"active_shards_percent_as_number":"NaN"}

What steps have you taken to install Kibana and Elasticsearch? What does your environment look like?

Hi,

Set up was done before i started working on kibana/elasticsearch and 2 instances of kibana and elastic search are running on different hosts that means 1 kibana and elasticsearch running on each of two hosts. 2 instances of elastic search are in cluster.

Why do you have 2 kibana servers for such a small cluster?

Hi,

We are started using kibana recently as the intial set up started with 2 instances

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.