Hi,
I have been using Kibana and Elastic Search, whereas elastic search service is up and running and kibana was down due to below error because of this unable to access the kibana and health URL shows red
"tags":["warning","stats-collection"],"pid":16426,"message":"Unable to fetch data from rollups collector"}
{"type":"error","@timestamp":"2019-05-09T08:44:56Z","tags":["warning","stats-collection"],"pid":16426,"level":"error","error":{"message":"[no_shard_available_action_exception] No shard available for [get [.kibana][doc][config:6.6.2]: routing [null]]","name":"Error","stack":"[no_shard_available_action_exception] No shard available for [get [.kibana][doc][config:6.6.2]: routing [null]] :: {"path":"/.kibana/doc/config%3A6.6.2","query":{},"statusCode":503,"response":"{\"error\":{\"root_cause\":[{\"type\":\"no_shard_available_action_exception\",\"reason\":\"No shard available for [get [.kibana][doc][config:6.6.2]: routing [null]]\"}],\"type\":\"no_shard_available_action_exception\",\"reason\":\"No shard available for [get [.kibana][doc][config:6.6.2]: routing [null]]\"},\"status\":503}"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector. (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[no_shard_available_action_exception] No shard available for [get [.kibana][doc][config:6.6.2]: routing [null]]"}
{"type":"log","@timestamp":"2019-05-09T08:44:56Z","tags":["warning","stats-collection"],"pid":16426,"message":"Unable to fetch data from kibana_settings collector"}
{"type":"error","@timestamp":"2019-05-09T08:44:56Z","tags":["warning","stats-collection"],"pid":16426,"level":"error","error":{"message":"[search_phase_execution_exception] all shards failed","name":"Error","stack":"[search_phase_execution_exception] all shards failed :: {"path":"/.kibana/_search","query":{"ignore_unavailable":true,"filter_path":"aggregations.types.buckets"},"body":"{\"size\":0,\"query\":{\"terms\":{\"type\":[\"dashboard\",\"visualization\",\"search\",\"index-pattern\",\"graph-workspace\",\"timelion-sheet\"]}},\"aggs\":{\"types\":{\"terms\":{\"field\":\"type\",\"size\":6}}}}","statusCode":503,"response":"{\"error\":{\"root_cause\":,\"type\":\"search_phase_execution_exception\",\"reason\":\"all shards failed\",\"phase\":\"query\",\"grouped\":true,\"failed_shards\":},\"status\":503}"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector. (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[search_phase_execution_exception] all shards failed"}
{"type":"log","@timestamp":"2019-05-09T08:44:56Z","tags":["warning","stats-collection"],"pid":16426,"message":"Unable to fetch data from kibana collector"}
{"type":"log","@timestamp":"2019-05-09T08:44:57Z","tags":["status","plugin:spaces@6.6.2","error"],"pid":16426,"state":"red","message":"Status changed from red to red - all shards failed: [search_phase_execution_exception] all shards failed","prevState":"red","prevMsg":"[data] Elasticsearch cluster did not respond with license information."}
{"type":"log","@timestamp":"2019-05-09T08:44:57Z","tags":["fatal","root"],"pid":16426,"message":"{ [search_phase_execution_exception] all shards failed :: {"path":"/.kibana/doc/_count","query":{},"body":"{\"query\":{\"bool\":{\"should\":[{\"bool\":{\"must\":[{\"exists\":{\"field\":\"index-pattern\"}},{\"bool\":{\"must_not\":{\"term\":{\"migrationVersion.index-pattern\":\"6.5.0\"}}}}]}}]}}}","statusCode":503,"response":"{\"error\":{\"root_cause\":,\"type\":\"search_phase_execution_exception\",\"reason\":\"all shards failed\",\"phase\":\"query\",\"grouped\":true,\"failed_shards\":},\"status\":503}"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector. (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)\n status: 503,\n displayName: 'ServiceUnavailable',\n message:\n 'all shards failed: [search_phase_execution_exception] all shards failed',\n path: '/.kibana/doc/_count',\n query: {},\n body:\n { error:\n { root_cause: ,\n type: 'search_phase_execution_exception',\n reason: 'all shards failed',\n phase: 'query',\n grouped: true,\n failed_shards: },\n status: 503 },\n statusCode: 503,\n response:\n '{"error":{"root_cause":,"type":"search_phase_execution_exception","reason":"all shards failed","phase":"query","grouped":true,"failed_shards":},"status":503}',\n toString: [Function],\n toJSON: [Function],\n isBoom: true,\n isServer: true,\n data: null,\n output:\n { statusCode: 503,\n payload:\n { message:\n 'all shards failed: [search_phase_execution_exception] all shards failed',\n statusCode: 503,\n error: 'Service Unavailable' },\n headers: {} },\n reformat: [Function],\n [Symbol(SavedObjectsClientErrorCode)]: 'SavedObjectsClient/esUnavailable' }"}
curl -X GET http://X.X.X.X:9200/_cluster/health
{"cluster_name":"XXX-apm-cluster","status":"red","timed_out":false,"number_of_nodes":2,"number_of_data_nodes":2,"active_primary_shards":0,"active_shards":0,"relocating_shards":0,"initializing_shards":0,"unassigned_shards":0,"delayed_unassigned_shards":0,"number_of_pending_tasks":0,"number_of_in_flight_fetch":0,"task_max_waiting_in_queue_millis":0,"active_shards_percent_as_number":"NaN"}