IndexNotFoundException

[vnallamothu (vamshi)]

I have installed Elasticsearch 5.5.3 and Kibana 5.5.3 on the same server and also installed Xpack and buckler plugins. Everything seems to be working but when I hit the monitoring tab in kibana, I see "No monitoring data found" error. When I see Elasticsearch logs, I see following exception:

[2018-10-09T05:17:38,879][WARN ][o.e.x.m.e.l.LocalExporter] unexpected error while indexing monitoring document
org.elasticsearch.xpack.monitoring.exporter.ExportException: [.monitoring-es-6-2018.10.09] IndexNotFoundException[no such index]
        at org.elasticsearch.xpack.monitoring.exporter.local.LocalBulk.lambda$throwExportException$2(LocalBulk.java:130) ~[?:?]
        at java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193) ~[?:1.8.0_181]
        at java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:175) ~[?:1.8.0_181]
        at java.util.Spliterators$ArraySpliterator.forEachRemaining(Spliterators.java:948) ~[?:1.8.0_181]
        at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:481) ~[?:1.8.0_181]
        at java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:471) ~[?:1.8.0_181]
        at java.util.stream.ForEachOps$ForEachOp.evaluateSequential(ForEachOps.java:151) ~[?:1.8.0_181]
        at java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(ForEachOps.java:174) ~[?:1.8.0_181]
        at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234) ~[?:1.8.0_181]
        at java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:418) ~[?:1.8.0_181]
        at org.elasticsearch.xpack.monitoring.exporter.local.LocalBulk.throwExportException(LocalBulk.java:131) ~[?:?]
        at org.elasticsearch.xpack.monitoring.exporter.local.LocalBulk.lambda$doFlush$0(LocalBulk.java:114) ~[?:?]
        at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:59) ~[elasticsearch-5.5.3.jar:5.5.3]
        at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:88) ~[elasticsearch-5.5.3.jar:5.5.3]
        at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:84) ~[elasticsearch-5.5.3.jar:5.5.3]
        at org.elasticsearch.action.bulk.TransportBulkAction$BulkRequestModifier.lambda$wrapActionListenerIfNeeded$0(TransportBulkAction.java:583) ~[elasticsearch-5.5.3.jar:5.5.3]
        at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:59) ~[elasticsearch-5.5.3.jar:5.5.3]
        at org.elasticsearch.action.bulk.TransportBulkAction$BulkOperation.doRun(TransportBulkAction.java:343) ~[elasticsearch-5.5.3.jar:5.5.3]
        at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-5.5.3.jar:5.5.3]
        at org.elasticsearch.action.bulk.TransportBulkAction.executeBulk(TransportBulkAction.java:474) ~[elasticsearch-5.5.3.jar:5.5.3]
        at org.elasticsearch.action.bulk.TransportBulkAction.doExecute(TransportBulkAction.java:205) ~[elasticsearch-5.5.3.jar:5.5.3]
        at org.elasticsearch.action.bulk.TransportBulkAction.lambda$processBulkIndexIngestRequest$3(TransportBulkAction.java:526) ~[elasticsearch-5.5.3.jar:5.5.3]
        at org.elasticsearch.ingest.PipelineExecutionService$2.doRun(PipelineExecutionService.java:98) [elasticsearch-5.5.3.jar:5.5.3]
        at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:638) [elasticsearch-5.5.3.jar:5.5.3]
        at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-5.5.3.jar:5.5.3]
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_181]
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_181]
        at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181]

Any ideas?

[vnallamothu (vamshi)]

kibana logs thows following error:

{"type":"log","@timestamp":"2018-10-08T05:03:32Z","tags":["error","monitoring-ui"],"pid":12393,"level":"error","message":"[export_exception] Exception when closing export bulk","error":{"message":"[export_exception] Exception when closing export bulk","name":"Error","stack":"[export_exception] Exception when closing export bulk :: {\"path\":\"/_xpack/monitoring/_bulk\",\"query\":{\"system_id\":\"kibana\",\"system_api_version\":\"6\",\"interval\":\"10000ms\"},\"body\":\"{\\\"index\\\":{\\\"_type\\\":\\\"kibana_stats\\\"}}\\n{\\\"kibana\\\":{\\\"uuid\\\":\\\"84ac5017-f022-41d7-a9aa-bde22c65719f\\\",\\\"name\\\":\\\"kibana01-bitbucket\\\",\\\"host\\\":\\\"cl0-o-dev-bbe01\\\",\\\"transport_address\\\":\\\"10.40.1.35:5601\\\",\\\"version\\\":\\\"5.5.3\\\",\\\"snapshot\\\":false,\\\"status\\\":\\\"green\\\"},\\\"concurrent_connections\\\":0,\\\"os\\\":{\\\"load\\\":{\\\"1m\\\":0.14208984375,\\\"5m\\\":0.392578125,\\\"15m\\\":0.24853515625},\\\"memory\\\":{\\\"total_in_bytes\\\":8202338304,\\\"free_in_bytes\\\":334426112,\\\"used_in_bytes\\\":7867912192},\\\"uptime_in_millis\\\":1777635000},\\\"process\\\":{\\\"event_loop_delay\\\":1.038787841796875,\\\"memory\\\":{\\\"heap\\\":{\\\"total_in_bytes\\\":109871104,\\\"used_in_bytes\\\":95045640,\\\"size_limit\\\":1501560832},\\\"resident_set_size_in_bytes\\\":156913664},\\\"uptime_in_millis\\\":209210918},\\\"requests\\\":{\\\"disconnects\\\":0,\\\"total\\\":0,\\\"status_codes\\\":{}},\\\"response_times\\\":{\\\"average\\\":0,\\\"max\\\":0},\\\"timestamp\\\":\\\"2018-10-08T05:03:27.405Z\\\"}\\n\",\"statusCode\":500,\"response\":\"{\\\"took\\\":3,\\\"errors\\\":true,\\\"error\\\":{\\\"type\\\":\\\"export_exception\\\",\\\"reason\\\":\\\"Exception when closing export bulk\\\",\\\"caused_by\\\":{\\\"type\\\":\\\"export_exception\\\",\\\"reason\\\":\\\"failed to flush export bulks\\\",\\\"caused_by\\\":{\\\"type\\\":\\\"export_exception\\\",\\\"reason\\\":\\\"bulk [default_local] reports failures when exporting documents\\\",\\\"exceptions\\\":[{\\\"type\\\":\\\"export_exception\\\",\\\"reason\\\":\\\"[.monitoring-kibana-6-2018.10.08] IndexNotFoundException[no such index]\\\",\\\"caused_by\\\":{\\\"type\\\":\\\"index_not_found_exception\\\",\\\"reason\\\":\\\"no such index\\\",\\\"resource.type\\\":\\\"index_expression\\\",\\\"resource.id\\\":\\\".monitoring-kibana-6-2018.10.08\\\",\\\"index_uuid\\\":\\\"_na_\\\",\\\"index\\\":\\\".monitoring-kibana-6-2018.10.08\\\"}}]}}}}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:295:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:254:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:159:7)\n at IncomingMessage.bound (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/dist/lodash.js:729:21)\n at emitNone (events.js:91:20)\n at IncomingMessage.emit (events.js:185:7)\n at endReadableNT (_stream_readable.js:974:12)\n at _combinedTickCallback (internal/process/next_tick.js:80:11)\n at process._tickDomainCallback (internal/process/next_tick.js:128:9)"}}