systemctl status kibana.service -l
kibana.service - Kibana
Loaded: loaded (/etc/systemd/system/kibana.service; enabled; vendor preset: disabled)
Active: active (running) since Mon 2020-11-23 18:24:19 IST; 2 days ago
Main PID: 9604 (node)
CGroup: /system.slice/kibana.service
└─9604 /usr/share/kibana/bin/../node/bin/node /usr/share/kibana/bin/../src/cli -c /etc/kibana/kibana.yml
Nov 25 20:56:37 kibana[9604]: {"type":"log","@timestamp":"2020-11-25T15:26:37Z","tags":["warning","monitoring","kibana-monitoring"],"pid":9604,"message":"[export_exception] failed to flush export bulks :: {"path":"/_monitoring/bulk","query":{"system_id":"kibana","system_api_version":"6","interval":"10000ms"},"body":"{\"index\":{\"_type\":\"kibana_stats\"}}\n{\"kibana\":{\"uuid\":\"1329e789-0ad5-49e2-8b65-bb4f31a71c73\",\"name\":\"kibana_1\",\"index\":\".kibana\",\"host\":\"0.0.0.0\",\"transport_address\":\"0.0.0.0:8601\",\"version\":\"7.4.0\",\"snapshot\":false,\"status\":\"green\"},\"usage\":{\"kql\":{\"optInCount\":0,\"optOutCount\":0,\"defaultQueryLanguage\":\"default-kuery\"},\"localization\":{\"locale\":\"en\",\"integrities\":{},\"labelsCount\":0},\"ui_metric\":{\"infra_metrics\":[{\"key\":\"pageview__inventory__delayed_15000ms\",\"value\":1},{\"key\":\"pageview__inventory\",\"value\":2}],\"uptime\":[{\"key\":\"pageview__overview\",\"value\":1},{\"key\":\"pageview__overview__delayed_15000ms\",\"value\":1}]},\"index\":\".kibana\",\"dashboard\":{\"total\":95},\"visualization\":{\"total\":581},\"search\":{\"total\":57},\"index_pattern\":{\"total\":5},\"graph_workspace\":{\"total\":0},\"timelion_sheet\":{\"total\":0},\"spaces\":{\"available\":true,\"enabled\":true,\"count\":1,\"usesFeatureControls\":false,\"disabledFeatures\":{\"discover\":0,\"visualize\":0,\"dashboard\":0,\"dev_tools\":0,\"advancedSettings\":0,\"indexPatterns\":0,\"savedObjectsManagement\":0,\"graph\":0,\"monitoring\":0,\"ml\":0,\"apm\":0,\"maps\":0,\"canvas\":0,\"infrastructure\":0,\"logs\":0,\"siem\":0,\"uptime\":0}},\"ml\":{\"file_data_visualizer\":{\"index_creation_count\":0}},\"apm\":{\"has_any_services\":false,\"services_per_agent\":{}},\"maps\":{\"mapsTotalCount\":0,\"timeCaptured\":\"2020-11-23T18:30:01.884Z\",\"attributesPerMap\":{\"dataSourcesCount\":{\"min\":0,\"max\":0,\"avg\":0},\"layersCount\":{\"min\":0,\"max\":0,\"avg\":0},\"layerTypesCount\":{},\"emsVectorLayersCount\":{}}},\"canvas\":{},\"cloud\":{\"isCloudEnabled\":false},\"infraops\":{\"last_24_hours\":{\"hits\":{\"infraops_hosts\":0,\"infraops_docker\":0,\"infraops_kubernetes\":0,\"logs\":9}}},\"rollups\":{\"index_patterns\":{\"total\":0},\"saved_searches\":{\"total\":0},\"visualizations\":{\"total\":0,\"saved_searches\":{\"total\":0}}},\"upgrade-assistant-telemetry\":{\"ui_open\":{\"overview\":0,\"cluster\":0,\"indices\":0},\"ui_reindex\":{\"close\":0,\"open\":0,\"start\":0,\"stop\":0},\"features\":{\"deprecation_logging\":{\"enabled\":true}}},\"uptime\":{\"last_24_hours\":{\"hits\":{\"overview_page\":1,\"monitor_page\":0}}},\"visualization_types\":{\"metrics\":{\"total\":237,\"spaces_min\":237,\"spaces_max\":237,\"spaces_avg\":237},\"metric\":{\"total\":39,\"spaces_min\":39,\"spaces_max\":39,\"spaces_avg\":39},\"pie\":{\"total\":87,\"spaces_min\":87,\"spaces_max\":87,\"spaces_avg\":87},\"histogram\":{\"total\":38,\"spaces_min\":38,\"spaces_max\":38,\"spaces_avg\":38},\"table\":{\"total\":42,\"spaces_min\":42,\"spaces_max\":42,\"spaces_avg\":42},\"tagcloud\":{\"total\":9,\"spaces_min\":9,\"spaces_max\":9,\"spaces_avg\":9},\"input_control_vis\":{\"total\":14,\"spaces_min\":14,\"spaces_max\":14,\"spaces_avg\":14},\"tile_map\":{\"total\":17,\"spaces_min\":17,\"spaces_max\":17,\"spaces_avg\":17},\"area\":{\"total\":13,\"spaces_min\":13,\"spaces_max\":13,\"spaces_avg\":13},\"timelion\":{\"total\":39,\"spaces_min\":39,\"spaces_max\":39,\"spaces_avg\":39},\"line\":{\"total\":33,\"spaces_min\":33,\"spaces_max\":33,\"spaces_avg\":33},\"horizontal_bar\":{\"total\":3,\"spaces_min\":3,\"spaces_max\":3,\"spaces_avg\":3},\"markdown\":{\"total\":8,\"spaces_min\":8,\"spaces_max\":8,\"spaces_avg\":8},\"gauge\":{\"total\":1,\"spaces_min\":1,\"spaces_max\":1,\"spaces_avg\":1},\"heatmap\":{\"total\":1,\"spaces_min\":1,\"spaces_max\":1,\"spaces_avg\":1}},\"csp\":{\"strict\":false,\"warnLegacyBrowsers\":true,\"rulesChangedFromDefault\":false},\"fileUploadTelemetry\":{\"filesUploadedTotalCount\":0},\"xpack\":{\"reporting\":{\"available\":true,\"browser_type\":\"chromium\",\"enabled\":true,\"lastDay\":{\"_all\":0,\"status\":{\"completed\":0,\"failed\":0},\"printable_pdf\":{\"available\":false,\"total\":0,\"app\":{\"dashboard\":0,\"visualization\":0},\"layout\":{\"preserve_layout\":0,\"print\":0}},\"csv\":{\"available\":true,\"total\":0},\"PNG\":{\"available\":false,\"total\":0}},\"last7Days\":{\"_all\":0,\"status\":{\"completed\":0,\"failed\":0},\"printable_pdf\":{\"available\":false,\"total\":0,\"app\":{\"dashboard\":0,\"visualization\":0},\"layout\":{\"preserve_layout\":0,\"print\":0}},\"csv\":{\"available\":true,\"total\":0},\"PNG\":{\"available\":false,\"total\":0}},\"_all\":0,\"status\":{\"completed\":0,\"failed\":0},\"printable_pdf\":{\"available\":false,\"total\":0,\"app\":{\"dashboard\":0,\"visualization\":0},\"layout\":{\"preserve_layout\":0,\"print\":0}},\"csv\":{\"available\":true,\"total\":0},\"PNG\":{\"available\":false,\"total\":0}}}},\"concurrent_connections\":0,\"os\":{\"load\":{\"1m\":0.21728515625,\"5m\":0.10888671875,\"15m\":0.12841796875},\"memory\":{\"total_in_bytes\":16825806848,\"free_in_bytes\":535048192,\"used_in_bytes\":16290758656},\"uptime_in_millis\":738997000,\"platform\":\"linux\",\"platformRelease\":\"linux-3.10.0-1062.el7.x86_64\",\"distro\":\"Red Hat Linux\",\"distroRelease\":\"Red Hat Linux-7.6\"},\"process\":{\"event_loop_delay\":1.3422919511795044,\"memory\":{\"heap\":{\"total_in_bytes\":259399680,\"used_in_bytes\":215711952,\"size_limit\":1526909922},\"resident_set_size_in_bytes\":296386560},\"uptime_in_millis\":181938497},\"requests\":{\"disconnects\":0,\"total\":0},\"response_times\":{\"average\":0,\"max\":0},\"timestamp\":\"2020-11-25T15:26:37.597Z\"}\n","statusCode":500,"response":"{\"took\":8,\"ignored\":false,\"errors\":true,\"error\":{\"type\":\"export_exception\",\"reason\":\"failed to flush export bulks\",\"caused_by\":{\"type\":\"export_exception\",\"reason\":\"bulk [default_local] reports failures when exporting documents\",\"exceptions\":[{\"type\":\"export_exception\",\"reason\":\"ClusterBlockException[index [.monitoring-kibana-7-2020.11.25] blocked by: [FORBIDDEN/12/index read-only / allow delete (api)];]\",\"caused_by\":{\"type\":\"cluster_block_exception\",\"reason\":\"index [.monitoring-kibana-7-2020.11.25] blocked by: [FORBIDDEN/12/index read-only / allow delete (api)];\"}}]}}}"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:349:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:306:7)\n at HttpConnector. (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:173:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4929:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"}
Nov 25 20:56:37 kibana[9604]: {"type":"log","@timestamp":"2020-11-25T15:26:37Z","tags":["warning","monitoring","kibana-monitoring"],"pid":9604,"message":"Unable to bulk upload the stats payload to the local cluster"}
Nov 25 20:56:39 kibana[9604]: {"type":"log","@timestamp":"2020-11-25T15:26:39Z","tags":["error","task_manager"],"pid":9604,"message":"Failed to poll for work: [cluster_block_exception] index [.kibana_task_manager_1] blocked by: [FORBIDDEN/12/index read-only / allow delete (api)]; :: {"path":"/.kibana_task_manager/_update/task%3AMaps-maps_telemetry","query":{"if_seq_no":23,"if_primary_term":16,"refresh":"wait_for"},"body":"{\"doc\":{\"task\":{\"taskType\":\"maps_telemetry\",\"state\":\"{\\\"runs\\\":1,\\\"stats\\\":{\\\"mapsTotalCount\\\":0,\\\"timeCaptured\\\":\\\"2020-11-23T18:30:01.884Z\\\",\\\"attributesPerMap\\\":{\\\"dataSourcesCount\\\":{\\\"min\\\":0,\\\"max\\\":0,\\\"avg\\\":0},\\\"layersCount\\\":{\\\"min\\\":0,\\\"max\\\":0,\\\"avg\\\":0},\\\"layerTypesCount\\\":{},\\\"emsVectorLayersCount\\\":{}}}}\",\"params\":\"{}\",\"attempts\":1,\"scheduledAt\":\"2020-11-19T12:45:52.098Z\",\"startedAt\":\"2020-11-25T15:26:39.643Z\",\"retryAt\":\"2020-11-25T15:32:39.643Z\",\"runAt\":\"2020-11-24T18:30:00.000Z\",\"status\":\"running\"},\"updated_at\":\"2020-11-25T15:26:39.643Z\",\"references\":}}","statusCode":403,"response":"{\"error\":{\"root_cause\":[{\"type\":\"cluster_block_exception\",\"reason\":\"index [.kibana_task_manager_1] blocked by: [FORBIDDEN/12/index read-only / allow delete (api)];\"}],\"type\":\"cluster_block_exception\",\"reason\":\"index [.kibana_task_manager_1] blocked by: [FORBIDDEN/12/index read-only / allow delete (api)];\"},\"status\":403}"}"}