Elasticsearch: 7.3.0
Kibana: 7.3.0
Hi
i am getting elasticsearch monitoring index bulk issues, it's keep repeating the same issue multiple times, can someone please suggest me, thanks
{[.monitoring-es-7-2019.09.11][_doc][ziQMIW0BB0dzJswY9UuK], source[{"cluster_uuid":"4FQR0XNKRPew8vGDV51MwQ","timestamp":"2019-09-11T15:59:17.051Z","interval_ms":10000,"type":"node_stats","source_node":{"uuid":"3dtmKtWdQ4qhxVJG_seRNA","host":"10.10.10.197","transport_address":"10.10.10.197:9300","ip":"10.10.10.197","name":"elastic1","timestamp":"2019-09-11T15:59:17.051Z"},"node_stats":{"node_id":"3dtmKtWdQ4qhxVJG_seRNA","node_master":false,"mlockall":true,"indices":{"docs":{"count":486577403},"store":{"size_in_bytes":128659683964},"indexing":{"index_total":239597994,"index_time_in_millis":46862361,"throttle_time_in_millis":0},"search":{"query_total":21413,"query_time_in_millis":6627079},"query_cache":{"memory_size_in_bytes":3925196,"hit_count":1602,"miss_count":137749,"evictions":69},"fielddata":{"memory_size_in_bytes":9008,"evictions":0},"segments":{"count":4193,"memory_in_bytes":331078149,"terms_memory_in_bytes":241172034,"stored_fields_memory_in_bytes":63549552,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":13300096,"points_memory_in_bytes":9932143,"doc_values_memory_in_bytes":3124324,"index_writer_memory_in_bytes":2421456104,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":627568},"request_cache":{"memory_size_in_bytes":397850,"evictions":0,"hit_count":15744,"miss_count":971}},"os":{"cpu":{"load_average":{"1m":6.78,"5m":6.21,"15m":6.19}},"cgroup":{"cpuacct":{"control_group":"/system.slice/elasticsearch.service","usage_nanos":93927780221431},"cpu":{"control_group":"/system.slice/elasticsearch.service","cfs_period_micros":100000,"cfs_quota_micros":-1,"stat":{"number_of_elapsed_periods":0,"number_of_times_throttled":0,"time_throttled_nanos":0}},"memory":{"control_group":"/system.slice/elasticsearch.service","limit_in_bytes":"9223372036854771712","usage_in_bytes":"31370838016"}}},"process":{"open_file_descriptors":5554,"max_file_descriptors":65535,"cpu":{"percent":84}},"jvm":{"mem":{"heap_used_in_bytes":12376577256,"heap_used_percent":48,"heap_max_in_bytes":25700073472},"gc":{"collectors":{"young":{"collection_count":31144,"collection_time_in_millis":1312383},"old":{"collection_count":37,"collection_time_in_millis":4901}}}},"thread_pool":{"generic":{"threads":66,"queue":0,"rejected":0},"get":{"threads":8,"queue":0,"rejected":0},"management":{"threads":5,"queue":0,"rejected":0},"search":{"threads":13,"queue":0,"rejected":0},"watcher":{"threads":0,"queue":0,"rejected":0},"write":{"threads":8,"queue":124,"rejected":5013}},"fs":{"total":{"total_in_bytes":1098974756864,"free_in_bytes":958553194496,"available_in_bytes":958553194496},"io_stats":{"total":{"operations":37970696,"read_operations":4769105,"write_operations":33201591,"read_kilobytes":292976016,"write_kilobytes":628498336}}}}}]}], target allocation id: MlWKPQCRTsux9a2KBUMulQ, primary term: 1 on EsThreadPoolExecutor[name = elastic3/write, queue capacity = 200, org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor@507f8e8d[Running, pool size = 8, active threads = 8, queued tasks = 298, completed tasks = 5870511]]
at org.elasticsearch.common.util.concurrent.EsAbortPolicy.rejectedExecution(EsAbortPolicy.java:48) ~[elasticsearch-7.3.0.jar:7.3.0]
at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:825) ~[?:?]
at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1355) ~[?:?]
at org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor.execute(EsThreadPoolExecutor.java:84) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.transport.TransportService.sendLocalRequest(TransportService.java:699) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.transport.TransportService.access$100(TransportService.java:74) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.transport.TransportService$3.sendRequest(TransportService.java:127) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.transport.TransportService.sendRequestInternal(TransportService.java:643) ~[elasticsearch-7.3.0.jar:7.3.0]
at com.floragunn.searchguard.transport.SearchGuardInterceptor.sendRequestDecorate(SearchGuardInterceptor.java:174) ~[?:?]
at com.floragunn.searchguard.SearchGuardPlugin$7$2.sendRequest(SearchGuardPlugin.java:635) ~[?:?]
at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:553) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:541) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.performAction(TransportReplicationAction.java:749) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.performLocalAction(TransportReplicationAction.java:700) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.doRun(TransportReplicationAction.java:687) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.action.support.replication.TransportReplicationAction.doExecute(TransportReplicationAction.java:165) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.action.support.replication.TransportReplicationAction.doExecute(TransportReplicationAction.java:97) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:145) ~[elasticsearch-7.3.0.jar:7.3.0]
at com.floragunn.searchguard.filter.SearchGuardFilter.apply0(SearchGuardFilter.java:208) ~[?:?]
at com.floragunn.searchguard.filter.SearchGuardFilter.apply(SearchGuardFilter.java:107) ~[?:?]