][ERROR][o.e.x.m.c.n.NodeStatsCollector] [es3] collector [node_stats] timed out when collecting data

Elastic Stack Elasticsearch
1

always get the errors

version ：6.2.2

[2018-05-18T11:15:20,964][ERROR][o.e.x.m.c.n.NodeStatsCollector] [es3] collector [node_stats] timed out when collecting data
[2018-05-18T11:15:40,964][ERROR][o.e.x.m.c.n.NodeStatsCollector] [es3] collector [node_stats] timed out when collecting data
[2018-05-18T11:16:00,978][ERROR][o.e.x.m.c.n.NodeStatsCollector] [es3] collector [node_stats] timed out when collecting data
[2018-05-18T11:16:21,003][ERROR][o.e.x.m.c.n.NodeStatsCollector] [es3] collector [node_stats] timed out when collecting data
[2018-05-18T11:16:40,965][ERROR][o.e.x.m.c.n.NodeStatsCollector] [es3] collector [node_stats] timed out when collecting data
[2018-05-18T11:17:00,965][ERROR][o.e.x.m.c.n.NodeStatsCollector] [es3] collector [node_stats] timed out when collecting data
[2018-05-18T11:17:20,966][ERROR][o.e.x.m.c.n.NodeStatsCollector] [es3] collector [node_stats] timed out when collecting data
[2018-05-18T11:17:40,966][ERROR][o.e.x.m.c.n.NodeStatsCollector] [es3] collector [node_stats] timed out when collecting data
[2018-05-18T11:18:01,003][ERROR][o.e.x.m.c.n.NodeStatsCollector] [es3] collector [node_stats] timed out when collecting data

how i avoid this error

2

:joy:

3

Can you provide the full output of the cluster stats API?

4 
{
  "_nodes" : {
    "total" : 3,
    "successful" : 3,
    "failed" : 0
  },
  "cluster_name" : "es-cluster",
  "timestamp" : 1526865843187,
  "status" : "green",
  "indices" : {
    "count" : 897,
    "shards" : {
      "total" : 8802,
      "primaries" : 4401,
      "replication" : 1.0,
      "index" : {
        "shards" : {
          "min" : 2,
          "max" : 10,
          "avg" : 9.812709030100335
        },
        "primaries" : {
          "min" : 1,
          "max" : 5,
          "avg" : 4.906354515050167
        },
        "replication" : {
          "min" : 1.0,
          "max" : 1.0,
          "avg" : 1.0
        }
      }
    },
    "docs" : {
      "count" : 972562100,
      "deleted" : 732502
    },
    "store" : {
      "size" : "486.4gb",
      "size_in_bytes" : 522324805311
    },
    "fielddata" : {
      "memory_size" : "194kb",
      "memory_size_in_bytes" : 198696,
      "evictions" : 0
    },
    "query_cache" : {
      "memory_size" : "44.8mb",
      "memory_size_in_bytes" : 47065175,
      "total_count" : 820507,
      "hit_count" : 105792,
      "miss_count" : 714715,
      "cache_size" : 2888,
      "cache_count" : 26380,
      "evictions" : 23492
    },
    "completion" : {
      "size" : "0b",
      "size_in_bytes" : 0
    },
    "segments" : {
      "count" : 62351,
      "memory" : "2.5gb",
      "memory_in_bytes" : 2755281461,
      "terms_memory" : "2.2gb",
      "terms_memory_in_bytes" : 2369491925,
      "stored_fields_memory" : "247.6mb",
      "stored_fields_memory_in_bytes" : 259632312,
      "term_vectors_memory" : "0b",
      "term_vectors_memory_in_bytes" : 0,
      "norms_memory" : "47.7mb",
      "norms_memory_in_bytes" : 50092800,
      "points_memory" : "43.3mb",
      "points_memory_in_bytes" : 45457388,
      "doc_values_memory" : "29.1mb",
      "doc_values_memory_in_bytes" : 30607036,
      "index_writer_memory" : "113.2mb",
      "index_writer_memory_in_bytes" : 118794544,
      "version_map_memory" : "0b",
      "version_map_memory_in_bytes" : 0,
      "fixed_bit_set" : "448b",
      "fixed_bit_set_memory_in_bytes" : 448,
      "max_unsafe_auto_id_timestamp" : 1526863401260,
      "file_sizes" : { }
    }
  },
  "nodes" : {
    "count" : {
      "total" : 3,
      "data" : 3,
      "coordinating_only" : 0,
      "master" : 3,
      "ingest" : 3
    },
    "versions" : [
      "6.2.2"
    ],
    "os" : {
      "available_processors" : 96,
      "allocated_processors" : 96,
      "names" : [
        {
          "name" : "Linux",
          "count" : 3
        }
      ],
      "mem" : {
        "total" : "93.4gb",
        "total_in_bytes" : 100312940544,
        "free" : "1.3gb",
        "free_in_bytes" : 1475432448,
        "used" : "92gb",
        "used_in_bytes" : 98837508096,
        "free_percent" : 1,
        "used_percent" : 99
      }
    },
    "process" : {
      "cpu" : {
        "percent" : 28
      },
      "open_file_descriptors" : {
        "min" : 9229,
        "max" : 11923,
        "avg" : 10537
      }
    },
    "jvm" : {
      "max_uptime" : "37.9d",
      "max_uptime_in_millis" : 3275675920,
      "versions" : [
        {
          "version" : "1.8.0_141",
          "vm_name" : "OpenJDK 64-Bit Server VM",
          "vm_version" : "25.141-b16",
          "vm_vendor" : "Oracle Corporation",
          "count" : 3
        }
      ],
      "mem" : {
        "heap_used" : "23.2gb",
        "heap_used_in_bytes" : 24928102712,
        "heap_max" : "47.4gb",
        "heap_max_in_bytes" : 50937790464
      },
5 
      "threads" : 1168
    },
    "fs" : {
      "total" : "26.8tb",
      "total_in_bytes" : 29467927904256,
      "free" : "25.9tb",
      "free_in_bytes" : 28517474603008,
      "available" : "25.9tb",
      "available_in_bytes" : 28517474603008
    },
    "plugins" : [
      {
        "name" : "x-pack-upgrade",
        "version" : "6.2.2",
        "description" : "Elasticsearch Expanded Pack Plugin - Upgrade",
        "classname" : "org.elasticsearch.xpack.upgrade.Upgrade",
        "extended_plugins" : [
          "x-pack-core"
        ],
        "has_native_controller" : false,
        "requires_keystore" : false
      },
      {
        "name" : "x-pack-ml",
        "version" : "6.2.2",
        "description" : "Elasticsearch Expanded Pack Plugin - Machine Learning",
        "classname" : "org.elasticsearch.xpack.ml.MachineLearning",
        "extended_plugins" : [
          "x-pack-core"
        ],
        "has_native_controller" : true,
        "requires_keystore" : false
      },
      {
        "name" : "x-pack-logstash",
        "version" : "6.2.2",
        "description" : "Elasticsearch Expanded Pack Plugin - Logstash",
        "classname" : "org.elasticsearch.xpack.logstash.Logstash",
        "extended_plugins" : [
          "x-pack-core"
        ],
        "has_native_controller" : false,
        "requires_keystore" : false
      },
      {
        "name" : "x-pack-deprecation",
        "version" : "6.2.2",
        "description" : "Elasticsearch Expanded Pack Plugin - Deprecation",
        "classname" : "org.elasticsearch.xpack.deprecation.Deprecation",
        "extended_plugins" : [
          "x-pack-core"
        ],
        "has_native_controller" : false,
        "requires_keystore" : false
      },
      {
        "name" : "x-pack-monitoring",
        "version" : "6.2.2",
        "description" : "Elasticsearch Expanded Pack Plugin - Monitoring",
        "classname" : "org.elasticsearch.xpack.monitoring.Monitoring",
        "extended_plugins" : [
          "x-pack-core"
        ],
        "has_native_controller" : false,
        "requires_keystore" : false
      },
      {
        "name" : "x-pack-core",
        "version" : "6.2.2",
        "description" : "Elasticsearch Expanded Pack Plugin - Core",
        "classname" : "org.elasticsearch.xpack.core.XPackPlugin",
        "extended_plugins" : [ ],
        "has_native_controller" : false,
        "requires_keystore" : false
      },
      {
        "name" : "x-pack-security",
        "version" : "6.2.2",
        "description" : "Elasticsearch Expanded Pack Plugin - Security",
        "classname" : "org.elasticsearch.xpack.security.Security",
        "extended_plugins" : [
          "x-pack-core"
        ],
        "has_native_controller" : false,
        "requires_keystore" : true
      },
      {
        "name" : "x-pack-watcher",
        "version" : "6.2.2",
        "description" : "Elasticsearch Expanded Pack Plugin - Watcher",
        "classname" : "org.elasticsearch.xpack.watcher.Watcher",
        "extended_plugins" : [
          "x-pack-core"
        ],
        "has_native_controller" : false,
        "requires_keystore" : false
      },
      {
        "name" : "x-pack-graph",
        "version" : "6.2.2",
        "description" : "Elasticsearch Expanded Pack Plugin - Graph",
        "classname" : "org.elasticsearch.xpack.graph.Graph",
        "extended_plugins" : [
          "x-pack-core"
        ],
        "has_native_controller" : false,
        "requires_keystore" : false
      }
    ],
    "network_types" : {
      "transport_types" : {
        "netty4" : 3
      },
      "http_types" : {
        "netty4" : 3
      }
    }
  }
}
6

It's too large,so I divide it into two segments

7

Given the data volume and size of the cluster, I believe you have far too many shards. Having a lot of small shards can be very inefficient. See this blog post for some further guidance on sharding practices.

8

thank you very much

9

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.