[ERROR][o.e.x.m.c.n.NodeStatsCollector] [data02] collector [node_stats] timed out when collecting data

Elastic Stack Elasticsearch
1

I am looking to get rid of this error.

[2018-07-10T16:36:42,048][ERROR][o.e.x.m.c.n.NodeStatsCollector] [data02] collector [node_stats] timed out when collecting data
[2018-07-10T16:37:02,049][ERROR][o.e.x.m.c.n.NodeStatsCollector] [data02] collector [node_stats] timed out when collecting data
[2018-07-10T16:37:22,053][ERROR][o.e.x.m.c.n.NodeStatsCollector] [data02] collector [node_stats] timed out when collecting data
[2018-07-10T16:37:42,053][ERROR][o.e.x.m.c.n.NodeStatsCollector] [data02] collector [node_stats] timed out when collecting data
[2018-07-10T16:38:02,056][ERROR][o.e.x.m.c.n.NodeStatsCollector] [data02] collector [node_stats] timed out when collecting data
[2018-07-10T16:38:22,057][ERROR][o.e.x.m.c.n.NodeStatsCollector] [data02] collector [node_stats] timed out when collecting data
[2018-07-10T16:38:42,060][ERROR][o.e.x.m.c.n.NodeStatsCollector] [data02] collector [node_stats] timed out when collecting data
[2018-07-10T16:39:02,062][ERROR][o.e.x.m.c.n.NodeStatsCollector] [data02] collector [node_stats] timed out when collecting data
[2018-07-10T16:39:22,062][ERROR][o.e.x.m.c.n.NodeStatsCollector] [data02] collector [node_stats] timed out when collecting data

I don't believe that I have too many shards, as in this post: ][ERROR][o.e.x.m.c.n.NodeStatsCollector] [es3] collector [node_stats] timed out when collecting data

Any other thoughts?

Here is the output from cluster stats API:
GET /_cluster/stats?human&pretty

{
  "_nodes": {
    "total": 8,
    "successful": 8,
    "failed": 0
  },
  "cluster_name": "ender",
  "timestamp": 1531241547811,
  "status": "green",
  "indices": {
    "count": 173,
    "shards": {
      "total": 975,
      "primaries": 487,
      "replication": 1.002053388090349,
      "index": {
        "shards": {
          "min": 2,
          "max": 6,
          "avg": 5.635838150289017
        },
        "primaries": {
          "min": 1,
          "max": 3,
          "avg": 2.815028901734104
        },
        "replication": {
          "min": 1,
          "max": 2,
          "avg": 1.0057803468208093
        }
      }
    },
    "docs": {
      "count": 1125582057,
      "deleted": 62430
    },
    "store": {
      "size": "3.5tb",
      "size_in_bytes": 3892547429714
    },
    "fielddata": {
      "memory_size": "82.5kb",
      "memory_size_in_bytes": 84512,
      "evictions": 0
    },
    "query_cache": {
      "memory_size": "199.9mb",
      "memory_size_in_bytes": 209626105,
      "total_count": 4546848,
      "hit_count": 1721283,
      "miss_count": 2825565,
      "cache_size": 2245,
      "cache_count": 32054,
      "evictions": 29809
    },
    "completion": {
      "size": "0b",
      "size_in_bytes": 0
    },
    "segments": {
      "count": 3340,
      "memory": "2.8gb",
      "memory_in_bytes": 3088943241,
      "terms_memory": "1.9gb",
      "terms_memory_in_bytes": 2079359998,
      "stored_fields_memory": "857.2mb",
      "stored_fields_memory_in_bytes": 898930440,
      "term_vectors_memory": "0b",
      "term_vectors_memory_in_bytes": 0,
      "norms_memory": "836.7kb",
      "norms_memory_in_bytes": 856832,
      "points_memory": "84mb",
      "points_memory_in_bytes": 88138059,
      "doc_values_memory": "20.6mb",
      "doc_values_memory_in_bytes": 21657912,
      "index_writer_memory": "1.6gb",
      "index_writer_memory_in_bytes": 1808651809,
      "version_map_memory": "16.6mb",
      "version_map_memory_in_bytes": 17413838,
      "fixed_bit_set": "4.7mb",
      "fixed_bit_set_memory_in_bytes": 5014808,
      "max_unsafe_auto_id_timestamp": 1531227032282,
      "file_sizes": {}
    }
  },
  "nodes": {
    "count": {
      "total": 8,
      "data": 3,
      "coordinating_only": 0,
      "master": 3,
      "ingest": 5
    },
    "versions": [
      "6.0.0"
    ],
    "os": {
      "available_processors": 76,
      "allocated_processors": 76,
      "names": [
        {
          "name": "Windows Server 2016",
          "count": 8
        }
      ],
      "mem": {
        "total": "291.9gb",
        "total_in_bytes": 313528844288,
        "free": "111.5gb",
        "free_in_bytes": 119825313792,
        "used": "180.4gb",
        "used_in_bytes": 193703530496,
        "free_percent": 38,
        "used_percent": 62
      }
    },
    "process": {
      "cpu": {
        "percent": 41
      },
      "open_file_descriptors": {
        "min": -1,
        "max": -1,
        "avg": 0
      }
    },
    "jvm": {
      "max_uptime": "18.8d",
      "max_uptime_in_millis": 1629853671,
      "versions": [
        {
          "version": "1.8.0_144",
          "vm_name": "Java HotSpot(TM) 64-Bit Server VM",
          "vm_version": "25.144-b01",
          "vm_vendor": "Oracle Corporation",
          "count": 7
        },
        {
          "version": "1.8.0_151",
          "vm_name": "Java HotSpot(TM) 64-Bit Server VM",
          "vm_version": "25.151-b12",
          "vm_vendor": "Oracle Corporation",
          "count": 1
        }
      ],
      "mem": {
        "heap_used": "39.4gb",
        "heap_used_in_bytes": 42393407872,
        "heap_max": "119.4gb",
        "heap_max_in_bytes": 128264830976
      },
      "threads": 1494
    },
2 
    "fs": {
      "total": "6.6tb",
      "total_in_bytes": 7272157630464,
      "free": "2.8tb",
      "free_in_bytes": 3082811850752,
      "available": "2.8tb",
      "available_in_bytes": 3082811850752
    },
    "plugins": [
      {
        "name": "repository-azure",
        "version": "6.0.0",
        "description": "The Azure Repository plugin adds support for Azure storage repositories.",
        "classname": "org.elasticsearch.plugin.repository.azure.AzureRepositoryPlugin",
        "has_native_controller": false,
        "requires_keystore": false
      },
      {
        "name": "repository-hdfs",
        "version": "6.0.0",
        "description": "The HDFS repository plugin adds support for Hadoop Distributed File-System (HDFS) repositories.",
        "classname": "org.elasticsearch.repositories.hdfs.HdfsPlugin",
        "has_native_controller": false,
        "requires_keystore": false
      },
      {
        "name": "x-pack",
        "version": "6.0.0",
        "description": "Elasticsearch Expanded Pack Plugin",
        "classname": "org.elasticsearch.xpack.XPackPlugin",
        "has_native_controller": true,
        "requires_keystore": true
      }
    ],
    "network_types": {
      "transport_types": {
        "security4": 8
      },
      "http_types": {
        "security4": 8
      }
    }
  }
}
3

I am also getting these error on the primary master node

[2018-07-10T17:07:59,311][ERROR][o.e.x.m.c.i.IndexStatsCollector] [data01] collector [index-stats] timed out when collecting data
[2018-07-10T17:08:09,315][ERROR][o.e.x.m.c.i.IndexRecoveryCollector] [data01] collector [index_recovery] timed out when collecting data
[2018-07-10T17:09:46,067][ERROR][o.e.x.m.c.n.NodeStatsCollector] [data01] collector [node_stats] timed out when collecting data
[2018-07-10T17:09:59,324][ERROR][o.e.x.m.c.i.IndexStatsCollector] [data01] collector [index-stats] timed out when collecting data
[2018-07-10T17:10:09,330][ERROR][o.e.x.m.c.i.IndexRecoveryCollector] [data01] collector [index_recovery] timed out when collecting data
[2018-07-10T17:10:19,331][ERROR][o.e.x.m.c.c.ClusterStatsCollector] [data01] collector [cluster_stats] timed out when collecting data
[2018-07-10T17:10:29,334][ERROR][o.e.x.m.c.n.NodeStatsCollector] [data01] collector [node_stats] timed out when collecting data
[2018-07-10T17:10:33,110][INFO ][o.e.m.j.JvmGcMonitorService] [data01] [gc][1617970] overhead, spent [254ms] collecting in the last [1s]
4

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.