Elasticsearch showing NodeDisconnectedException followed by Outofmemory?

Elastic Stack Elasticsearch
1

Hi All,

I am using ES 2.4.1 Cluster hosted in Azure . I have 3 Nodes which are running in 3 VM's each has 4 cores with 14gb RAM and in that i had given 7021m to Elasticsearch HEAP. I am getting NodeDisconnectedException followed by Outofmemory exception in my elasticsearch logs . Eventhough i had given 7021m i dont why i am getting Outofmemory exception . Node 0 logs are here . I can see same errors with different index names in all the other nodes too.

Why i am getting these erros and How can i resolve this ?

Please let me know if you want any further information.

Thanks

2

I saw this in the logs:

delaying allocation for [419] unassigned shards

How much data do you have in the cluster? How many indices/shards? Do you have any non-default settings on the nodes?

3

Thanks @Christian_Dahlqvist

This is my Cluster stats

   {
  "timestamp": 1498557360368,
  "cluster_name": "EScluster",
  "status": "green",
  "indices": {
    "count": 133,
    "shards": {
      "total": 1258,
      "primaries": 629,
      "replication": 1,
      "index": {
        "shards": {
          "min": 2,
          "max": 10,
          "avg": 9.458646616541353
        },
        "primaries": {
          "min": 1,
          "max": 5,
          "avg": 4.7293233082706765
        },
        "replication": {
          "min": 1,
          "max": 1,
          "avg": 1
        }
      }
    },
    "docs": {
      "count": 14152372,
      "deleted": 168431
    },
    "store": {
      "size": "21.6gb",
      "size_in_bytes": 23216340974,
      "throttle_time": "0s",
      "throttle_time_in_millis": 0
    },
    "fielddata": {
      "memory_size": "0b",
      "memory_size_in_bytes": 0,
      "evictions": 0
    },
    "query_cache": {
      "memory_size": "15.9mb",
      "memory_size_in_bytes": 16703264,
      "total_count": 6701668,
      "hit_count": 540086,
      "miss_count": 6161582,
      "cache_size": 5757,
      "cache_count": 54483,
      "evictions": 48726
    },
    "completion": {
      "size": "0b",
      "size_in_bytes": 0
    },
    "segments": {
      "count": 6203,
      "memory": "151.9mb",
      "memory_in_bytes": 159379563,
      "terms_memory": "114.5mb",
      "terms_memory_in_bytes": 120076703,
      "stored_fields_memory": "6.9mb",
      "stored_fields_memory_in_bytes": 7287912,
      "term_vectors_memory": "0b",
      "term_vectors_memory_in_bytes": 0,
      "norms_memory": "6.9mb",
      "norms_memory_in_bytes": 7335808,
      "doc_values_memory": "23.5mb",
      "doc_values_memory_in_bytes": 24679140,
      "index_writer_memory": "0b",
      "index_writer_memory_in_bytes": 0,
      "index_writer_max_memory": "4.5gb",
      "index_writer_max_memory_in_bytes": 4834409173,
      "version_map_memory": "0b",
      "version_map_memory_in_bytes": 0,
      "fixed_bit_set": "222.1kb",
      "fixed_bit_set_memory_in_bytes": 227496
    },
    "percolate": {
      "total": 0,
      "time": "0s",
      "time_in_millis": 0,
      "current": 0,
      "memory_size_in_bytes": -1,
      "memory_size": "-1b",
      "queries": 0
    }
  },
  "nodes": {
    "count": {
      "total": 3,
      "master_only": 0,
      "data_only": 0,
      "master_data": 3,
      "client": 0
    },
    "versions": [
      "2.4.1"
    ],
    "os": {
      "available_processors": 12,
      "allocated_processors": 12,
      "mem": {
        "total": "6.6gb",
        "total_in_bytes": 7162269696
      },
      "names": [
        {
          "name": "Linux",
          "count": 3
        }
      ]
    },
    "process": {
      "cpu": {
        "percent": 9
      },
      "open_file_descriptors": {
        "min": 4113,
        "max": 4170,
        "avg": 4149
      }
    },
    "jvm": {
      "max_uptime": "5.1d",
      "max_uptime_in_millis": 442925266,
      "versions": [
        {
          "version": "1.8.0_121",
          "vm_name": "Java HotSpot(TM) 64-Bit Server VM",
          "vm_version": "25.121-b13",
          "vm_vendor": "Oracle Corporation",
          "count": 3
        }
      ],
      "mem": {
        "heap_used": "11.1gb",
        "heap_used_in_bytes": 11984195640,
        "heap_max": "20.4gb",
        "heap_max_in_bytes": 21953249280
      },
      "threads": 218
    },
    "fs": {
      "total": "5.8tb",
      "total_in_bytes": 6486802710528,
      "free": "5.8tb",
      "free_in_bytes": 6462915895296,
      "available": "5.5tb",
      "available_in_bytes": 6133334679552,
      "spins": "true"
    },

Regarding indices and total data

image.png1123×75 6.06 KB

BTW all are default settings except for the slowlogs

Thanks