Elasticsearch showing NodeDisconnectedException followed by Outofmemory?

Hi All,

I am using ES 2.4.1 Cluster hosted in Azure . I have 3 Nodes which are running in 3 VM's each has 4 cores with 14gb RAM and in that i had given 7021m to Elasticsearch HEAP. I am getting NodeDisconnectedException followed by Outofmemory exception in my elasticsearch logs . Eventhough i had given 7021m i dont why i am getting Outofmemory exception . Node 0 logs are here . I can see same errors with different index names in all the other nodes too.

Why i am getting these erros and How can i resolve this ?

Please let me know if you want any further information.

Thanks

I saw this in the logs:

delaying allocation for [419] unassigned shards

How much data do you have in the cluster? How many indices/shards? Do you have any non-default settings on the nodes?

Thanks @Christian_Dahlqvist

This is my Cluster stats

   {
  "timestamp": 1498557360368,
  "cluster_name": "EScluster",
  "status": "green",
  "indices": {
    "count": 133,
    "shards": {
      "total": 1258,
      "primaries": 629,
      "replication": 1,
      "index": {
        "shards": {
          "min": 2,
          "max": 10,
          "avg": 9.458646616541353
        },
        "primaries": {
          "min": 1,
          "max": 5,
          "avg": 4.7293233082706765
        },
        "replication": {
          "min": 1,
          "max": 1,
          "avg": 1
        }
      }
    },
    "docs": {
      "count": 14152372,
      "deleted": 168431
    },
    "store": {
      "size": "21.6gb",
      "size_in_bytes": 23216340974,
      "throttle_time": "0s",
      "throttle_time_in_millis": 0
    },
    "fielddata": {
      "memory_size": "0b",
      "memory_size_in_bytes": 0,
      "evictions": 0
    },
    "query_cache": {
      "memory_size": "15.9mb",
      "memory_size_in_bytes": 16703264,
      "total_count": 6701668,
      "hit_count": 540086,
      "miss_count": 6161582,
      "cache_size": 5757,
      "cache_count": 54483,
      "evictions": 48726
    },
    "completion": {
      "size": "0b",
      "size_in_bytes": 0
    },
    "segments": {
      "count": 6203,
      "memory": "151.9mb",
      "memory_in_bytes": 159379563,
      "terms_memory": "114.5mb",
      "terms_memory_in_bytes": 120076703,
      "stored_fields_memory": "6.9mb",
      "stored_fields_memory_in_bytes": 7287912,
      "term_vectors_memory": "0b",
      "term_vectors_memory_in_bytes": 0,
      "norms_memory": "6.9mb",
      "norms_memory_in_bytes": 7335808,
      "doc_values_memory": "23.5mb",
      "doc_values_memory_in_bytes": 24679140,
      "index_writer_memory": "0b",
      "index_writer_memory_in_bytes": 0,
      "index_writer_max_memory": "4.5gb",
      "index_writer_max_memory_in_bytes": 4834409173,
      "version_map_memory": "0b",
      "version_map_memory_in_bytes": 0,
      "fixed_bit_set": "222.1kb",
      "fixed_bit_set_memory_in_bytes": 227496
    },
    "percolate": {
      "total": 0,
      "time": "0s",
      "time_in_millis": 0,
      "current": 0,
      "memory_size_in_bytes": -1,
      "memory_size": "-1b",
      "queries": 0
    }
  },
  "nodes": {
    "count": {
      "total": 3,
      "master_only": 0,
      "data_only": 0,
      "master_data": 3,
      "client": 0
    },
    "versions": [
      "2.4.1"
    ],
    "os": {
      "available_processors": 12,
      "allocated_processors": 12,
      "mem": {
        "total": "6.6gb",
        "total_in_bytes": 7162269696
      },
      "names": [
        {
          "name": "Linux",
          "count": 3
        }
      ]
    },
    "process": {
      "cpu": {
        "percent": 9
      },
      "open_file_descriptors": {
        "min": 4113,
        "max": 4170,
        "avg": 4149
      }
    },
    "jvm": {
      "max_uptime": "5.1d",
      "max_uptime_in_millis": 442925266,
      "versions": [
        {
          "version": "1.8.0_121",
          "vm_name": "Java HotSpot(TM) 64-Bit Server VM",
          "vm_version": "25.121-b13",
          "vm_vendor": "Oracle Corporation",
          "count": 3
        }
      ],
      "mem": {
        "heap_used": "11.1gb",
        "heap_used_in_bytes": 11984195640,
        "heap_max": "20.4gb",
        "heap_max_in_bytes": 21953249280
      },
      "threads": 218
    },
    "fs": {
      "total": "5.8tb",
      "total_in_bytes": 6486802710528,
      "free": "5.8tb",
      "free_in_bytes": 6462915895296,
      "available": "5.5tb",
      "available_in_bytes": 6133334679552,
      "spins": "true"
    },

Regarding indices and total data

BTW all are default settings except for the slowlogs

Thanks

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.