Memory usage (Lucene Memory)

Elastic Stack Elasticsearch
1

Hi ,

We have two node cluster each node has 32gb memory.
One of the node(Master and data node) is using around 85% memory and the other node (only data) is using around 80% memory.
On the both the nodes MAX heap size is 16gb. On both the nodes JVM memory doing at 50% around 7gb.
We are assuming that rest 16gb on both the nodes is being used by Lucene.
We also checked fielddata , filtercache memory they account for 3gb menory. Not sure what else is consuming the rest of the memory. We know 50% is being used by JVM not sure where the rest 50% is being used.
When memory hits 85% we restart the nodes and after 9-10 days again memory usage crawls to 85%.
I have place few stats below , which may give some clue,

Thanks,
Pavan

Cluster stats
{

"docs" : {
  "count" : 12438729,
  "deleted" : 303455
},
"store" : {
  "size" : "99.6gb",
  "size_in_bytes" : 106967522413,
  "throttle_time" : "3.5h",
  "throttle_time_in_millis" : 12859587
},
"fielddata" : {
  "memory_size" : "1.9gb",
  "memory_size_in_bytes" : 2136445224,
  "evictions" : 0
},
"filter_cache" : {
  "memory_size" : "655.9mb",
  "memory_size_in_bytes" : 687859112,
  "evictions" : 0
},
"id_cache" : {
  "memory_size" : "0b",
  "memory_size_in_bytes" : 0
},
"completion" : {
  "size" : "0b",
  "size_in_bytes" : 0
},
"segments" : {
  "count" : 3549,
  "memory" : "835.6mb",
  "memory_in_bytes" : 876224314,
  "index_writer_memory" : "0b",
  "index_writer_memory_in_bytes" : 0,
  "index_writer_max_memory" : "3.2gb",
  "index_writer_max_memory_in_bytes" : 3440447182,
  "version_map_memory" : "6.5mb",
  "version_map_memory_in_bytes" : 6890072,
  "fixed_bit_set" : "0b",
  "fixed_bit_set_memory_in_bytes" : 0
},
"percolate" : {
  "total" : 0,
  "get_time" : "0s",
  "time_in_millis" : 0,
  "current" : 0,
  "memory_size_in_bytes" : -1,
  "memory_size" : "-1b",
  "queries" : 0
}

},
"nodes" : {
"count" : {
"total" : 2,
"master_only" : 0,
"data_only" : 1,
"master_data" : 1,
"client" : 0
},

  "cpu" : [ {
    "vendor" : "Intel",
    "model" : "Xeon",
    "mhz" : 2600,
    "total_cores" : 4,
    "total_sockets" : 1,
    "cores_per_socket" : 4,
    "cache_size" : "-1b",
    "cache_size_in_bytes" : -1,
    "count" : 2
  } ]
},
"process" : {
  "cpu" : {
    "percent" : 0
  },
  "open_file_descriptors" : {
    "min" : 5226,
    "max" : 5235,
    "avg" : 5230
  }
},
"jvm" : {
  "max_uptime" : "9d",
  "max_uptime_in_millis" : 785035464,
  "versions" : [ {
    "version" : "1.8.0_25",
    "vm_name" : "Java HotSpot(TM) 64-Bit Server VM",
    "vm_version" : "25.25-b02",
    "vm_vendor" : "Oracle Corporation",
    "count" : 2
  } ],
  "mem" : {
    "heap_used" : "16gb",
    "heap_used_in_bytes" : 17202867568,
    "heap_max" : "31.9gb",
    "heap_max_in_bytes" : 34290008064
  },
  "threads" : 151
},
"fs" : {
  "total" : "399.7gb",
  "total_in_bytes" : 429224091648,
  "free" : "200.4gb",
  "free_in_bytes" : 215275565056,
  "available" : "200.4gb",
  "available_in_bytes" : 215275565056,
  "disk_reads" : 9841894,
  "disk_writes" : 348214185,
  "disk_io_op" : 358056079,
  "disk_read_size" : "3.6gb",
  "disk_read_size_in_bytes" : 3952797184,
  "disk_write_size" : "5.4gb",
  "disk_write_size_in_bytes" : 5829136384,
  "disk_io_size" : "9.1gb",
  "disk_io_size_in_bytes" : 9781933568,
  "disk_queue" : "0"
},
"plugins" : [ {
  "name" : "head",
  "version" : "NA",
  "description" : "No description found.",
  "url" : "/_plugin/head/",
  "jvm" : false,
  "site" : true
} ]

}
}

2

Are you talking heap size or OS memory stats?

3

Hi thanks for the reply,

This is about overall OS memory our heap memory is doing good its around 50% but the OS memory is 85%.

Thanks,
Pavan

4

Basically this is the OS caching files that ES/Lucene is opening.

Is it a problem or are you just interested?

5

It is a problem as the server memory is nearing 100%.

Thanks,
Pavan

6

It's not a problem though, the OS will handle it and release memory for other things as it needs.

There's a number of other threads on this exact thing, I'd encourage you to have a look around.

7

Hi Mark , thanks for the reply.

i would be looking at other threads.
Could you please mention any relevant thread if you remember.

Thanks again,
Pavan