# Elastic Search nodes / servers with constant high CPU utilization in idle state

**URL:** <https://discuss.elastic.co/t/elastic-search-nodes-servers-with-constant-high-cpu-utilization-in-idle-state/18544>\
**Category:** Elasticsearch\
**Created:** [July 9, 2014, 4:54am UTC](https://discuss.elastic.co/t/elastic-search-nodes-servers-with-constant-high-cpu-utilization-in-idle-state/18544 "2014-07-09T04:54:07Z")\
**Posts on this page:** 9\
**Page:** 1

<div class="post-metadata">

**Author:** ![webish](https://avatars.discourse-cdn.com/v4/letter/w/919ad9/32.png) [@webish](https://discuss.elastic.co/u/webish)\
**Post date:** [July 9, 2014, 4:54am UTC](https://discuss.elastic.co/t/elastic-search-nodes-servers-with-constant-high-cpu-utilization-in-idle-state/18544/1 "2014-07-09T04:54:07Z")

</div>

I'm seeing very high CPU utilization in an idle or close to idle state. I  
haven't seen it use less than 40% on each node over the course of days.  
I'm not certain how long it has been like this but could be weeks.

Any help resolving this would be greatly appreciated!

_Details:_

There are 2 nodes in the cluster.

AWS instances are:  
vCPUECUMemory (GiB)Instance Storage (GB)Linux/UNIX Usage

r3.2xlarge826611 x 160 SSD

curl -XGET '[http://localhost:9200/](http://localhost:9200/)'  
{  
"status" : 200,  
"name" : "Eddie Brock",  
"version" : {  
"number" : "1.1.0",  
"build\_hash" : "2181e113dea80b4a9e31e58e9686658a2d46e363",  
"build\_timestamp" : "2014-03-25T15:59:51Z",  
"build\_snapshot" : false,  
"lucene\_version" : "4.7"  
},  
"tagline" : "You Know, for Search"  
}

% java -version  
java version "1.7.0\_55"  
OpenJDK Runtime Environment (IcedTea 2.4.7) (7u55-2.4.7-1ubuntu1~0.12.04.2)  
OpenJDK 64-Bit Server VM (build 24.51-b03, mixed mode)

curl -XGET '[http://localhost:9200/\_nodes/stats](http://localhost:9200/_nodes/stats)'  
{  
"cluster\_name":"ab\_elastic",  
"nodes":{  
"kpHPVkBCyjOkRaSpS5Q":{  
"timestamp":1404880773859,  
"name":"Milan",  
"transport\_address":"removed",  
"host":"removed",  
"ip":[  
"removed",  
"NONE"  
],  
"indices":{  
"docs":{  
"count":17793781,  
"deleted":466537  
},  
"store":{  
"size\_in\_bytes":9902722189,  
"throttle\_time\_in\_millis":8777423  
},  
"indexing":{  
"index\_total":3548638,  
"index\_time\_in\_millis":2610694,  
"index\_current":1,  
"delete\_total":0,  
"delete\_time\_in\_millis":0,  
"delete\_current":0  
},  
"get":{  
"total":0,  
"time\_in\_millis":0,  
"exists\_total":0,  
"exists\_time\_in\_millis":0,  
"missing\_total":0,  
"missing\_time\_in\_millis":0,  
"current":0  
},  
"search":{  
"open\_contexts":0,  
"query\_total":155361213,  
"query\_time\_in\_millis":817989500,  
"query\_current":0,  
"fetch\_total":4119309,  
"fetch\_time\_in\_millis":1967525,  
"fetch\_current":0  
},  
"merges":{  
"current":0,  
"current\_docs":0,  
"current\_size\_in\_bytes":0,  
"total":564,  
"total\_time\_in\_millis":76065,  
"total\_docs":1093579,  
"total\_size\_in\_bytes":470304093  
},  
"refresh":{  
"total":15117234,  
"total\_time\_in\_millis":50581020  
},  
"flush":{  
"total":2336,  
"total\_time\_in\_millis":99980  
},  
"warmer":{  
"current":0,  
"total":2719640,  
"total\_time\_in\_millis":5980882  
},  
"filter\_cache":{  
"memory\_size\_in\_bytes":722232396,  
"evictions":1179740  
},  
"id\_cache":{  
"memory\_size\_in\_bytes":131672871  
},  
"fielddata":{  
"memory\_size\_in\_bytes":133728950,  
"evictions":0  
},  
"percolate":{  
"total":0,  
"time\_in\_millis":0,  
"current":0,  
"memory\_size\_in\_bytes":-1,  
"memory\_size":"-1b",  
"queries":0  
},  
"completion":{  
"size\_in\_bytes":0  
},  
"segments":{  
"count":278,  
"memory\_in\_bytes":220799236  
},  
"translog":{  
"operations":68,  
"size\_in\_bytes":0  
}  
},  
"os":{  
"timestamp":1404880773910,  
"uptime\_in\_millis":915746,  
"load\_average":[  
2.68,  
3.17,  
3.35  
],  
"cpu":{  
"sys":0,  
"user":38,  
"idle":61,  
"usage":38,  
"stolen":0  
},  
"mem":{  
"free\_in\_bytes":12301606912,  
"used\_in\_bytes":51857985536,  
"free\_percent":35,  
"used\_percent":64,  
"actual\_free\_in\_bytes":23005929472,  
"actual\_used\_in\_bytes":41153662976  
},  
"swap":{  
"used\_in\_bytes":0,  
"free\_in\_bytes":0  
}  
},  
"process":{  
"timestamp":1404880773911,  
"open\_file\_descriptors":300,  
"cpu":{  
"percent":313,  
"sys\_in\_millis":14021810,  
"user\_in\_millis":794249870,  
"total\_in\_millis":808271680  
},  
"mem":{  
"resident\_in\_bytes":32342851584,  
"share\_in\_bytes":712302592,  
"total\_virtual\_in\_bytes":47406919680  
}  
},  
"jvm":{  
"timestamp":1404880773911,  
"uptime\_in\_millis":309661546,  
"mem":{  
"heap\_used\_in\_bytes":18409783280,  
"heap\_used\_percent":57,  
"heap\_committed\_in\_bytes":32142524416,  
"heap\_max\_in\_bytes":32142524416,  
"non\_heap\_used\_in\_bytes":65012920,  
"non\_heap\_committed\_in\_bytes":98009088,  
"pools":{  
"young":{  
"used\_in\_bytes":538302112,  
"max\_in\_bytes":558432256,  
"peak\_used\_in\_bytes":558432256,  
"peak\_max\_in\_bytes":558432256  
},  
"survivor":{  
"used\_in\_bytes":11403704,  
"max\_in\_bytes":69730304,  
"peak\_used\_in\_bytes":69730304,  
"peak\_max\_in\_bytes":69730304  
},  
"old":{  
"used\_in\_bytes":17860077464,  
"max\_in\_bytes":31514361856,  
"peak\_used\_in\_bytes":23694417088,  
"peak\_max\_in\_bytes":31514361856  
}  
}  
},  
"threads":{  
"count":117,  
"peak\_count":133  
},  
"gc":{  
"collectors":{  
"young":{  
"collection\_count":200644,  
"collection\_time\_in\_millis":10870850  
},  
"old":{  
"collection\_count":9,  
"collection\_time\_in\_millis":2987  
}  
}  
},  
"buffer\_pools":{  
"direct":{  
"count":154,  
"used\_in\_bytes":78176363,  
"total\_capacity\_in\_bytes":78176363  
},  
"mapped":{  
"count":1293,  
"used\_in\_bytes":9831615457,  
"total\_capacity\_in\_bytes":9831615457  
}  
}  
},  
"thread\_pool":{  
"generic":{  
"threads":1,  
"queue":0,  
"active":0,  
"rejected":0,  
"largest":14,  
"completed":56826  
},  
"index":{  
"threads":8,  
"queue":0,  
"active":0,  
"rejected":0,  
"largest":8,  
"completed":3607948  
},  
"get":{  
"threads":0,  
"queue":0,  
"active":0,  
"rejected":0,  
"largest":0,  
"completed":0  
},  
"snapshot":{  
"threads":4,  
"queue":0,  
"active":0,  
"rejected":0,  
"largest":4,  
"completed":95697  
},  
"merge":{  
"threads":2,  
"queue":0,  
"active":0,  
"rejected":0,  
"largest":4,  
"completed":282257  
},  
"suggest":{  
"threads":0,  
"queue":0,  
"active":0,  
"rejected":0,  
"largest":0,  
"completed":0  
},  
"bulk":{  
"threads":8,  
"queue":0,  
"active":0,  
"rejected":0,  
"largest":8,  
"completed":1081  
},  
"optimize":{  
"threads":0,  
"queue":0,  
"active":0,  
"rejected":0,  
"largest":0,  
"completed":0  
},  
"warmer":{  
"threads":2,  
"queue":0,  
"active":0,  
"rejected":0,  
"largest":4,  
"completed":5556773  
},  
"flush":{  
"threads":1,  
"queue":0,  
"active":0,  
"rejected":0,  
"largest":2,  
"completed":2336  
},  
"search":{  
"threads":24,  
"queue":0,  
"active":0,  
"rejected":0,  
"largest":24,  
"completed":160139043  
},  
"percolate":{  
"threads":0,  
"queue":0,  
"active":0,  
"rejected":0,  
"largest":0,  
"completed":0  
},  
"management":{  
"threads":1,  
"queue":0,  
"active":1,  
"rejected":0,  
"largest":5,  
"completed":1064  
},  
"refresh":{  
"threads":4,  
"queue":0,  
"active":0,  
"rejected":0,  
"largest":4,  
"completed":15126156  
}  
},  
"network":{  
"tcp":{  
"active\_opens":1513869,  
"passive\_opens":224206,  
"curr\_estab":52,  
"in\_segs":1929140868,  
"out\_segs":2005198310,  
"retrans\_segs":175635,  
"estab\_resets":37,  
"attempt\_fails":669,  
"in\_errs":3,  
"out\_rsts":88  
}  
},  
"fs":{  
"timestamp":1404880773912,  
"total":{  
"total\_in\_bytes":158534184960,  
"free\_in\_bytes":148425240576,  
"available\_in\_bytes":140372176896,  
"disk\_reads":1993,  
"disk\_writes":16068284,  
"disk\_io\_op":16070277,  
"disk\_read\_size\_in\_bytes":7445504,  
"disk\_write\_size\_in\_bytes":650206560256,  
"disk\_io\_size\_in\_bytes":650214005760  
},  
"data":[  
{  
"path":"/mnt/esdata/ab\_elastic/nodes/0",  
"mount":"/mnt/esdata",  
"dev":"/dev/xvdb",  
"total\_in\_bytes":158534184960,  
"free\_in\_bytes":148425240576,  
"available\_in\_bytes":140372176896,  
"disk\_reads":1993,  
"disk\_writes":16068284,  
"disk\_io\_op":16070277,  
"disk\_read\_size\_in\_bytes":7445504,  
"disk\_write\_size\_in\_bytes":650206560256,  
"disk\_io\_size\_in\_bytes":650214005760  
}  
]  
},  
"transport":{  
"server\_open":26,  
"rx\_count":214082686,  
"rx\_size\_in\_bytes":148941344505,  
"tx\_count":209856841,  
"tx\_size\_in\_bytes":90356130749  
},  
"http":{  
"current\_open":0,  
"total\_opened":0  
},  
"fielddata\_breaker":{  
"maximum\_size\_in\_bytes":25714019532,  
"maximum\_size":"23.9gb",  
"estimated\_size\_in\_bytes":265401821,  
"estimated\_size":"253.1mb",  
"overhead":1.03  
}  
},  
"erc5l7dQai3mKBIX6sw":{  
"timestamp":1404880733754,  
"name":"Eddie Brock",  
"transport\_address":"removed",  
"host":"removed",  
"ip":[  
"removed",  
"NONE"  
],  
"indices":{  
"docs":{  
"count":17793781,  
"deleted":529675  
},  
"store":{  
"size\_in\_bytes":9969074949,  
"throttle\_time\_in\_millis":9235478  
},  
"indexing":{  
"index\_total":3531955,  
"index\_time\_in\_millis":2783791,  
"index\_current":0,  
"delete\_total":0,  
"delete\_time\_in\_millis":0,  
"delete\_current":0  
},  
"get":{  
"total":2947020,  
"time\_in\_millis":217006,  
"exists\_total":2946589,  
"exists\_time\_in\_millis":216977,  
"missing\_total":431,  
"missing\_time\_in\_millis":29,  
"current":0  
},  
"search":{  
"open\_contexts":0,  
"query\_total":155416200,  
"query\_time\_in\_millis":1076023052,  
"query\_current":0,  
"fetch\_total":4101442,  
"fetch\_time\_in\_millis":2337321,  
"fetch\_current":0  
},  
"merges":{  
"current":0,  
"current\_docs":0,  
"current\_size\_in\_bytes":0,  
"total":563,  
"total\_time\_in\_millis":89211,  
"total\_docs":1099389,  
"total\_size\_in\_bytes":470013054  
},  
"refresh":{  
"total":15123966,  
"total\_time\_in\_millis":63368992  
},  
"flush":{  
"total":2335,  
"total\_time\_in\_millis":414790  
},  
"warmer":{  
"current":0,  
"total":2690030,  
"total\_time\_in\_millis":7339293  
},  
"filter\_cache":{  
"memory\_size\_in\_bytes":695354935,  
"evictions":1377260  
},  
"id\_cache":{  
"memory\_size\_in\_bytes":131817497  
},  
"fielddata":{  
"memory\_size\_in\_bytes":137907189,  
"evictions":0  
},  
"percolate":{  
"total":0,  
"time\_in\_millis":0,  
"current":0,  
"memory\_size\_in\_bytes":-1,  
"memory\_size":"-1b",  
"queries":0  
},  
"completion":{  
"size\_in\_bytes":0  
},  
"segments":{  
"count":282,  
"memory\_in\_bytes":217468955  
},  
"translog":{  
"operations":86,  
"size\_in\_bytes":0  
}  
},  
"os":{  
"timestamp":1404880733801,  
"uptime\_in\_millis":923467,  
"load\_average":[  
3.71,  
4.57,  
4.46  
],  
"cpu":{  
"sys":0,  
"user":46,  
"idle":52,  
"usage":46,  
"stolen":0  
},  
"mem":{  
"free\_in\_bytes":4438659072,  
"used\_in\_bytes":59720933376,  
"free\_percent":32,  
"used\_percent":67,  
"actual\_free\_in\_bytes":21124698112,  
"actual\_used\_in\_bytes":43034894336  
},  
"swap":{  
"used\_in\_bytes":0,  
"free\_in\_bytes":0  
}  
},  
"process":{  
"timestamp":1404880733801,  
"open\_file\_descriptors":301,  
"cpu":{  
"percent":377,  
"sys\_in\_millis":26524340,  
"user\_in\_millis":947578840,  
"total\_in\_millis":974103180  
},  
"mem":{  
"resident\_in\_bytes":33945767936,  
"share\_in\_bytes":692912128,  
"total\_virtual\_in\_bytes":47473528832  
}  
},  
"jvm":{  
"timestamp":1404880733802,  
"uptime\_in\_millis":310301938,  
"mem":{  
"heap\_used\_in\_bytes":10113402832,  
"heap\_used\_percent":31,  
"heap\_committed\_in\_bytes":32142524416,  
"heap\_max\_in\_bytes":32142524416,  
"non\_heap\_used\_in\_bytes":71762640,  
"non\_heap\_committed\_in\_bytes":106397696,  
"pools":{  
"young":{  
"used\_in\_bytes":218720808,  
"max\_in\_bytes":558432256,  
"peak\_used\_in\_bytes":558432256,  
"peak\_max\_in\_bytes":558432256  
},  
"survivor":{  
"used\_in\_bytes":9630920,  
"max\_in\_bytes":69730304,  
"peak\_used\_in\_bytes":69730304,  
"peak\_max\_in\_bytes":69730304  
},  
"old":{  
"used\_in\_bytes":9885051104,  
"max\_in\_bytes":31514361856,  
"peak\_used\_in\_bytes":23868903928,  
"peak\_max\_in\_bytes":31514361856  
}  
}  
},  
"threads":{  
"count":121,  
"peak\_count":144  
},  
"gc":{  
"collectors":{  
"young":{  
"collection\_count":209658,  
"collection\_time\_in\_millis":12089132  
},  
"old":{  
"collection\_count":11,  
"collection\_time\_in\_millis":6628  
}  
}  
},  
"buffer\_pools":{  
"direct":{  
"count":162,  
"used\_in\_bytes":195159665,  
"total\_capacity\_in\_bytes":195159665  
},  
"mapped":{  
"count":1292,  
"used\_in\_bytes":9897821921,  
"total\_capacity\_in\_bytes":9897821921  
}  
}  
},  
"thread\_pool":{  
"generic":{  
"threads":1,  
"queue":0,  
"active":0,  
"rejected":0,  
"largest":23,  
"completed":67214  
},  
"index":{  
"threads":8,  
"queue":0,  
"active":0,  
"rejected":0,  
"largest":8,  
"completed":3611159  
},  
"get":{  
"threads":0,  
"queue":0,  
"active":0,  
"rejected":0,  
"largest":0,  
"completed":0  
},  
"snapshot":{  
"threads":2,  
"queue":0,  
"active":0,  
"rejected":0,  
"largest":4,  
"completed":95672  
},  
"merge":{  
"threads":3,  
"queue":0,  
"active":0,  
"rejected":0,  
"largest":4,  
"completed":285955  
},  
"suggest":{  
"threads":0,  
"queue":0,  
"active":0,  
"rejected":0,  
"largest":0,  
"completed":0  
},  
"bulk":{  
"threads":8,  
"queue":0,  
"active":0,  
"rejected":0,  
"largest":8,  
"completed":1125  
},  
"optimize":{  
"threads":0,  
"queue":0,  
"active":0,  
"rejected":0,  
"largest":0,  
"completed":0  
},  
"warmer":{  
"threads":2,  
"queue":0,  
"active":0,  
"rejected":0,  
"largest":4,  
"completed":5487739  
},  
"flush":{  
"threads":1,  
"queue":0,  
"active":0,  
"rejected":0,  
"largest":2,  
"completed":2335  
},  
"search":{  
"threads":24,  
"queue":0,  
"active":0,  
"rejected":0,  
"largest":24,  
"completed":160510703  
},  
"percolate":{  
"threads":0,  
"queue":0,  
"active":0,  
"rejected":0,  
"largest":0,  
"completed":0  
},  
"management":{  
"threads":5,  
"queue":0,  
"active":1,  
"rejected":0,  
"largest":5,  
"completed":22893  
},  
"refresh":{  
"threads":4,  
"queue":0,  
"active":0,  
"rejected":0,  
"largest":4,  
"completed":15135553  
}  
},  
"network":{  
"tcp":{  
"active\_opens":1025664,  
"passive\_opens":105715571,  
"curr\_estab":55,  
"in\_segs":2557757673,  
"out\_segs":2502369803,  
"retrans\_segs":205765,  
"estab\_resets":68,  
"attempt\_fails":207,  
"in\_errs":0,  
"out\_rsts":1641  
}  
},  
"fs":{  
"timestamp":1404880733802,  
"total":{  
"total\_in\_bytes":158533136384,  
"free\_in\_bytes":137526714368,  
"available\_in\_bytes":129473703936,  
"disk\_reads":8637,  
"disk\_writes":17332161,  
"disk\_io\_op":17340798,  
"disk\_read\_size\_in\_bytes":139357184,  
"disk\_write\_size\_in\_bytes":706635259904,  
"disk\_io\_size\_in\_bytes":706774617088  
},  
"data":[  
{  
"path":"/mnt/esdata/ab\_elastic/nodes/0",  
"mount":"/mnt/esdata",  
"dev":"/dev/xvdb",  
"total\_in\_bytes":158533136384,  
"free\_in\_bytes":137526714368,  
"available\_in\_bytes":129473703936,  
"disk\_reads":8637,  
"disk\_writes":17332161,  
"disk\_io\_op":17340798,  
"disk\_read\_size\_in\_bytes":139357184,  
"disk\_write\_size\_in\_bytes":706635259904,  
"disk\_io\_size\_in\_bytes":706774617088  
}  
]  
},  
"transport":{  
"server\_open":26,  
"rx\_count":214086149,  
"rx\_size\_in\_bytes":90356374759,  
"tx\_count":210167189,  
"tx\_size\_in\_bytes":149348401545  
},  
"http":{  
"current\_open":1,  
"total\_opened":62412134  
},  
"fielddata\_breaker":{  
"maximum\_size\_in\_bytes":25714019532,  
"maximum\_size":"23.9gb",  
"estimated\_size\_in\_bytes":269724686,  
"estimated\_size":"257.2mb",  
"overhead":1.03  
}  
}  
}  
}

--  
You received this message because you are subscribed to the Google Groups "elasticsearch" group.  
To unsubscribe from this group and stop receiving emails from it, send an email to [elasticsearch+unsubscribe@googlegroups.com](mailto:elasticsearch+unsubscribe@googlegroups.com).  
To view this discussion on the web visit [https://groups.google.com/d/msgid/elasticsearch/75d11657-d3c7-4a0a-ba54-ca8691f9b14d%40googlegroups.com](https://groups.google.com/d/msgid/elasticsearch/75d11657-d3c7-4a0a-ba54-ca8691f9b14d%40googlegroups.com).  
For more options, visit [https://groups.google.com/d/optout](https://groups.google.com/d/optout).

---

<div class="post-metadata">

**Author:** ![warkolm](https://sea2.discourse-cdn.com/elastic/user_avatar/discuss.elastic.co/warkolm/32/39224_2.png) [@warkolm](https://discuss.elastic.co/u/warkolm)\
**Post date:** [July 9, 2014, 5:34am UTC](https://discuss.elastic.co/t/elastic-search-nodes-servers-with-constant-high-cpu-utilization-in-idle-state/18544/2 "2014-07-09T05:34:49Z")

</div>

It's better to pop long output like that into a gist/pastebin, it makes it  
easier to read the thread.  
It's also worth installing a monitoring plugin like ElasticHQ or marvel, as  
they provide graphical insight into what is happening and will extrapolate  
some of the raw figures out.

If you can change to Oracle java you will get better performance. Are you  
using TTL?

Regards,  
Mark Walkom

Infrastructure Engineer  
Campaign Monitor  
email: [markw@campaignmonitor.com](mailto:markw@campaignmonitor.com)  
web: [www.campaignmonitor.com](http://www.campaignmonitor.com)

On 9 July 2014 14:54, webish [gregory@yoursports.com](mailto:gregory@yoursports.com) wrote:

> I'm seeing very high CPU utilization in an idle or close to idle state. I  
> haven't seen it use less than 40% on each node over the course of days.  
> I'm not certain how long it has been like this but could be weeks.
> 
> Any help resolving this would be greatly appreciated!
> 
> _Details:_
> 
> There are 2 nodes in the cluster.
> 
> AWS instances are:  
> vCPU ECUMemory (GiB) Instance Storage (GB) Linux/UNIX Usage
> 
> r3.2xlarge 82661 1 x 160 SSD
> 
> curl -XGET '[http://localhost:9200/](http://localhost:9200/)'  
> {  
> "status" : 200,  
> "name" : "Eddie Brock",  
> "version" : {  
> "number" : "1.1.0",  
> "build\_hash" : "2181e113dea80b4a9e31e58e9686658a2d46e363",  
> "build\_timestamp" : "2014-03-25T15:59:51Z",  
> "build\_snapshot" : false,  
> "lucene\_version" : "4.7"  
> },  
> "tagline" : "You Know, for Search"  
> }
> 
> % java -version  
> java version "1.7.0\_55"  
> OpenJDK Runtime Environment (IcedTea 2.4.7) (7u55-2.4.7-1ubuntu1~0.12.04.2)  
> OpenJDK 64-Bit Server VM (build 24.51-b03, mixed mode)
> 
> curl -XGET '[http://localhost:9200/\_nodes/stats](http://localhost:9200/_nodes/stats)'  
> {  
> "cluster\_name":"ab\_elastic",  
> "nodes":{  
> "kpHPVkBCyjOkRaSpS5Q":{  
> "timestamp":1404880773859,  
> "name":"Milan",  
> "transport\_address":"removed",  
> "host":"removed",  
> "ip":[  
> "removed",  
> "NONE"  
> ],  
> "indices":{  
> "docs":{  
> "count":17793781,  
> "deleted":466537  
> },  
> "store":{  
> "size\_in\_bytes":9902722189,  
> "throttle\_time\_in\_millis":8777423  
> },  
> "indexing":{  
> "index\_total":3548638,  
> "index\_time\_in\_millis":2610694,  
> "index\_current":1,  
> "delete\_total":0,  
> "delete\_time\_in\_millis":0,  
> "delete\_current":0  
> },  
> "get":{  
> "total":0,  
> "time\_in\_millis":0,  
> "exists\_total":0,  
> "exists\_time\_in\_millis":0,  
> "missing\_total":0,  
> "missing\_time\_in\_millis":0,  
> "current":0  
> },  
> "search":{  
> "open\_contexts":0,  
> "query\_total":155361213,  
> "query\_time\_in\_millis":817989500,  
> "query\_current":0,  
> "fetch\_total":4119309,  
> "fetch\_time\_in\_millis":1967525,  
> "fetch\_current":0  
> },  
> "merges":{  
> "current":0,  
> "current\_docs":0,  
> "current\_size\_in\_bytes":0,  
> "total":564,  
> "total\_time\_in\_millis":76065,  
> "total\_docs":1093579,  
> "total\_size\_in\_bytes":470304093  
> },  
> "refresh":{  
> "total":15117234,  
> "total\_time\_in\_millis":50581020  
> },  
> "flush":{  
> "total":2336,  
> "total\_time\_in\_millis":99980  
> },  
> "warmer":{  
> "current":0,  
> "total":2719640,  
> "total\_time\_in\_millis":5980882  
> },  
> "filter\_cache":{  
> "memory\_size\_in\_bytes":722232396,  
> "evictions":1179740  
> },  
> "id\_cache":{  
> "memory\_size\_in\_bytes":131672871  
> },  
> "fielddata":{  
> "memory\_size\_in\_bytes":133728950,  
> "evictions":0  
> },  
> "percolate":{  
> "total":0,  
> "time\_in\_millis":0,  
> "current":0,  
> "memory\_size\_in\_bytes":-1,  
> "memory\_size":"-1b",  
> "queries":0  
> },  
> "completion":{  
> "size\_in\_bytes":0  
> },  
> "segments":{  
> "count":278,  
> "memory\_in\_bytes":220799236  
> },  
> "translog":{  
> "operations":68,  
> "size\_in\_bytes":0  
> }  
> },  
> "os":{  
> "timestamp":1404880773910,  
> "uptime\_in\_millis":915746,  
> "load\_average":[  
> 2.68,  
> 3.17,  
> 3.35  
> ],  
> "cpu":{  
> "sys":0,  
> "user":38,  
> "idle":61,  
> "usage":38,  
> "stolen":0  
> },  
> "mem":{  
> "free\_in\_bytes":12301606912,  
> "used\_in\_bytes":51857985536,  
> "free\_percent":35,  
> "used\_percent":64,  
> "actual\_free\_in\_bytes":23005929472,  
> "actual\_used\_in\_bytes":41153662976  
> },  
> "swap":{  
> "used\_in\_bytes":0,  
> "free\_in\_bytes":0  
> }  
> },  
> "process":{  
> "timestamp":1404880773911,  
> "open\_file\_descriptors":300,  
> "cpu":{  
> "percent":313,  
> "sys\_in\_millis":14021810,  
> "user\_in\_millis":794249870,  
> "total\_in\_millis":808271680  
> },  
> "mem":{  
> "resident\_in\_bytes":32342851584,  
> "share\_in\_bytes":712302592,  
> "total\_virtual\_in\_bytes":47406919680  
> }  
> },  
> "jvm":{  
> "timestamp":1404880773911,  
> "uptime\_in\_millis":309661546,  
> "mem":{  
> "heap\_used\_in\_bytes":18409783280,  
> "heap\_used\_percent":57,  
> "heap\_committed\_in\_bytes":32142524416,  
> "heap\_max\_in\_bytes":32142524416,  
> "non\_heap\_used\_in\_bytes":65012920,  
> "non\_heap\_committed\_in\_bytes":98009088,  
> "pools":{  
> "young":{  
> "used\_in\_bytes":538302112,  
> "max\_in\_bytes":558432256,  
> "peak\_used\_in\_bytes":558432256,  
> "peak\_max\_in\_bytes":558432256  
> },  
> "survivor":{  
> "used\_in\_bytes":11403704,  
> "max\_in\_bytes":69730304,  
> "peak\_used\_in\_bytes":69730304,  
> "peak\_max\_in\_bytes":69730304  
> },  
> "old":{  
> "used\_in\_bytes":17860077464,  
> "max\_in\_bytes":31514361856,  
> "peak\_used\_in\_bytes":23694417088,  
> "peak\_max\_in\_bytes":31514361856  
> }  
> }  
> },  
> "threads":{  
> "count":117,  
> "peak\_count":133  
> },  
> "gc":{  
> "collectors":{  
> "young":{  
> "collection\_count":200644,  
> "collection\_time\_in\_millis":10870850  
> },  
> "old":{  
> "collection\_count":9,  
> "collection\_time\_in\_millis":2987  
> }  
> }  
> },  
> "buffer\_pools":{  
> "direct":{  
> "count":154,  
> "used\_in\_bytes":78176363,  
> "total\_capacity\_in\_bytes":78176363  
> },  
> "mapped":{  
> "count":1293,  
> "used\_in\_bytes":9831615457,  
> "total\_capacity\_in\_bytes":9831615457  
> }  
> }  
> },  
> "thread\_pool":{  
> "generic":{  
> "threads":1,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":14,  
> "completed":56826  
> },  
> "index":{  
> "threads":8,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":8,  
> "completed":3607948  
> },  
> "get":{  
> "threads":0,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":0,  
> "completed":0  
> },  
> "snapshot":{  
> "threads":4,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":4,  
> "completed":95697  
> },  
> "merge":{  
> "threads":2,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":4,  
> "completed":282257  
> },  
> "suggest":{  
> "threads":0,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":0,  
> "completed":0  
> },  
> "bulk":{  
> "threads":8,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":8,  
> "completed":1081  
> },  
> "optimize":{  
> "threads":0,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":0,  
> "completed":0  
> },  
> "warmer":{  
> "threads":2,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":4,  
> "completed":5556773  
> },  
> "flush":{  
> "threads":1,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":2,  
> "completed":2336  
> },  
> "search":{  
> "threads":24,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":24,  
> "completed":160139043  
> },  
> "percolate":{  
> "threads":0,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":0,  
> "completed":0  
> },  
> "management":{  
> "threads":1,  
> "queue":0,  
> "active":1,  
> "rejected":0,  
> "largest":5,  
> "completed":1064  
> },  
> "refresh":{  
> "threads":4,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":4,  
> "completed":15126156  
> }  
> },  
> "network":{  
> "tcp":{  
> "active\_opens":1513869,  
> "passive\_opens":224206,  
> "curr\_estab":52,  
> "in\_segs":1929140868,  
> "out\_segs":2005198310,  
> "retrans\_segs":175635,  
> "estab\_resets":37,  
> "attempt\_fails":669,  
> "in\_errs":3,  
> "out\_rsts":88  
> }  
> },  
> "fs":{  
> "timestamp":1404880773912,  
> "total":{  
> "total\_in\_bytes":158534184960,  
> "free\_in\_bytes":148425240576,  
> "available\_in\_bytes":140372176896,  
> "disk\_reads":1993,  
> "disk\_writes":16068284,  
> "disk\_io\_op":16070277,  
> "disk\_read\_size\_in\_bytes":7445504,  
> "disk\_write\_size\_in\_bytes":650206560256,  
> "disk\_io\_size\_in\_bytes":650214005760  
> },  
> "data":[  
> {  
> "path":"/mnt/esdata/ab\_elastic/nodes/0",  
> "mount":"/mnt/esdata",  
> "dev":"/dev/xvdb",  
> "total\_in\_bytes":158534184960,  
> "free\_in\_bytes":148425240576,  
> "available\_in\_bytes":140372176896,  
> "disk\_reads":1993,  
> "disk\_writes":16068284,  
> "disk\_io\_op":16070277,  
> "disk\_read\_size\_in\_bytes":7445504,  
> "disk\_write\_size\_in\_bytes":650206560256,  
> "disk\_io\_size\_in\_bytes":650214005760  
> }  
> ]  
> },  
> "transport":{  
> "server\_open":26,  
> "rx\_count":214082686,  
> "rx\_size\_in\_bytes":148941344505,  
> "tx\_count":209856841,  
> "tx\_size\_in\_bytes":90356130749  
> },  
> "http":{  
> "current\_open":0,  
> "total\_opened":0  
> },  
> "fielddata\_breaker":{  
> "maximum\_size\_in\_bytes":25714019532,  
> "maximum\_size":"23.9gb",  
> "estimated\_size\_in\_bytes":265401821,  
> "estimated\_size":"253.1mb",  
> "overhead":1.03  
> }  
> },  
> "erc5l7dQai3mKBIX6sw":{  
> "timestamp":1404880733754,  
> "name":"Eddie Brock",  
> "transport\_address":"removed",  
> "host":"removed",  
> "ip":[  
> "removed",  
> "NONE"  
> ],  
> "indices":{  
> "docs":{  
> "count":17793781,  
> "deleted":529675  
> },  
> "store":{  
> "size\_in\_bytes":9969074949,  
> "throttle\_time\_in\_millis":9235478  
> },  
> "indexing":{  
> "index\_total":3531955,  
> "index\_time\_in\_millis":2783791,  
> "index\_current":0,  
> "delete\_total":0,  
> "delete\_time\_in\_millis":0,  
> "delete\_current":0  
> },  
> "get":{  
> "total":2947020,  
> "time\_in\_millis":217006,  
> "exists\_total":2946589,  
> "exists\_time\_in\_millis":216977,  
> "missing\_total":431,  
> "missing\_time\_in\_millis":29,  
> "current":0  
> },  
> "search":{  
> "open\_contexts":0,  
> "query\_total":155416200,  
> "query\_time\_in\_millis":1076023052,  
> "query\_current":0,  
> "fetch\_total":4101442,  
> "fetch\_time\_in\_millis":2337321,  
> "fetch\_current":0  
> },  
> "merges":{  
> "current":0,  
> "current\_docs":0,  
> "current\_size\_in\_bytes":0,  
> "total":563,  
> "total\_time\_in\_millis":89211,  
> "total\_docs":1099389,  
> "total\_size\_in\_bytes":470013054  
> },  
> "refresh":{  
> "total":15123966,  
> "total\_time\_in\_millis":63368992  
> },  
> "flush":{  
> "total":2335,  
> "total\_time\_in\_millis":414790  
> },  
> "warmer":{  
> "current":0,  
> "total":2690030,  
> "total\_time\_in\_millis":7339293  
> },  
> "filter\_cache":{  
> "memory\_size\_in\_bytes":695354935,  
> "evictions":1377260  
> },  
> "id\_cache":{  
> "memory\_size\_in\_bytes":131817497  
> },  
> "fielddata":{  
> "memory\_size\_in\_bytes":137907189,  
> "evictions":0  
> },  
> "percolate":{  
> "total":0,  
> "time\_in\_millis":0,  
> "current":0,  
> "memory\_size\_in\_bytes":-1,  
> "memory\_size":"-1b",  
> "queries":0  
> },  
> "completion":{  
> "size\_in\_bytes":0  
> },  
> "segments":{  
> "count":282,  
> "memory\_in\_bytes":217468955  
> },  
> "translog":{  
> "operations":86,  
> "size\_in\_bytes":0  
> }  
> },  
> "os":{  
> "timestamp":1404880733801,  
> "uptime\_in\_millis":923467,  
> "load\_average":[  
> 3.71,  
> 4.57,  
> 4.46  
> ],  
> "cpu":{  
> "sys":0,  
> "user":46,  
> "idle":52,  
> "usage":46,  
> "stolen":0  
> },  
> "mem":{  
> "free\_in\_bytes":4438659072,  
> "used\_in\_bytes":59720933376,  
> "free\_percent":32,  
> "used\_percent":67,  
> "actual\_free\_in\_bytes":21124698112,  
> "actual\_used\_in\_bytes":43034894336  
> },  
> "swap":{  
> "used\_in\_bytes":0,  
> "free\_in\_bytes":0  
> }  
> },  
> "process":{  
> "timestamp":1404880733801,  
> "open\_file\_descriptors":301,  
> "cpu":{  
> "percent":377,  
> "sys\_in\_millis":26524340,  
> "user\_in\_millis":947578840,  
> "total\_in\_millis":974103180  
> },  
> "mem":{  
> "resident\_in\_bytes":33945767936,  
> "share\_in\_bytes":692912128,  
> "total\_virtual\_in\_bytes":47473528832  
> }  
> },  
> "jvm":{  
> "timestamp":1404880733802,  
> "uptime\_in\_millis":310301938,  
> "mem":{  
> "heap\_used\_in\_bytes":10113402832,  
> "heap\_used\_percent":31,  
> "heap\_committed\_in\_bytes":32142524416,  
> "heap\_max\_in\_bytes":32142524416,  
> "non\_heap\_used\_in\_bytes":71762640,  
> "non\_heap\_committed\_in\_bytes":106397696,  
> "pools":{  
> "young":{  
> "used\_in\_bytes":218720808,  
> "max\_in\_bytes":558432256,  
> "peak\_used\_in\_bytes":558432256,  
> "peak\_max\_in\_bytes":558432256  
> },  
> "survivor":{  
> "used\_in\_bytes":9630920,  
> "max\_in\_bytes":69730304,  
> "peak\_used\_in\_bytes":69730304,  
> "peak\_max\_in\_bytes":69730304  
> },  
> "old":{  
> "used\_in\_bytes":9885051104,  
> "max\_in\_bytes":31514361856,  
> "peak\_used\_in\_bytes":23868903928,  
> "peak\_max\_in\_bytes":31514361856  
> }  
> }  
> },  
> "threads":{  
> "count":121,  
> "peak\_count":144  
> },  
> "gc":{  
> "collectors":{  
> "young":{  
> "collection\_count":209658,  
> "collection\_time\_in\_millis":12089132  
> },  
> "old":{  
> "collection\_count":11,  
> "collection\_time\_in\_millis":6628  
> }  
> }  
> },  
> "buffer\_pools":{  
> "direct":{  
> "count":162,  
> "used\_in\_bytes":195159665,  
> "total\_capacity\_in\_bytes":195159665  
> },  
> "mapped":{  
> "count":1292,  
> "used\_in\_bytes":9897821921,  
> "total\_capacity\_in\_bytes":9897821921  
> }  
> }  
> },  
> "thread\_pool":{  
> "generic":{  
> "threads":1,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":23,  
> "completed":67214  
> },  
> "index":{  
> "threads":8,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":8,  
> "completed":3611159  
> },  
> "get":{  
> "threads":0,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":0,  
> "completed":0  
> },  
> "snapshot":{  
> "threads":2,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":4,  
> "completed":95672  
> },  
> "merge":{  
> "threads":3,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":4,  
> "completed":285955  
> },  
> "suggest":{  
> "threads":0,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":0,  
> "completed":0  
> },  
> "bulk":{  
> "threads":8,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":8,  
> "completed":1125  
> },  
> "optimize":{  
> "threads":0,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":0,  
> "completed":0  
> },  
> "warmer":{  
> "threads":2,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":4,  
> "completed":5487739  
> },  
> "flush":{  
> "threads":1,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":2,  
> "completed":2335  
> },  
> "search":{  
> "threads":24,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":24,  
> "completed":160510703  
> },  
> "percolate":{  
> "threads":0,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":0,  
> "completed":0  
> },  
> "management":{  
> "threads":5,  
> "queue":0,  
> "active":1,  
> "rejected":0,  
> "largest":5,  
> "completed":22893  
> },  
> "refresh":{  
> "threads":4,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":4,  
> "completed":15135553  
> }  
> },  
> "network":{  
> "tcp":{  
> "active\_opens":1025664,  
> "passive\_opens":105715571,  
> "curr\_estab":55,  
> "in\_segs":2557757673,  
> "out\_segs":2502369803,  
> "retrans\_segs":205765,  
> "estab\_resets":68,  
> "attempt\_fails":207,  
> "in\_errs":0,  
> "out\_rsts":1641  
> }  
> },  
> "fs":{  
> "timestamp":1404880733802,  
> "total":{  
> "total\_in\_bytes":158533136384,  
> "free\_in\_bytes":137526714368,  
> "available\_in\_bytes":129473703936,  
> "disk\_reads":8637,  
> "disk\_writes":17332161,  
> "disk\_io\_op":17340798,  
> "disk\_read\_size\_in\_bytes":139357184,  
> "disk\_write\_size\_in\_bytes":706635259904,  
> "disk\_io\_size\_in\_bytes":706774617088  
> },  
> "data":[  
> {  
> "path":"/mnt/esdata/ab\_elastic/nodes/0",  
> "mount":"/mnt/esdata",  
> "dev":"/dev/xvdb",  
> "total\_in\_bytes":158533136384,  
> "free\_in\_bytes":137526714368,  
> "available\_in\_bytes":129473703936,  
> "disk\_reads":8637,  
> "disk\_writes":17332161,  
> "disk\_io\_op":17340798,  
> "disk\_read\_size\_in\_bytes":139357184,  
> "disk\_write\_size\_in\_bytes":706635259904,  
> "disk\_io\_size\_in\_bytes":706774617088  
> }  
> ]  
> },  
> "transport":{  
> "server\_open":26,  
> "rx\_count":214086149,  
> "rx\_size\_in\_bytes":90356374759,  
> "tx\_count":210167189,  
> "tx\_size\_in\_bytes":149348401545  
> },  
> "http":{  
> "current\_open":1,  
> "total\_opened":62412134  
> },  
> "fielddata\_breaker":{  
> "maximum\_size\_in\_bytes":25714019532,  
> "maximum\_size":"23.9gb",  
> "estimated\_size\_in\_bytes":269724686,  
> "estimated\_size":"257.2mb",  
> "overhead":1.03  
> }  
> }  
> }  
> }
> 
> --  
> You received this message because you are subscribed to the Google Groups  
> "elasticsearch" group.  
> To unsubscribe from this group and stop receiving emails from it, send an  
> email to [elasticsearch+unsubscribe@googlegroups.com](mailto:elasticsearch+unsubscribe@googlegroups.com).  
> To view this discussion on the web visit  
> [https://groups.google.com/d/msgid/elasticsearch/75d11657-d3c7-4a0a-ba54-ca8691f9b14d%40googlegroups.com](https://groups.google.com/d/msgid/elasticsearch/75d11657-d3c7-4a0a-ba54-ca8691f9b14d%40googlegroups.com)  
> [https://groups.google.com/d/msgid/elasticsearch/75d11657-d3c7-4a0a-ba54-ca8691f9b14d%40googlegroups.com?utm\_medium=email&utm\_source=footer](https://groups.google.com/d/msgid/elasticsearch/75d11657-d3c7-4a0a-ba54-ca8691f9b14d%40googlegroups.com?utm_medium=email&utm_source=footer)  
> .  
> For more options, visit [https://groups.google.com/d/optout](https://groups.google.com/d/optout).

--  
You received this message because you are subscribed to the Google Groups "elasticsearch" group.  
To unsubscribe from this group and stop receiving emails from it, send an email to [elasticsearch+unsubscribe@googlegroups.com](mailto:elasticsearch+unsubscribe@googlegroups.com).  
To view this discussion on the web visit [https://groups.google.com/d/msgid/elasticsearch/CAEM624aRDRDV6\_QbBRZQ13HgNVsaqp9qCwhfJdB9\_Vo0K9xPGQ%40mail.gmail.com](https://groups.google.com/d/msgid/elasticsearch/CAEM624aRDRDV6_QbBRZQ13HgNVsaqp9qCwhfJdB9_Vo0K9xPGQ%40mail.gmail.com).  
For more options, visit [https://groups.google.com/d/optout](https://groups.google.com/d/optout).

---

<div class="post-metadata">

**Author:** ![webish](https://avatars.discourse-cdn.com/v4/letter/w/919ad9/32.png) [@webish](https://discuss.elastic.co/u/webish)\
**Post date:** [July 9, 2014, 5:54am UTC](https://discuss.elastic.co/t/elastic-search-nodes-servers-with-constant-high-cpu-utilization-in-idle-state/18544/3 "2014-07-09T05:54:47Z")

</div>

Thanks Mark. My mistake.

I can try switching to Oracle Java. There is no TTL. I've used Marvel for  
development testing. Perhaps I can install the plugin...

On Wednesday, July 9, 2014 1:35:20 AM UTC-4, Mark Walkom wrote:

> It's better to pop long output like that into a gist/pastebin, it makes it  
> easier to read the thread.  
> It's also worth installing a monitoring plugin like ElasticHQ or marvel,  
> as they provide graphical insight into what is happening and will  
> extrapolate some of the raw figures out.
> 
> If you can change to Oracle java you will get better performance. Are you  
> using TTL?
> 
> Regards,  
> Mark Walkom
> 
> Infrastructure Engineer  
> Campaign Monitor  
> email: [ma...@campaignmonitor.com](mailto:ma...@campaignmonitor.com) \<javascript:\>  
> web: [www.campaignmonitor.com](http://www.campaignmonitor.com)
> 
> On 9 July 2014 14:54, webish \<[gre...@yoursports.com](mailto:gre...@yoursports.com) \<javascript:\>\> wrote:
> 
> I'm seeing very high CPU utilization in an idle or close to idle state. I  
> haven't seen it use less than 40% on each node over the course of days.  
> I'm not certain how long it has been like this but could be weeks.
> 
> Any help resolving this would be greatly appreciated!
> 
> _Details:_
> 
> There are 2 nodes in the cluster.
> 
> AWS instances are:  
> vCPU ECUMemory (GiB) Instance Storage (GB) Linux/UNIX Usage
> 
> r3.2xlarge 82661 1 x 160 SSD
> 
> curl -XGET '[http://localhost:9200/](http://localhost:9200/)'  
> {  
> "status" : 200,  
> "name" : "Eddie Brock",  
> "version" : {  
> "number" : "1.1.0",  
> "build\_hash" : "2181e113dea80b4a9e31e58e9686658a2d46e363",  
> "build\_timestamp" : "2014-03-25T15:59:51Z",  
> "build\_snapshot" : false,  
> "lucene\_version" : "4.7"  
> },  
> "tagline" : "You Know, for Search"  
> }
> 
> % java -version  
> java version "1.7.0\_55"  
> OpenJDK Runtime Environment (IcedTea 2.4.7) (7u55-2.4.7-1ubuntu1~0.12.04.2)  
> OpenJDK 64-Bit Server VM (build 24.51-b03, mixed mode)
> 
> curl -XGET '[http://localhost:9200/\_nodes/stats](http://localhost:9200/_nodes/stats)'  
> {  
> "cluster\_name":"ab\_elastic",  
> "nodes":{  
> "kpHPVkBCyjOkRaSpS5Q":{  
> "timestamp":1404880773859,  
> "name":"Milan",  
> "transport\_address":"removed",  
> "host":"removed",  
> "ip":[  
> "removed",  
> "NONE"  
> ],  
> "indices":{  
> "docs":{  
> "count":17793781,  
> "deleted":466537  
> },  
> "store":{  
> "size\_in\_bytes":9902722189,  
> "throttle\_time\_in\_millis":8777423  
> },  
> "indexing":{  
> "index\_total":3548638,  
> "index\_time\_in\_millis":2610694,  
> "index\_current":1,  
> "delete\_total":0,  
> "delete\_time\_in\_millis":0,  
> "delete\_current":0  
> },  
> "get":{  
> "total":0,  
> "time\_in\_millis":0,  
> "exists\_total":0,  
> "exists\_time\_in\_millis":0,  
> "missing\_total":0,  
> "missing\_time\_in\_millis":0,  
> "current":0  
> },  
> "search":{  
> "open\_contexts":0,  
> "query\_total":155361213,  
> "query\_time\_in\_millis":817989500,  
> "query\_current":0,  
> "fetch\_total":4119309,  
> "fetch\_time\_in\_millis":1967525,  
> "fetch\_current":0  
> },  
> "merges":{  
> "current":0,  
> "current\_docs":0,  
> "current\_size\_in\_bytes":0,  
> "total":564,  
> "total\_time\_in\_millis":76065,  
> "total\_docs":1093579,  
> "total\_size\_in\_bytes":470304093  
> },  
> "refresh":{  
> "total":15117234,  
> "total\_time\_in\_millis":50581020  
> },  
> "flush":{  
> "total":2336,  
> "total\_time\_in\_millis":99980  
> },  
> "warmer":{  
> "current":0,  
> "total":2719640,  
> "total\_time\_in\_millis":5980882  
> },  
> "filter\_cache":{  
> "memory\_size\_in\_bytes":722232396,  
> "evictions":1179740  
> },  
> "id\_cache":{  
> "memory\_size\_in\_bytes":131672871  
> },  
> "fielddata":{  
> "memory\_size\_in\_bytes":133728950,  
> "evictions":0  
> },  
> "percolate":{  
> "total":0,  
> "time\_in\_millis":0,  
> "current":0,  
> "memory\_size\_in\_bytes":-1,  
> "memory\_size":"-1b",  
> "queries":0  
> },  
> "completion":{  
> "size\_in\_bytes":0  
> },  
> "segments":{  
> "count":278,  
> "memory\_in\_bytes":220799236  
> },  
> "translog":{  
> "operations":68,  
> "size\_in\_bytes":0  
> }  
> },  
> "os":{  
> "timestamp":1404880773910,  
> "uptime\_in\_millis":915746,  
> "load\_average":[  
> 2.68,  
> 3.17,  
> 3.35  
> ],  
> "cpu":{  
> "sys":0,  
> "user":38,  
> "idle":61,  
> "usage":38,  
> "stolen":0  
> },  
> "mem":{  
> "free\_in\_bytes":12301606912,  
> "used\_in\_bytes":51857985536,  
> "free\_percent":35,  
> "used\_percent":64,  
> "actual\_free\_in\_bytes":23005929472,  
> "actual\_used\_in\_bytes":41153662976  
> },  
> "swap":{  
> "used\_in\_bytes":0,  
> "free\_in\_bytes":0  
> }  
> },  
> "process":{  
> "timestamp":1404880773911,  
> "open\_file\_descriptors":300,  
> "cpu":{  
> "percent":313,  
> "sys\_in\_millis":14021810,  
> "user\_in\_millis":794249870,  
> "total\_in\_millis":808271680  
> },  
> "mem":{  
> "resident\_in\_bytes":32342851584,  
> "share\_in\_bytes":712302592,  
> "total\_virtual\_in\_bytes":47406919680  
> }  
> },  
> "jvm":{  
> "timestamp":1404880773911,  
> "uptime\_in\_millis":309661546,  
> "mem":{  
> "heap\_used\_in\_bytes":18409783280,  
> "heap\_used\_percent":57,  
> "heap\_committed\_in\_bytes":32142524416,  
> "heap\_max\_in\_bytes":32142524416,  
> "non\_heap\_used\_in\_bytes":65012920,  
> "non\_heap\_committed\_in\_bytes":98009088,  
> "pools":{  
> "young":{  
> "used\_in\_bytes":538302112,  
> "max\_in\_bytes":558432256,  
> "peak\_used\_in\_bytes":558432256,  
> "peak\_max\_in\_bytes":558432256  
> },  
> "survivor":{  
> "used\_in\_bytes":11403704,  
> "max\_in\_bytes":69730304,  
> "peak\_used\_in\_bytes":69730304,  
> "peak\_max\_in\_bytes":69730304  
> },  
> "old":{  
> "used\_in\_bytes":17860077464,  
> "max\_in\_bytes":31514361856,  
> "peak\_used\_in\_bytes":23694417088,  
> "peak\_max\_in\_bytes":31514361856  
> }  
> }  
> },  
> "threads":{  
> "count":117,  
> "peak\_count":133  
> },  
> "gc":{  
> "collectors":{  
> "young":{  
> "collection\_count":200644,  
> "collection\_time\_in\_millis":10870850  
> },  
> "old":{  
> "collection\_count":9,  
> "collection\_time\_in\_millis":2987  
> }  
> }  
> },  
> "buffer\_pools":{  
> "direct":{  
> "count":154,  
> "used\_in\_bytes":78176363,  
> "total\_capacity\_in\_bytes":78176363  
> },  
> "mapped":{  
> "count":1293,  
> "used\_in\_bytes":9831615457,  
> "total\_capacity\_in\_bytes":9831615457  
> }  
> }  
> },  
> "thread\_pool":{  
> "generic":{  
> "threads":1,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":14,  
> "completed":56826  
> },  
> "index":{  
> "threads":8,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":8,  
> "completed":3607948  
> },  
> "get":{  
> "threads":0,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":0,  
> "completed":0  
> },  
> "snapshot":{  
> "threads":4,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":4,  
> "completed":95697  
> },  
> "merge":{  
> "threads":2,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":4,  
> "completed":282257  
> },  
> "suggest":{  
> "threads":0,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":0,  
> "completed":0  
> },  
> "bulk":{  
> "threads":8,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":8,  
> "completed":1081  
> },  
> "optimize":{  
> "threads":0,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":0,  
> "completed":0  
> },  
> "warmer":{  
> "threads":2,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":4,  
> "completed":5556773  
> },  
> "flush":{  
> "threads":1,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":2,  
> "completed":2336  
> },  
> "search":{  
> "threads":24,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":24,  
> "completed":160139043  
> },  
> "percolate":{  
> "threads":0,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":0,  
> "completed":0  
> },  
> "management":{  
> "threads":1,  
> "queue":0,  
> "active":1,  
> "rejected":0,  
> "largest":5,  
> "completed":1064  
> },  
> "refresh":{  
> "threads":4,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":4,  
> "completed":15126156  
> }  
> },  
> "network":{  
> "tcp":{  
> "active\_opens":1513869,  
> "passive\_opens":224206,  
> "curr\_estab":52,  
> "in\_segs":1929140868,  
> "out\_segs":2005198310,  
> "retrans\_segs":175635,  
> "estab\_resets":37,  
> "attempt\_fails":669,  
> "in\_errs":3,  
> "out\_rsts":88  
> }  
> },  
> "fs":{  
> "timestamp":1404880773912,  
> "total":{  
> "total\_in\_bytes":158534184960,  
> "free\_in\_bytes":148425240576,  
> "available\_in\_bytes":140372176896,  
> "disk\_reads":1993,  
> "disk\_writes":16068284,  
> "disk\_io\_op":16070277,  
> "disk\_read\_size\_in\_bytes":7445504,  
> "disk\_write\_size\_in\_bytes":650206560256,  
> "disk\_io\_size\_in\_bytes":650214005760  
> },  
> "data":[  
> {  
> "path":"/mnt/esdata/ab\_elastic/nodes/0",  
> "mount":"/mnt/esdata",  
> "dev":"/dev/xvdb",  
> "total\_in\_bytes":158534184960,  
> "free\_in\_bytes":148425240576,  
> "available\_in\_bytes":140372176896,  
> "disk\_reads":1993,  
> "disk\_writes":16068284,  
> "disk\_io\_op":16070277,  
> "disk\_read\_size\_in\_bytes":7445504,  
> "disk\_write\_size\_in\_bytes":650206560256,  
> "disk\_io\_size\_in\_bytes":650214005760  
> }  
> ]  
> },  
> "transport":{  
> "server\_open":26,  
> "rx\_count":214082686,  
> "rx\_size\_in\_bytes":148941344505,  
> "tx\_count":209856841,  
> "tx\_size\_in\_bytes":90356130749  
> },  
> "http":{  
> "current\_open":0,  
> "total\_opened":0  
> },  
> "fielddata\_breaker":{  
> "maximum\_size\_in\_bytes":25714019532,  
> "maximum\_size":"23.9gb",  
> "estimated\_size\_in\_bytes":265401821,  
> "estimated\_size":"253.1mb",  
> "overhead":1.03  
> }  
> },  
> "erc5l7dQai3mKBIX6sw":{  
> "timestamp":1404880733754,  
> "name":"Eddie Brock",  
> "transport\_address":"removed",  
> "host":"removed",  
> "ip":[  
> "removed",  
> "NONE"  
> ],  
> "indices":{  
> "docs":{  
> "count":17793781,  
> "deleted":529675  
> },  
> "store":{  
> "size\_in\_bytes":9969074949,  
> "throttle\_time\_in\_millis":9235478  
> },  
> "indexing":{  
> "index\_total":3531955,  
> "index\_time\_in\_millis":2783791,  
> "index\_current":0,  
> "delete\_total":0,  
> "delete\_time\_in\_millis":0,  
> "delete\_current":0  
> },  
> "get":{  
> "total":2947020,  
> "time\_in\_millis":217006,  
> "exists\_total":2946589,  
> "exists\_time\_in\_millis":216977,  
> "missing\_total":431,  
> "missing\_time\_in\_millis":29,  
> "current":0  
> },  
> "search":{  
> "open\_contexts":0,  
> "query\_total":155416200,  
> "query\_time\_in\_millis":1076023052,  
> "query\_current":0,  
> "fetch\_total":4101442,  
> "fetch\_time\_in\_millis":2337321,  
> "fetch\_current":0  
> },  
> "merges":{  
> "current":0,  
> "current\_docs":0,  
> "current\_size\_in\_bytes":0,  
> "total":563,  
> "total\_time\_in\_millis":89211,  
> "total\_docs":1099389,  
> "total\_size\_in\_bytes":470013054  
> },  
> "refresh":{  
> "total":15123966,  
> "total\_time\_in\_millis":63368992  
> },  
> "flush":{  
> "total":2335,  
> "total\_time\_in\_millis":414790  
> },  
> "warmer":{  
> "current":0,  
> "total":2690030,  
> "total\_time\_in\_millis":7339293  
> },  
> "filter\_cache":{  
> "memory\_size\_in\_bytes":695354935,  
> "evictions":1377260  
> },  
> "id\_cache":{  
> "memory\_size\_in\_bytes":131817497  
> },  
> "fielddata":{  
> "memory\_size\_in\_bytes":137907189,  
> "evictions":0  
> },  
> "percolate":{  
> "total":0,  
> "time\_in\_millis":0,  
> "current":0,  
> "memory\_size\_in\_bytes":-1,  
> "memory\_size":"-1b",  
> "queries":0  
> },  
> "completion":{  
> "size\_in\_bytes":0  
> },  
> "segments":{  
> "count":282,  
> "memory\_in\_bytes":217468955  
> },  
> "translog":{  
> "operations":86,  
> "size\_in\_bytes":0  
> }  
> },  
> "os":{  
> "timestamp":1404880733801,  
> "uptime\_in\_millis":923467,  
> "load\_average":[  
> 3.71,  
> 4.57,  
> 4.46  
> ],  
> "cpu":{  
> "sys":0,  
> "user":46,  
> "idle":52,  
> "usage":46,  
> "stolen":0  
> },  
> "mem":{  
> "free\_in\_bytes":4438659072,  
> "used\_in\_bytes":59720933376,  
> "free\_percent":32,  
> "used\_percent":67,  
> "actual\_free\_in\_bytes":21124698112,  
> "actual\_used\_in\_bytes":43034894336  
> },  
> "swap":{  
> "used\_in\_bytes":0,  
> "free\_in\_bytes":0  
> }  
> },  
> "process":{  
> "timestamp":1404880733801,  
> "open\_file\_descriptors":301,  
> "cpu":{  
> "percent":377,  
> "sys\_in\_millis":26524340,  
> "user\_in\_millis":947578840,  
> "total\_in\_millis":974103180  
> },  
> "mem":{  
> "resident\_in\_bytes":33945767936,  
> "share\_in\_bytes":692912128,  
> "total\_virtual\_in\_bytes":47473528832  
> }  
> },  
> "jvm":{  
> "timestamp":1404880733802,  
> "uptime\_in\_millis":310301938,  
> "mem":{  
> "heap\_used\_in\_bytes":10113402832,  
> "heap\_used\_percent":31,  
> "heap\_committed\_in\_bytes":32142524416,  
> "heap\_max\_in\_bytes":32142524416,  
> "non\_heap\_used\_in\_bytes":71762640,  
> "non\_heap\_committed\_in\_bytes":106397696,  
> "pools":{  
> "young":{  
> "used\_in\_bytes":218720808,  
> "max\_in\_bytes":558432256,  
> "peak\_used\_in\_bytes":558432256,  
> "peak\_max\_in\_bytes":558432256  
> },  
> "survivor":{  
> "used\_in\_bytes":9630920,  
> "max\_in\_bytes":69730304,  
> "peak\_used\_in\_bytes":69730304,  
> "peak\_max\_in\_bytes":69730304  
> },  
> "old":{  
> "used\_in\_bytes":9885051104,  
> "max\_in\_bytes":31514361856,  
> "peak\_used\_in\_bytes":23868903928,  
> "peak\_max\_in\_bytes":31514361856  
> }  
> }  
> },  
> "threads":{  
> "count":121,  
> "peak\_count":144  
> },  
> "gc":{  
> "collectors":{  
> "young":{  
> "collection\_count":209658,  
> "collection\_time\_in\_millis":12089132  
> },  
> "old":{  
> "collection\_count":11,  
> "collection\_time\_in\_millis":6628  
> }  
> }  
> },  
> "buffer\_pools":{  
> "direct":{  
> "count":162,  
> "used\_in\_bytes":195159665,  
> "total\_capacity\_in\_bytes":195159665  
> },  
> "mapped":{  
> "count":1292,  
> "used\_in\_bytes":9897821921,  
> "total\_capacity\_in\_bytes":9897821921  
> }  
> }  
> },  
> "thread\_pool":{  
> "generic":{  
> "threads":1,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":23,  
> "completed":67214  
> },  
> "index":{  
> "threads":8,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":8,  
> "completed":3611159  
> },  
> "get":{  
> "threads":0,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":0,  
> "completed":0  
> },  
> "snapshot":{  
> "threads":2,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":4,  
> "completed":95672  
> },  
> "merge":{  
> "threads":3,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":4,  
> "completed":285955  
> },  
> "suggest":{  
> "threads":0,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":0,  
> "completed":0  
> },  
> "bulk":{  
> "threads":8,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":8,  
> "completed":1125  
> },  
> "optimize":{  
> "threads":0,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":0,  
> "completed":0  
> },  
> "warmer":{  
> "threads":2,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":4,  
> "completed":5487739  
> },  
> "flush":{  
> "threads":1,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":2,  
> "completed":2335  
> },  
> "search":{  
> "threads":24,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":24,  
> "completed":160510703  
> },  
> "percolate":{  
> "threads":0,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":0,  
> "completed":0  
> },  
> "management":{  
> "threads":5,  
> "queue":0,  
> "active":1,  
> "rejected":0,  
> "largest":5,  
> "completed":22893  
> },  
> "refresh":{  
> "threads":4,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":4,  
> "completed":15135553  
> }  
> },  
> "network":{  
> "tcp":{  
> "active\_opens":1025664,  
> "passive\_opens":105715571,  
> "curr\_estab":55,  
> "in\_segs":2557757673,  
> "out\_segs":2502369803,  
> "retrans\_segs":205765,  
> "estab\_resets":68,  
> "attempt\_fails":207,  
> "in\_errs":0,  
> "out\_rsts":1641  
> }  
> },  
> "fs":{  
> &qu
> 
> ...

--  
You received this message because you are subscribed to the Google Groups "elasticsearch" group.  
To unsubscribe from this group and stop receiving emails from it, send an email to [elasticsearch+unsubscribe@googlegroups.com](mailto:elasticsearch+unsubscribe@googlegroups.com).  
To view this discussion on the web visit [https://groups.google.com/d/msgid/elasticsearch/5140dad5-60e6-4a51-8986-c3f1c0da4c0f%40googlegroups.com](https://groups.google.com/d/msgid/elasticsearch/5140dad5-60e6-4a51-8986-c3f1c0da4c0f%40googlegroups.com).  
For more options, visit [https://groups.google.com/d/optout](https://groups.google.com/d/optout).

---

<div class="post-metadata">

**Author:** ![Ivan](https://avatars.discourse-cdn.com/v4/letter/i/df788c/32.png) [@Ivan](https://discuss.elastic.co/u/Ivan)\
**Post date:** [July 9, 2014, 7:11am UTC](https://discuss.elastic.co/t/elastic-search-nodes-servers-with-constant-high-cpu-utilization-in-idle-state/18544/4 "2014-07-09T07:11:09Z")

</div>

Yes, please use gist/pastebin in the future.

Try using the hot threads API [1] to see if there are any threads that are  
truly busy. If your system is truly idle, your thread pools should be  
almost empty [2]. Your output above has only one management thread in use,  
which could be simply the thread serving up the output.

[1]

> **[Elasticsearch Platform — Find real-time answers at scale](https://www.elastic.co)**
>
> Power insights and outcomes with the Elasticsearch Platform and AI. See into your data and find answers that matter with enterprise solutions designed to help you build, observe, and protect. Try Elasticsearch free today.

[2]

> **[Elasticsearch Platform — Find real-time answers at scale](https://www.elastic.co)**
>
> Power insights and outcomes with the Elasticsearch Platform and AI. See into your data and find answers that matter with enterprise solutions designed to help you build, observe, and protect. Try Elasticsearch free today.

Cheers,

Ivan

On Tue, Jul 8, 2014 at 10:54 PM, webish [gregory@yoursports.com](mailto:gregory@yoursports.com) wrote:

> Thanks Mark. My mistake.
> 
> I can try switching to Oracle Java. There is no TTL. I've used Marvel  
> for development testing. Perhaps I can install the plugin...
> 
> On Wednesday, July 9, 2014 1:35:20 AM UTC-4, Mark Walkom wrote:
> 
> > It's better to pop long output like that into a gist/pastebin, it makes  
> > it easier to read the thread.  
> > It's also worth installing a monitoring plugin like ElasticHQ or marvel,  
> > as they provide graphical insight into what is happening and will  
> > extrapolate some of the raw figures out.
> > 
> > If you can change to Oracle java you will get better performance. Are you  
> > using TTL?
> > 
> > Regards,  
> > Mark Walkom
> > 
> > Infrastructure Engineer  
> > Campaign Monitor  
> > email: [ma...@campaignmonitor.com](mailto:ma...@campaignmonitor.com)  
> > web: [www.campaignmonitor.com](http://www.campaignmonitor.com)
> > 
> > On 9 July 2014 14:54, webish [gre...@yoursports.com](mailto:gre...@yoursports.com) wrote:
> > 
> > I'm seeing very high CPU utilization in an idle or close to idle state.  
> > I haven't seen it use less than 40% on each node over the course of days.  
> > I'm not certain how long it has been like this but could be weeks.
> > 
> > Any help resolving this would be greatly appreciated!
> > 
> > _Details:_
> > 
> > There are 2 nodes in the cluster.
> > 
> > AWS instances are:  
> > vCPU ECUMemory (GiB) Instance Storage (GB) Linux/UNIX Usage
> > 
> > r3.2xlarge 82661 1 x 160 SSD
> > 
> > curl -XGET '[http://localhost:9200/](http://localhost:9200/)'  
> > {  
> > "status" : 200,  
> > "name" : "Eddie Brock",  
> > "version" : {  
> > "number" : "1.1.0",  
> > "build\_hash" : "2181e113dea80b4a9e31e58e9686658a2d46e363",  
> > "build\_timestamp" : "2014-03-25T15:59:51Z",  
> > "build\_snapshot" : false,  
> > "lucene\_version" : "4.7"  
> > },  
> > "tagline" : "You Know, for Search"  
> > }
> > 
> > % java -version  
> > java version "1.7.0\_55"  
> > OpenJDK Runtime Environment (IcedTea 2.4.7) (7u55-2.4.7-1ubuntu1~0.12.04.  
> > 2)  
> > OpenJDK 64-Bit Server VM (build 24.51-b03, mixed mode)
> > 
> > curl -XGET '[http://localhost:9200/\_nodes/stats](http://localhost:9200/_nodes/stats)'  
> > {  
> > "cluster\_name":"ab\_elastic",  
> > "nodes":{  
> > "kpHPVkBCyjOkRaSpS5Q":{  
> > "timestamp":1404880773859,  
> > "name":"Milan",  
> > "transport\_address":"removed",  
> > "host":"removed",  
> > "ip":[  
> > "removed",  
> > "NONE"  
> > ],  
> > "indices":{  
> > "docs":{  
> > "count":17793781,  
> > "deleted":466537  
> > },  
> > "store":{  
> > "size\_in\_bytes":9902722189,  
> > "throttle\_time\_in\_millis":8777423  
> > },  
> > "indexing":{  
> > "index\_total":3548638,  
> > "index\_time\_in\_millis":2610694,  
> > "index\_current":1,  
> > "delete\_total":0,  
> > "delete\_time\_in\_millis":0,  
> > "delete\_current":0  
> > },  
> > "get":{  
> > "total":0,  
> > "time\_in\_millis":0,  
> > "exists\_total":0,  
> > "exists\_time\_in\_millis":0,  
> > "missing\_total":0,  
> > "missing\_time\_in\_millis":0,  
> > "current":0  
> > },  
> > "search":{  
> > "open\_contexts":0,  
> > "query\_total":155361213,  
> > "query\_time\_in\_millis":817989500,  
> > "query\_current":0,  
> > "fetch\_total":4119309,  
> > "fetch\_time\_in\_millis":1967525,  
> > "fetch\_current":0  
> > },  
> > "merges":{  
> > "current":0,  
> > "current\_docs":0,  
> > "current\_size\_in\_bytes":0,  
> > "total":564,  
> > "total\_time\_in\_millis":76065,  
> > "total\_docs":1093579,  
> > "total\_size\_in\_bytes":470304093  
> > },  
> > "refresh":{  
> > "total":15117234,  
> > "total\_time\_in\_millis":50581020  
> > },  
> > "flush":{  
> > "total":2336,  
> > "total\_time\_in\_millis":99980  
> > },  
> > "warmer":{  
> > "current":0,  
> > "total":2719640,  
> > "total\_time\_in\_millis":5980882  
> > },  
> > "filter\_cache":{  
> > "memory\_size\_in\_bytes":722232396,  
> > "evictions":1179740  
> > },  
> > "id\_cache":{  
> > "memory\_size\_in\_bytes":131672871  
> > },  
> > "fielddata":{  
> > "memory\_size\_in\_bytes":133728950,  
> > "evictions":0  
> > },  
> > "percolate":{  
> > "total":0,  
> > "time\_in\_millis":0,  
> > "current":0,  
> > "memory\_size\_in\_bytes":-1,  
> > "memory\_size":"-1b",  
> > "queries":0  
> > },  
> > "completion":{  
> > "size\_in\_bytes":0  
> > },  
> > "segments":{  
> > "count":278,  
> > "memory\_in\_bytes":220799236  
> > },  
> > "translog":{  
> > "operations":68,  
> > "size\_in\_bytes":0  
> > }  
> > },  
> > "os":{  
> > "timestamp":1404880773910,  
> > "uptime\_in\_millis":915746,  
> > "load\_average":[  
> > 2.68,  
> > 3.17,  
> > 3.35  
> > ],  
> > "cpu":{  
> > "sys":0,  
> > "user":38,  
> > "idle":61,  
> > "usage":38,  
> > "stolen":0  
> > },  
> > "mem":{  
> > "free\_in\_bytes":12301606912,  
> > "used\_in\_bytes":51857985536,  
> > "free\_percent":35,  
> > "used\_percent":64,  
> > "actual\_free\_in\_bytes":23005929472,  
> > "actual\_used\_in\_bytes":41153662976  
> > },  
> > "swap":{  
> > "used\_in\_bytes":0,  
> > "free\_in\_bytes":0  
> > }  
> > },  
> > "process":{  
> > "timestamp":1404880773911,  
> > "open\_file\_descriptors":300,  
> > "cpu":{  
> > "percent":313,  
> > "sys\_in\_millis":14021810,  
> > "user\_in\_millis":794249870,  
> > "total\_in\_millis":808271680  
> > },  
> > "mem":{  
> > "resident\_in\_bytes":32342851584,  
> > "share\_in\_bytes":712302592,  
> > "total\_virtual\_in\_bytes":47406919680  
> > }  
> > },  
> > "jvm":{  
> > "timestamp":1404880773911,  
> > "uptime\_in\_millis":309661546,  
> > "mem":{  
> > "heap\_used\_in\_bytes":18409783280,  
> > "heap\_used\_percent":57,  
> > "heap\_committed\_in\_bytes":32142524416,  
> > "heap\_max\_in\_bytes":32142524416,  
> > "non\_heap\_used\_in\_bytes":65012920,  
> > "non\_heap\_committed\_in\_bytes":98009088,  
> > "pools":{  
> > "young":{  
> > "used\_in\_bytes":538302112,  
> > "max\_in\_bytes":558432256,  
> > "peak\_used\_in\_bytes":558432256,  
> > "peak\_max\_in\_bytes":558432256  
> > },  
> > "survivor":{  
> > "used\_in\_bytes":11403704,  
> > "max\_in\_bytes":69730304,  
> > "peak\_used\_in\_bytes":69730304,  
> > "peak\_max\_in\_bytes":69730304  
> > },  
> > "old":{  
> > "used\_in\_bytes":17860077464,  
> > "max\_in\_bytes":31514361856,  
> > "peak\_used\_in\_bytes":23694417088,  
> > "peak\_max\_in\_bytes":31514361856  
> > }  
> > }  
> > },  
> > "threads":{  
> > "count":117,  
> > "peak\_count":133  
> > },  
> > "gc":{  
> > "collectors":{  
> > "young":{  
> > "collection\_count":200644,  
> > "collection\_time\_in\_millis":10870850  
> > },  
> > "old":{  
> > "collection\_count":9,  
> > "collection\_time\_in\_millis":2987  
> > }  
> > }  
> > },  
> > "buffer\_pools":{  
> > "direct":{  
> > "count":154,  
> > "used\_in\_bytes":78176363,  
> > "total\_capacity\_in\_bytes":78176363  
> > },  
> > "mapped":{  
> > "count":1293,  
> > "used\_in\_bytes":9831615457,  
> > "total\_capacity\_in\_bytes":9831615457  
> > }  
> > }  
> > },  
> > "thread\_pool":{  
> > "generic":{  
> > "threads":1,  
> > "queue":0,  
> > "active":0,  
> > "rejected":0,  
> > "largest":14,  
> > "completed":56826  
> > },  
> > "index":{  
> > "threads":8,  
> > "queue":0,  
> > "active":0,  
> > "rejected":0,  
> > "largest":8,  
> > "completed":3607948  
> > },  
> > "get":{  
> > "threads":0,  
> > "queue":0,  
> > "active":0,  
> > "rejected":0,  
> > "largest":0,  
> > "completed":0  
> > },  
> > "snapshot":{  
> > "threads":4,  
> > "queue":0,  
> > "active":0,  
> > "rejected":0,  
> > "largest":4,  
> > "completed":95697  
> > },  
> > "merge":{  
> > "threads":2,  
> > "queue":0,  
> > "active":0,  
> > "rejected":0,  
> > "largest":4,  
> > "completed":282257  
> > },  
> > "suggest":{  
> > "threads":0,  
> > "queue":0,  
> > "active":0,  
> > "rejected":0,  
> > "largest":0,  
> > "completed":0  
> > },  
> > "bulk":{  
> > "threads":8,  
> > "queue":0,  
> > "active":0,  
> > "rejected":0,  
> > "largest":8,  
> > "completed":1081  
> > },  
> > "optimize":{  
> > "threads":0,  
> > "queue":0,  
> > "active":0,  
> > "rejected":0,  
> > "largest":0,  
> > "completed":0  
> > },  
> > "warmer":{  
> > "threads":2,  
> > "queue":0,  
> > "active":0,  
> > "rejected":0,  
> > "largest":4,  
> > "completed":5556773  
> > },  
> > "flush":{  
> > "threads":1,  
> > "queue":0,  
> > "active":0,  
> > "rejected":0,  
> > "largest":2,  
> > "completed":2336  
> > },  
> > "search":{  
> > "threads":24,  
> > "queue":0,  
> > "active":0,  
> > "rejected":0,  
> > "largest":24,  
> > "completed":160139043  
> > },  
> > "percolate":{  
> > "threads":0,  
> > "queue":0,  
> > "active":0,  
> > "rejected":0,  
> > "largest":0,  
> > "completed":0  
> > },  
> > "management":{  
> > "threads":1,  
> > "queue":0,  
> > "active":1,  
> > "rejected":0,  
> > "largest":5,  
> > "completed":1064  
> > },  
> > "refresh":{  
> > "threads":4,  
> > "queue":0,  
> > "active":0,  
> > "rejected":0,  
> > "largest":4,  
> > "completed":15126156  
> > }  
> > },  
> > "network":{  
> > "tcp":{  
> > "active\_opens":1513869,  
> > "passive\_opens":224206,  
> > "curr\_estab":52,  
> > "in\_segs":1929140868,  
> > "out\_segs":2005198310,  
> > "retrans\_segs":175635,  
> > "estab\_resets":37,  
> > "attempt\_fails":669,  
> > "in\_errs":3,  
> > "out\_rsts":88  
> > }  
> > },  
> > "fs":{  
> > "timestamp":1404880773912,  
> > "total":{  
> > "total\_in\_bytes":158534184960,  
> > "free\_in\_bytes":148425240576,  
> > "available\_in\_bytes":140372176896,  
> > "disk\_reads":1993,  
> > "disk\_writes":16068284,  
> > "disk\_io\_op":16070277,  
> > "disk\_read\_size\_in\_bytes":7445504,  
> > "disk\_write\_size\_in\_bytes":650206560256,  
> > "disk\_io\_size\_in\_bytes":650214005760  
> > },  
> > "data":[  
> > {  
> > "path":"/mnt/esdata/ab\_elastic/nodes/0",  
> > "mount":"/mnt/esdata",  
> > "dev":"/dev/xvdb",  
> > "total\_in\_bytes":158534184960,  
> > "free\_in\_bytes":148425240576,  
> > "available\_in\_bytes":140372176896,  
> > "disk\_reads":1993,  
> > "disk\_writes":16068284,  
> > "disk\_io\_op":16070277,  
> > "disk\_read\_size\_in\_bytes":7445504,  
> > "disk\_write\_size\_in\_bytes":650206560256,  
> > "disk\_io\_size\_in\_bytes":650214005760  
> > }  
> > ]  
> > },  
> > "transport":{  
> > "server\_open":26,  
> > "rx\_count":214082686,  
> > "rx\_size\_in\_bytes":148941344505,  
> > "tx\_count":209856841,  
> > "tx\_size\_in\_bytes":90356130749  
> > },  
> > "http":{  
> > "current\_open":0,  
> > "total\_opened":0  
> > },  
> > "fielddata\_breaker":{  
> > "maximum\_size\_in\_bytes":25714019532,  
> > "maximum\_size":"23.9gb",  
> > "estimated\_size\_in\_bytes":265401821,  
> > "estimated\_size":"253.1mb",  
> > "overhead":1.03  
> > }  
> > },  
> > "erc5l7dQai3mKBIX6sw":{  
> > "timestamp":1404880733754,  
> > "name":"Eddie Brock",  
> > "transport\_address":"removed",  
> > "host":"removed",  
> > "ip":[  
> > "removed",  
> > "NONE"  
> > ],  
> > "indices":{  
> > "docs":{  
> > "count":17793781,  
> > "deleted":529675  
> > },  
> > "store":{  
> > "size\_in\_bytes":9969074949,  
> > "throttle\_time\_in\_millis":9235478  
> > },  
> > "indexing":{  
> > "index\_total":3531955,  
> > "index\_time\_in\_millis":2783791,  
> > "index\_current":0,  
> > "delete\_total":0,  
> > "delete\_time\_in\_millis":0,  
> > "delete\_current":0  
> > },  
> > "get":{  
> > "total":2947020,  
> > "time\_in\_millis":217006,  
> > "exists\_total":2946589,  
> > "exists\_time\_in\_millis":216977,  
> > "missing\_total":431,  
> > "missing\_time\_in\_millis":29,  
> > "current":0  
> > },  
> > "search":{  
> > "open\_contexts":0,  
> > "query\_total":155416200,  
> > "query\_time\_in\_millis":1076023052,  
> > "query\_current":0,  
> > "fetch\_total":4101442,  
> > "fetch\_time\_in\_millis":2337321,  
> > "fetch\_current":0  
> > },  
> > "merges":{  
> > "current":0,  
> > "current\_docs":0,  
> > "current\_size\_in\_bytes":0,  
> > "total":563,  
> > "total\_time\_in\_millis":89211,  
> > "total\_docs":1099389,  
> > "total\_size\_in\_bytes":470013054  
> > },  
> > "refresh":{  
> > "total":15123966,  
> > "total\_time\_in\_millis":63368992  
> > },  
> > "flush":{  
> > "total":2335,  
> > "total\_time\_in\_millis":414790  
> > },  
> > "warmer":{  
> > "current":0,  
> > "total":2690030,  
> > "total\_time\_in\_millis":7339293  
> > },  
> > "filter\_cache":{  
> > "memory\_size\_in\_bytes":695354935,  
> > "evictions":1377260  
> > },  
> > "id\_cache":{  
> > "memory\_size\_in\_bytes":131817497  
> > },  
> > "fielddata":{  
> > "memory\_size\_in\_bytes":137907189,  
> > "evictions":0  
> > },  
> > "percolate":{  
> > "total":0,  
> > "time\_in\_millis":0,  
> > "current":0,  
> > "memory\_size\_in\_bytes":-1,  
> > "memory\_size":"-1b",  
> > "queries":0  
> > },  
> > "completion":{  
> > "size\_in\_bytes":0  
> > },  
> > "segments":{  
> > "count":282,  
> > "memory\_in\_bytes":217468955  
> > },  
> > "translog":{  
> > "operations":86,  
> > "size\_in\_bytes":0  
> > }  
> > },  
> > "os":{  
> > "timestamp":1404880733801,  
> > "uptime\_in\_millis":923467,  
> > "load\_average":[  
> > 3.71,  
> > 4.57,  
> > 4.46  
> > ],  
> > "cpu":{  
> > "sys":0,  
> > "user":46,  
> > "idle":52,  
> > "usage":46,  
> > "stolen":0  
> > },  
> > "mem":{  
> > "free\_in\_bytes":4438659072,  
> > "used\_in\_bytes":59720933376,  
> > "free\_percent":32,  
> > "used\_percent":67,  
> > "actual\_free\_in\_bytes":21124698112,  
> > "actual\_used\_in\_bytes":43034894336  
> > },  
> > "swap":{  
> > "used\_in\_bytes":0,  
> > "free\_in\_bytes":0  
> > }  
> > },  
> > "process":{  
> > "timestamp":1404880733801,  
> > "open\_file\_descriptors":301,  
> > "cpu":{  
> > "percent":377,  
> > "sys\_in\_millis":26524340,  
> > "user\_in\_millis":947578840,  
> > "total\_in\_millis":974103180  
> > },  
> > "mem":{  
> > "resident\_in\_bytes":33945767936,  
> > "share\_in\_bytes":692912128,  
> > "total\_virtual\_in\_bytes":47473528832  
> > }  
> > },  
> > "jvm":{  
> > "timestamp":1404880733802,  
> > "uptime\_in\_millis":310301938,  
> > "mem":{  
> > "heap\_used\_in\_bytes":10113402832,  
> > "heap\_used\_percent":31,  
> > "heap\_committed\_in\_bytes":32142524416,  
> > "heap\_max\_in\_bytes":32142524416,  
> > "non\_heap\_used\_in\_bytes":71762640,  
> > "non\_heap\_committed\_in\_bytes":106397696,  
> > "pools":{  
> > "young":{  
> > "used\_in\_bytes":218720808,  
> > "max\_in\_bytes":558432256,  
> > "peak\_used\_in\_bytes":558432256,  
> > "peak\_max\_in\_bytes":558432256  
> > },  
> > "survivor":{  
> > "used\_in\_bytes":9630920,  
> > "max\_in\_bytes":69730304,  
> > "peak\_used\_in\_bytes":69730304,  
> > "peak\_max\_in\_bytes":69730304  
> > },  
> > "old":{  
> > "used\_in\_bytes":9885051104,  
> > "max\_in\_bytes":31514361856,  
> > "peak\_used\_in\_bytes":23868903928,  
> > "peak\_max\_in\_bytes":31514361856  
> > }  
> > }  
> > },  
> > "threads":{  
> > "count":121,  
> > "peak\_count":144  
> > },  
> > "gc":{  
> > "collectors":{  
> > "young":{  
> > "collection\_count":209658,  
> > "collection\_time\_in\_millis":12089132  
> > },  
> > "old":{  
> > "collection\_count":11,  
> > "collection\_time\_in\_millis":6628  
> > }  
> > }  
> > },  
> > "buffer\_pools":{  
> > "direct":{  
> > "count":162,  
> > "used\_in\_bytes":195159665,  
> > "total\_capacity\_in\_bytes":195159665  
> > },  
> > "mapped":{  
> > "count":1292,  
> > "used\_in\_bytes":9897821921,  
> > "total\_capacity\_in\_bytes":9897821921  
> > }  
> > }  
> > },  
> > "thread\_pool":{  
> > "generic":{  
> > "threads":1,  
> > "queue":0,  
> > "active":0,  
> > "rejected":0,  
> > "largest":23,  
> > "completed":67214  
> > },  
> > "index":{  
> > "threads":8,  
> > "queue":0,  
> > "active":0,  
> > "rejected":0,  
> > "largest":8,  
> > "completed":3611159  
> > },  
> > "get":{  
> > "threads":0,  
> > "queue":0,  
> > "active":0,  
> > "rejected":0,  
> > "largest":0,  
> > "completed":0  
> > },  
> > "snapshot":{  
> > "threads":2,  
> > "queue":0,  
> > "active":0,  
> > "rejected":0,  
> > "largest":4,  
> > "completed":95672  
> > },  
> > "merge":{  
> > "threads":3,  
> > "queue":0,  
> > "active":0,  
> > "rejected":0,  
> > "largest":4,  
> > "completed":285955  
> > },  
> > "suggest":{  
> > "threads":0,  
> > "queue":0,  
> > "active":0,  
> > "rejected":0,  
> > "largest":0,  
> > "completed":0  
> > },  
> > "bulk":{  
> > "threads":8,  
> > "queue":0,  
> > "active":0,  
> > "rejected":0,  
> > "largest":8,  
> > "completed":1125  
> > },  
> > "optimize":{  
> > "threads":0,  
> > "queue":0,  
> > "active":0,  
> > "rejected":0,  
> > "largest":0,  
> > "completed":0  
> > },  
> > "warmer":{  
> > "threads":2,  
> > "queue":0,  
> > "active":0,  
> > "rejected":0,  
> > "largest":4,  
> > "completed":5487739  
> > },  
> > "flush":{  
> > "threads":1,  
> > "queue":0,  
> > "active":0,  
> > "rejected":0,  
> > "largest":2,  
> > "completed":2335  
> > },  
> > "search":{  
> > "threads":24,  
> > "queue":0,  
> > "active":0,  
> > "rejected":0,  
> > "largest":24,  
> > "completed":160510703  
> > },  
> > "percolate":{  
> > "threads":0,  
> > "queue":0,  
> > "active":0,  
> > "rejected":0,  
> > "largest":0,  
> > "completed":0  
> > },  
> > "management":{  
> > "threads":5,  
> > "queue":0,  
> > "active":1,  
> > "rejected":0,  
> > "largest":5,  
> > "completed":22893  
> > },  
> > "refresh":{  
> > "threads":4,  
> > "queue":0,  
> > "active":0,  
> > "rejected":0,  
> > "largest":4,  
> > "completed":15135553  
> > }  
> > },  
> > "network":{  
> > "tcp":{  
> > "active\_opens":1025664,  
> > "passive\_opens":105715571,  
> > "curr\_estab":55,  
> > "in\_segs":2557757673,  
> > "out\_segs":2502369803,  
> > "retrans\_segs":205765,  
> > "estab\_resets":68,  
> > "attempt\_fails":207,  
> > "in\_errs":0,  
> > "out\_rsts":1641  
> > }  
> > },  
> > "fs":{  
> > &qu
> > 
> > ...
> 
> --  
> You received this message because you are subscribed to the Google Groups  
> "elasticsearch" group.  
> To unsubscribe from this group and stop receiving emails from it, send an  
> email to [elasticsearch+unsubscribe@googlegroups.com](mailto:elasticsearch+unsubscribe@googlegroups.com).  
> To view this discussion on the web visit  
> [https://groups.google.com/d/msgid/elasticsearch/5140dad5-60e6-4a51-8986-c3f1c0da4c0f%40googlegroups.com](https://groups.google.com/d/msgid/elasticsearch/5140dad5-60e6-4a51-8986-c3f1c0da4c0f%40googlegroups.com)  
> [https://groups.google.com/d/msgid/elasticsearch/5140dad5-60e6-4a51-8986-c3f1c0da4c0f%40googlegroups.com?utm\_medium=email&utm\_source=footer](https://groups.google.com/d/msgid/elasticsearch/5140dad5-60e6-4a51-8986-c3f1c0da4c0f%40googlegroups.com?utm_medium=email&utm_source=footer)  
> .
> 
> For more options, visit [https://groups.google.com/d/optout](https://groups.google.com/d/optout).

--  
You received this message because you are subscribed to the Google Groups "elasticsearch" group.  
To unsubscribe from this group and stop receiving emails from it, send an email to [elasticsearch+unsubscribe@googlegroups.com](mailto:elasticsearch+unsubscribe@googlegroups.com).  
To view this discussion on the web visit [https://groups.google.com/d/msgid/elasticsearch/CALY%3DcQABy%3DV%3DG4wuKcqT3Bj6e6P4cDnuddS8\_%3DOMeA30MEt02g%40mail.gmail.com](https://groups.google.com/d/msgid/elasticsearch/CALY%3DcQABy%3DV%3DG4wuKcqT3Bj6e6P4cDnuddS8_%3DOMeA30MEt02g%40mail.gmail.com).  
For more options, visit [https://groups.google.com/d/optout](https://groups.google.com/d/optout).

---

<div class="post-metadata">

**Author:** ![webish](https://avatars.discourse-cdn.com/v4/letter/w/919ad9/32.png) [@webish](https://discuss.elastic.co/u/webish)\
**Post date:** [July 9, 2014, 7:32am UTC](https://discuss.elastic.co/t/elastic-search-nodes-servers-with-constant-high-cpu-utilization-in-idle-state/18544/5 "2014-07-09T07:32:14Z")

</div>

Definitely.

Here are all the captures thus far in gists.

_CLUSTER:_  
curl -XGET '[http://localhost:9200/\_nodes/stats](http://localhost:9200/_nodes/stats)'  
[curl -XGET 'http://localhost:9200/\_nodes/stats' · GitHub](https://gist.github.com/w3b1sh/a4759e5aa4efbe780fa7)

curl -XGET '[http://localhost:9200/](http://localhost:9200/)'  
[curl -XGET 'http://localhost:9200/' · GitHub](https://gist.github.com/w3b1sh/f7a727b1bae53772e56d)

_THREADS:_  
curl -XGET '[http://localhost:9200/\_nodes/hot\_threads](http://localhost:9200/_nodes/hot_threads)'  
[nodeA\> curl -XGET 'http://localhost:9200/\_nodes/hot\_threads' · GitHub](https://gist.github.com/w3b1sh/ea6a2b3fbfc837d5d9d8)

curl localhost:9200/\_cat/thread\_pool

> <https://gist.github.com/w3b1sh/778194c19ec3ed724f1e>

On Wednesday, July 9, 2014 3:11:25 AM UTC-4, Ivan Brusic wrote:

> Yes, please use gist/pastebin in the future.
> 
> Try using the hot threads API [1] to see if there are any threads that are  
> truly busy. If your system is truly idle, your thread pools should be  
> almost empty [2]. Your output above has only one management thread in use,  
> which could be simply the thread serving up the output.
> 
> [1]  
> [Elasticsearch Platform — Find real-time answers at scale | Elastic](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/cluster-nodes-hot-threads.html)  
> [2]  
> [Elasticsearch Platform — Find real-time answers at scale | Elastic](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/cat-thread-pool.html)
> 
> Cheers,
> 
> Ivan
> 
> On Tue, Jul 8, 2014 at 10:54 PM, webish \<[gre...@yoursports.com](mailto:gre...@yoursports.com)  
> \<javascript:\>\> wrote:
> 
> Thanks Mark. My mistake.
> 
> I can try switching to Oracle Java. There is no TTL. I've used Marvel  
> for development testing. Perhaps I can install the plugin...
> 
> On Wednesday, July 9, 2014 1:35:20 AM UTC-4, Mark Walkom wrote:
> 
> It's better to pop long output like that into a gist/pastebin, it makes it  
> easier to read the thread.  
> It's also worth installing a monitoring plugin like ElasticHQ or marvel,  
> as they provide graphical insight into what is happening and will  
> extrapolate some of the raw figures out.
> 
> If you can change to Oracle java you will get better performance. Are you  
> using TTL?
> 
> Regards,  
> Mark Walkom
> 
> Infrastructure Engineer  
> Campaign Monitor  
> email: [ma...@campaignmonitor.com](mailto:ma...@campaignmonitor.com)  
> web: [www.campaignmonitor.com](http://www.campaignmonitor.com)
> 
> On 9 July 2014 14:54, webish [gre...@yoursports.com](mailto:gre...@yoursports.com) wrote:
> 
> I'm seeing very high CPU utilization in an idle or close to idle state. I  
> haven't seen it use less than 40% on each node over the course of days.  
> I'm not certain how long it has been like this but could be weeks.
> 
> Any help resolving this would be greatly appreciated!
> 
> _Details:_
> 
> There are 2 nodes in the cluster.
> 
> AWS instances are:  
> vCPU ECUMemory (GiB) Instance Storage (GB) Linux/UNIX Usage
> 
> r3.2xlarge 82661 1 x 160 SSD
> 
> curl -XGET '[http://localhost:9200/](http://localhost:9200/)'  
> {  
> "status" : 200,  
> "name" : "Eddie Brock",  
> "version" : {  
> "number" : "1.1.0",  
> "build\_hash" : "2181e113dea80b4a9e31e58e9686658a2d46e363",  
> "build\_timestamp" : "2014-03-25T15:59:51Z",  
> "build\_snapshot" : false,  
> "lucene\_version" : "4.7"  
> },  
> "tagline" : "You Know, for Search"  
> }
> 
> % java -version  
> java version "1.7.0\_55"  
> OpenJDK Runtime Environment (IcedTea 2.4.7) (7u55-2.4.7-1ubuntu1~0.12.04.  
> 2)  
> OpenJDK 64-Bit Server VM (build 24.51-b03, mixed mode)
> 
> curl -XGET '[http://localhost:9200/\_nodes/stats](http://localhost:9200/_nodes/stats)'  
> {  
> "cluster\_name":"ab\_elastic",  
> "nodes":{  
> "kpHPVkBCyjOkRaSpS5Q":{  
> "timestamp":1404880773859,  
> "name":"Milan",  
> "transport\_address":"removed",  
> "host":"removed",  
> "ip":[  
> "removed",  
> "NONE"  
> ],  
> "indices":{  
> "docs":{  
> "count":17793781,  
> "deleted":466537  
> },  
> "store":{  
> "size\_in\_bytes":9902722189,  
> "throttle\_time\_in\_millis":8777423  
> },  
> "indexing":{  
> "index\_total":3548638,  
> "index\_time\_in\_millis":2610694,  
> "index\_current":1,  
> "delete\_total":0,  
> "delete\_time\_in\_millis":0,  
> "delete\_current":0  
> },  
> "get":{  
> "total":0,  
> "time\_in\_millis":0,  
> "exists\_total":0,  
> "exists\_time\_in\_millis":0,  
> "missing\_total":0,  
> "missing\_time\_in\_millis":0,  
> "current":0  
> },  
> "search":{  
> "open\_contexts":0,  
> "query\_total":155361213,  
> "query\_time\_in\_millis":817989500,  
> "query\_current":0,  
> "fetch\_total":4119309,  
> "fetch\_time\_in\_millis":1967525,  
> "fetch\_current":0  
> },  
> "merges":{  
> "current":0,  
> "current\_docs":0,  
> "current\_size\_in\_bytes":0,  
> "total":564,  
> "total\_time\_in\_millis":76065,  
> "total\_docs":1093579,  
> "total\_size\_in\_bytes":470304093  
> },  
> "refresh":{  
> "total":15117234,  
> "total\_time\_in\_millis":50581020  
> },  
> "flush":{  
> "total":2336,  
> "total\_time\_in\_millis":99980  
> },  
> "warmer":{  
> "current":0,  
> "total":2719640,  
> "total\_time\_in\_millis":5980882  
> },  
> "filter\_cache":{  
> "memory\_size\_in\_bytes":722232396,  
> "evictions":1179740  
> },  
> "id\_cache":{  
> "memory\_size\_in\_bytes":131672871  
> },  
> "fielddata":{  
> "memory\_size\_in\_bytes":133728950,  
> "evictions":0  
> },  
> "percolate":{  
> "total":0,  
> "time\_in\_millis":0,  
> "current":0,  
> "memory\_size\_in\_bytes":-1,  
> "memory\_size":"-1b",  
> "queries":0  
> },  
> "completion":{  
> "size\_in\_bytes":0  
> },  
> "segments":{  
> "count":278,  
> "memory\_in\_bytes":220799236  
> },  
> "translog":{  
> "operations":68,  
> "size\_in\_bytes":0  
> }  
> },  
> "os":{  
> "timestamp":1404880773910,  
> "uptime\_in\_millis":915746,  
> "load\_average":[  
> 2.68,  
> 3.17,  
> 3.35  
> ],  
> "cpu":{  
> "sys":0,  
> "user":38,  
> "idle":61,  
> "usage":38,  
> "stolen":0  
> },  
> "mem":{  
> "free\_in\_bytes":12301606912,  
> "used\_in\_bytes":51857985536,  
> "free\_percent":35,  
> "used\_percent":64,  
> "actual\_free\_in\_bytes":23005929472,  
> "actual\_used\_in\_bytes":41153662976  
> },  
> "swap":{  
> "used\_in\_bytes":0,  
> "free\_in\_bytes":0  
> }  
> },  
> "process":{  
> "timestamp":1404880773911,  
> "open\_file\_descriptors":300,  
> "cpu":{  
> "percent":313,  
> "sys\_in\_millis":14021810,  
> "user\_in\_millis":794249870,  
> "total\_in\_millis":808271680  
> },  
> "mem":{  
> "resident\_in\_bytes":32342851584,  
> "share\_in\_bytes":712302592,  
> "total\_virtual\_in\_bytes":47406919680  
> }  
> },  
> "jvm":{  
> "timestamp":1404880773911,  
> "uptime\_in\_millis":309661546,  
> "mem":{  
> "heap\_used\_in\_bytes":18409783280,  
> "heap\_used\_percent":57,  
> "heap\_committed\_in\_bytes":32142524416,  
> "heap\_max\_in\_bytes":32142524416,  
> "non\_heap\_used\_in\_bytes":65012920,  
> "non\_heap\_committed\_in\_bytes":98009088,  
> "pools":{  
> "young":{  
> "used\_in\_bytes":538302112,  
> "max\_in\_bytes":558432256,  
> "peak\_used\_in\_bytes":558432256,  
> "peak\_max\_in\_bytes":558432256  
> },  
> "survivor":{  
> "used\_in\_bytes":11403704,  
> "max\_in\_bytes":69730304,  
> "peak\_used\_in\_bytes":69730304,  
> "peak\_max\_in\_bytes":69730304  
> },  
> "old":{  
> "used\_in\_bytes":17860077464,  
> "max\_in\_bytes":31514361856,  
> "peak\_used\_in\_bytes":23694417088,  
> "peak\_max\_in\_bytes":31514361856  
> }  
> }  
> },  
> "threads":{  
> "count":117,  
> "peak\_count":133  
> },  
> "gc":{  
> "collectors":{  
> "young":{  
> "collection\_count":200644,  
> "collection\_time\_in\_millis":10870850  
> },  
> "old":{  
> "collection\_count":9,  
> "collection\_time\_in\_millis":2987  
> }  
> }  
> },  
> "buffer\_pools":{  
> "direct":{  
> "count":154,  
> "used\_in\_bytes":78176363,  
> "total\_capacity\_in\_bytes":78176363  
> },  
> "mapped":{  
> "count":1293,  
> "used\_in\_bytes":9831615457,  
> "total\_capacity\_in\_bytes":9831615457  
> }  
> }  
> },  
> "thread\_pool":{  
> "generic":{  
> "threads":1,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":14,  
> "completed":56826  
> },  
> "index":{  
> "threads":8,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":8,  
> "completed":3607948  
> },  
> "get":{  
> "threads":0,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":0,  
> "completed":0  
> },  
> "snapshot":{  
> "threads":4,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":4,  
> "completed":95697  
> },  
> "merge":{  
> "threads":2,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":4,  
> "completed":282257  
> },  
> "suggest":{  
> "threads":0,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":0,  
> "completed":0  
> },  
> "bulk":{  
> "threads":8,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":8,  
> "completed":1081  
> },  
> "optimize":{  
> "threads":0,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":0,  
> "completed":0  
> },  
> "warmer":{  
> "threads":2,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":4,  
> "completed":5556773  
> },  
> "flush":{  
> "threads":1,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":2,  
> "completed":2336  
> },  
> "search":{  
> "threads":24,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":24,  
> "completed":160139043  
> },  
> "percolate":{  
> "threads":0,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":0,  
> "completed":0  
> },  
> "management":{  
> "threads":1,  
> "queue":0,  
> "active":1,  
> "rejected":0,  
> "largest":5,  
> "completed":1064  
> },  
> "refresh":{  
> "threads":4,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":4,  
> "completed":15126156  
> }  
> },  
> "network":{  
> "tcp":{  
> "active\_opens":1513869,  
> "passive\_opens":224206,  
> "curr\_estab":52,  
> "in\_segs":1929140868,  
> "out\_segs":2005198310,  
> "retrans\_segs":175635,  
> "estab\_resets":37,  
> "attempt\_fails":669,  
> "in\_errs":3,  
> "out\_rsts":88  
> }  
> },  
> "fs":{  
> "timestamp":1404880773912,  
> "total":{  
> "total\_in\_bytes":158534184960,  
> "free\_in\_bytes":148425240576,  
> "available\_in\_bytes":140372176896,  
> "disk\_reads":1993,  
> "disk\_writes":16068284,  
> "disk\_io\_op":16070277,  
> "disk\_read\_size\_in\_bytes":7445504,  
> "disk\_write\_size\_in\_bytes":650206560256,  
> "disk\_io\_size\_in\_bytes":650214005760  
> },  
> "data":[  
> {  
> "path":"/mnt/esdata/ab\_elastic/nodes/0",  
> "mount":"/mnt/esdata",  
> "dev":"/dev/xvdb",  
> "total\_in\_bytes":158534184960,  
> "free\_in\_bytes":148425240576,  
> "available\_in\_bytes":140372176896,  
> "disk\_reads":1993,  
> "disk\_writes":16068284,  
> "disk\_io\_op":16070277,  
> "disk\_read\_size\_in\_bytes":7445504,  
> "disk\_write\_size\_in\_bytes":650206560256,  
> "disk\_io\_size\_in\_bytes":650214005760  
> }  
> ]  
> },  
> "transport":{  
> "server\_open":26,  
> "rx\_count":214082686,  
> "rx\_size\_in\_bytes":148941344505,  
> "tx\_count":209856841,  
> "tx\_size\_in\_bytes":90356130749  
> },  
> "http":{  
> "current\_open":0,  
> "total\_opened":0  
> },  
> "fielddata\_breaker":{  
> "maximum\_size\_in\_bytes":25714019532,  
> "maximum\_size":"23.9gb",  
> "estimated\_size\_in\_bytes":265401821,  
> "estimated\_size":"253.1mb",  
> "overhead":1.03  
> }  
> },  
> "erc5l7dQai3mKBIX6sw":{  
> "timestamp":1404880733754,  
> "name":"Eddie Brock",  
> "transport\_address":"removed",  
> "host":"removed",  
> "ip":[  
> "removed",  
> "NONE"  
> ],  
> "indices":{  
> "docs":{  
> "count":17793781,  
> "deleted":529675  
> },  
> "store":{  
> "size\_in\_bytes":9969074949,  
> "throttle\_time\_in\_millis":9235478  
> },  
> "indexing":{  
> "index\_total":3531955,  
> "index\_time\_in\_millis":2783791,  
> "index\_current":0,  
> "delete\_total":0,  
> "delete\_time\_in\_millis":0,  
> "delete\_current":0  
> },  
> "get":{  
> "total":2947020,  
> "time\_in\_millis":217006,  
> "exists\_total":2946589,  
> "exists\_time\_in\_millis":216977,  
> "missing\_total":431,  
> "missing\_time\_in\_millis":29,  
> "current":0  
> },  
> "search":{  
> "open\_contexts":0,  
> "query\_total":155416200,  
> "query\_time\_in\_millis":1076023052,  
> "query\_current":0,  
> "fetch\_total":4101442,  
> "fetch\_time\_in\_millis":2337321,  
> "fetch\_current":0  
> },  
> "merges":{  
> "current":0,  
> "current\_docs":0,  
> "current\_size\_in\_bytes":0,  
> "total":563,  
> "total\_time\_in\_millis":89211,  
> "total\_docs":1099389,  
> "total\_size\_in\_bytes":470013054  
> },  
> "refresh":{  
> "total":15123966,  
> "total\_time\_in\_millis":63368992  
> },  
> "flush":{  
> "total":2335,  
> "total\_time\_in\_millis":414790  
> },  
> "warmer":{  
> "current":0,  
> "total":2690030,  
> "total\_time\_in\_millis":7339293  
> },  
> "filter\_cache":{  
> "memory\_size\_in\_bytes":695354935,  
> "evictions":1377260  
> },  
> "id\_cache":{  
> "memory\_size\_in\_bytes":131817497  
> },  
> "fielddata":{  
> "memory\_size\_in\_bytes":137907189,  
> "evictions":0  
> },  
> "percolate":{  
> "total":0,  
> "time\_in\_millis":0,  
> "current":0,  
> "memory\_size\_in\_bytes":-1,  
> "memory\_size":"-1b",  
> "queries":0  
> },  
> "completion":{  
> "size\_in\_bytes":0  
> },  
> "segments":{  
> "count":282,  
> "memory\_in\_bytes":217468955  
> },  
> "translog":{  
> "operations":86,  
> "size\_in\_bytes":0  
> }  
> },  
> "os":{  
> "timestamp":1404880733801,  
> "uptime\_in\_millis":923467,  
> "load\_average":[  
> 3.71,  
> 4.57,  
> 4.46  
> ],  
> "cpu":{  
> "sys":0,  
> "user":46,  
> "idle":52,  
> "usage":46,  
> "stolen":0  
> },  
> "mem":{  
> "free\_in\_bytes":4438659072,  
> "used\_in\_bytes":59720933376,  
> "free\_percent":32,  
> "used\_percent":67,  
> "actual\_free\_in\_bytes":21124698112,  
> "actual\_used\_in\_bytes":43034894336  
> },  
> "swap":{  
> "used\_in\_bytes":0,  
> "free\_in\_bytes":0  
> }  
> },  
> "process":{  
> "timestamp":1404880733801,  
> "open\_file\_descriptors":301,  
> "cpu":{  
> "percent":377,  
> "sys\_in\_millis":26524340,  
> "user\_in\_millis":947578840,  
> "total\_in\_millis":974103180  
> },  
> "mem":{  
> "resident\_in\_bytes":33945767936,  
> "share\_in\_bytes":692912128,  
> "total\_virtual\_in\_bytes":47473528832  
> }  
> },  
> "jvm":{  
> "timestamp":1404880733802,  
> "uptime\_in\_millis":310301938,  
> "mem":{  
> "heap\_used\_in\_bytes":10113402832,  
> "heap\_used\_percent":31,  
> "heap\_committed\_in\_bytes":32142524416,  
> "heap\_max\_in\_bytes":32142524416,  
> "non\_heap\_used\_in\_bytes":71762640,  
> "non\_heap\_committed\_in\_bytes":106397696,  
> "pools":{  
> "young":{  
> "used\_in\_bytes":218720808,  
> "max\_in\_bytes":558432256,  
> "peak\_used\_in\_bytes":558432256,  
> "peak\_max\_in\_bytes":558432256  
> },  
> "survivor":{  
> "used\_in\_bytes":9630920,  
> "max\_in\_bytes":69730304,  
> "peak\_used\_in\_bytes":69730304,  
> "peak\_max\_in\_bytes":69730304  
> },  
> "old":{  
> "used\_in\_bytes":9885051104,  
> "max\_in\_bytes":31514361856,  
> "peak\_used\_in\_bytes":23868903928,  
> "peak\_max\_in\_bytes":31514361856  
> }  
> }  
> },  
> "threads":{  
> "count":121,  
> "peak\_count":144  
> },  
> "gc":{  
> "collectors":{  
> "young":{  
> "collection\_count":209658,  
> "collection\_time\_in\_millis":12089132  
> },  
> "old":{  
> "collection\_count":11,  
> "collection\_time\_in\_millis":6628  
> }  
> }  
> },  
> "buffer\_pools":{  
> "direct":{  
> "count":162,  
> "used\_in\_bytes":195159665,  
> "total\_capacity\_in\_bytes":195159665  
> },  
> "mapped":{  
> "count":1292,  
> "used\_in\_bytes":9897821921,  
> "total\_capacity\_in\_bytes":9897821921  
> }  
> }  
> },  
> "thread\_pool":{  
> "generic":{  
> "threads":1,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":23,  
> "completed":67214  
> },  
> "index":{  
> "threads":8,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":8,  
> "completed":3611159  
> },  
> "get":{  
> "threads":0,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":0,  
> "completed":0  
> },  
> "snapshot":{  
> "threads":2,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":4,  
> "completed":95672  
> },  
> "merge":{  
> "threads":3,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":4,  
> "completed":285955  
> },  
> "suggest":{  
> "threads":0,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":0,  
> "completed":0  
> },  
> "bulk":{  
> "threads":8,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":8,  
> "completed":1125  
> },  
> "optimize":{  
> "threads":0,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":0,  
> "completed":0  
> },  
> "warmer":{  
> "threads":2,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":4,  
> "completed":5487739  
> },  
> "flush":{  
> "threads":1,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":2,  
> "completed":2335  
> },  
> "search":{
> 
> ...

--  
You received this message because you are subscribed to the Google Groups "elasticsearch" group.  
To unsubscribe from this group and stop receiving emails from it, send an email to [elasticsearch+unsubscribe@googlegroups.com](mailto:elasticsearch+unsubscribe@googlegroups.com).  
To view this discussion on the web visit [https://groups.google.com/d/msgid/elasticsearch/ed76cee8-1c01-43b4-810a-1b203a80ad04%40googlegroups.com](https://groups.google.com/d/msgid/elasticsearch/ed76cee8-1c01-43b4-810a-1b203a80ad04%40googlegroups.com).  
For more options, visit [https://groups.google.com/d/optout](https://groups.google.com/d/optout).

---

<div class="post-metadata">

**Author:** ![warkolm](https://sea2.discourse-cdn.com/elastic/user_avatar/discuss.elastic.co/warkolm/32/39224_2.png) [@warkolm](https://discuss.elastic.co/u/warkolm)\
**Post date:** [July 9, 2014, 7:36am UTC](https://discuss.elastic.co/t/elastic-search-nodes-servers-with-constant-high-cpu-utilization-in-idle-state/18544/6 "2014-07-09T07:36:20Z")

</div>

The output of hot threads is only from the one node, ie requesting it isn't  
at a cluster level, if you can add the other node it'd help.

Regards,  
Mark Walkom

Infrastructure Engineer  
Campaign Monitor  
email: [markw@campaignmonitor.com](mailto:markw@campaignmonitor.com)  
web: [www.campaignmonitor.com](http://www.campaignmonitor.com)

On 9 July 2014 17:32, webish [gregory@yoursports.com](mailto:gregory@yoursports.com) wrote:

> Definitely.
> 
> Here are all the captures thus far in gists.
> 
> _CLUSTER:_  
> curl -XGET '[http://localhost:9200/\_nodes/stats](http://localhost:9200/_nodes/stats)'  
> [curl -XGET 'http://localhost:9200/\_nodes/stats' · GitHub](https://gist.github.com/w3b1sh/a4759e5aa4efbe780fa7)
> 
> curl -XGET '[http://localhost:9200/](http://localhost:9200/)'  
> [curl -XGET 'http://localhost:9200/' · GitHub](https://gist.github.com/w3b1sh/f7a727b1bae53772e56d)
> 
> _THREADS:_  
> curl -XGET '[http://localhost:9200/\_nodes/hot\_threads](http://localhost:9200/_nodes/hot_threads)'  
> [nodeA\> curl -XGET 'http://localhost:9200/\_nodes/hot\_threads' · GitHub](https://gist.github.com/w3b1sh/ea6a2b3fbfc837d5d9d8)
> 
> curl localhost:9200/\_cat/thread\_pool  
> [curl localhost:9200/\_cat/thread\_pool · GitHub](https://gist.github.com/w3b1sh/778194c19ec3ed724f1e)
> 
> On Wednesday, July 9, 2014 3:11:25 AM UTC-4, Ivan Brusic wrote:
> 
> > Yes, please use gist/pastebin in the future.
> > 
> > Try using the hot threads API [1] to see if there are any threads that  
> > are truly busy. If your system is truly idle, your thread pools should be  
> > almost empty [2]. Your output above has only one management thread in use,  
> > which could be simply the thread serving up the output.
> > 
> > [1] [Elasticsearch Platform — Find real-time answers at scale | Elastic](http://www.elasticsearch.org/guide/en/elasticsearch/)  
> > reference/current/cluster-nodes-hot-threads.html  
> > [2] [Elasticsearch Platform — Find real-time answers at scale | Elastic](http://www.elasticsearch.org/guide/en/elasticsearch/)  
> > reference/current/cat-thread-pool.html
> > 
> > Cheers,
> > 
> > Ivan
> > 
> > On Tue, Jul 8, 2014 at 10:54 PM, webish [gre...@yoursports.com](mailto:gre...@yoursports.com) wrote:
> > 
> > Thanks Mark. My mistake.
> > 
> > I can try switching to Oracle Java. There is no TTL. I've used Marvel  
> > for development testing. Perhaps I can install the plugin...
> > 
> > On Wednesday, July 9, 2014 1:35:20 AM UTC-4, Mark Walkom wrote:
> > 
> > It's better to pop long output like that into a gist/pastebin, it makes  
> > it easier to read the thread.  
> > It's also worth installing a monitoring plugin like ElasticHQ or marvel,  
> > as they provide graphical insight into what is happening and will  
> > extrapolate some of the raw figures out.
> > 
> > If you can change to Oracle java you will get better performance. Are you  
> > using TTL?
> > 
> > Regards,  
> > Mark Walkom
> > 
> > Infrastructure Engineer  
> > Campaign Monitor  
> > email: [ma...@campaignmonitor.com](mailto:ma...@campaignmonitor.com)  
> > web: [www.campaignmonitor.com](http://www.campaignmonitor.com)
> > 
> > On 9 July 2014 14:54, webish [gre...@yoursports.com](mailto:gre...@yoursports.com) wrote:
> > 
> > I'm seeing very high CPU utilization in an idle or close to idle state.  
> > I haven't seen it use less than 40% on each node over the course of days.  
> > I'm not certain how long it has been like this but could be weeks.
> > 
> > Any help resolving this would be greatly appreciated!
> > 
> > _Details:_
> > 
> > There are 2 nodes in the cluster.
> > 
> > AWS instances are:  
> > vCPU ECUMemory (GiB) Instance Storage (GB) Linux/UNIX Usage
> > 
> > r3.2xlarge 82661 1 x 160 SSD
> > 
> > curl -XGET '[http://localhost:9200/](http://localhost:9200/)'  
> > {  
> > "status" : 200,  
> > "name" : "Eddie Brock",  
> > "version" : {  
> > "number" : "1.1.0",  
> > "build\_hash" : "2181e113dea80b4a9e31e58e9686658a2d46e363",  
> > "build\_timestamp" : "2014-03-25T15:59:51Z",  
> > "build\_snapshot" : false,  
> > "lucene\_version" : "4.7"  
> > },  
> > "tagline" : "You Know, for Search"  
> > }
> > 
> > % java -version  
> > java version "1.7.0\_55"  
> > OpenJDK Runtime Environment (IcedTea 2.4.7) (7u55-2.4.7-1ubuntu1~0.12.04.  
> > 2)  
> > OpenJDK 64-Bit Server VM (build 24.51-b03, mixed mode)
> > 
> > curl -XGET '[http://localhost:9200/\_nodes/stats](http://localhost:9200/_nodes/stats)'  
> > {  
> > "cluster\_name":"ab\_elastic",  
> > "nodes":{  
> > "kpHPVkBCyjOkRaSpS5Q":{  
> > "timestamp":1404880773859,  
> > "name":"Milan",  
> > "transport\_address":"removed",  
> > "host":"removed",  
> > "ip":[  
> > "removed",  
> > "NONE"  
> > ],  
> > "indices":{  
> > "docs":{  
> > "count":17793781,  
> > "deleted":466537  
> > },  
> > "store":{  
> > "size\_in\_bytes":9902722189,  
> > "throttle\_time\_in\_millis":8777423  
> > },  
> > "indexing":{  
> > "index\_total":3548638,  
> > "index\_time\_in\_millis":2610694,  
> > "index\_current":1,  
> > "delete\_total":0,  
> > "delete\_time\_in\_millis":0,  
> > "delete\_current":0  
> > },  
> > "get":{  
> > "total":0,  
> > "time\_in\_millis":0,  
> > "exists\_total":0,  
> > "exists\_time\_in\_millis":0,  
> > "missing\_total":0,  
> > "missing\_time\_in\_millis":0,  
> > "current":0  
> > },  
> > "search":{  
> > "open\_contexts":0,  
> > "query\_total":155361213,  
> > "query\_time\_in\_millis":817989500,  
> > "query\_current":0,  
> > "fetch\_total":4119309,  
> > "fetch\_time\_in\_millis":1967525,  
> > "fetch\_current":0  
> > },  
> > "merges":{  
> > "current":0,  
> > "current\_docs":0,  
> > "current\_size\_in\_bytes":0,  
> > "total":564,  
> > "total\_time\_in\_millis":76065,  
> > "total\_docs":1093579,  
> > "total\_size\_in\_bytes":470304093  
> > },  
> > "refresh":{  
> > "total":15117234,  
> > "total\_time\_in\_millis":50581020  
> > },  
> > "flush":{  
> > "total":2336,  
> > "total\_time\_in\_millis":99980  
> > },  
> > "warmer":{  
> > "current":0,  
> > "total":2719640,  
> > "total\_time\_in\_millis":5980882  
> > },  
> > "filter\_cache":{  
> > "memory\_size\_in\_bytes":722232396,  
> > "evictions":1179740  
> > },  
> > "id\_cache":{  
> > "memory\_size\_in\_bytes":131672871  
> > },  
> > "fielddata":{  
> > "memory\_size\_in\_bytes":133728950,  
> > "evictions":0  
> > },  
> > "percolate":{  
> > "total":0,  
> > "time\_in\_millis":0,  
> > "current":0,  
> > "memory\_size\_in\_bytes":-1,  
> > "memory\_size":"-1b",  
> > "queries":0  
> > },  
> > "completion":{  
> > "size\_in\_bytes":0  
> > },  
> > "segments":{  
> > "count":278,  
> > "memory\_in\_bytes":220799236  
> > },  
> > "translog":{  
> > "operations":68,  
> > "size\_in\_bytes":0  
> > }  
> > },  
> > "os":{  
> > "timestamp":1404880773910,  
> > "uptime\_in\_millis":915746,  
> > "load\_average":[  
> > 2.68,  
> > 3.17,  
> > 3.35  
> > ],  
> > "cpu":{  
> > "sys":0,  
> > "user":38,  
> > "idle":61,  
> > "usage":38,  
> > "stolen":0  
> > },  
> > "mem":{  
> > "free\_in\_bytes":12301606912,  
> > "used\_in\_bytes":51857985536,  
> > "free\_percent":35,  
> > "used\_percent":64,  
> > "actual\_free\_in\_bytes":23005929472,  
> > "actual\_used\_in\_bytes":41153662976  
> > },  
> > "swap":{  
> > "used\_in\_bytes":0,  
> > "free\_in\_bytes":0  
> > }  
> > },  
> > "process":{  
> > "timestamp":1404880773911,  
> > "open\_file\_descriptors":300,  
> > "cpu":{  
> > "percent":313,  
> > "sys\_in\_millis":14021810,  
> > "user\_in\_millis":794249870,  
> > "total\_in\_millis":808271680  
> > },  
> > "mem":{  
> > "resident\_in\_bytes":32342851584,  
> > "share\_in\_bytes":712302592,  
> > "total\_virtual\_in\_bytes":47406919680  
> > }  
> > },  
> > "jvm":{  
> > "timestamp":1404880773911,  
> > "uptime\_in\_millis":309661546,  
> > "mem":{  
> > "heap\_used\_in\_bytes":18409783280,  
> > "heap\_used\_percent":57,  
> > "heap\_committed\_in\_bytes":32142524416,  
> > "heap\_max\_in\_bytes":32142524416,  
> > "non\_heap\_used\_in\_bytes":65012920,  
> > "non\_heap\_committed\_in\_bytes":98009088,  
> > "pools":{  
> > "young":{  
> > "used\_in\_bytes":538302112,  
> > "max\_in\_bytes":558432256,  
> > "peak\_used\_in\_bytes":558432256,  
> > "peak\_max\_in\_bytes":558432256  
> > },  
> > "survivor":{  
> > "used\_in\_bytes":11403704,  
> > "max\_in\_bytes":69730304,  
> > "peak\_used\_in\_bytes":69730304,  
> > "peak\_max\_in\_bytes":69730304  
> > },  
> > "old":{  
> > "used\_in\_bytes":17860077464,  
> > "max\_in\_bytes":31514361856,  
> > "peak\_used\_in\_bytes":23694417088,  
> > "peak\_max\_in\_bytes":31514361856  
> > }  
> > }  
> > },  
> > "threads":{  
> > "count":117,  
> > "peak\_count":133  
> > },  
> > "gc":{  
> > "collectors":{  
> > "young":{  
> > "collection\_count":200644,  
> > "collection\_time\_in\_millis":10870850  
> > },  
> > "old":{  
> > "collection\_count":9,  
> > "collection\_time\_in\_millis":2987  
> > }  
> > }  
> > },  
> > "buffer\_pools":{  
> > "direct":{  
> > "count":154,  
> > "used\_in\_bytes":78176363,  
> > "total\_capacity\_in\_bytes":78176363  
> > },  
> > "mapped":{  
> > "count":1293,  
> > "used\_in\_bytes":9831615457,  
> > "total\_capacity\_in\_bytes":9831615457  
> > }  
> > }  
> > },  
> > "thread\_pool":{  
> > "generic":{  
> > "threads":1,  
> > "queue":0,  
> > "active":0,  
> > "rejected":0,  
> > "largest":14,  
> > "completed":56826  
> > },  
> > "index":{  
> > "threads":8,  
> > "queue":0,  
> > "active":0,  
> > "rejected":0,  
> > "largest":8,  
> > "completed":3607948  
> > },  
> > "get":{  
> > "threads":0,  
> > "queue":0,  
> > "active":0,  
> > "rejected":0,  
> > "largest":0,  
> > "completed":0  
> > },  
> > "snapshot":{  
> > "threads":4,  
> > "queue":0,  
> > "active":0,  
> > "rejected":0,  
> > "largest":4,  
> > "completed":95697  
> > },  
> > "merge":{  
> > "threads":2,  
> > "queue":0,  
> > "active":0,  
> > "rejected":0,  
> > "largest":4,  
> > "completed":282257  
> > },  
> > "suggest":{  
> > "threads":0,  
> > "queue":0,  
> > "active":0,  
> > "rejected":0,  
> > "largest":0,  
> > "completed":0  
> > },  
> > "bulk":{  
> > "threads":8,  
> > "queue":0,  
> > "active":0,  
> > "rejected":0,  
> > "largest":8,  
> > "completed":1081  
> > },  
> > "optimize":{  
> > "threads":0,  
> > "queue":0,  
> > "active":0,  
> > "rejected":0,  
> > "largest":0,  
> > "completed":0  
> > },  
> > "warmer":{  
> > "threads":2,  
> > "queue":0,  
> > "active":0,  
> > "rejected":0,  
> > "largest":4,  
> > "completed":5556773  
> > },  
> > "flush":{  
> > "threads":1,  
> > "queue":0,  
> > "active":0,  
> > "rejected":0,  
> > "largest":2,  
> > "completed":2336  
> > },  
> > "search":{  
> > "threads":24,  
> > "queue":0,  
> > "active":0,  
> > "rejected":0,  
> > "largest":24,  
> > "completed":160139043  
> > },  
> > "percolate":{  
> > "threads":0,  
> > "queue":0,  
> > "active":0,  
> > "rejected":0,  
> > "largest":0,  
> > "completed":0  
> > },  
> > "management":{  
> > "threads":1,  
> > "queue":0,  
> > "active":1,  
> > "rejected":0,  
> > "largest":5,  
> > "completed":1064  
> > },  
> > "refresh":{  
> > "threads":4,  
> > "queue":0,  
> > "active":0,  
> > "rejected":0,  
> > "largest":4,  
> > "completed":15126156  
> > }  
> > },  
> > "network":{  
> > "tcp":{  
> > "active\_opens":1513869,  
> > "passive\_opens":224206,  
> > "curr\_estab":52,  
> > "in\_segs":1929140868,  
> > "out\_segs":2005198310,  
> > "retrans\_segs":175635,  
> > "estab\_resets":37,  
> > "attempt\_fails":669,  
> > "in\_errs":3,  
> > "out\_rsts":88  
> > }  
> > },  
> > "fs":{  
> > "timestamp":1404880773912,  
> > "total":{  
> > "total\_in\_bytes":158534184960,  
> > "free\_in\_bytes":148425240576,  
> > "available\_in\_bytes":140372176896,  
> > "disk\_reads":1993,  
> > "disk\_writes":16068284,  
> > "disk\_io\_op":16070277,  
> > "disk\_read\_size\_in\_bytes":7445504,  
> > "disk\_write\_size\_in\_bytes":650206560256,  
> > "disk\_io\_size\_in\_bytes":650214005760  
> > },  
> > "data":[  
> > {  
> > "path":"/mnt/esdata/ab\_elastic/nodes/0",  
> > "mount":"/mnt/esdata",  
> > "dev":"/dev/xvdb",  
> > "total\_in\_bytes":158534184960,  
> > "free\_in\_bytes":148425240576,  
> > "available\_in\_bytes":140372176896,  
> > "disk\_reads":1993,  
> > "disk\_writes":16068284,  
> > "disk\_io\_op":16070277,  
> > "disk\_read\_size\_in\_bytes":7445504,  
> > "disk\_write\_size\_in\_bytes":650206560256,  
> > "disk\_io\_size\_in\_bytes":650214005760  
> > }  
> > ]  
> > },  
> > "transport":{  
> > "server\_open":26,  
> > "rx\_count":214082686,  
> > "rx\_size\_in\_bytes":148941344505,  
> > "tx\_count":209856841,  
> > "tx\_size\_in\_bytes":90356130749  
> > },  
> > "http":{  
> > "current\_open":0,  
> > "total\_opened":0  
> > },  
> > "fielddata\_breaker":{  
> > "maximum\_size\_in\_bytes":25714019532,  
> > "maximum\_size":"23.9gb",  
> > "estimated\_size\_in\_bytes":265401821,  
> > "estimated\_size":"253.1mb",  
> > "overhead":1.03  
> > }  
> > },  
> > "erc5l7dQai3mKBIX6sw":{  
> > "timestamp":1404880733754,  
> > "name":"Eddie Brock",  
> > "transport\_address":"removed",  
> > "host":"removed",  
> > "ip":[  
> > "removed",  
> > "NONE"  
> > ],  
> > "indices":{  
> > "docs":{  
> > "count":17793781,  
> > "deleted":529675  
> > },  
> > "store":{  
> > "size\_in\_bytes":9969074949,  
> > "throttle\_time\_in\_millis":9235478  
> > },  
> > "indexing":{  
> > "index\_total":3531955,  
> > "index\_time\_in\_millis":2783791,  
> > "index\_current":0,  
> > "delete\_total":0,  
> > "delete\_time\_in\_millis":0,  
> > "delete\_current":0  
> > },  
> > "get":{  
> > "total":2947020,  
> > "time\_in\_millis":217006,  
> > "exists\_total":2946589,  
> > "exists\_time\_in\_millis":216977,  
> > "missing\_total":431,  
> > "missing\_time\_in\_millis":29,  
> > "current":0  
> > },  
> > "search":{  
> > "open\_contexts":0,  
> > "query\_total":155416200,  
> > "query\_time\_in\_millis":1076023052,  
> > "query\_current":0,  
> > "fetch\_total":4101442,  
> > "fetch\_time\_in\_millis":2337321,  
> > "fetch\_current":0  
> > },  
> > "merges":{  
> > "current":0,  
> > "current\_docs":0,  
> > "current\_size\_in\_bytes":0,  
> > "total":563,  
> > "total\_time\_in\_millis":89211,  
> > "total\_docs":1099389,  
> > "total\_size\_in\_bytes":470013054  
> > },  
> > "refresh":{  
> > "total":15123966,  
> > "total\_time\_in\_millis":63368992  
> > },  
> > "flush":{  
> > "total":2335,  
> > "total\_time\_in\_millis":414790  
> > },  
> > "warmer":{  
> > "current":0,  
> > "total":2690030,  
> > "total\_time\_in\_millis":7339293  
> > },  
> > "filter\_cache":{  
> > "memory\_size\_in\_bytes":695354935,  
> > "evictions":1377260  
> > },  
> > "id\_cache":{  
> > "memory\_size\_in\_bytes":131817497  
> > },  
> > "fielddata":{  
> > "memory\_size\_in\_bytes":137907189,  
> > "evictions":0  
> > },  
> > "percolate":{  
> > "total":0,  
> > "time\_in\_millis":0,  
> > "current":0,  
> > "memory\_size\_in\_bytes":-1,  
> > "memory\_size":"-1b",  
> > "queries":0  
> > },  
> > "completion":{  
> > "size\_in\_bytes":0  
> > },  
> > "segments":{  
> > "count":282,  
> > "memory\_in\_bytes":217468955  
> > },  
> > "translog":{  
> > "operations":86,  
> > "size\_in\_bytes":0  
> > }  
> > },  
> > "os":{  
> > "timestamp":1404880733801,  
> > "uptime\_in\_millis":923467,  
> > "load\_average":[  
> > 3.71,  
> > 4.57,  
> > 4.46  
> > ],  
> > "cpu":{  
> > "sys":0,  
> > "user":46,  
> > "idle":52,  
> > "usage":46,  
> > "stolen":0  
> > },  
> > "mem":{  
> > "free\_in\_bytes":4438659072,  
> > "used\_in\_bytes":59720933376,  
> > "free\_percent":32,  
> > "used\_percent":67,  
> > "actual\_free\_in\_bytes":21124698112,  
> > "actual\_used\_in\_bytes":43034894336  
> > },  
> > "swap":{  
> > "used\_in\_bytes":0,  
> > "free\_in\_bytes":0  
> > }  
> > },  
> > "process":{  
> > "timestamp":1404880733801,  
> > "open\_file\_descriptors":301,  
> > "cpu":{  
> > "percent":377,  
> > "sys\_in\_millis":26524340,  
> > "user\_in\_millis":947578840,  
> > "total\_in\_millis":974103180  
> > },  
> > "mem":{  
> > "resident\_in\_bytes":33945767936,  
> > "share\_in\_bytes":692912128,  
> > "total\_virtual\_in\_bytes":47473528832  
> > }  
> > },  
> > "jvm":{  
> > "timestamp":1404880733802,  
> > "uptime\_in\_millis":310301938,  
> > "mem":{  
> > "heap\_used\_in\_bytes":10113402832,  
> > "heap\_used\_percent":31,  
> > "heap\_committed\_in\_bytes":32142524416,  
> > "heap\_max\_in\_bytes":32142524416,  
> > "non\_heap\_used\_in\_bytes":71762640,  
> > "non\_heap\_committed\_in\_bytes":106397696,  
> > "pools":{  
> > "young":{  
> > "used\_in\_bytes":218720808,  
> > "max\_in\_bytes":558432256,  
> > "peak\_used\_in\_bytes":558432256,  
> > "peak\_max\_in\_bytes":558432256  
> > },  
> > "survivor":{  
> > "used\_in\_bytes":9630920,  
> > "max\_in\_bytes":69730304,  
> > "peak\_used\_in\_bytes":69730304,  
> > "peak\_max\_in\_bytes":69730304  
> > },  
> > "old":{  
> > "used\_in\_bytes":9885051104,  
> > "max\_in\_bytes":31514361856,  
> > "peak\_used\_in\_bytes":23868903928,  
> > "peak\_max\_in\_bytes":31514361856  
> > }  
> > }  
> > },  
> > "threads":{  
> > "count":121,  
> > "peak\_count":144  
> > },  
> > "gc":{  
> > "collectors":{  
> > "young":{  
> > "collection\_count":209658,  
> > "collection\_time\_in\_millis":12089132  
> > },  
> > "old":{  
> > "collection\_count":11,  
> > "collection\_time\_in\_millis":6628  
> > }  
> > }  
> > },  
> > "buffer\_pools":{  
> > "direct":{  
> > "count":162,  
> > "used\_in\_bytes":195159665,  
> > "total\_capacity\_in\_bytes":195159665  
> > },  
> > "mapped":{  
> > "count":1292,  
> > "used\_in\_bytes":9897821921,  
> > "total\_capacity\_in\_bytes":9897821921  
> > }  
> > }  
> > },  
> > "thread\_pool":{  
> > "generic":{  
> > "threads":1,  
> > "queue":0,  
> > "active":0,  
> > "rejected":0,  
> > "largest":23,  
> > "completed":67214  
> > },  
> > "index":{  
> > "threads":8,  
> > "queue":0,  
> > "active":0,  
> > "rejected":0,  
> > "largest":8,  
> > "completed":3611159  
> > },  
> > "get":{  
> > "threads":0,  
> > "queue":0,  
> > "active":0,  
> > "rejected":0,  
> > "largest":0,  
> > "completed":0  
> > },  
> > "snapshot":{  
> > "threads":2,  
> > "queue":0,  
> > "active":0,  
> > "rejected":0,  
> > "largest":4,  
> > "completed":95672  
> > },  
> > "merge":{  
> > "threads":3,  
> > "queue":0,  
> > "active":0,  
> > "rejected":0,  
> > "largest":4,  
> > "completed":285955  
> > },  
> > "suggest":{  
> > "threads":0,  
> > "queue":0,  
> > "active":0,  
> > "rejected":0,  
> > "largest":0,  
> > "completed":0  
> > },  
> > "bulk":{  
> > "threads":8,  
> > "queue":0,  
> > "active":0,  
> > "rejected":0,  
> > "largest":8,  
> > "completed":1125  
> > },  
> > "optimize":{  
> > "threads":0,  
> > "queue":0,  
> > "active":0,  
> > "rejected":0,  
> > "largest":0,  
> > "completed":0  
> > },  
> > "warmer":{  
> > "threads":2,  
> > "queue":0,  
> > "active":0,  
> > "rejected":0,  
> > "largest":4,  
> > "completed":5487739  
> > },  
> > "flush":{  
> > "threads":1,  
> > "queue":0,  
> > "active":0,  
> > "rejected":0,  
> > "largest":2,  
> > "completed":2335  
> > },  
> > "search":{
> > 
> > ...
> 
> --  
> You received this message because you are subscribed to the Google Groups  
> "elasticsearch" group.  
> To unsubscribe from this group and stop receiving emails from it, send an  
> email to [elasticsearch+unsubscribe@googlegroups.com](mailto:elasticsearch+unsubscribe@googlegroups.com).  
> To view this discussion on the web visit  
> [https://groups.google.com/d/msgid/elasticsearch/ed76cee8-1c01-43b4-810a-1b203a80ad04%40googlegroups.com](https://groups.google.com/d/msgid/elasticsearch/ed76cee8-1c01-43b4-810a-1b203a80ad04%40googlegroups.com)  
> [https://groups.google.com/d/msgid/elasticsearch/ed76cee8-1c01-43b4-810a-1b203a80ad04%40googlegroups.com?utm\_medium=email&utm\_source=footer](https://groups.google.com/d/msgid/elasticsearch/ed76cee8-1c01-43b4-810a-1b203a80ad04%40googlegroups.com?utm_medium=email&utm_source=footer)  
> .  
> For more options, visit [https://groups.google.com/d/optout](https://groups.google.com/d/optout).

--  
You received this message because you are subscribed to the Google Groups "elasticsearch" group.  
To unsubscribe from this group and stop receiving emails from it, send an email to [elasticsearch+unsubscribe@googlegroups.com](mailto:elasticsearch+unsubscribe@googlegroups.com).  
To view this discussion on the web visit [https://groups.google.com/d/msgid/elasticsearch/CAEM624a4xhLW922qecSOCrhbBkCx3WFc-uz\_Y0%3Da2%2B2hNAhR0Q%40mail.gmail.com](https://groups.google.com/d/msgid/elasticsearch/CAEM624a4xhLW922qecSOCrhbBkCx3WFc-uz_Y0%3Da2%2B2hNAhR0Q%40mail.gmail.com).  
For more options, visit [https://groups.google.com/d/optout](https://groups.google.com/d/optout).

---

<div class="post-metadata">

**Author:** ![webish](https://avatars.discourse-cdn.com/v4/letter/w/919ad9/32.png) [@webish](https://discuss.elastic.co/u/webish)\
**Post date:** [July 9, 2014, 7:47am UTC](https://discuss.elastic.co/t/elastic-search-nodes-servers-with-constant-high-cpu-utilization-in-idle-state/18544/7 "2014-07-09T07:47:12Z")

</div>

Ahh, ok. Sorry.

NodeA:

> <https://gist.github.com/w3b1sh/ea6a2b3fbfc837d5d9d8>

NodeB:

> <https://gist.github.com/w3b1sh/2ca12bd920ebf20644ef>

On Wednesday, July 9, 2014 3:36:53 AM UTC-4, Mark Walkom wrote:

> The output of hot threads is only from the one node, ie requesting it  
> isn't at a cluster level, if you can add the other node it'd help.
> 
> Regards,  
> Mark Walkom
> 
> Infrastructure Engineer  
> Campaign Monitor  
> email: [ma...@campaignmonitor.com](mailto:ma...@campaignmonitor.com) \<javascript:\>  
> web: [www.campaignmonitor.com](http://www.campaignmonitor.com)
> 
> On 9 July 2014 17:32, webish \<[gre...@yoursports.com](mailto:gre...@yoursports.com) \<javascript:\>\> wrote:
> 
> Definitely.
> 
> Here are all the captures thus far in gists.
> 
> _CLUSTER:_  
> curl -XGET '[http://localhost:9200/\_nodes/stats](http://localhost:9200/_nodes/stats)'  
> [curl -XGET 'http://localhost:9200/\_nodes/stats' · GitHub](https://gist.github.com/w3b1sh/a4759e5aa4efbe780fa7)
> 
> curl -XGET '[http://localhost:9200/](http://localhost:9200/)'  
> [curl -XGET 'http://localhost:9200/' · GitHub](https://gist.github.com/w3b1sh/f7a727b1bae53772e56d)
> 
> _THREADS:_  
> curl -XGET '[http://localhost:9200/\_nodes/hot\_threads](http://localhost:9200/_nodes/hot_threads)'  
> [nodeA\> curl -XGET 'http://localhost:9200/\_nodes/hot\_threads' · GitHub](https://gist.github.com/w3b1sh/ea6a2b3fbfc837d5d9d8)
> 
> curl localhost:9200/\_cat/thread\_pool  
> [curl localhost:9200/\_cat/thread\_pool · GitHub](https://gist.github.com/w3b1sh/778194c19ec3ed724f1e)
> 
> On Wednesday, July 9, 2014 3:11:25 AM UTC-4, Ivan Brusic wrote:
> 
> Yes, please use gist/pastebin in the future.
> 
> Try using the hot threads API [1] to see if there are any threads that are  
> truly busy. If your system is truly idle, your thread pools should be  
> almost empty [2]. Your output above has only one management thread in use,  
> which could be simply the thread serving up the output.
> 
> [1] [Elasticsearch Platform — Find real-time answers at scale | Elastic](http://www.elasticsearch.org/guide/en/elasticsearch/)  
> reference/current/cluster-nodes-hot-threads.html  
> [2] [Elasticsearch Platform — Find real-time answers at scale | Elastic](http://www.elasticsearch.org/guide/en/elasticsearch/)  
> reference/current/cat-thread-pool.html
> 
> Cheers,
> 
> Ivan
> 
> On Tue, Jul 8, 2014 at 10:54 PM, webish [gre...@yoursports.com](mailto:gre...@yoursports.com) wrote:
> 
> Thanks Mark. My mistake.
> 
> I can try switching to Oracle Java. There is no TTL. I've used Marvel  
> for development testing. Perhaps I can install the plugin...
> 
> On Wednesday, July 9, 2014 1:35:20 AM UTC-4, Mark Walkom wrote:
> 
> It's better to pop long output like that into a gist/pastebin, it makes it  
> easier to read the thread.  
> It's also worth installing a monitoring plugin like ElasticHQ or marvel,  
> as they provide graphical insight into what is happening and will  
> extrapolate some of the raw figures out.
> 
> If you can change to Oracle java you will get better performance. Are you  
> using TTL?
> 
> Regards,  
> Mark Walkom
> 
> Infrastructure Engineer  
> Campaign Monitor  
> email: [ma...@campaignmonitor.com](mailto:ma...@campaignmonitor.com)  
> web: [www.campaignmonitor.com](http://www.campaignmonitor.com)
> 
> On 9 July 2014 14:54, webish [gre...@yoursports.com](mailto:gre...@yoursports.com) wrote:
> 
> I'm seeing very high CPU utilization in an idle or close to idle state. I  
> haven't seen it use less than 40% on each node over the course of days.  
> I'm not certain how long it has been like this but could be weeks.
> 
> Any help resolving this would be greatly appreciated!
> 
> _Details:_
> 
> There are 2 nodes in the cluster.
> 
> AWS instances are:  
> vCPU ECUMemory (GiB) Instance Storage (GB) Linux/UNIX Usage
> 
> r3.2xlarge 82661 1 x 160 SSD
> 
> curl -XGET '[http://localhost:9200/](http://localhost:9200/)'  
> {  
> "status" : 200,  
> "name" : "Eddie Brock",  
> "version" : {  
> "number" : "1.1.0",  
> "build\_hash" : "2181e113dea80b4a9e31e58e9686658a2d46e363",  
> "build\_timestamp" : "2014-03-25T15:59:51Z",  
> "build\_snapshot" : false,  
> "lucene\_version" : "4.7"  
> },  
> "tagline" : "You Know, for Search"  
> }
> 
> % java -version  
> java version "1.7.0\_55"  
> OpenJDK Runtime Environment (IcedTea 2.4.7) (7u55-2.4.7-1ubuntu1~0.12.04.2  
> )  
> OpenJDK 64-Bit Server VM (build 24.51-b03, mixed mode)
> 
> curl -XGET '[http://localhost:9200/\_nodes/stats](http://localhost:9200/_nodes/stats)'  
> {  
> "cluster\_name":"ab\_elastic",  
> "nodes":{  
> "kpHPVkBCyjOkRaSpS5Q":{  
> "timestamp":1404880773859,  
> "name":"Milan",  
> "transport\_address":"removed",  
> "host":"removed",  
> "ip":[  
> "removed",  
> "NONE"  
> ],  
> "indices":{  
> "docs":{  
> "count":17793781,  
> "deleted":466537  
> },  
> "store":{  
> "size\_in\_bytes":9902722189,  
> "throttle\_time\_in\_millis":8777423  
> },  
> "indexing":{  
> "index\_total":3548638,  
> "index\_time\_in\_millis":2610694,  
> "index\_current":1,  
> "delete\_total":0,  
> "delete\_time\_in\_millis":0,  
> "delete\_current":0  
> },  
> "get":{  
> "total":0,  
> "time\_in\_millis":0,  
> "exists\_total":0,  
> "exists\_time\_in\_millis":0,  
> "missing\_total":0,  
> "missing\_time\_in\_millis":0,  
> "current":0  
> },  
> "search":{  
> "open\_contexts":0,  
> "query\_total":155361213,  
> "query\_time\_in\_millis":817989500,  
> "query\_current":0,  
> "fetch\_total":4119309,  
> "fetch\_time\_in\_millis":1967525,  
> "fetch\_current":0  
> },  
> "merges":{  
> "current":0,  
> "current\_docs":0,  
> "current\_size\_in\_bytes":0,  
> "total":564,  
> "total\_time\_in\_millis":76065,  
> "total\_docs":1093579,  
> "total\_size\_in\_bytes":470304093  
> },  
> "refresh":{  
> "total":15117234,  
> "total\_time\_in\_millis":50581020  
> },  
> "flush":{  
> "total":2336,  
> "total\_time\_in\_millis":99980  
> },  
> "warmer":{  
> "current":0,  
> "total":2719640,  
> "total\_time\_in\_millis":5980882  
> },  
> "filter\_cache":{  
> "memory\_size\_in\_bytes":722232396,  
> "evictions":1179740  
> },  
> "id\_cache":{  
> "memory\_size\_in\_bytes":131672871  
> },  
> "fielddata":{  
> "memory\_size\_in\_bytes":133728950,  
> "evictions":0  
> },  
> "percolate":{  
> "total":0,  
> "time\_in\_millis":0,  
> "current":0,  
> "memory\_size\_in\_bytes":-1,  
> "memory\_size":"-1b",  
> "queries":0  
> },  
> "completion":{  
> "size\_in\_bytes":0  
> },  
> "segments":{  
> "count":278,  
> "memory\_in\_bytes":220799236  
> },  
> "translog":{  
> "operations":68,  
> "size\_in\_bytes":0  
> }  
> },  
> "os":{  
> "timestamp":1404880773910,  
> "uptime\_in\_millis":915746,  
> "load\_average":[  
> 2.68,  
> 3.17,  
> 3.35  
> ],  
> "cpu":{  
> "sys":0,  
> "user":38,  
> "idle":61,  
> "usage":38,  
> "stolen":0  
> },  
> "mem":{  
> "free\_in\_bytes":12301606912,  
> "used\_in\_bytes":51857985536,  
> "free\_percent":35,  
> "used\_percent":64,  
> "actual\_free\_in\_bytes":23005929472,  
> "actual\_used\_in\_bytes":41153662976  
> },  
> "swap":{  
> "used\_in\_bytes":0,  
> "free\_in\_bytes":0  
> }  
> },  
> "process":{  
> "timestamp":1404880773911,  
> "open\_file\_descriptors":300,  
> "cpu":{  
> "percent":313,  
> "sys\_in\_millis":14021810,  
> "user\_in\_millis":794249870,  
> "total\_in\_millis":808271680  
> },  
> "mem":{  
> "resident\_in\_bytes":32342851584,  
> "share\_in\_bytes":712302592,  
> "total\_virtual\_in\_bytes":47406919680  
> }  
> },  
> "jvm":{  
> "timestamp":1404880773911,  
> "uptime\_in\_millis":309661546,  
> "mem":{  
> "heap\_used\_in\_bytes":18409783280,  
> "heap\_used\_percent":57,  
> "heap\_committed\_in\_bytes":32142524416,  
> "heap\_max\_in\_bytes":32142524416,  
> "non\_heap\_used\_in\_bytes":65012920,  
> "non\_heap\_committed\_in\_bytes":98009088,  
> "pools":{  
> "young":{  
> "used\_in\_bytes":538302112,  
> "max\_in\_bytes":558432256,  
> "peak\_used\_in\_bytes":558432256,  
> "peak\_max\_in\_bytes":558432256  
> },  
> "survivor":{  
> "used\_in\_bytes":11403704,  
> "max\_in\_bytes":69730304,  
> "peak\_used\_in\_bytes":69730304,  
> "peak\_max\_in\_bytes":69730304  
> },  
> "old":{  
> "used\_in\_bytes":17860077464,  
> "max\_in\_bytes":31514361856,  
> "peak\_used\_in\_bytes":23694417088,  
> "peak\_max\_in\_bytes":31514361856  
> }  
> }  
> },  
> "threads":{  
> "count":117,  
> "peak\_count":133  
> },  
> "gc":{  
> "collectors":{  
> "young":{  
> "collection\_count":200644,  
> "collection\_time\_in\_millis":10870850  
> },  
> "old":{  
> "collection\_count":9,  
> "collection\_time\_in\_millis":2987  
> }  
> }  
> },  
> "buffer\_pools":{  
> "direct":{  
> "count":154,  
> "used\_in\_bytes":78176363,  
> "total\_capacity\_in\_bytes":78176363  
> },  
> "mapped":{  
> "count":1293,  
> "used\_in\_bytes":9831615457,  
> "total\_capacity\_in\_bytes":9831615457  
> }  
> }  
> },  
> "thread\_pool":{  
> "generic":{  
> "threads":1,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":14,  
> "completed":56826  
> },  
> "index":{  
> "threads":8,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":8,  
> "completed":3607948  
> },  
> "get":{  
> "threads":0,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":0,  
> "completed":0  
> },  
> "snapshot":{  
> "threads":4,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":4,  
> "completed":95697  
> },  
> "merge":{  
> "threads":2,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":4,  
> "completed":282257  
> },  
> "suggest":{  
> "threads":0,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":0,  
> "completed":0  
> },  
> "bulk":{  
> "threads":8,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":8,  
> "completed":1081  
> },  
> "optimize":{  
> "threads":0,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":0,  
> "completed":0  
> },  
> "warmer":{  
> "threads":2,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":4,  
> "completed":5556773  
> },  
> "flush":{  
> "threads":1,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":2,  
> "completed":2336  
> },  
> "search":{  
> "threads":24,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":24,  
> "completed":160139043  
> },  
> "percolate":{  
> "threads":0,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":0,  
> "completed":0  
> },  
> "management":{  
> "threads":1,  
> "queue":0,  
> "active":1,  
> "rejected":0,  
> "largest":5,  
> "completed":1064  
> },  
> "refresh":{  
> "threads":4,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":4,  
> "completed":15126156  
> }  
> },  
> "network":{  
> "tcp":{  
> "active\_opens":1513869,  
> "passive\_opens":224206,  
> "curr\_estab":52,  
> "in\_segs":1929140868,  
> "out\_segs":2005198310,  
> "retrans\_segs":175635,  
> "estab\_resets":37,  
> "attempt\_fails":669,  
> "in\_errs":3,  
> "out\_rsts":88  
> }  
> },  
> "fs":{  
> "timestamp":1404880773912,  
> "total":{  
> "total\_in\_bytes":158534184960,  
> "free\_in\_bytes":148425240576,  
> "available\_in\_bytes":140372176896,  
> "disk\_reads":1993,  
> "disk\_writes":16068284,  
> "disk\_io\_op":16070277,  
> "disk\_read\_size\_in\_bytes":7445504,  
> "disk\_write\_size\_in\_bytes":650206560256,  
> "disk\_io\_size\_in\_bytes":650214005760  
> },  
> "data":[  
> {  
> "path":"/mnt/esdata/ab\_elastic/nodes/0",  
> "mount":"/mnt/esdata",  
> "dev":"/dev/xvdb",  
> "total\_in\_bytes":158534184960,  
> "free\_in\_bytes":148425240576,  
> "available\_in\_bytes":140372176896,  
> "disk\_reads":1993,  
> "disk\_writes":16068284,  
> "disk\_io\_op":16070277,  
> "disk\_read\_size\_in\_bytes":7445504,  
> "disk\_write\_size\_in\_bytes":650206560256,  
> "disk\_io\_size\_in\_bytes":650214005760  
> }  
> ]  
> },  
> "transport":{  
> "server\_open":26,  
> "rx\_count":214082686,  
> "rx\_size\_in\_bytes":148941344505,  
> "tx\_count":209856841,  
> "tx\_size\_in\_bytes":90356130749  
> },  
> "http":{  
> "current\_open":0,  
> "total\_opened":0  
> },  
> "fielddata\_breaker":{  
> "maximum\_size\_in\_bytes":25714019532,  
> "maximum\_size":"23.9gb",  
> "estimated\_size\_in\_bytes":265401821,  
> "estimated\_size":"253.1mb",  
> "overhead":1.03  
> }  
> },  
> "erc5l7dQai3mKBIX6sw":{  
> "timestamp":1404880733754,  
> "name":"Eddie Brock",  
> "transport\_address":"removed",  
> "host":"removed",  
> "ip":[  
> "removed",  
> "NONE"  
> ],  
> "indices":{  
> "docs":{  
> "count":17793781,  
> "deleted":529675  
> },  
> "store":{  
> "size\_in\_bytes":9969074949,  
> "throttle\_time\_in\_millis":9235478  
> },  
> "indexing":{  
> "index\_total":3531955,  
> "index\_time\_in\_millis":2783791,  
> "index\_current":0,  
> "delete\_total":0,  
> "delete\_time\_in\_millis":0,  
> "delete\_current":0  
> },  
> "get":{  
> "total":2947020,  
> "time\_in\_millis":217006,  
> "exists\_total":2946589,  
> "exists\_time\_in\_millis":216977,  
> "missing\_total":431,  
> "missing\_time\_in\_millis":29,  
> "current":0  
> },  
> "search":{  
> "open\_contexts":0,  
> "query\_total":155416200,  
> "query\_time\_in\_millis":1076023052,  
> "query\_current":0,  
> "fetch\_total":4101442,  
> "fetch\_time\_in\_millis":2337321,  
> "fetch\_current":0  
> },  
> "merges":{  
> "current":0,  
> "current\_docs":0,  
> "current\_size\_in\_bytes":0,  
> "total":563,  
> "total\_time\_in\_millis":89211,  
> "total\_docs":1099389,  
> "total\_size\_in\_bytes":470013054  
> },  
> "refresh":{  
> "total":15123966,  
> "total\_time\_in\_millis":63368992  
> },  
> "flush":{  
> "total":2335,  
> "total\_time\_in\_millis":414790  
> },  
> "warmer":{  
> "current":0,  
> "total":2690030,  
> "total\_time\_in\_millis":7339293  
> },  
> "filter\_cache":{  
> "memory\_size\_in\_bytes":695354935,  
> "evictions":1377260  
> },  
> "id\_cache":{  
> "memory\_size\_in\_bytes":131817497  
> },  
> "fielddata":{  
> "memory\_size\_in\_bytes":137907189,  
> "evictions":0  
> },  
> "percolate":{  
> "total":0,  
> "time\_in\_millis":0,  
> "current":0,  
> "memory\_size\_in\_bytes":-1,  
> "memory\_size":"-1b",  
> "queries":0  
> },  
> "completion":{  
> "size\_in\_bytes":0  
> },  
> "segments":{  
> "count":282,  
> "memory\_in\_bytes":217468955  
> },  
> "translog":{  
> "operations":86,  
> "size\_in\_bytes":0  
> }  
> },  
> "os":{  
> "timestamp":1404880733801,  
> "uptime\_in\_millis":923467,  
> "load\_average":[  
> 3.71,  
> 4.57,  
> 4.46  
> ],  
> "cpu":{  
> "sys":0,  
> "user":46,  
> "idle":52,  
> "usage":46,  
> "stolen":0  
> },  
> "mem":{  
> "free\_in\_bytes":4438659072,  
> "used\_in\_bytes":59720933376,  
> "free\_percent":32,  
> "used\_percent":67,  
> "actual\_free\_in\_bytes":21124698112,  
> "actual\_used\_in\_bytes":43034894336  
> },  
> "swap":{  
> "used\_in\_bytes":0,  
> "free\_in\_bytes":0  
> }  
> },  
> "process":{  
> "timestamp":1404880733801,  
> "open\_file\_descriptors":301,  
> "cpu":{  
> "percent":377,  
> "sys\_in\_millis":26524340,  
> "user\_in\_millis":947578840,  
> "total\_in\_millis":974103180  
> },  
> "mem":{  
> "resident\_in\_bytes":33945767936,  
> "share\_in\_bytes":692912128,  
> "total\_virtual\_in\_bytes":47473528832  
> }  
> },  
> "jvm":{  
> "timestamp":1404880733802,  
> "uptime\_in\_millis":310301938,  
> "mem":{  
> "heap\_used\_in\_bytes":10113402832,  
> "heap\_used\_percent":31,  
> "heap\_committed\_in\_bytes":32142524416,  
> "heap\_max\_in\_bytes":32142524416,  
> "non\_heap\_used\_in\_bytes":71762640,  
> "non\_heap\_committed\_in\_bytes":106397696,  
> "pools":{  
> "young":{  
> "used\_in\_bytes":218720808,  
> "max\_in\_bytes":558432256,  
> "peak\_used\_in\_bytes":558432256,  
> "peak\_max\_in\_bytes":558432256  
> },  
> "survivor":{  
> "used\_in\_bytes":9630920,  
> "max\_in\_bytes":69730304,  
> "peak\_used\_in\_bytes":69730304,  
> "peak\_max\_in\_bytes":69730304  
> },  
> "old":{  
> "used\_in\_bytes":9885051104,  
> "max\_in\_bytes":31514361856,  
> "peak\_used\_in\_bytes":23868903928,  
> "peak\_max\_in\_bytes":31514361856  
> }  
> }  
> },  
> "threads":{  
> "count":121,  
> "peak\_count":144  
> },  
> "gc":{  
> "collectors":{  
> "young":{  
> "collection\_count":209658,  
> "collection\_time\_in\_millis":12089132  
> },  
> "old":{  
> "collection\_count":11,  
> "collection\_time\_in\_millis":6628  
> }  
> }  
> },  
> "buffer\_pools":{  
> "direct":{  
> "count":162,  
> "used\_in\_bytes":195159665,  
> "total\_capacity\_in\_bytes":195159665  
> },  
> "mapped":{  
> "count":1292,  
> "used\_in\_bytes":9897821921,  
> "total\_capacity\_in\_bytes":9897821921  
> }  
> }  
> },  
> "thread\_pool":{  
> "generic":{  
> "threads":1,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":23,  
> "completed":67214  
> },  
> "index":{  
> "threads":8,
> 
> ...

--  
You received this message because you are subscribed to the Google Groups "elasticsearch" group.  
To unsubscribe from this group and stop receiving emails from it, send an email to [elasticsearch+unsubscribe@googlegroups.com](mailto:elasticsearch+unsubscribe@googlegroups.com).  
To view this discussion on the web visit [https://groups.google.com/d/msgid/elasticsearch/904664f8-2d70-4983-8d92-38064989d781%40googlegroups.com](https://groups.google.com/d/msgid/elasticsearch/904664f8-2d70-4983-8d92-38064989d781%40googlegroups.com).  
For more options, visit [https://groups.google.com/d/optout](https://groups.google.com/d/optout).

---

<div class="post-metadata">

**Author:** ![webish](https://avatars.discourse-cdn.com/v4/letter/w/919ad9/32.png) [@webish](https://discuss.elastic.co/u/webish)\
**Post date:** [July 12, 2014, 4:51am UTC](https://discuss.elastic.co/t/elastic-search-nodes-servers-with-constant-high-cpu-utilization-in-idle-state/18544/8 "2014-07-12T04:51:44Z")

</div>

After changing to Oracle Java it seemed the problem was resolved. CPU on  
both nodes dropped to 5% or less during the same state; however, in the  
last 24 hrs history is repeating itself.

CPU on one node is 75% and on the other is 50%....

On Wednesday, July 9, 2014 3:47:12 AM UTC-4, webish wrote:

> Ahh, ok. Sorry.
> 
> NodeA:  
> [nodeA\> curl -XGET 'http://localhost:9200/\_nodes/hot\_threads' · GitHub](https://gist.github.com/w3b1sh/ea6a2b3fbfc837d5d9d8)
> 
> NodeB:  
> [nodeB\> curl -XGET 'http://localhost:9200/\_nodes/hot\_threads' · GitHub](https://gist.github.com/w3b1sh/2ca12bd920ebf20644ef)
> 
> On Wednesday, July 9, 2014 3:36:53 AM UTC-4, Mark Walkom wrote:
> 
> The output of hot threads is only from the one node, ie requesting it  
> isn't at a cluster level, if you can add the other node it'd help.
> 
> Regards,  
> Mark Walkom
> 
> Infrastructure Engineer  
> Campaign Monitor  
> email: [ma...@campaignmonitor.com](mailto:ma...@campaignmonitor.com)  
> web: [www.campaignmonitor.com](http://www.campaignmonitor.com)
> 
> On 9 July 2014 17:32, webish [gre...@yoursports.com](mailto:gre...@yoursports.com) wrote:
> 
> Definitely.
> 
> Here are all the captures thus far in gists.
> 
> _CLUSTER:_  
> curl -XGET '[http://localhost:9200/\_nodes/stats](http://localhost:9200/_nodes/stats)'  
> [curl -XGET 'http://localhost:9200/\_nodes/stats' · GitHub](https://gist.github.com/w3b1sh/a4759e5aa4efbe780fa7)
> 
> curl -XGET '[http://localhost:9200/](http://localhost:9200/)'  
> [curl -XGET 'http://localhost:9200/' · GitHub](https://gist.github.com/w3b1sh/f7a727b1bae53772e56d)
> 
> _THREADS:_  
> curl -XGET '[http://localhost:9200/\_nodes/hot\_threads](http://localhost:9200/_nodes/hot_threads)'  
> [nodeA\> curl -XGET 'http://localhost:9200/\_nodes/hot\_threads' · GitHub](https://gist.github.com/w3b1sh/ea6a2b3fbfc837d5d9d8)
> 
> curl localhost:9200/\_cat/thread\_pool  
> [curl localhost:9200/\_cat/thread\_pool · GitHub](https://gist.github.com/w3b1sh/778194c19ec3ed724f1e)
> 
> On Wednesday, July 9, 2014 3:11:25 AM UTC-4, Ivan Brusic wrote:
> 
> Yes, please use gist/pastebin in the future.
> 
> Try using the hot threads API [1] to see if there are any threads that are  
> truly busy. If your system is truly idle, your thread pools should be  
> almost empty [2]. Your output above has only one management thread in use,  
> which could be simply the thread serving up the output.
> 
> [1] [Elasticsearch Platform — Find real-time answers at scale | Elastic](http://www.elasticsearch.org/guide/en/elasticsearch/)  
> reference/current/cluster-nodes-hot-threads.html  
> [2] [Elasticsearch Platform — Find real-time answers at scale | Elastic](http://www.elasticsearch.org/guide/en/elasticsearch/)  
> reference/current/cat-thread-pool.html
> 
> Cheers,
> 
> Ivan
> 
> On Tue, Jul 8, 2014 at 10:54 PM, webish [gre...@yoursports.com](mailto:gre...@yoursports.com) wrote:
> 
> Thanks Mark. My mistake.
> 
> I can try switching to Oracle Java. There is no TTL. I've used Marvel  
> for development testing. Perhaps I can install the plugin...
> 
> On Wednesday, July 9, 2014 1:35:20 AM UTC-4, Mark Walkom wrote:
> 
> It's better to pop long output like that into a gist/pastebin, it makes it  
> easier to read the thread.  
> It's also worth installing a monitoring plugin like ElasticHQ or marvel,  
> as they provide graphical insight into what is happening and will  
> extrapolate some of the raw figures out.
> 
> If you can change to Oracle java you will get better performance. Are you  
> using TTL?
> 
> Regards,  
> Mark Walkom
> 
> Infrastructure Engineer  
> Campaign Monitor  
> email: [ma...@campaignmonitor.com](mailto:ma...@campaignmonitor.com)  
> web: [www.campaignmonitor.com](http://www.campaignmonitor.com)
> 
> On 9 July 2014 14:54, webish [gre...@yoursports.com](mailto:gre...@yoursports.com) wrote:
> 
> I'm seeing very high CPU utilization in an idle or close to idle state. I  
> haven't seen it use less than 40% on each node over the course of days.  
> I'm not certain how long it has been like this but could be weeks.
> 
> Any help resolving this would be greatly appreciated!
> 
> _Details:_
> 
> There are 2 nodes in the cluster.
> 
> AWS instances are:  
> vCPU ECUMemory (GiB) Instance Storage (GB) Linux/UNIX Usage
> 
> r3.2xlarge 82661 1 x 160 SSD
> 
> curl -XGET '[http://localhost:9200/](http://localhost:9200/)'  
> {  
> "status" : 200,  
> "name" : "Eddie Brock",  
> "version" : {  
> "number" : "1.1.0",  
> "build\_hash" : "2181e113dea80b4a9e31e58e9686658a2d46e363",  
> "build\_timestamp" : "2014-03-25T15:59:51Z",  
> "build\_snapshot" : false,  
> "lucene\_version" : "4.7"  
> },  
> "tagline" : "You Know, for Search"  
> }
> 
> % java -version  
> java version "1.7.0\_55"  
> OpenJDK Runtime Environment (IcedTea 2.4.7) (7u55-2.4.7-1ubuntu1~0.12.04.2  
> )  
> OpenJDK 64-Bit Server VM (build 24.51-b03, mixed mode)
> 
> curl -XGET '[http://localhost:9200/\_nodes/stats](http://localhost:9200/_nodes/stats)'  
> {  
> "cluster\_name":"ab\_elastic",  
> "nodes":{  
> "kpHPVkBCyjOkRaSpS5Q":{  
> "timestamp":1404880773859,  
> "name":"Milan",  
> "transport\_address":"removed",  
> "host":"removed",  
> "ip":[  
> "removed",  
> "NONE"  
> ],  
> "indices":{  
> "docs":{  
> "count":17793781,  
> "deleted":466537  
> },  
> "store":{  
> "size\_in\_bytes":9902722189,  
> "throttle\_time\_in\_millis":8777423  
> },  
> "indexing":{  
> "index\_total":3548638,  
> "index\_time\_in\_millis":2610694,  
> "index\_current":1,  
> "delete\_total":0,  
> "delete\_time\_in\_millis":0,  
> "delete\_current":0  
> },  
> "get":{  
> "total":0,  
> "time\_in\_millis":0,  
> "exists\_total":0,  
> "exists\_time\_in\_millis":0,  
> "missing\_total":0,  
> "missing\_time\_in\_millis":0,  
> "current":0  
> },  
> "search":{  
> "open\_contexts":0,  
> "query\_total":155361213,  
> "query\_time\_in\_millis":817989500,  
> "query\_current":0,  
> "fetch\_total":4119309,  
> "fetch\_time\_in\_millis":1967525,  
> "fetch\_current":0  
> },  
> "merges":{  
> "current":0,  
> "current\_docs":0,  
> "current\_size\_in\_bytes":0,  
> "total":564,  
> "total\_time\_in\_millis":76065,  
> "total\_docs":1093579,  
> "total\_size\_in\_bytes":470304093  
> },  
> "refresh":{  
> "total":15117234,  
> "total\_time\_in\_millis":50581020  
> },  
> "flush":{  
> "total":2336,  
> "total\_time\_in\_millis":99980  
> },  
> "warmer":{  
> "current":0,  
> "total":2719640,  
> "total\_time\_in\_millis":5980882  
> },  
> "filter\_cache":{  
> "memory\_size\_in\_bytes":722232396,  
> "evictions":1179740  
> },  
> "id\_cache":{  
> "memory\_size\_in\_bytes":131672871  
> },  
> "fielddata":{  
> "memory\_size\_in\_bytes":133728950,  
> "evictions":0  
> },  
> "percolate":{  
> "total":0,  
> "time\_in\_millis":0,  
> "current":0,  
> "memory\_size\_in\_bytes":-1,  
> "memory\_size":"-1b",  
> "queries":0  
> },  
> "completion":{  
> "size\_in\_bytes":0  
> },  
> "segments":{  
> "count":278,  
> "memory\_in\_bytes":220799236  
> },  
> "translog":{  
> "operations":68,  
> "size\_in\_bytes":0  
> }  
> },  
> "os":{  
> "timestamp":1404880773910,  
> "uptime\_in\_millis":915746,  
> "load\_average":[  
> 2.68,  
> 3.17,  
> 3.35  
> ],  
> "cpu":{  
> "sys":0,  
> "user":38,  
> "idle":61,  
> "usage":38,  
> "stolen":0  
> },  
> "mem":{  
> "free\_in\_bytes":12301606912,  
> "used\_in\_bytes":51857985536,  
> "free\_percent":35,  
> "used\_percent":64,  
> "actual\_free\_in\_bytes":23005929472,  
> "actual\_used\_in\_bytes":41153662976  
> },  
> "swap":{  
> "used\_in\_bytes":0,  
> "free\_in\_bytes":0  
> }  
> },  
> "process":{  
> "timestamp":1404880773911,  
> "open\_file\_descriptors":300,  
> "cpu":{  
> "percent":313,  
> "sys\_in\_millis":14021810,  
> "user\_in\_millis":794249870,  
> "total\_in\_millis":808271680  
> },  
> "mem":{  
> "resident\_in\_bytes":32342851584,  
> "share\_in\_bytes":712302592,  
> "total\_virtual\_in\_bytes":47406919680  
> }  
> },  
> "jvm":{  
> "timestamp":1404880773911,  
> "uptime\_in\_millis":309661546,  
> "mem":{  
> "heap\_used\_in\_bytes":18409783280,  
> "heap\_used\_percent":57,  
> "heap\_committed\_in\_bytes":32142524416,  
> "heap\_max\_in\_bytes":32142524416,  
> "non\_heap\_used\_in\_bytes":65012920,  
> "non\_heap\_committed\_in\_bytes":98009088,  
> "pools":{  
> "young":{  
> "used\_in\_bytes":538302112,  
> "max\_in\_bytes":558432256,  
> "peak\_used\_in\_bytes":558432256,  
> "peak\_max\_in\_bytes":558432256  
> },  
> "survivor":{  
> "used\_in\_bytes":11403704,  
> "max\_in\_bytes":69730304,  
> "peak\_used\_in\_bytes":69730304,  
> "peak\_max\_in\_bytes":69730304  
> },  
> "old":{  
> "used\_in\_bytes":17860077464,  
> "max\_in\_bytes":31514361856,  
> "peak\_used\_in\_bytes":23694417088,  
> "peak\_max\_in\_bytes":31514361856  
> }  
> }  
> },  
> "threads":{  
> "count":117,  
> "peak\_count":133  
> },  
> "gc":{  
> "collectors":{  
> "young":{  
> "collection\_count":200644,  
> "collection\_time\_in\_millis":10870850  
> },  
> "old":{  
> "collection\_count":9,  
> "collection\_time\_in\_millis":2987  
> }  
> }  
> },  
> "buffer\_pools":{  
> "direct":{  
> "count":154,  
> "used\_in\_bytes":78176363,  
> "total\_capacity\_in\_bytes":78176363  
> },  
> "mapped":{  
> "count":1293,  
> "used\_in\_bytes":9831615457,  
> "total\_capacity\_in\_bytes":9831615457  
> }  
> }  
> },  
> "thread\_pool":{  
> "generic":{  
> "threads":1,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":14,  
> "completed":56826  
> },  
> "index":{  
> "threads":8,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":8,  
> "completed":3607948  
> },  
> "get":{  
> "threads":0,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":0,  
> "completed":0  
> },  
> "snapshot":{  
> "threads":4,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":4,  
> "completed":95697  
> },  
> "merge":{  
> "threads":2,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":4,  
> "completed":282257  
> },  
> "suggest":{  
> "threads":0,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":0,  
> "completed":0  
> },  
> "bulk":{  
> "threads":8,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":8,  
> "completed":1081  
> },  
> "optimize":{  
> "threads":0,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":0,  
> "completed":0  
> },  
> "warmer":{  
> "threads":2,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":4,  
> "completed":5556773  
> },  
> "flush":{  
> "threads":1,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":2,  
> "completed":2336  
> },  
> "search":{  
> "threads":24,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":24,  
> "completed":160139043  
> },  
> "percolate":{  
> "threads":0,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":0,  
> "completed":0  
> },  
> "management":{  
> "threads":1,  
> "queue":0,  
> "active":1,  
> "rejected":0,  
> "largest":5,  
> "completed":1064  
> },  
> "refresh":{  
> "threads":4,  
> "queue":0,  
> "active":0,  
> "rejected":0,  
> "largest":4,  
> "completed":15126156  
> }  
> },  
> "network":{  
> "tcp":{  
> "active\_opens":1513869,
> 
> ...

--  
You received this message because you are subscribed to the Google Groups "elasticsearch" group.  
To unsubscribe from this group and stop receiving emails from it, send an email to [elasticsearch+unsubscribe@googlegroups.com](mailto:elasticsearch+unsubscribe@googlegroups.com).  
To view this discussion on the web visit [https://groups.google.com/d/msgid/elasticsearch/712110a5-a554-46c6-a9e5-4eba8c27c2a5%40googlegroups.com](https://groups.google.com/d/msgid/elasticsearch/712110a5-a554-46c6-a9e5-4eba8c27c2a5%40googlegroups.com).  
For more options, visit [https://groups.google.com/d/optout](https://groups.google.com/d/optout).

---

<div class="post-metadata">

**Author:** ![system](https://us1.discourse-cdn.com/elastic/original/3X/1/a/1ac57faf039f6b580b3f104ef42a2a89e41014de.png) [@system](https://discuss.elastic.co/u/system)\
**Post date:** [July 6, 2017, 1:16am UTC](https://discuss.elastic.co/t/elastic-search-nodes-servers-with-constant-high-cpu-utilization-in-idle-state/18544/9 "2017-07-06T01:16:20Z")

</div>