Elastic Search nodes / servers with constant high CPU utilization in idle state

[webish]

I'm seeing very high CPU utilization in an idle or close to idle state. I
haven't seen it use less than 40% on each node over the course of days.
I'm not certain how long it has been like this but could be weeks.

Any help resolving this would be greatly appreciated!

Details:

There are 2 nodes in the cluster.

AWS instances are:
vCPUECUMemory (GiB)Instance Storage (GB)Linux/UNIX Usage

r3.2xlarge826611 x 160 SSD

curl -XGET 'http://localhost:9200/'
{
"status" : 200,
"name" : "Eddie Brock",
"version" : {
"number" : "1.1.0",
"build_hash" : "2181e113dea80b4a9e31e58e9686658a2d46e363",
"build_timestamp" : "2014-03-25T15:59:51Z",
"build_snapshot" : false,
"lucene_version" : "4.7"
},
"tagline" : "You Know, for Search"
}

% java -version
java version "1.7.0_55"
OpenJDK Runtime Environment (IcedTea 2.4.7) (7u55-2.4.7-1ubuntu1~0.12.04.2)
OpenJDK 64-Bit Server VM (build 24.51-b03, mixed mode)

curl -XGET 'http://localhost:9200/_nodes/stats'
{
"cluster_name":"ab_elastic",
"nodes":{
"kpHPVkBCyjOkRaSpS5Q":{
"timestamp":1404880773859,
"name":"Milan",
"transport_address":"removed",
"host":"removed",
"ip":[
"removed",
"NONE"
],
"indices":{
"docs":{
"count":17793781,
"deleted":466537
},
"store":{
"size_in_bytes":9902722189,
"throttle_time_in_millis":8777423
},
"indexing":{
"index_total":3548638,
"index_time_in_millis":2610694,
"index_current":1,
"delete_total":0,
"delete_time_in_millis":0,
"delete_current":0
},
"get":{
"total":0,
"time_in_millis":0,
"exists_total":0,
"exists_time_in_millis":0,
"missing_total":0,
"missing_time_in_millis":0,
"current":0
},
"search":{
"open_contexts":0,
"query_total":155361213,
"query_time_in_millis":817989500,
"query_current":0,
"fetch_total":4119309,
"fetch_time_in_millis":1967525,
"fetch_current":0
},
"merges":{
"current":0,
"current_docs":0,
"current_size_in_bytes":0,
"total":564,
"total_time_in_millis":76065,
"total_docs":1093579,
"total_size_in_bytes":470304093
},
"refresh":{
"total":15117234,
"total_time_in_millis":50581020
},
"flush":{
"total":2336,
"total_time_in_millis":99980
},
"warmer":{
"current":0,
"total":2719640,
"total_time_in_millis":5980882
},
"filter_cache":{
"memory_size_in_bytes":722232396,
"evictions":1179740
},
"id_cache":{
"memory_size_in_bytes":131672871
},
"fielddata":{
"memory_size_in_bytes":133728950,
"evictions":0
},
"percolate":{
"total":0,
"time_in_millis":0,
"current":0,
"memory_size_in_bytes":-1,
"memory_size":"-1b",
"queries":0
},
"completion":{
"size_in_bytes":0
},
"segments":{
"count":278,
"memory_in_bytes":220799236
},
"translog":{
"operations":68,
"size_in_bytes":0
}
},
"os":{
"timestamp":1404880773910,
"uptime_in_millis":915746,
"load_average":[
2.68,
3.17,
3.35
],
"cpu":{
"sys":0,
"user":38,
"idle":61,
"usage":38,
"stolen":0
},
"mem":{
"free_in_bytes":12301606912,
"used_in_bytes":51857985536,
"free_percent":35,
"used_percent":64,
"actual_free_in_bytes":23005929472,
"actual_used_in_bytes":41153662976
},
"swap":{
"used_in_bytes":0,
"free_in_bytes":0
}
},
"process":{
"timestamp":1404880773911,
"open_file_descriptors":300,
"cpu":{
"percent":313,
"sys_in_millis":14021810,
"user_in_millis":794249870,
"total_in_millis":808271680
},
"mem":{
"resident_in_bytes":32342851584,
"share_in_bytes":712302592,
"total_virtual_in_bytes":47406919680
}
},
"jvm":{
"timestamp":1404880773911,
"uptime_in_millis":309661546,
"mem":{
"heap_used_in_bytes":18409783280,
"heap_used_percent":57,
"heap_committed_in_bytes":32142524416,
"heap_max_in_bytes":32142524416,
"non_heap_used_in_bytes":65012920,
"non_heap_committed_in_bytes":98009088,
"pools":{
"young":{
"used_in_bytes":538302112,
"max_in_bytes":558432256,
"peak_used_in_bytes":558432256,
"peak_max_in_bytes":558432256
},
"survivor":{
"used_in_bytes":11403704,
"max_in_bytes":69730304,
"peak_used_in_bytes":69730304,
"peak_max_in_bytes":69730304
},
"old":{
"used_in_bytes":17860077464,
"max_in_bytes":31514361856,
"peak_used_in_bytes":23694417088,
"peak_max_in_bytes":31514361856
}
}
},
"threads":{
"count":117,
"peak_count":133
},
"gc":{
"collectors":{
"young":{
"collection_count":200644,
"collection_time_in_millis":10870850
},
"old":{
"collection_count":9,
"collection_time_in_millis":2987
}
}
},
"buffer_pools":{
"direct":{
"count":154,
"used_in_bytes":78176363,
"total_capacity_in_bytes":78176363
},
"mapped":{
"count":1293,
"used_in_bytes":9831615457,
"total_capacity_in_bytes":9831615457
}
}
},
"thread_pool":{
"generic":{
"threads":1,
"queue":0,
"active":0,
"rejected":0,
"largest":14,
"completed":56826
},
"index":{
"threads":8,
"queue":0,
"active":0,
"rejected":0,
"largest":8,
"completed":3607948
},
"get":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"snapshot":{
"threads":4,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":95697
},
"merge":{
"threads":2,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":282257
},
"suggest":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"bulk":{
"threads":8,
"queue":0,
"active":0,
"rejected":0,
"largest":8,
"completed":1081
},
"optimize":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"warmer":{
"threads":2,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":5556773
},
"flush":{
"threads":1,
"queue":0,
"active":0,
"rejected":0,
"largest":2,
"completed":2336
},
"search":{
"threads":24,
"queue":0,
"active":0,
"rejected":0,
"largest":24,
"completed":160139043
},
"percolate":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"management":{
"threads":1,
"queue":0,
"active":1,
"rejected":0,
"largest":5,
"completed":1064
},
"refresh":{
"threads":4,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":15126156
}
},
"network":{
"tcp":{
"active_opens":1513869,
"passive_opens":224206,
"curr_estab":52,
"in_segs":1929140868,
"out_segs":2005198310,
"retrans_segs":175635,
"estab_resets":37,
"attempt_fails":669,
"in_errs":3,
"out_rsts":88
}
},
"fs":{
"timestamp":1404880773912,
"total":{
"total_in_bytes":158534184960,
"free_in_bytes":148425240576,
"available_in_bytes":140372176896,
"disk_reads":1993,
"disk_writes":16068284,
"disk_io_op":16070277,
"disk_read_size_in_bytes":7445504,
"disk_write_size_in_bytes":650206560256,
"disk_io_size_in_bytes":650214005760
},
"data":[
{
"path":"/mnt/esdata/ab_elastic/nodes/0",
"mount":"/mnt/esdata",
"dev":"/dev/xvdb",
"total_in_bytes":158534184960,
"free_in_bytes":148425240576,
"available_in_bytes":140372176896,
"disk_reads":1993,
"disk_writes":16068284,
"disk_io_op":16070277,
"disk_read_size_in_bytes":7445504,
"disk_write_size_in_bytes":650206560256,
"disk_io_size_in_bytes":650214005760
}
]
},
"transport":{
"server_open":26,
"rx_count":214082686,
"rx_size_in_bytes":148941344505,
"tx_count":209856841,
"tx_size_in_bytes":90356130749
},
"http":{
"current_open":0,
"total_opened":0
},
"fielddata_breaker":{
"maximum_size_in_bytes":25714019532,
"maximum_size":"23.9gb",
"estimated_size_in_bytes":265401821,
"estimated_size":"253.1mb",
"overhead":1.03
}
},
"erc5l7dQai3mKBIX6sw":{
"timestamp":1404880733754,
"name":"Eddie Brock",
"transport_address":"removed",
"host":"removed",
"ip":[
"removed",
"NONE"
],
"indices":{
"docs":{
"count":17793781,
"deleted":529675
},
"store":{
"size_in_bytes":9969074949,
"throttle_time_in_millis":9235478
},
"indexing":{
"index_total":3531955,
"index_time_in_millis":2783791,
"index_current":0,
"delete_total":0,
"delete_time_in_millis":0,
"delete_current":0
},
"get":{
"total":2947020,
"time_in_millis":217006,
"exists_total":2946589,
"exists_time_in_millis":216977,
"missing_total":431,
"missing_time_in_millis":29,
"current":0
},
"search":{
"open_contexts":0,
"query_total":155416200,
"query_time_in_millis":1076023052,
"query_current":0,
"fetch_total":4101442,
"fetch_time_in_millis":2337321,
"fetch_current":0
},
"merges":{
"current":0,
"current_docs":0,
"current_size_in_bytes":0,
"total":563,
"total_time_in_millis":89211,
"total_docs":1099389,
"total_size_in_bytes":470013054
},
"refresh":{
"total":15123966,
"total_time_in_millis":63368992
},
"flush":{
"total":2335,
"total_time_in_millis":414790
},
"warmer":{
"current":0,
"total":2690030,
"total_time_in_millis":7339293
},
"filter_cache":{
"memory_size_in_bytes":695354935,
"evictions":1377260
},
"id_cache":{
"memory_size_in_bytes":131817497
},
"fielddata":{
"memory_size_in_bytes":137907189,
"evictions":0
},
"percolate":{
"total":0,
"time_in_millis":0,
"current":0,
"memory_size_in_bytes":-1,
"memory_size":"-1b",
"queries":0
},
"completion":{
"size_in_bytes":0
},
"segments":{
"count":282,
"memory_in_bytes":217468955
},
"translog":{
"operations":86,
"size_in_bytes":0
}
},
"os":{
"timestamp":1404880733801,
"uptime_in_millis":923467,
"load_average":[
3.71,
4.57,
4.46
],
"cpu":{
"sys":0,
"user":46,
"idle":52,
"usage":46,
"stolen":0
},
"mem":{
"free_in_bytes":4438659072,
"used_in_bytes":59720933376,
"free_percent":32,
"used_percent":67,
"actual_free_in_bytes":21124698112,
"actual_used_in_bytes":43034894336
},
"swap":{
"used_in_bytes":0,
"free_in_bytes":0
}
},
"process":{
"timestamp":1404880733801,
"open_file_descriptors":301,
"cpu":{
"percent":377,
"sys_in_millis":26524340,
"user_in_millis":947578840,
"total_in_millis":974103180
},
"mem":{
"resident_in_bytes":33945767936,
"share_in_bytes":692912128,
"total_virtual_in_bytes":47473528832
}
},
"jvm":{
"timestamp":1404880733802,
"uptime_in_millis":310301938,
"mem":{
"heap_used_in_bytes":10113402832,
"heap_used_percent":31,
"heap_committed_in_bytes":32142524416,
"heap_max_in_bytes":32142524416,
"non_heap_used_in_bytes":71762640,
"non_heap_committed_in_bytes":106397696,
"pools":{
"young":{
"used_in_bytes":218720808,
"max_in_bytes":558432256,
"peak_used_in_bytes":558432256,
"peak_max_in_bytes":558432256
},
"survivor":{
"used_in_bytes":9630920,
"max_in_bytes":69730304,
"peak_used_in_bytes":69730304,
"peak_max_in_bytes":69730304
},
"old":{
"used_in_bytes":9885051104,
"max_in_bytes":31514361856,
"peak_used_in_bytes":23868903928,
"peak_max_in_bytes":31514361856
}
}
},
"threads":{
"count":121,
"peak_count":144
},
"gc":{
"collectors":{
"young":{
"collection_count":209658,
"collection_time_in_millis":12089132
},
"old":{
"collection_count":11,
"collection_time_in_millis":6628
}
}
},
"buffer_pools":{
"direct":{
"count":162,
"used_in_bytes":195159665,
"total_capacity_in_bytes":195159665
},
"mapped":{
"count":1292,
"used_in_bytes":9897821921,
"total_capacity_in_bytes":9897821921
}
}
},
"thread_pool":{
"generic":{
"threads":1,
"queue":0,
"active":0,
"rejected":0,
"largest":23,
"completed":67214
},
"index":{
"threads":8,
"queue":0,
"active":0,
"rejected":0,
"largest":8,
"completed":3611159
},
"get":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"snapshot":{
"threads":2,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":95672
},
"merge":{
"threads":3,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":285955
},
"suggest":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"bulk":{
"threads":8,
"queue":0,
"active":0,
"rejected":0,
"largest":8,
"completed":1125
},
"optimize":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"warmer":{
"threads":2,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":5487739
},
"flush":{
"threads":1,
"queue":0,
"active":0,
"rejected":0,
"largest":2,
"completed":2335
},
"search":{
"threads":24,
"queue":0,
"active":0,
"rejected":0,
"largest":24,
"completed":160510703
},
"percolate":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"management":{
"threads":5,
"queue":0,
"active":1,
"rejected":0,
"largest":5,
"completed":22893
},
"refresh":{
"threads":4,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":15135553
}
},
"network":{
"tcp":{
"active_opens":1025664,
"passive_opens":105715571,
"curr_estab":55,
"in_segs":2557757673,
"out_segs":2502369803,
"retrans_segs":205765,
"estab_resets":68,
"attempt_fails":207,
"in_errs":0,
"out_rsts":1641
}
},
"fs":{
"timestamp":1404880733802,
"total":{
"total_in_bytes":158533136384,
"free_in_bytes":137526714368,
"available_in_bytes":129473703936,
"disk_reads":8637,
"disk_writes":17332161,
"disk_io_op":17340798,
"disk_read_size_in_bytes":139357184,
"disk_write_size_in_bytes":706635259904,
"disk_io_size_in_bytes":706774617088
},
"data":[
{
"path":"/mnt/esdata/ab_elastic/nodes/0",
"mount":"/mnt/esdata",
"dev":"/dev/xvdb",
"total_in_bytes":158533136384,
"free_in_bytes":137526714368,
"available_in_bytes":129473703936,
"disk_reads":8637,
"disk_writes":17332161,
"disk_io_op":17340798,
"disk_read_size_in_bytes":139357184,
"disk_write_size_in_bytes":706635259904,
"disk_io_size_in_bytes":706774617088
}
]
},
"transport":{
"server_open":26,
"rx_count":214086149,
"rx_size_in_bytes":90356374759,
"tx_count":210167189,
"tx_size_in_bytes":149348401545
},
"http":{
"current_open":1,
"total_opened":62412134
},
"fielddata_breaker":{
"maximum_size_in_bytes":25714019532,
"maximum_size":"23.9gb",
"estimated_size_in_bytes":269724686,
"estimated_size":"257.2mb",
"overhead":1.03
}
}
}
}

--
You received this message because you are subscribed to the Google Groups "elasticsearch" group.
To unsubscribe from this group and stop receiving emails from it, send an email to elasticsearch+unsubscribe@googlegroups.com.
To view this discussion on the web visit https://groups.google.com/d/msgid/elasticsearch/75d11657-d3c7-4a0a-ba54-ca8691f9b14d%40googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[warkolm (Mark Walkom)]

It's better to pop long output like that into a gist/pastebin, it makes it
easier to read the thread.
It's also worth installing a monitoring plugin like ElasticHQ or marvel, as
they provide graphical insight into what is happening and will extrapolate
some of the raw figures out.

If you can change to Oracle java you will get better performance. Are you
using TTL?

Regards,
Mark Walkom

Infrastructure Engineer
Campaign Monitor
email: markw@campaignmonitor.com
web: www.campaignmonitor.com

On 9 July 2014 14:54, webish gregory@yoursports.com wrote:

I'm seeing very high CPU utilization in an idle or close to idle state. I
haven't seen it use less than 40% on each node over the course of days.
I'm not certain how long it has been like this but could be weeks.

Any help resolving this would be greatly appreciated!

Details:

There are 2 nodes in the cluster.

AWS instances are:
vCPU ECUMemory (GiB) Instance Storage (GB) Linux/UNIX Usage

r3.2xlarge 82661 1 x 160 SSD

curl -XGET 'http://localhost:9200/'
{
"status" : 200,
"name" : "Eddie Brock",
"version" : {
"number" : "1.1.0",
"build_hash" : "2181e113dea80b4a9e31e58e9686658a2d46e363",
"build_timestamp" : "2014-03-25T15:59:51Z",
"build_snapshot" : false,
"lucene_version" : "4.7"
},
"tagline" : "You Know, for Search"
}

% java -version
java version "1.7.0_55"
OpenJDK Runtime Environment (IcedTea 2.4.7) (7u55-2.4.7-1ubuntu1~0.12.04.2)
OpenJDK 64-Bit Server VM (build 24.51-b03, mixed mode)

curl -XGET 'http://localhost:9200/_nodes/stats'
{
"cluster_name":"ab_elastic",
"nodes":{
"kpHPVkBCyjOkRaSpS5Q":{
"timestamp":1404880773859,
"name":"Milan",
"transport_address":"removed",
"host":"removed",
"ip":[
"removed",
"NONE"
],
"indices":{
"docs":{
"count":17793781,
"deleted":466537
},
"store":{
"size_in_bytes":9902722189,
"throttle_time_in_millis":8777423
},
"indexing":{
"index_total":3548638,
"index_time_in_millis":2610694,
"index_current":1,
"delete_total":0,
"delete_time_in_millis":0,
"delete_current":0
},
"get":{
"total":0,
"time_in_millis":0,
"exists_total":0,
"exists_time_in_millis":0,
"missing_total":0,
"missing_time_in_millis":0,
"current":0
},
"search":{
"open_contexts":0,
"query_total":155361213,
"query_time_in_millis":817989500,
"query_current":0,
"fetch_total":4119309,
"fetch_time_in_millis":1967525,
"fetch_current":0
},
"merges":{
"current":0,
"current_docs":0,
"current_size_in_bytes":0,
"total":564,
"total_time_in_millis":76065,
"total_docs":1093579,
"total_size_in_bytes":470304093
},
"refresh":{
"total":15117234,
"total_time_in_millis":50581020
},
"flush":{
"total":2336,
"total_time_in_millis":99980
},
"warmer":{
"current":0,
"total":2719640,
"total_time_in_millis":5980882
},
"filter_cache":{
"memory_size_in_bytes":722232396,
"evictions":1179740
},
"id_cache":{
"memory_size_in_bytes":131672871
},
"fielddata":{
"memory_size_in_bytes":133728950,
"evictions":0
},
"percolate":{
"total":0,
"time_in_millis":0,
"current":0,
"memory_size_in_bytes":-1,
"memory_size":"-1b",
"queries":0
},
"completion":{
"size_in_bytes":0
},
"segments":{
"count":278,
"memory_in_bytes":220799236
},
"translog":{
"operations":68,
"size_in_bytes":0
}
},
"os":{
"timestamp":1404880773910,
"uptime_in_millis":915746,
"load_average":[
2.68,
3.17,
3.35
],
"cpu":{
"sys":0,
"user":38,
"idle":61,
"usage":38,
"stolen":0
},
"mem":{
"free_in_bytes":12301606912,
"used_in_bytes":51857985536,
"free_percent":35,
"used_percent":64,
"actual_free_in_bytes":23005929472,
"actual_used_in_bytes":41153662976
},
"swap":{
"used_in_bytes":0,
"free_in_bytes":0
}
},
"process":{
"timestamp":1404880773911,
"open_file_descriptors":300,
"cpu":{
"percent":313,
"sys_in_millis":14021810,
"user_in_millis":794249870,
"total_in_millis":808271680
},
"mem":{
"resident_in_bytes":32342851584,
"share_in_bytes":712302592,
"total_virtual_in_bytes":47406919680
}
},
"jvm":{
"timestamp":1404880773911,
"uptime_in_millis":309661546,
"mem":{
"heap_used_in_bytes":18409783280,
"heap_used_percent":57,
"heap_committed_in_bytes":32142524416,
"heap_max_in_bytes":32142524416,
"non_heap_used_in_bytes":65012920,
"non_heap_committed_in_bytes":98009088,
"pools":{
"young":{
"used_in_bytes":538302112,
"max_in_bytes":558432256,
"peak_used_in_bytes":558432256,
"peak_max_in_bytes":558432256
},
"survivor":{
"used_in_bytes":11403704,
"max_in_bytes":69730304,
"peak_used_in_bytes":69730304,
"peak_max_in_bytes":69730304
},
"old":{
"used_in_bytes":17860077464,
"max_in_bytes":31514361856,
"peak_used_in_bytes":23694417088,
"peak_max_in_bytes":31514361856
}
}
},
"threads":{
"count":117,
"peak_count":133
},
"gc":{
"collectors":{
"young":{
"collection_count":200644,
"collection_time_in_millis":10870850
},
"old":{
"collection_count":9,
"collection_time_in_millis":2987
}
}
},
"buffer_pools":{
"direct":{
"count":154,
"used_in_bytes":78176363,
"total_capacity_in_bytes":78176363
},
"mapped":{
"count":1293,
"used_in_bytes":9831615457,
"total_capacity_in_bytes":9831615457
}
}
},
"thread_pool":{
"generic":{
"threads":1,
"queue":0,
"active":0,
"rejected":0,
"largest":14,
"completed":56826
},
"index":{
"threads":8,
"queue":0,
"active":0,
"rejected":0,
"largest":8,
"completed":3607948
},
"get":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"snapshot":{
"threads":4,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":95697
},
"merge":{
"threads":2,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":282257
},
"suggest":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"bulk":{
"threads":8,
"queue":0,
"active":0,
"rejected":0,
"largest":8,
"completed":1081
},
"optimize":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"warmer":{
"threads":2,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":5556773
},
"flush":{
"threads":1,
"queue":0,
"active":0,
"rejected":0,
"largest":2,
"completed":2336
},
"search":{
"threads":24,
"queue":0,
"active":0,
"rejected":0,
"largest":24,
"completed":160139043
},
"percolate":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"management":{
"threads":1,
"queue":0,
"active":1,
"rejected":0,
"largest":5,
"completed":1064
},
"refresh":{
"threads":4,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":15126156
}
},
"network":{
"tcp":{
"active_opens":1513869,
"passive_opens":224206,
"curr_estab":52,
"in_segs":1929140868,
"out_segs":2005198310,
"retrans_segs":175635,
"estab_resets":37,
"attempt_fails":669,
"in_errs":3,
"out_rsts":88
}
},
"fs":{
"timestamp":1404880773912,
"total":{
"total_in_bytes":158534184960,
"free_in_bytes":148425240576,
"available_in_bytes":140372176896,
"disk_reads":1993,
"disk_writes":16068284,
"disk_io_op":16070277,
"disk_read_size_in_bytes":7445504,
"disk_write_size_in_bytes":650206560256,
"disk_io_size_in_bytes":650214005760
},
"data":[
{
"path":"/mnt/esdata/ab_elastic/nodes/0",
"mount":"/mnt/esdata",
"dev":"/dev/xvdb",
"total_in_bytes":158534184960,
"free_in_bytes":148425240576,
"available_in_bytes":140372176896,
"disk_reads":1993,
"disk_writes":16068284,
"disk_io_op":16070277,
"disk_read_size_in_bytes":7445504,
"disk_write_size_in_bytes":650206560256,
"disk_io_size_in_bytes":650214005760
}
]
},
"transport":{
"server_open":26,
"rx_count":214082686,
"rx_size_in_bytes":148941344505,
"tx_count":209856841,
"tx_size_in_bytes":90356130749
},
"http":{
"current_open":0,
"total_opened":0
},
"fielddata_breaker":{
"maximum_size_in_bytes":25714019532,
"maximum_size":"23.9gb",
"estimated_size_in_bytes":265401821,
"estimated_size":"253.1mb",
"overhead":1.03
}
},
"erc5l7dQai3mKBIX6sw":{
"timestamp":1404880733754,
"name":"Eddie Brock",
"transport_address":"removed",
"host":"removed",
"ip":[
"removed",
"NONE"
],
"indices":{
"docs":{
"count":17793781,
"deleted":529675
},
"store":{
"size_in_bytes":9969074949,
"throttle_time_in_millis":9235478
},
"indexing":{
"index_total":3531955,
"index_time_in_millis":2783791,
"index_current":0,
"delete_total":0,
"delete_time_in_millis":0,
"delete_current":0
},
"get":{
"total":2947020,
"time_in_millis":217006,
"exists_total":2946589,
"exists_time_in_millis":216977,
"missing_total":431,
"missing_time_in_millis":29,
"current":0
},
"search":{
"open_contexts":0,
"query_total":155416200,
"query_time_in_millis":1076023052,
"query_current":0,
"fetch_total":4101442,
"fetch_time_in_millis":2337321,
"fetch_current":0
},
"merges":{
"current":0,
"current_docs":0,
"current_size_in_bytes":0,
"total":563,
"total_time_in_millis":89211,
"total_docs":1099389,
"total_size_in_bytes":470013054
},
"refresh":{
"total":15123966,
"total_time_in_millis":63368992
},
"flush":{
"total":2335,
"total_time_in_millis":414790
},
"warmer":{
"current":0,
"total":2690030,
"total_time_in_millis":7339293
},
"filter_cache":{
"memory_size_in_bytes":695354935,
"evictions":1377260
},
"id_cache":{
"memory_size_in_bytes":131817497
},
"fielddata":{
"memory_size_in_bytes":137907189,
"evictions":0
},
"percolate":{
"total":0,
"time_in_millis":0,
"current":0,
"memory_size_in_bytes":-1,
"memory_size":"-1b",
"queries":0
},
"completion":{
"size_in_bytes":0
},
"segments":{
"count":282,
"memory_in_bytes":217468955
},
"translog":{
"operations":86,
"size_in_bytes":0
}
},
"os":{
"timestamp":1404880733801,
"uptime_in_millis":923467,
"load_average":[
3.71,
4.57,
4.46
],
"cpu":{
"sys":0,
"user":46,
"idle":52,
"usage":46,
"stolen":0
},
"mem":{
"free_in_bytes":4438659072,
"used_in_bytes":59720933376,
"free_percent":32,
"used_percent":67,
"actual_free_in_bytes":21124698112,
"actual_used_in_bytes":43034894336
},
"swap":{
"used_in_bytes":0,
"free_in_bytes":0
}
},
"process":{
"timestamp":1404880733801,
"open_file_descriptors":301,
"cpu":{
"percent":377,
"sys_in_millis":26524340,
"user_in_millis":947578840,
"total_in_millis":974103180
},
"mem":{
"resident_in_bytes":33945767936,
"share_in_bytes":692912128,
"total_virtual_in_bytes":47473528832
}
},
"jvm":{
"timestamp":1404880733802,
"uptime_in_millis":310301938,
"mem":{
"heap_used_in_bytes":10113402832,
"heap_used_percent":31,
"heap_committed_in_bytes":32142524416,
"heap_max_in_bytes":32142524416,
"non_heap_used_in_bytes":71762640,
"non_heap_committed_in_bytes":106397696,
"pools":{
"young":{
"used_in_bytes":218720808,
"max_in_bytes":558432256,
"peak_used_in_bytes":558432256,
"peak_max_in_bytes":558432256
},
"survivor":{
"used_in_bytes":9630920,
"max_in_bytes":69730304,
"peak_used_in_bytes":69730304,
"peak_max_in_bytes":69730304
},
"old":{
"used_in_bytes":9885051104,
"max_in_bytes":31514361856,
"peak_used_in_bytes":23868903928,
"peak_max_in_bytes":31514361856
}
}
},
"threads":{
"count":121,
"peak_count":144
},
"gc":{
"collectors":{
"young":{
"collection_count":209658,
"collection_time_in_millis":12089132
},
"old":{
"collection_count":11,
"collection_time_in_millis":6628
}
}
},
"buffer_pools":{
"direct":{
"count":162,
"used_in_bytes":195159665,
"total_capacity_in_bytes":195159665
},
"mapped":{
"count":1292,
"used_in_bytes":9897821921,
"total_capacity_in_bytes":9897821921
}
}
},
"thread_pool":{
"generic":{
"threads":1,
"queue":0,
"active":0,
"rejected":0,
"largest":23,
"completed":67214
},
"index":{
"threads":8,
"queue":0,
"active":0,
"rejected":0,
"largest":8,
"completed":3611159
},
"get":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"snapshot":{
"threads":2,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":95672
},
"merge":{
"threads":3,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":285955
},
"suggest":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"bulk":{
"threads":8,
"queue":0,
"active":0,
"rejected":0,
"largest":8,
"completed":1125
},
"optimize":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"warmer":{
"threads":2,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":5487739
},
"flush":{
"threads":1,
"queue":0,
"active":0,
"rejected":0,
"largest":2,
"completed":2335
},
"search":{
"threads":24,
"queue":0,
"active":0,
"rejected":0,
"largest":24,
"completed":160510703
},
"percolate":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"management":{
"threads":5,
"queue":0,
"active":1,
"rejected":0,
"largest":5,
"completed":22893
},
"refresh":{
"threads":4,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":15135553
}
},
"network":{
"tcp":{
"active_opens":1025664,
"passive_opens":105715571,
"curr_estab":55,
"in_segs":2557757673,
"out_segs":2502369803,
"retrans_segs":205765,
"estab_resets":68,
"attempt_fails":207,
"in_errs":0,
"out_rsts":1641
}
},
"fs":{
"timestamp":1404880733802,
"total":{
"total_in_bytes":158533136384,
"free_in_bytes":137526714368,
"available_in_bytes":129473703936,
"disk_reads":8637,
"disk_writes":17332161,
"disk_io_op":17340798,
"disk_read_size_in_bytes":139357184,
"disk_write_size_in_bytes":706635259904,
"disk_io_size_in_bytes":706774617088
},
"data":[
{
"path":"/mnt/esdata/ab_elastic/nodes/0",
"mount":"/mnt/esdata",
"dev":"/dev/xvdb",
"total_in_bytes":158533136384,
"free_in_bytes":137526714368,
"available_in_bytes":129473703936,
"disk_reads":8637,
"disk_writes":17332161,
"disk_io_op":17340798,
"disk_read_size_in_bytes":139357184,
"disk_write_size_in_bytes":706635259904,
"disk_io_size_in_bytes":706774617088
}
]
},
"transport":{
"server_open":26,
"rx_count":214086149,
"rx_size_in_bytes":90356374759,
"tx_count":210167189,
"tx_size_in_bytes":149348401545
},
"http":{
"current_open":1,
"total_opened":62412134
},
"fielddata_breaker":{
"maximum_size_in_bytes":25714019532,
"maximum_size":"23.9gb",
"estimated_size_in_bytes":269724686,
"estimated_size":"257.2mb",
"overhead":1.03
}
}
}
}

--
You received this message because you are subscribed to the Google Groups
"elasticsearch" group.
To unsubscribe from this group and stop receiving emails from it, send an
email to elasticsearch+unsubscribe@googlegroups.com.
To view this discussion on the web visit
https://groups.google.com/d/msgid/elasticsearch/75d11657-d3c7-4a0a-ba54-ca8691f9b14d%40googlegroups.com
https://groups.google.com/d/msgid/elasticsearch/75d11657-d3c7-4a0a-ba54-ca8691f9b14d%40googlegroups.com?utm_medium=email&utm_source=footer
.
For more options, visit https://groups.google.com/d/optout.

--
You received this message because you are subscribed to the Google Groups "elasticsearch" group.
To unsubscribe from this group and stop receiving emails from it, send an email to elasticsearch+unsubscribe@googlegroups.com.
To view this discussion on the web visit https://groups.google.com/d/msgid/elasticsearch/CAEM624aRDRDV6_QbBRZQ13HgNVsaqp9qCwhfJdB9_Vo0K9xPGQ%40mail.gmail.com.
For more options, visit https://groups.google.com/d/optout.

[webish]

Thanks Mark. My mistake.

I can try switching to Oracle Java. There is no TTL. I've used Marvel for
development testing. Perhaps I can install the plugin...

On Wednesday, July 9, 2014 1:35:20 AM UTC-4, Mark Walkom wrote:

It's better to pop long output like that into a gist/pastebin, it makes it
easier to read the thread.
It's also worth installing a monitoring plugin like ElasticHQ or marvel,
as they provide graphical insight into what is happening and will
extrapolate some of the raw figures out.

If you can change to Oracle java you will get better performance. Are you
using TTL?

Regards,
Mark Walkom

Infrastructure Engineer
Campaign Monitor
email: ma...@campaignmonitor.com <javascript:>
web: www.campaignmonitor.com

On 9 July 2014 14:54, webish <gre...@yoursports.com <javascript:>> wrote:

I'm seeing very high CPU utilization in an idle or close to idle state. I
haven't seen it use less than 40% on each node over the course of days.
I'm not certain how long it has been like this but could be weeks.

Any help resolving this would be greatly appreciated!

Details:

There are 2 nodes in the cluster.

AWS instances are:
vCPU ECUMemory (GiB) Instance Storage (GB) Linux/UNIX Usage

r3.2xlarge 82661 1 x 160 SSD

curl -XGET 'http://localhost:9200/'
{
"status" : 200,
"name" : "Eddie Brock",
"version" : {
"number" : "1.1.0",
"build_hash" : "2181e113dea80b4a9e31e58e9686658a2d46e363",
"build_timestamp" : "2014-03-25T15:59:51Z",
"build_snapshot" : false,
"lucene_version" : "4.7"
},
"tagline" : "You Know, for Search"
}

% java -version
java version "1.7.0_55"
OpenJDK Runtime Environment (IcedTea 2.4.7) (7u55-2.4.7-1ubuntu1~0.12.04.2)
OpenJDK 64-Bit Server VM (build 24.51-b03, mixed mode)

curl -XGET 'http://localhost:9200/_nodes/stats'
{
"cluster_name":"ab_elastic",
"nodes":{
"kpHPVkBCyjOkRaSpS5Q":{
"timestamp":1404880773859,
"name":"Milan",
"transport_address":"removed",
"host":"removed",
"ip":[
"removed",
"NONE"
],
"indices":{
"docs":{
"count":17793781,
"deleted":466537
},
"store":{
"size_in_bytes":9902722189,
"throttle_time_in_millis":8777423
},
"indexing":{
"index_total":3548638,
"index_time_in_millis":2610694,
"index_current":1,
"delete_total":0,
"delete_time_in_millis":0,
"delete_current":0
},
"get":{
"total":0,
"time_in_millis":0,
"exists_total":0,
"exists_time_in_millis":0,
"missing_total":0,
"missing_time_in_millis":0,
"current":0
},
"search":{
"open_contexts":0,
"query_total":155361213,
"query_time_in_millis":817989500,
"query_current":0,
"fetch_total":4119309,
"fetch_time_in_millis":1967525,
"fetch_current":0
},
"merges":{
"current":0,
"current_docs":0,
"current_size_in_bytes":0,
"total":564,
"total_time_in_millis":76065,
"total_docs":1093579,
"total_size_in_bytes":470304093
},
"refresh":{
"total":15117234,
"total_time_in_millis":50581020
},
"flush":{
"total":2336,
"total_time_in_millis":99980
},
"warmer":{
"current":0,
"total":2719640,
"total_time_in_millis":5980882
},
"filter_cache":{
"memory_size_in_bytes":722232396,
"evictions":1179740
},
"id_cache":{
"memory_size_in_bytes":131672871
},
"fielddata":{
"memory_size_in_bytes":133728950,
"evictions":0
},
"percolate":{
"total":0,
"time_in_millis":0,
"current":0,
"memory_size_in_bytes":-1,
"memory_size":"-1b",
"queries":0
},
"completion":{
"size_in_bytes":0
},
"segments":{
"count":278,
"memory_in_bytes":220799236
},
"translog":{
"operations":68,
"size_in_bytes":0
}
},
"os":{
"timestamp":1404880773910,
"uptime_in_millis":915746,
"load_average":[
2.68,
3.17,
3.35
],
"cpu":{
"sys":0,
"user":38,
"idle":61,
"usage":38,
"stolen":0
},
"mem":{
"free_in_bytes":12301606912,
"used_in_bytes":51857985536,
"free_percent":35,
"used_percent":64,
"actual_free_in_bytes":23005929472,
"actual_used_in_bytes":41153662976
},
"swap":{
"used_in_bytes":0,
"free_in_bytes":0
}
},
"process":{
"timestamp":1404880773911,
"open_file_descriptors":300,
"cpu":{
"percent":313,
"sys_in_millis":14021810,
"user_in_millis":794249870,
"total_in_millis":808271680
},
"mem":{
"resident_in_bytes":32342851584,
"share_in_bytes":712302592,
"total_virtual_in_bytes":47406919680
}
},
"jvm":{
"timestamp":1404880773911,
"uptime_in_millis":309661546,
"mem":{
"heap_used_in_bytes":18409783280,
"heap_used_percent":57,
"heap_committed_in_bytes":32142524416,
"heap_max_in_bytes":32142524416,
"non_heap_used_in_bytes":65012920,
"non_heap_committed_in_bytes":98009088,
"pools":{
"young":{
"used_in_bytes":538302112,
"max_in_bytes":558432256,
"peak_used_in_bytes":558432256,
"peak_max_in_bytes":558432256
},
"survivor":{
"used_in_bytes":11403704,
"max_in_bytes":69730304,
"peak_used_in_bytes":69730304,
"peak_max_in_bytes":69730304
},
"old":{
"used_in_bytes":17860077464,
"max_in_bytes":31514361856,
"peak_used_in_bytes":23694417088,
"peak_max_in_bytes":31514361856
}
}
},
"threads":{
"count":117,
"peak_count":133
},
"gc":{
"collectors":{
"young":{
"collection_count":200644,
"collection_time_in_millis":10870850
},
"old":{
"collection_count":9,
"collection_time_in_millis":2987
}
}
},
"buffer_pools":{
"direct":{
"count":154,
"used_in_bytes":78176363,
"total_capacity_in_bytes":78176363
},
"mapped":{
"count":1293,
"used_in_bytes":9831615457,
"total_capacity_in_bytes":9831615457
}
}
},
"thread_pool":{
"generic":{
"threads":1,
"queue":0,
"active":0,
"rejected":0,
"largest":14,
"completed":56826
},
"index":{
"threads":8,
"queue":0,
"active":0,
"rejected":0,
"largest":8,
"completed":3607948
},
"get":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"snapshot":{
"threads":4,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":95697
},
"merge":{
"threads":2,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":282257
},
"suggest":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"bulk":{
"threads":8,
"queue":0,
"active":0,
"rejected":0,
"largest":8,
"completed":1081
},
"optimize":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"warmer":{
"threads":2,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":5556773
},
"flush":{
"threads":1,
"queue":0,
"active":0,
"rejected":0,
"largest":2,
"completed":2336
},
"search":{
"threads":24,
"queue":0,
"active":0,
"rejected":0,
"largest":24,
"completed":160139043
},
"percolate":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"management":{
"threads":1,
"queue":0,
"active":1,
"rejected":0,
"largest":5,
"completed":1064
},
"refresh":{
"threads":4,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":15126156
}
},
"network":{
"tcp":{
"active_opens":1513869,
"passive_opens":224206,
"curr_estab":52,
"in_segs":1929140868,
"out_segs":2005198310,
"retrans_segs":175635,
"estab_resets":37,
"attempt_fails":669,
"in_errs":3,
"out_rsts":88
}
},
"fs":{
"timestamp":1404880773912,
"total":{
"total_in_bytes":158534184960,
"free_in_bytes":148425240576,
"available_in_bytes":140372176896,
"disk_reads":1993,
"disk_writes":16068284,
"disk_io_op":16070277,
"disk_read_size_in_bytes":7445504,
"disk_write_size_in_bytes":650206560256,
"disk_io_size_in_bytes":650214005760
},
"data":[
{
"path":"/mnt/esdata/ab_elastic/nodes/0",
"mount":"/mnt/esdata",
"dev":"/dev/xvdb",
"total_in_bytes":158534184960,
"free_in_bytes":148425240576,
"available_in_bytes":140372176896,
"disk_reads":1993,
"disk_writes":16068284,
"disk_io_op":16070277,
"disk_read_size_in_bytes":7445504,
"disk_write_size_in_bytes":650206560256,
"disk_io_size_in_bytes":650214005760
}
]
},
"transport":{
"server_open":26,
"rx_count":214082686,
"rx_size_in_bytes":148941344505,
"tx_count":209856841,
"tx_size_in_bytes":90356130749
},
"http":{
"current_open":0,
"total_opened":0
},
"fielddata_breaker":{
"maximum_size_in_bytes":25714019532,
"maximum_size":"23.9gb",
"estimated_size_in_bytes":265401821,
"estimated_size":"253.1mb",
"overhead":1.03
}
},
"erc5l7dQai3mKBIX6sw":{
"timestamp":1404880733754,
"name":"Eddie Brock",
"transport_address":"removed",
"host":"removed",
"ip":[
"removed",
"NONE"
],
"indices":{
"docs":{
"count":17793781,
"deleted":529675
},
"store":{
"size_in_bytes":9969074949,
"throttle_time_in_millis":9235478
},
"indexing":{
"index_total":3531955,
"index_time_in_millis":2783791,
"index_current":0,
"delete_total":0,
"delete_time_in_millis":0,
"delete_current":0
},
"get":{
"total":2947020,
"time_in_millis":217006,
"exists_total":2946589,
"exists_time_in_millis":216977,
"missing_total":431,
"missing_time_in_millis":29,
"current":0
},
"search":{
"open_contexts":0,
"query_total":155416200,
"query_time_in_millis":1076023052,
"query_current":0,
"fetch_total":4101442,
"fetch_time_in_millis":2337321,
"fetch_current":0
},
"merges":{
"current":0,
"current_docs":0,
"current_size_in_bytes":0,
"total":563,
"total_time_in_millis":89211,
"total_docs":1099389,
"total_size_in_bytes":470013054
},
"refresh":{
"total":15123966,
"total_time_in_millis":63368992
},
"flush":{
"total":2335,
"total_time_in_millis":414790
},
"warmer":{
"current":0,
"total":2690030,
"total_time_in_millis":7339293
},
"filter_cache":{
"memory_size_in_bytes":695354935,
"evictions":1377260
},
"id_cache":{
"memory_size_in_bytes":131817497
},
"fielddata":{
"memory_size_in_bytes":137907189,
"evictions":0
},
"percolate":{
"total":0,
"time_in_millis":0,
"current":0,
"memory_size_in_bytes":-1,
"memory_size":"-1b",
"queries":0
},
"completion":{
"size_in_bytes":0
},
"segments":{
"count":282,
"memory_in_bytes":217468955
},
"translog":{
"operations":86,
"size_in_bytes":0
}
},
"os":{
"timestamp":1404880733801,
"uptime_in_millis":923467,
"load_average":[
3.71,
4.57,
4.46
],
"cpu":{
"sys":0,
"user":46,
"idle":52,
"usage":46,
"stolen":0
},
"mem":{
"free_in_bytes":4438659072,
"used_in_bytes":59720933376,
"free_percent":32,
"used_percent":67,
"actual_free_in_bytes":21124698112,
"actual_used_in_bytes":43034894336
},
"swap":{
"used_in_bytes":0,
"free_in_bytes":0
}
},
"process":{
"timestamp":1404880733801,
"open_file_descriptors":301,
"cpu":{
"percent":377,
"sys_in_millis":26524340,
"user_in_millis":947578840,
"total_in_millis":974103180
},
"mem":{
"resident_in_bytes":33945767936,
"share_in_bytes":692912128,
"total_virtual_in_bytes":47473528832
}
},
"jvm":{
"timestamp":1404880733802,
"uptime_in_millis":310301938,
"mem":{
"heap_used_in_bytes":10113402832,
"heap_used_percent":31,
"heap_committed_in_bytes":32142524416,
"heap_max_in_bytes":32142524416,
"non_heap_used_in_bytes":71762640,
"non_heap_committed_in_bytes":106397696,
"pools":{
"young":{
"used_in_bytes":218720808,
"max_in_bytes":558432256,
"peak_used_in_bytes":558432256,
"peak_max_in_bytes":558432256
},
"survivor":{
"used_in_bytes":9630920,
"max_in_bytes":69730304,
"peak_used_in_bytes":69730304,
"peak_max_in_bytes":69730304
},
"old":{
"used_in_bytes":9885051104,
"max_in_bytes":31514361856,
"peak_used_in_bytes":23868903928,
"peak_max_in_bytes":31514361856
}
}
},
"threads":{
"count":121,
"peak_count":144
},
"gc":{
"collectors":{
"young":{
"collection_count":209658,
"collection_time_in_millis":12089132
},
"old":{
"collection_count":11,
"collection_time_in_millis":6628
}
}
},
"buffer_pools":{
"direct":{
"count":162,
"used_in_bytes":195159665,
"total_capacity_in_bytes":195159665
},
"mapped":{
"count":1292,
"used_in_bytes":9897821921,
"total_capacity_in_bytes":9897821921
}
}
},
"thread_pool":{
"generic":{
"threads":1,
"queue":0,
"active":0,
"rejected":0,
"largest":23,
"completed":67214
},
"index":{
"threads":8,
"queue":0,
"active":0,
"rejected":0,
"largest":8,
"completed":3611159
},
"get":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"snapshot":{
"threads":2,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":95672
},
"merge":{
"threads":3,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":285955
},
"suggest":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"bulk":{
"threads":8,
"queue":0,
"active":0,
"rejected":0,
"largest":8,
"completed":1125
},
"optimize":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"warmer":{
"threads":2,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":5487739
},
"flush":{
"threads":1,
"queue":0,
"active":0,
"rejected":0,
"largest":2,
"completed":2335
},
"search":{
"threads":24,
"queue":0,
"active":0,
"rejected":0,
"largest":24,
"completed":160510703
},
"percolate":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"management":{
"threads":5,
"queue":0,
"active":1,
"rejected":0,
"largest":5,
"completed":22893
},
"refresh":{
"threads":4,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":15135553
}
},
"network":{
"tcp":{
"active_opens":1025664,
"passive_opens":105715571,
"curr_estab":55,
"in_segs":2557757673,
"out_segs":2502369803,
"retrans_segs":205765,
"estab_resets":68,
"attempt_fails":207,
"in_errs":0,
"out_rsts":1641
}
},
"fs":{
&qu

...

--
You received this message because you are subscribed to the Google Groups "elasticsearch" group.
To unsubscribe from this group and stop receiving emails from it, send an email to elasticsearch+unsubscribe@googlegroups.com.
To view this discussion on the web visit https://groups.google.com/d/msgid/elasticsearch/5140dad5-60e6-4a51-8986-c3f1c0da4c0f%40googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[Ivan (Ivan Brusic)]

Yes, please use gist/pastebin in the future.

Try using the hot threads API [1] to see if there are any threads that are
truly busy. If your system is truly idle, your thread pools should be
almost empty [2]. Your output above has only one management thread in use,
which could be simply the thread serving up the output.

[1]

Elasticsearch Platform — Find real-time answers at scale

[2]

Elasticsearch Platform — Find real-time answers at scale

Cheers,

Ivan

On Tue, Jul 8, 2014 at 10:54 PM, webish gregory@yoursports.com wrote:

Thanks Mark. My mistake.

I can try switching to Oracle Java. There is no TTL. I've used Marvel
for development testing. Perhaps I can install the plugin...

On Wednesday, July 9, 2014 1:35:20 AM UTC-4, Mark Walkom wrote:

It's better to pop long output like that into a gist/pastebin, it makes
it easier to read the thread.
It's also worth installing a monitoring plugin like ElasticHQ or marvel,
as they provide graphical insight into what is happening and will
extrapolate some of the raw figures out.

If you can change to Oracle java you will get better performance. Are you
using TTL?

Regards,
Mark Walkom

Infrastructure Engineer
Campaign Monitor
email: ma...@campaignmonitor.com
web: www.campaignmonitor.com

On 9 July 2014 14:54, webish gre...@yoursports.com wrote:

I'm seeing very high CPU utilization in an idle or close to idle state.
I haven't seen it use less than 40% on each node over the course of days.
I'm not certain how long it has been like this but could be weeks.

Any help resolving this would be greatly appreciated!

Details:

There are 2 nodes in the cluster.

AWS instances are:
vCPU ECUMemory (GiB) Instance Storage (GB) Linux/UNIX Usage

r3.2xlarge 82661 1 x 160 SSD

curl -XGET 'http://localhost:9200/'
{
"status" : 200,
"name" : "Eddie Brock",
"version" : {
"number" : "1.1.0",
"build_hash" : "2181e113dea80b4a9e31e58e9686658a2d46e363",
"build_timestamp" : "2014-03-25T15:59:51Z",
"build_snapshot" : false,
"lucene_version" : "4.7"
},
"tagline" : "You Know, for Search"
}

% java -version
java version "1.7.0_55"
OpenJDK Runtime Environment (IcedTea 2.4.7) (7u55-2.4.7-1ubuntu1~0.12.04.
2)
OpenJDK 64-Bit Server VM (build 24.51-b03, mixed mode)

curl -XGET 'http://localhost:9200/_nodes/stats'
{
"cluster_name":"ab_elastic",
"nodes":{
"kpHPVkBCyjOkRaSpS5Q":{
"timestamp":1404880773859,
"name":"Milan",
"transport_address":"removed",
"host":"removed",
"ip":[
"removed",
"NONE"
],
"indices":{
"docs":{
"count":17793781,
"deleted":466537
},
"store":{
"size_in_bytes":9902722189,
"throttle_time_in_millis":8777423
},
"indexing":{
"index_total":3548638,
"index_time_in_millis":2610694,
"index_current":1,
"delete_total":0,
"delete_time_in_millis":0,
"delete_current":0
},
"get":{
"total":0,
"time_in_millis":0,
"exists_total":0,
"exists_time_in_millis":0,
"missing_total":0,
"missing_time_in_millis":0,
"current":0
},
"search":{
"open_contexts":0,
"query_total":155361213,
"query_time_in_millis":817989500,
"query_current":0,
"fetch_total":4119309,
"fetch_time_in_millis":1967525,
"fetch_current":0
},
"merges":{
"current":0,
"current_docs":0,
"current_size_in_bytes":0,
"total":564,
"total_time_in_millis":76065,
"total_docs":1093579,
"total_size_in_bytes":470304093
},
"refresh":{
"total":15117234,
"total_time_in_millis":50581020
},
"flush":{
"total":2336,
"total_time_in_millis":99980
},
"warmer":{
"current":0,
"total":2719640,
"total_time_in_millis":5980882
},
"filter_cache":{
"memory_size_in_bytes":722232396,
"evictions":1179740
},
"id_cache":{
"memory_size_in_bytes":131672871
},
"fielddata":{
"memory_size_in_bytes":133728950,
"evictions":0
},
"percolate":{
"total":0,
"time_in_millis":0,
"current":0,
"memory_size_in_bytes":-1,
"memory_size":"-1b",
"queries":0
},
"completion":{
"size_in_bytes":0
},
"segments":{
"count":278,
"memory_in_bytes":220799236
},
"translog":{
"operations":68,
"size_in_bytes":0
}
},
"os":{
"timestamp":1404880773910,
"uptime_in_millis":915746,
"load_average":[
2.68,
3.17,
3.35
],
"cpu":{
"sys":0,
"user":38,
"idle":61,
"usage":38,
"stolen":0
},
"mem":{
"free_in_bytes":12301606912,
"used_in_bytes":51857985536,
"free_percent":35,
"used_percent":64,
"actual_free_in_bytes":23005929472,
"actual_used_in_bytes":41153662976
},
"swap":{
"used_in_bytes":0,
"free_in_bytes":0
}
},
"process":{
"timestamp":1404880773911,
"open_file_descriptors":300,
"cpu":{
"percent":313,
"sys_in_millis":14021810,
"user_in_millis":794249870,
"total_in_millis":808271680
},
"mem":{
"resident_in_bytes":32342851584,
"share_in_bytes":712302592,
"total_virtual_in_bytes":47406919680
}
},
"jvm":{
"timestamp":1404880773911,
"uptime_in_millis":309661546,
"mem":{
"heap_used_in_bytes":18409783280,
"heap_used_percent":57,
"heap_committed_in_bytes":32142524416,
"heap_max_in_bytes":32142524416,
"non_heap_used_in_bytes":65012920,
"non_heap_committed_in_bytes":98009088,
"pools":{
"young":{
"used_in_bytes":538302112,
"max_in_bytes":558432256,
"peak_used_in_bytes":558432256,
"peak_max_in_bytes":558432256
},
"survivor":{
"used_in_bytes":11403704,
"max_in_bytes":69730304,
"peak_used_in_bytes":69730304,
"peak_max_in_bytes":69730304
},
"old":{
"used_in_bytes":17860077464,
"max_in_bytes":31514361856,
"peak_used_in_bytes":23694417088,
"peak_max_in_bytes":31514361856
}
}
},
"threads":{
"count":117,
"peak_count":133
},
"gc":{
"collectors":{
"young":{
"collection_count":200644,
"collection_time_in_millis":10870850
},
"old":{
"collection_count":9,
"collection_time_in_millis":2987
}
}
},
"buffer_pools":{
"direct":{
"count":154,
"used_in_bytes":78176363,
"total_capacity_in_bytes":78176363
},
"mapped":{
"count":1293,
"used_in_bytes":9831615457,
"total_capacity_in_bytes":9831615457
}
}
},
"thread_pool":{
"generic":{
"threads":1,
"queue":0,
"active":0,
"rejected":0,
"largest":14,
"completed":56826
},
"index":{
"threads":8,
"queue":0,
"active":0,
"rejected":0,
"largest":8,
"completed":3607948
},
"get":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"snapshot":{
"threads":4,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":95697
},
"merge":{
"threads":2,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":282257
},
"suggest":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"bulk":{
"threads":8,
"queue":0,
"active":0,
"rejected":0,
"largest":8,
"completed":1081
},
"optimize":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"warmer":{
"threads":2,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":5556773
},
"flush":{
"threads":1,
"queue":0,
"active":0,
"rejected":0,
"largest":2,
"completed":2336
},
"search":{
"threads":24,
"queue":0,
"active":0,
"rejected":0,
"largest":24,
"completed":160139043
},
"percolate":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"management":{
"threads":1,
"queue":0,
"active":1,
"rejected":0,
"largest":5,
"completed":1064
},
"refresh":{
"threads":4,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":15126156
}
},
"network":{
"tcp":{
"active_opens":1513869,
"passive_opens":224206,
"curr_estab":52,
"in_segs":1929140868,
"out_segs":2005198310,
"retrans_segs":175635,
"estab_resets":37,
"attempt_fails":669,
"in_errs":3,
"out_rsts":88
}
},
"fs":{
"timestamp":1404880773912,
"total":{
"total_in_bytes":158534184960,
"free_in_bytes":148425240576,
"available_in_bytes":140372176896,
"disk_reads":1993,
"disk_writes":16068284,
"disk_io_op":16070277,
"disk_read_size_in_bytes":7445504,
"disk_write_size_in_bytes":650206560256,
"disk_io_size_in_bytes":650214005760
},
"data":[
{
"path":"/mnt/esdata/ab_elastic/nodes/0",
"mount":"/mnt/esdata",
"dev":"/dev/xvdb",
"total_in_bytes":158534184960,
"free_in_bytes":148425240576,
"available_in_bytes":140372176896,
"disk_reads":1993,
"disk_writes":16068284,
"disk_io_op":16070277,
"disk_read_size_in_bytes":7445504,
"disk_write_size_in_bytes":650206560256,
"disk_io_size_in_bytes":650214005760
}
]
},
"transport":{
"server_open":26,
"rx_count":214082686,
"rx_size_in_bytes":148941344505,
"tx_count":209856841,
"tx_size_in_bytes":90356130749
},
"http":{
"current_open":0,
"total_opened":0
},
"fielddata_breaker":{
"maximum_size_in_bytes":25714019532,
"maximum_size":"23.9gb",
"estimated_size_in_bytes":265401821,
"estimated_size":"253.1mb",
"overhead":1.03
}
},
"erc5l7dQai3mKBIX6sw":{
"timestamp":1404880733754,
"name":"Eddie Brock",
"transport_address":"removed",
"host":"removed",
"ip":[
"removed",
"NONE"
],
"indices":{
"docs":{
"count":17793781,
"deleted":529675
},
"store":{
"size_in_bytes":9969074949,
"throttle_time_in_millis":9235478
},
"indexing":{
"index_total":3531955,
"index_time_in_millis":2783791,
"index_current":0,
"delete_total":0,
"delete_time_in_millis":0,
"delete_current":0
},
"get":{
"total":2947020,
"time_in_millis":217006,
"exists_total":2946589,
"exists_time_in_millis":216977,
"missing_total":431,
"missing_time_in_millis":29,
"current":0
},
"search":{
"open_contexts":0,
"query_total":155416200,
"query_time_in_millis":1076023052,
"query_current":0,
"fetch_total":4101442,
"fetch_time_in_millis":2337321,
"fetch_current":0
},
"merges":{
"current":0,
"current_docs":0,
"current_size_in_bytes":0,
"total":563,
"total_time_in_millis":89211,
"total_docs":1099389,
"total_size_in_bytes":470013054
},
"refresh":{
"total":15123966,
"total_time_in_millis":63368992
},
"flush":{
"total":2335,
"total_time_in_millis":414790
},
"warmer":{
"current":0,
"total":2690030,
"total_time_in_millis":7339293
},
"filter_cache":{
"memory_size_in_bytes":695354935,
"evictions":1377260
},
"id_cache":{
"memory_size_in_bytes":131817497
},
"fielddata":{
"memory_size_in_bytes":137907189,
"evictions":0
},
"percolate":{
"total":0,
"time_in_millis":0,
"current":0,
"memory_size_in_bytes":-1,
"memory_size":"-1b",
"queries":0
},
"completion":{
"size_in_bytes":0
},
"segments":{
"count":282,
"memory_in_bytes":217468955
},
"translog":{
"operations":86,
"size_in_bytes":0
}
},
"os":{
"timestamp":1404880733801,
"uptime_in_millis":923467,
"load_average":[
3.71,
4.57,
4.46
],
"cpu":{
"sys":0,
"user":46,
"idle":52,
"usage":46,
"stolen":0
},
"mem":{
"free_in_bytes":4438659072,
"used_in_bytes":59720933376,
"free_percent":32,
"used_percent":67,
"actual_free_in_bytes":21124698112,
"actual_used_in_bytes":43034894336
},
"swap":{
"used_in_bytes":0,
"free_in_bytes":0
}
},
"process":{
"timestamp":1404880733801,
"open_file_descriptors":301,
"cpu":{
"percent":377,
"sys_in_millis":26524340,
"user_in_millis":947578840,
"total_in_millis":974103180
},
"mem":{
"resident_in_bytes":33945767936,
"share_in_bytes":692912128,
"total_virtual_in_bytes":47473528832
}
},
"jvm":{
"timestamp":1404880733802,
"uptime_in_millis":310301938,
"mem":{
"heap_used_in_bytes":10113402832,
"heap_used_percent":31,
"heap_committed_in_bytes":32142524416,
"heap_max_in_bytes":32142524416,
"non_heap_used_in_bytes":71762640,
"non_heap_committed_in_bytes":106397696,
"pools":{
"young":{
"used_in_bytes":218720808,
"max_in_bytes":558432256,
"peak_used_in_bytes":558432256,
"peak_max_in_bytes":558432256
},
"survivor":{
"used_in_bytes":9630920,
"max_in_bytes":69730304,
"peak_used_in_bytes":69730304,
"peak_max_in_bytes":69730304
},
"old":{
"used_in_bytes":9885051104,
"max_in_bytes":31514361856,
"peak_used_in_bytes":23868903928,
"peak_max_in_bytes":31514361856
}
}
},
"threads":{
"count":121,
"peak_count":144
},
"gc":{
"collectors":{
"young":{
"collection_count":209658,
"collection_time_in_millis":12089132
},
"old":{
"collection_count":11,
"collection_time_in_millis":6628
}
}
},
"buffer_pools":{
"direct":{
"count":162,
"used_in_bytes":195159665,
"total_capacity_in_bytes":195159665
},
"mapped":{
"count":1292,
"used_in_bytes":9897821921,
"total_capacity_in_bytes":9897821921
}
}
},
"thread_pool":{
"generic":{
"threads":1,
"queue":0,
"active":0,
"rejected":0,
"largest":23,
"completed":67214
},
"index":{
"threads":8,
"queue":0,
"active":0,
"rejected":0,
"largest":8,
"completed":3611159
},
"get":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"snapshot":{
"threads":2,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":95672
},
"merge":{
"threads":3,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":285955
},
"suggest":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"bulk":{
"threads":8,
"queue":0,
"active":0,
"rejected":0,
"largest":8,
"completed":1125
},
"optimize":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"warmer":{
"threads":2,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":5487739
},
"flush":{
"threads":1,
"queue":0,
"active":0,
"rejected":0,
"largest":2,
"completed":2335
},
"search":{
"threads":24,
"queue":0,
"active":0,
"rejected":0,
"largest":24,
"completed":160510703
},
"percolate":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"management":{
"threads":5,
"queue":0,
"active":1,
"rejected":0,
"largest":5,
"completed":22893
},
"refresh":{
"threads":4,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":15135553
}
},
"network":{
"tcp":{
"active_opens":1025664,
"passive_opens":105715571,
"curr_estab":55,
"in_segs":2557757673,
"out_segs":2502369803,
"retrans_segs":205765,
"estab_resets":68,
"attempt_fails":207,
"in_errs":0,
"out_rsts":1641
}
},
"fs":{
&qu

...

--
You received this message because you are subscribed to the Google Groups
"elasticsearch" group.
To unsubscribe from this group and stop receiving emails from it, send an
email to elasticsearch+unsubscribe@googlegroups.com.
To view this discussion on the web visit
https://groups.google.com/d/msgid/elasticsearch/5140dad5-60e6-4a51-8986-c3f1c0da4c0f%40googlegroups.com
https://groups.google.com/d/msgid/elasticsearch/5140dad5-60e6-4a51-8986-c3f1c0da4c0f%40googlegroups.com?utm_medium=email&utm_source=footer
.

For more options, visit https://groups.google.com/d/optout.

--
You received this message because you are subscribed to the Google Groups "elasticsearch" group.
To unsubscribe from this group and stop receiving emails from it, send an email to elasticsearch+unsubscribe@googlegroups.com.
To view this discussion on the web visit https://groups.google.com/d/msgid/elasticsearch/CALY%3DcQABy%3DV%3DG4wuKcqT3Bj6e6P4cDnuddS8_%3DOMeA30MEt02g%40mail.gmail.com.
For more options, visit https://groups.google.com/d/optout.

[webish]

Definitely.

Here are all the captures thus far in gists.

CLUSTER:
curl -XGET 'http://localhost:9200/_nodes/stats'
curl -XGET 'http://localhost:9200/_nodes/stats' · GitHub

curl -XGET 'http://localhost:9200/'
curl -XGET 'http://localhost:9200/' · GitHub

THREADS:
curl -XGET 'http://localhost:9200/_nodes/hot_threads'
nodeA> curl -XGET 'http://localhost:9200/_nodes/hot_threads' · GitHub

curl localhost:9200/_cat/thread_pool

https://gist.github.com/w3b1sh/778194c19ec3ed724f1e

On Wednesday, July 9, 2014 3:11:25 AM UTC-4, Ivan Brusic wrote:

Yes, please use gist/pastebin in the future.

Try using the hot threads API [1] to see if there are any threads that are
truly busy. If your system is truly idle, your thread pools should be
almost empty [2]. Your output above has only one management thread in use,
which could be simply the thread serving up the output.

[1]
Elasticsearch Platform — Find real-time answers at scale | Elastic
[2]
Elasticsearch Platform — Find real-time answers at scale | Elastic

Cheers,

Ivan

On Tue, Jul 8, 2014 at 10:54 PM, webish <gre...@yoursports.com
<javascript:>> wrote:

Thanks Mark. My mistake.

I can try switching to Oracle Java. There is no TTL. I've used Marvel
for development testing. Perhaps I can install the plugin...

On Wednesday, July 9, 2014 1:35:20 AM UTC-4, Mark Walkom wrote:

It's better to pop long output like that into a gist/pastebin, it makes it
easier to read the thread.
It's also worth installing a monitoring plugin like ElasticHQ or marvel,
as they provide graphical insight into what is happening and will
extrapolate some of the raw figures out.

If you can change to Oracle java you will get better performance. Are you
using TTL?

Regards,
Mark Walkom

Infrastructure Engineer
Campaign Monitor
email: ma...@campaignmonitor.com
web: www.campaignmonitor.com

On 9 July 2014 14:54, webish gre...@yoursports.com wrote:

I'm seeing very high CPU utilization in an idle or close to idle state. I
haven't seen it use less than 40% on each node over the course of days.
I'm not certain how long it has been like this but could be weeks.

Any help resolving this would be greatly appreciated!

Details:

There are 2 nodes in the cluster.

AWS instances are:
vCPU ECUMemory (GiB) Instance Storage (GB) Linux/UNIX Usage

r3.2xlarge 82661 1 x 160 SSD

curl -XGET 'http://localhost:9200/'
{
"status" : 200,
"name" : "Eddie Brock",
"version" : {
"number" : "1.1.0",
"build_hash" : "2181e113dea80b4a9e31e58e9686658a2d46e363",
"build_timestamp" : "2014-03-25T15:59:51Z",
"build_snapshot" : false,
"lucene_version" : "4.7"
},
"tagline" : "You Know, for Search"
}

% java -version
java version "1.7.0_55"
OpenJDK Runtime Environment (IcedTea 2.4.7) (7u55-2.4.7-1ubuntu1~0.12.04.
2)
OpenJDK 64-Bit Server VM (build 24.51-b03, mixed mode)

curl -XGET 'http://localhost:9200/_nodes/stats'
{
"cluster_name":"ab_elastic",
"nodes":{
"kpHPVkBCyjOkRaSpS5Q":{
"timestamp":1404880773859,
"name":"Milan",
"transport_address":"removed",
"host":"removed",
"ip":[
"removed",
"NONE"
],
"indices":{
"docs":{
"count":17793781,
"deleted":466537
},
"store":{
"size_in_bytes":9902722189,
"throttle_time_in_millis":8777423
},
"indexing":{
"index_total":3548638,
"index_time_in_millis":2610694,
"index_current":1,
"delete_total":0,
"delete_time_in_millis":0,
"delete_current":0
},
"get":{
"total":0,
"time_in_millis":0,
"exists_total":0,
"exists_time_in_millis":0,
"missing_total":0,
"missing_time_in_millis":0,
"current":0
},
"search":{
"open_contexts":0,
"query_total":155361213,
"query_time_in_millis":817989500,
"query_current":0,
"fetch_total":4119309,
"fetch_time_in_millis":1967525,
"fetch_current":0
},
"merges":{
"current":0,
"current_docs":0,
"current_size_in_bytes":0,
"total":564,
"total_time_in_millis":76065,
"total_docs":1093579,
"total_size_in_bytes":470304093
},
"refresh":{
"total":15117234,
"total_time_in_millis":50581020
},
"flush":{
"total":2336,
"total_time_in_millis":99980
},
"warmer":{
"current":0,
"total":2719640,
"total_time_in_millis":5980882
},
"filter_cache":{
"memory_size_in_bytes":722232396,
"evictions":1179740
},
"id_cache":{
"memory_size_in_bytes":131672871
},
"fielddata":{
"memory_size_in_bytes":133728950,
"evictions":0
},
"percolate":{
"total":0,
"time_in_millis":0,
"current":0,
"memory_size_in_bytes":-1,
"memory_size":"-1b",
"queries":0
},
"completion":{
"size_in_bytes":0
},
"segments":{
"count":278,
"memory_in_bytes":220799236
},
"translog":{
"operations":68,
"size_in_bytes":0
}
},
"os":{
"timestamp":1404880773910,
"uptime_in_millis":915746,
"load_average":[
2.68,
3.17,
3.35
],
"cpu":{
"sys":0,
"user":38,
"idle":61,
"usage":38,
"stolen":0
},
"mem":{
"free_in_bytes":12301606912,
"used_in_bytes":51857985536,
"free_percent":35,
"used_percent":64,
"actual_free_in_bytes":23005929472,
"actual_used_in_bytes":41153662976
},
"swap":{
"used_in_bytes":0,
"free_in_bytes":0
}
},
"process":{
"timestamp":1404880773911,
"open_file_descriptors":300,
"cpu":{
"percent":313,
"sys_in_millis":14021810,
"user_in_millis":794249870,
"total_in_millis":808271680
},
"mem":{
"resident_in_bytes":32342851584,
"share_in_bytes":712302592,
"total_virtual_in_bytes":47406919680
}
},
"jvm":{
"timestamp":1404880773911,
"uptime_in_millis":309661546,
"mem":{
"heap_used_in_bytes":18409783280,
"heap_used_percent":57,
"heap_committed_in_bytes":32142524416,
"heap_max_in_bytes":32142524416,
"non_heap_used_in_bytes":65012920,
"non_heap_committed_in_bytes":98009088,
"pools":{
"young":{
"used_in_bytes":538302112,
"max_in_bytes":558432256,
"peak_used_in_bytes":558432256,
"peak_max_in_bytes":558432256
},
"survivor":{
"used_in_bytes":11403704,
"max_in_bytes":69730304,
"peak_used_in_bytes":69730304,
"peak_max_in_bytes":69730304
},
"old":{
"used_in_bytes":17860077464,
"max_in_bytes":31514361856,
"peak_used_in_bytes":23694417088,
"peak_max_in_bytes":31514361856
}
}
},
"threads":{
"count":117,
"peak_count":133
},
"gc":{
"collectors":{
"young":{
"collection_count":200644,
"collection_time_in_millis":10870850
},
"old":{
"collection_count":9,
"collection_time_in_millis":2987
}
}
},
"buffer_pools":{
"direct":{
"count":154,
"used_in_bytes":78176363,
"total_capacity_in_bytes":78176363
},
"mapped":{
"count":1293,
"used_in_bytes":9831615457,
"total_capacity_in_bytes":9831615457
}
}
},
"thread_pool":{
"generic":{
"threads":1,
"queue":0,
"active":0,
"rejected":0,
"largest":14,
"completed":56826
},
"index":{
"threads":8,
"queue":0,
"active":0,
"rejected":0,
"largest":8,
"completed":3607948
},
"get":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"snapshot":{
"threads":4,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":95697
},
"merge":{
"threads":2,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":282257
},
"suggest":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"bulk":{
"threads":8,
"queue":0,
"active":0,
"rejected":0,
"largest":8,
"completed":1081
},
"optimize":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"warmer":{
"threads":2,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":5556773
},
"flush":{
"threads":1,
"queue":0,
"active":0,
"rejected":0,
"largest":2,
"completed":2336
},
"search":{
"threads":24,
"queue":0,
"active":0,
"rejected":0,
"largest":24,
"completed":160139043
},
"percolate":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"management":{
"threads":1,
"queue":0,
"active":1,
"rejected":0,
"largest":5,
"completed":1064
},
"refresh":{
"threads":4,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":15126156
}
},
"network":{
"tcp":{
"active_opens":1513869,
"passive_opens":224206,
"curr_estab":52,
"in_segs":1929140868,
"out_segs":2005198310,
"retrans_segs":175635,
"estab_resets":37,
"attempt_fails":669,
"in_errs":3,
"out_rsts":88
}
},
"fs":{
"timestamp":1404880773912,
"total":{
"total_in_bytes":158534184960,
"free_in_bytes":148425240576,
"available_in_bytes":140372176896,
"disk_reads":1993,
"disk_writes":16068284,
"disk_io_op":16070277,
"disk_read_size_in_bytes":7445504,
"disk_write_size_in_bytes":650206560256,
"disk_io_size_in_bytes":650214005760
},
"data":[
{
"path":"/mnt/esdata/ab_elastic/nodes/0",
"mount":"/mnt/esdata",
"dev":"/dev/xvdb",
"total_in_bytes":158534184960,
"free_in_bytes":148425240576,
"available_in_bytes":140372176896,
"disk_reads":1993,
"disk_writes":16068284,
"disk_io_op":16070277,
"disk_read_size_in_bytes":7445504,
"disk_write_size_in_bytes":650206560256,
"disk_io_size_in_bytes":650214005760
}
]
},
"transport":{
"server_open":26,
"rx_count":214082686,
"rx_size_in_bytes":148941344505,
"tx_count":209856841,
"tx_size_in_bytes":90356130749
},
"http":{
"current_open":0,
"total_opened":0
},
"fielddata_breaker":{
"maximum_size_in_bytes":25714019532,
"maximum_size":"23.9gb",
"estimated_size_in_bytes":265401821,
"estimated_size":"253.1mb",
"overhead":1.03
}
},
"erc5l7dQai3mKBIX6sw":{
"timestamp":1404880733754,
"name":"Eddie Brock",
"transport_address":"removed",
"host":"removed",
"ip":[
"removed",
"NONE"
],
"indices":{
"docs":{
"count":17793781,
"deleted":529675
},
"store":{
"size_in_bytes":9969074949,
"throttle_time_in_millis":9235478
},
"indexing":{
"index_total":3531955,
"index_time_in_millis":2783791,
"index_current":0,
"delete_total":0,
"delete_time_in_millis":0,
"delete_current":0
},
"get":{
"total":2947020,
"time_in_millis":217006,
"exists_total":2946589,
"exists_time_in_millis":216977,
"missing_total":431,
"missing_time_in_millis":29,
"current":0
},
"search":{
"open_contexts":0,
"query_total":155416200,
"query_time_in_millis":1076023052,
"query_current":0,
"fetch_total":4101442,
"fetch_time_in_millis":2337321,
"fetch_current":0
},
"merges":{
"current":0,
"current_docs":0,
"current_size_in_bytes":0,
"total":563,
"total_time_in_millis":89211,
"total_docs":1099389,
"total_size_in_bytes":470013054
},
"refresh":{
"total":15123966,
"total_time_in_millis":63368992
},
"flush":{
"total":2335,
"total_time_in_millis":414790
},
"warmer":{
"current":0,
"total":2690030,
"total_time_in_millis":7339293
},
"filter_cache":{
"memory_size_in_bytes":695354935,
"evictions":1377260
},
"id_cache":{
"memory_size_in_bytes":131817497
},
"fielddata":{
"memory_size_in_bytes":137907189,
"evictions":0
},
"percolate":{
"total":0,
"time_in_millis":0,
"current":0,
"memory_size_in_bytes":-1,
"memory_size":"-1b",
"queries":0
},
"completion":{
"size_in_bytes":0
},
"segments":{
"count":282,
"memory_in_bytes":217468955
},
"translog":{
"operations":86,
"size_in_bytes":0
}
},
"os":{
"timestamp":1404880733801,
"uptime_in_millis":923467,
"load_average":[
3.71,
4.57,
4.46
],
"cpu":{
"sys":0,
"user":46,
"idle":52,
"usage":46,
"stolen":0
},
"mem":{
"free_in_bytes":4438659072,
"used_in_bytes":59720933376,
"free_percent":32,
"used_percent":67,
"actual_free_in_bytes":21124698112,
"actual_used_in_bytes":43034894336
},
"swap":{
"used_in_bytes":0,
"free_in_bytes":0
}
},
"process":{
"timestamp":1404880733801,
"open_file_descriptors":301,
"cpu":{
"percent":377,
"sys_in_millis":26524340,
"user_in_millis":947578840,
"total_in_millis":974103180
},
"mem":{
"resident_in_bytes":33945767936,
"share_in_bytes":692912128,
"total_virtual_in_bytes":47473528832
}
},
"jvm":{
"timestamp":1404880733802,
"uptime_in_millis":310301938,
"mem":{
"heap_used_in_bytes":10113402832,
"heap_used_percent":31,
"heap_committed_in_bytes":32142524416,
"heap_max_in_bytes":32142524416,
"non_heap_used_in_bytes":71762640,
"non_heap_committed_in_bytes":106397696,
"pools":{
"young":{
"used_in_bytes":218720808,
"max_in_bytes":558432256,
"peak_used_in_bytes":558432256,
"peak_max_in_bytes":558432256
},
"survivor":{
"used_in_bytes":9630920,
"max_in_bytes":69730304,
"peak_used_in_bytes":69730304,
"peak_max_in_bytes":69730304
},
"old":{
"used_in_bytes":9885051104,
"max_in_bytes":31514361856,
"peak_used_in_bytes":23868903928,
"peak_max_in_bytes":31514361856
}
}
},
"threads":{
"count":121,
"peak_count":144
},
"gc":{
"collectors":{
"young":{
"collection_count":209658,
"collection_time_in_millis":12089132
},
"old":{
"collection_count":11,
"collection_time_in_millis":6628
}
}
},
"buffer_pools":{
"direct":{
"count":162,
"used_in_bytes":195159665,
"total_capacity_in_bytes":195159665
},
"mapped":{
"count":1292,
"used_in_bytes":9897821921,
"total_capacity_in_bytes":9897821921
}
}
},
"thread_pool":{
"generic":{
"threads":1,
"queue":0,
"active":0,
"rejected":0,
"largest":23,
"completed":67214
},
"index":{
"threads":8,
"queue":0,
"active":0,
"rejected":0,
"largest":8,
"completed":3611159
},
"get":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"snapshot":{
"threads":2,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":95672
},
"merge":{
"threads":3,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":285955
},
"suggest":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"bulk":{
"threads":8,
"queue":0,
"active":0,
"rejected":0,
"largest":8,
"completed":1125
},
"optimize":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"warmer":{
"threads":2,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":5487739
},
"flush":{
"threads":1,
"queue":0,
"active":0,
"rejected":0,
"largest":2,
"completed":2335
},
"search":{

...

--
You received this message because you are subscribed to the Google Groups "elasticsearch" group.
To unsubscribe from this group and stop receiving emails from it, send an email to elasticsearch+unsubscribe@googlegroups.com.
To view this discussion on the web visit https://groups.google.com/d/msgid/elasticsearch/ed76cee8-1c01-43b4-810a-1b203a80ad04%40googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[warkolm (Mark Walkom)]

The output of hot threads is only from the one node, ie requesting it isn't
at a cluster level, if you can add the other node it'd help.

Regards,
Mark Walkom

Infrastructure Engineer
Campaign Monitor
email: markw@campaignmonitor.com
web: www.campaignmonitor.com

On 9 July 2014 17:32, webish gregory@yoursports.com wrote:

Definitely.

Here are all the captures thus far in gists.

CLUSTER:
curl -XGET 'http://localhost:9200/_nodes/stats'
curl -XGET 'http://localhost:9200/_nodes/stats' · GitHub

curl -XGET 'http://localhost:9200/'
curl -XGET 'http://localhost:9200/' · GitHub

THREADS:
curl -XGET 'http://localhost:9200/_nodes/hot_threads'
nodeA> curl -XGET 'http://localhost:9200/_nodes/hot_threads' · GitHub

curl localhost:9200/_cat/thread_pool
curl localhost:9200/_cat/thread_pool · GitHub

On Wednesday, July 9, 2014 3:11:25 AM UTC-4, Ivan Brusic wrote:

Yes, please use gist/pastebin in the future.

Try using the hot threads API [1] to see if there are any threads that
are truly busy. If your system is truly idle, your thread pools should be
almost empty [2]. Your output above has only one management thread in use,
which could be simply the thread serving up the output.

[1] Elasticsearch Platform — Find real-time answers at scale | Elastic
reference/current/cluster-nodes-hot-threads.html
[2] Elasticsearch Platform — Find real-time answers at scale | Elastic
reference/current/cat-thread-pool.html

Cheers,

Ivan

On Tue, Jul 8, 2014 at 10:54 PM, webish gre...@yoursports.com wrote:

Thanks Mark. My mistake.

I can try switching to Oracle Java. There is no TTL. I've used Marvel
for development testing. Perhaps I can install the plugin...

On Wednesday, July 9, 2014 1:35:20 AM UTC-4, Mark Walkom wrote:

It's better to pop long output like that into a gist/pastebin, it makes
it easier to read the thread.
It's also worth installing a monitoring plugin like ElasticHQ or marvel,
as they provide graphical insight into what is happening and will
extrapolate some of the raw figures out.

If you can change to Oracle java you will get better performance. Are you
using TTL?

Regards,
Mark Walkom

Infrastructure Engineer
Campaign Monitor
email: ma...@campaignmonitor.com
web: www.campaignmonitor.com

On 9 July 2014 14:54, webish gre...@yoursports.com wrote:

I'm seeing very high CPU utilization in an idle or close to idle state.
I haven't seen it use less than 40% on each node over the course of days.
I'm not certain how long it has been like this but could be weeks.

Any help resolving this would be greatly appreciated!

Details:

There are 2 nodes in the cluster.

AWS instances are:
vCPU ECUMemory (GiB) Instance Storage (GB) Linux/UNIX Usage

r3.2xlarge 82661 1 x 160 SSD

curl -XGET 'http://localhost:9200/'
{
"status" : 200,
"name" : "Eddie Brock",
"version" : {
"number" : "1.1.0",
"build_hash" : "2181e113dea80b4a9e31e58e9686658a2d46e363",
"build_timestamp" : "2014-03-25T15:59:51Z",
"build_snapshot" : false,
"lucene_version" : "4.7"
},
"tagline" : "You Know, for Search"
}

% java -version
java version "1.7.0_55"
OpenJDK Runtime Environment (IcedTea 2.4.7) (7u55-2.4.7-1ubuntu1~0.12.04.
2)
OpenJDK 64-Bit Server VM (build 24.51-b03, mixed mode)

curl -XGET 'http://localhost:9200/_nodes/stats'
{
"cluster_name":"ab_elastic",
"nodes":{
"kpHPVkBCyjOkRaSpS5Q":{
"timestamp":1404880773859,
"name":"Milan",
"transport_address":"removed",
"host":"removed",
"ip":[
"removed",
"NONE"
],
"indices":{
"docs":{
"count":17793781,
"deleted":466537
},
"store":{
"size_in_bytes":9902722189,
"throttle_time_in_millis":8777423
},
"indexing":{
"index_total":3548638,
"index_time_in_millis":2610694,
"index_current":1,
"delete_total":0,
"delete_time_in_millis":0,
"delete_current":0
},
"get":{
"total":0,
"time_in_millis":0,
"exists_total":0,
"exists_time_in_millis":0,
"missing_total":0,
"missing_time_in_millis":0,
"current":0
},
"search":{
"open_contexts":0,
"query_total":155361213,
"query_time_in_millis":817989500,
"query_current":0,
"fetch_total":4119309,
"fetch_time_in_millis":1967525,
"fetch_current":0
},
"merges":{
"current":0,
"current_docs":0,
"current_size_in_bytes":0,
"total":564,
"total_time_in_millis":76065,
"total_docs":1093579,
"total_size_in_bytes":470304093
},
"refresh":{
"total":15117234,
"total_time_in_millis":50581020
},
"flush":{
"total":2336,
"total_time_in_millis":99980
},
"warmer":{
"current":0,
"total":2719640,
"total_time_in_millis":5980882
},
"filter_cache":{
"memory_size_in_bytes":722232396,
"evictions":1179740
},
"id_cache":{
"memory_size_in_bytes":131672871
},
"fielddata":{
"memory_size_in_bytes":133728950,
"evictions":0
},
"percolate":{
"total":0,
"time_in_millis":0,
"current":0,
"memory_size_in_bytes":-1,
"memory_size":"-1b",
"queries":0
},
"completion":{
"size_in_bytes":0
},
"segments":{
"count":278,
"memory_in_bytes":220799236
},
"translog":{
"operations":68,
"size_in_bytes":0
}
},
"os":{
"timestamp":1404880773910,
"uptime_in_millis":915746,
"load_average":[
2.68,
3.17,
3.35
],
"cpu":{
"sys":0,
"user":38,
"idle":61,
"usage":38,
"stolen":0
},
"mem":{
"free_in_bytes":12301606912,
"used_in_bytes":51857985536,
"free_percent":35,
"used_percent":64,
"actual_free_in_bytes":23005929472,
"actual_used_in_bytes":41153662976
},
"swap":{
"used_in_bytes":0,
"free_in_bytes":0
}
},
"process":{
"timestamp":1404880773911,
"open_file_descriptors":300,
"cpu":{
"percent":313,
"sys_in_millis":14021810,
"user_in_millis":794249870,
"total_in_millis":808271680
},
"mem":{
"resident_in_bytes":32342851584,
"share_in_bytes":712302592,
"total_virtual_in_bytes":47406919680
}
},
"jvm":{
"timestamp":1404880773911,
"uptime_in_millis":309661546,
"mem":{
"heap_used_in_bytes":18409783280,
"heap_used_percent":57,
"heap_committed_in_bytes":32142524416,
"heap_max_in_bytes":32142524416,
"non_heap_used_in_bytes":65012920,
"non_heap_committed_in_bytes":98009088,
"pools":{
"young":{
"used_in_bytes":538302112,
"max_in_bytes":558432256,
"peak_used_in_bytes":558432256,
"peak_max_in_bytes":558432256
},
"survivor":{
"used_in_bytes":11403704,
"max_in_bytes":69730304,
"peak_used_in_bytes":69730304,
"peak_max_in_bytes":69730304
},
"old":{
"used_in_bytes":17860077464,
"max_in_bytes":31514361856,
"peak_used_in_bytes":23694417088,
"peak_max_in_bytes":31514361856
}
}
},
"threads":{
"count":117,
"peak_count":133
},
"gc":{
"collectors":{
"young":{
"collection_count":200644,
"collection_time_in_millis":10870850
},
"old":{
"collection_count":9,
"collection_time_in_millis":2987
}
}
},
"buffer_pools":{
"direct":{
"count":154,
"used_in_bytes":78176363,
"total_capacity_in_bytes":78176363
},
"mapped":{
"count":1293,
"used_in_bytes":9831615457,
"total_capacity_in_bytes":9831615457
}
}
},
"thread_pool":{
"generic":{
"threads":1,
"queue":0,
"active":0,
"rejected":0,
"largest":14,
"completed":56826
},
"index":{
"threads":8,
"queue":0,
"active":0,
"rejected":0,
"largest":8,
"completed":3607948
},
"get":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"snapshot":{
"threads":4,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":95697
},
"merge":{
"threads":2,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":282257
},
"suggest":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"bulk":{
"threads":8,
"queue":0,
"active":0,
"rejected":0,
"largest":8,
"completed":1081
},
"optimize":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"warmer":{
"threads":2,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":5556773
},
"flush":{
"threads":1,
"queue":0,
"active":0,
"rejected":0,
"largest":2,
"completed":2336
},
"search":{
"threads":24,
"queue":0,
"active":0,
"rejected":0,
"largest":24,
"completed":160139043
},
"percolate":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"management":{
"threads":1,
"queue":0,
"active":1,
"rejected":0,
"largest":5,
"completed":1064
},
"refresh":{
"threads":4,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":15126156
}
},
"network":{
"tcp":{
"active_opens":1513869,
"passive_opens":224206,
"curr_estab":52,
"in_segs":1929140868,
"out_segs":2005198310,
"retrans_segs":175635,
"estab_resets":37,
"attempt_fails":669,
"in_errs":3,
"out_rsts":88
}
},
"fs":{
"timestamp":1404880773912,
"total":{
"total_in_bytes":158534184960,
"free_in_bytes":148425240576,
"available_in_bytes":140372176896,
"disk_reads":1993,
"disk_writes":16068284,
"disk_io_op":16070277,
"disk_read_size_in_bytes":7445504,
"disk_write_size_in_bytes":650206560256,
"disk_io_size_in_bytes":650214005760
},
"data":[
{
"path":"/mnt/esdata/ab_elastic/nodes/0",
"mount":"/mnt/esdata",
"dev":"/dev/xvdb",
"total_in_bytes":158534184960,
"free_in_bytes":148425240576,
"available_in_bytes":140372176896,
"disk_reads":1993,
"disk_writes":16068284,
"disk_io_op":16070277,
"disk_read_size_in_bytes":7445504,
"disk_write_size_in_bytes":650206560256,
"disk_io_size_in_bytes":650214005760
}
]
},
"transport":{
"server_open":26,
"rx_count":214082686,
"rx_size_in_bytes":148941344505,
"tx_count":209856841,
"tx_size_in_bytes":90356130749
},
"http":{
"current_open":0,
"total_opened":0
},
"fielddata_breaker":{
"maximum_size_in_bytes":25714019532,
"maximum_size":"23.9gb",
"estimated_size_in_bytes":265401821,
"estimated_size":"253.1mb",
"overhead":1.03
}
},
"erc5l7dQai3mKBIX6sw":{
"timestamp":1404880733754,
"name":"Eddie Brock",
"transport_address":"removed",
"host":"removed",
"ip":[
"removed",
"NONE"
],
"indices":{
"docs":{
"count":17793781,
"deleted":529675
},
"store":{
"size_in_bytes":9969074949,
"throttle_time_in_millis":9235478
},
"indexing":{
"index_total":3531955,
"index_time_in_millis":2783791,
"index_current":0,
"delete_total":0,
"delete_time_in_millis":0,
"delete_current":0
},
"get":{
"total":2947020,
"time_in_millis":217006,
"exists_total":2946589,
"exists_time_in_millis":216977,
"missing_total":431,
"missing_time_in_millis":29,
"current":0
},
"search":{
"open_contexts":0,
"query_total":155416200,
"query_time_in_millis":1076023052,
"query_current":0,
"fetch_total":4101442,
"fetch_time_in_millis":2337321,
"fetch_current":0
},
"merges":{
"current":0,
"current_docs":0,
"current_size_in_bytes":0,
"total":563,
"total_time_in_millis":89211,
"total_docs":1099389,
"total_size_in_bytes":470013054
},
"refresh":{
"total":15123966,
"total_time_in_millis":63368992
},
"flush":{
"total":2335,
"total_time_in_millis":414790
},
"warmer":{
"current":0,
"total":2690030,
"total_time_in_millis":7339293
},
"filter_cache":{
"memory_size_in_bytes":695354935,
"evictions":1377260
},
"id_cache":{
"memory_size_in_bytes":131817497
},
"fielddata":{
"memory_size_in_bytes":137907189,
"evictions":0
},
"percolate":{
"total":0,
"time_in_millis":0,
"current":0,
"memory_size_in_bytes":-1,
"memory_size":"-1b",
"queries":0
},
"completion":{
"size_in_bytes":0
},
"segments":{
"count":282,
"memory_in_bytes":217468955
},
"translog":{
"operations":86,
"size_in_bytes":0
}
},
"os":{
"timestamp":1404880733801,
"uptime_in_millis":923467,
"load_average":[
3.71,
4.57,
4.46
],
"cpu":{
"sys":0,
"user":46,
"idle":52,
"usage":46,
"stolen":0
},
"mem":{
"free_in_bytes":4438659072,
"used_in_bytes":59720933376,
"free_percent":32,
"used_percent":67,
"actual_free_in_bytes":21124698112,
"actual_used_in_bytes":43034894336
},
"swap":{
"used_in_bytes":0,
"free_in_bytes":0
}
},
"process":{
"timestamp":1404880733801,
"open_file_descriptors":301,
"cpu":{
"percent":377,
"sys_in_millis":26524340,
"user_in_millis":947578840,
"total_in_millis":974103180
},
"mem":{
"resident_in_bytes":33945767936,
"share_in_bytes":692912128,
"total_virtual_in_bytes":47473528832
}
},
"jvm":{
"timestamp":1404880733802,
"uptime_in_millis":310301938,
"mem":{
"heap_used_in_bytes":10113402832,
"heap_used_percent":31,
"heap_committed_in_bytes":32142524416,
"heap_max_in_bytes":32142524416,
"non_heap_used_in_bytes":71762640,
"non_heap_committed_in_bytes":106397696,
"pools":{
"young":{
"used_in_bytes":218720808,
"max_in_bytes":558432256,
"peak_used_in_bytes":558432256,
"peak_max_in_bytes":558432256
},
"survivor":{
"used_in_bytes":9630920,
"max_in_bytes":69730304,
"peak_used_in_bytes":69730304,
"peak_max_in_bytes":69730304
},
"old":{
"used_in_bytes":9885051104,
"max_in_bytes":31514361856,
"peak_used_in_bytes":23868903928,
"peak_max_in_bytes":31514361856
}
}
},
"threads":{
"count":121,
"peak_count":144
},
"gc":{
"collectors":{
"young":{
"collection_count":209658,
"collection_time_in_millis":12089132
},
"old":{
"collection_count":11,
"collection_time_in_millis":6628
}
}
},
"buffer_pools":{
"direct":{
"count":162,
"used_in_bytes":195159665,
"total_capacity_in_bytes":195159665
},
"mapped":{
"count":1292,
"used_in_bytes":9897821921,
"total_capacity_in_bytes":9897821921
}
}
},
"thread_pool":{
"generic":{
"threads":1,
"queue":0,
"active":0,
"rejected":0,
"largest":23,
"completed":67214
},
"index":{
"threads":8,
"queue":0,
"active":0,
"rejected":0,
"largest":8,
"completed":3611159
},
"get":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"snapshot":{
"threads":2,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":95672
},
"merge":{
"threads":3,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":285955
},
"suggest":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"bulk":{
"threads":8,
"queue":0,
"active":0,
"rejected":0,
"largest":8,
"completed":1125
},
"optimize":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"warmer":{
"threads":2,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":5487739
},
"flush":{
"threads":1,
"queue":0,
"active":0,
"rejected":0,
"largest":2,
"completed":2335
},
"search":{

...

--
You received this message because you are subscribed to the Google Groups
"elasticsearch" group.
To unsubscribe from this group and stop receiving emails from it, send an
email to elasticsearch+unsubscribe@googlegroups.com.
To view this discussion on the web visit
https://groups.google.com/d/msgid/elasticsearch/ed76cee8-1c01-43b4-810a-1b203a80ad04%40googlegroups.com
https://groups.google.com/d/msgid/elasticsearch/ed76cee8-1c01-43b4-810a-1b203a80ad04%40googlegroups.com?utm_medium=email&utm_source=footer
.
For more options, visit https://groups.google.com/d/optout.

--
You received this message because you are subscribed to the Google Groups "elasticsearch" group.
To unsubscribe from this group and stop receiving emails from it, send an email to elasticsearch+unsubscribe@googlegroups.com.
To view this discussion on the web visit https://groups.google.com/d/msgid/elasticsearch/CAEM624a4xhLW922qecSOCrhbBkCx3WFc-uz_Y0%3Da2%2B2hNAhR0Q%40mail.gmail.com.
For more options, visit https://groups.google.com/d/optout.

[webish]

Ahh, ok. Sorry.

NodeA:

https://gist.github.com/w3b1sh/ea6a2b3fbfc837d5d9d8

NodeB:

https://gist.github.com/w3b1sh/2ca12bd920ebf20644ef

On Wednesday, July 9, 2014 3:36:53 AM UTC-4, Mark Walkom wrote:

The output of hot threads is only from the one node, ie requesting it
isn't at a cluster level, if you can add the other node it'd help.

Regards,
Mark Walkom

Infrastructure Engineer
Campaign Monitor
email: ma...@campaignmonitor.com <javascript:>
web: www.campaignmonitor.com

On 9 July 2014 17:32, webish <gre...@yoursports.com <javascript:>> wrote:

Definitely.

Here are all the captures thus far in gists.

CLUSTER:
curl -XGET 'http://localhost:9200/_nodes/stats'
curl -XGET 'http://localhost:9200/_nodes/stats' · GitHub

curl -XGET 'http://localhost:9200/'
curl -XGET 'http://localhost:9200/' · GitHub

THREADS:
curl -XGET 'http://localhost:9200/_nodes/hot_threads'
nodeA> curl -XGET 'http://localhost:9200/_nodes/hot_threads' · GitHub

curl localhost:9200/_cat/thread_pool
curl localhost:9200/_cat/thread_pool · GitHub

On Wednesday, July 9, 2014 3:11:25 AM UTC-4, Ivan Brusic wrote:

Yes, please use gist/pastebin in the future.

Try using the hot threads API [1] to see if there are any threads that are
truly busy. If your system is truly idle, your thread pools should be
almost empty [2]. Your output above has only one management thread in use,
which could be simply the thread serving up the output.

[1] Elasticsearch Platform — Find real-time answers at scale | Elastic
reference/current/cluster-nodes-hot-threads.html
[2] Elasticsearch Platform — Find real-time answers at scale | Elastic
reference/current/cat-thread-pool.html

Cheers,

Ivan

On Tue, Jul 8, 2014 at 10:54 PM, webish gre...@yoursports.com wrote:

Thanks Mark. My mistake.

I can try switching to Oracle Java. There is no TTL. I've used Marvel
for development testing. Perhaps I can install the plugin...

On Wednesday, July 9, 2014 1:35:20 AM UTC-4, Mark Walkom wrote:

It's better to pop long output like that into a gist/pastebin, it makes it
easier to read the thread.
It's also worth installing a monitoring plugin like ElasticHQ or marvel,
as they provide graphical insight into what is happening and will
extrapolate some of the raw figures out.

If you can change to Oracle java you will get better performance. Are you
using TTL?

Regards,
Mark Walkom

Infrastructure Engineer
Campaign Monitor
email: ma...@campaignmonitor.com
web: www.campaignmonitor.com

On 9 July 2014 14:54, webish gre...@yoursports.com wrote:

I'm seeing very high CPU utilization in an idle or close to idle state. I
haven't seen it use less than 40% on each node over the course of days.
I'm not certain how long it has been like this but could be weeks.

Any help resolving this would be greatly appreciated!

Details:

There are 2 nodes in the cluster.

AWS instances are:
vCPU ECUMemory (GiB) Instance Storage (GB) Linux/UNIX Usage

r3.2xlarge 82661 1 x 160 SSD

curl -XGET 'http://localhost:9200/'
{
"status" : 200,
"name" : "Eddie Brock",
"version" : {
"number" : "1.1.0",
"build_hash" : "2181e113dea80b4a9e31e58e9686658a2d46e363",
"build_timestamp" : "2014-03-25T15:59:51Z",
"build_snapshot" : false,
"lucene_version" : "4.7"
},
"tagline" : "You Know, for Search"
}

% java -version
java version "1.7.0_55"
OpenJDK Runtime Environment (IcedTea 2.4.7) (7u55-2.4.7-1ubuntu1~0.12.04.2
)
OpenJDK 64-Bit Server VM (build 24.51-b03, mixed mode)

curl -XGET 'http://localhost:9200/_nodes/stats'
{
"cluster_name":"ab_elastic",
"nodes":{
"kpHPVkBCyjOkRaSpS5Q":{
"timestamp":1404880773859,
"name":"Milan",
"transport_address":"removed",
"host":"removed",
"ip":[
"removed",
"NONE"
],
"indices":{
"docs":{
"count":17793781,
"deleted":466537
},
"store":{
"size_in_bytes":9902722189,
"throttle_time_in_millis":8777423
},
"indexing":{
"index_total":3548638,
"index_time_in_millis":2610694,
"index_current":1,
"delete_total":0,
"delete_time_in_millis":0,
"delete_current":0
},
"get":{
"total":0,
"time_in_millis":0,
"exists_total":0,
"exists_time_in_millis":0,
"missing_total":0,
"missing_time_in_millis":0,
"current":0
},
"search":{
"open_contexts":0,
"query_total":155361213,
"query_time_in_millis":817989500,
"query_current":0,
"fetch_total":4119309,
"fetch_time_in_millis":1967525,
"fetch_current":0
},
"merges":{
"current":0,
"current_docs":0,
"current_size_in_bytes":0,
"total":564,
"total_time_in_millis":76065,
"total_docs":1093579,
"total_size_in_bytes":470304093
},
"refresh":{
"total":15117234,
"total_time_in_millis":50581020
},
"flush":{
"total":2336,
"total_time_in_millis":99980
},
"warmer":{
"current":0,
"total":2719640,
"total_time_in_millis":5980882
},
"filter_cache":{
"memory_size_in_bytes":722232396,
"evictions":1179740
},
"id_cache":{
"memory_size_in_bytes":131672871
},
"fielddata":{
"memory_size_in_bytes":133728950,
"evictions":0
},
"percolate":{
"total":0,
"time_in_millis":0,
"current":0,
"memory_size_in_bytes":-1,
"memory_size":"-1b",
"queries":0
},
"completion":{
"size_in_bytes":0
},
"segments":{
"count":278,
"memory_in_bytes":220799236
},
"translog":{
"operations":68,
"size_in_bytes":0
}
},
"os":{
"timestamp":1404880773910,
"uptime_in_millis":915746,
"load_average":[
2.68,
3.17,
3.35
],
"cpu":{
"sys":0,
"user":38,
"idle":61,
"usage":38,
"stolen":0
},
"mem":{
"free_in_bytes":12301606912,
"used_in_bytes":51857985536,
"free_percent":35,
"used_percent":64,
"actual_free_in_bytes":23005929472,
"actual_used_in_bytes":41153662976
},
"swap":{
"used_in_bytes":0,
"free_in_bytes":0
}
},
"process":{
"timestamp":1404880773911,
"open_file_descriptors":300,
"cpu":{
"percent":313,
"sys_in_millis":14021810,
"user_in_millis":794249870,
"total_in_millis":808271680
},
"mem":{
"resident_in_bytes":32342851584,
"share_in_bytes":712302592,
"total_virtual_in_bytes":47406919680
}
},
"jvm":{
"timestamp":1404880773911,
"uptime_in_millis":309661546,
"mem":{
"heap_used_in_bytes":18409783280,
"heap_used_percent":57,
"heap_committed_in_bytes":32142524416,
"heap_max_in_bytes":32142524416,
"non_heap_used_in_bytes":65012920,
"non_heap_committed_in_bytes":98009088,
"pools":{
"young":{
"used_in_bytes":538302112,
"max_in_bytes":558432256,
"peak_used_in_bytes":558432256,
"peak_max_in_bytes":558432256
},
"survivor":{
"used_in_bytes":11403704,
"max_in_bytes":69730304,
"peak_used_in_bytes":69730304,
"peak_max_in_bytes":69730304
},
"old":{
"used_in_bytes":17860077464,
"max_in_bytes":31514361856,
"peak_used_in_bytes":23694417088,
"peak_max_in_bytes":31514361856
}
}
},
"threads":{
"count":117,
"peak_count":133
},
"gc":{
"collectors":{
"young":{
"collection_count":200644,
"collection_time_in_millis":10870850
},
"old":{
"collection_count":9,
"collection_time_in_millis":2987
}
}
},
"buffer_pools":{
"direct":{
"count":154,
"used_in_bytes":78176363,
"total_capacity_in_bytes":78176363
},
"mapped":{
"count":1293,
"used_in_bytes":9831615457,
"total_capacity_in_bytes":9831615457
}
}
},
"thread_pool":{
"generic":{
"threads":1,
"queue":0,
"active":0,
"rejected":0,
"largest":14,
"completed":56826
},
"index":{
"threads":8,
"queue":0,
"active":0,
"rejected":0,
"largest":8,
"completed":3607948
},
"get":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"snapshot":{
"threads":4,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":95697
},
"merge":{
"threads":2,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":282257
},
"suggest":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"bulk":{
"threads":8,
"queue":0,
"active":0,
"rejected":0,
"largest":8,
"completed":1081
},
"optimize":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"warmer":{
"threads":2,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":5556773
},
"flush":{
"threads":1,
"queue":0,
"active":0,
"rejected":0,
"largest":2,
"completed":2336
},
"search":{
"threads":24,
"queue":0,
"active":0,
"rejected":0,
"largest":24,
"completed":160139043
},
"percolate":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"management":{
"threads":1,
"queue":0,
"active":1,
"rejected":0,
"largest":5,
"completed":1064
},
"refresh":{
"threads":4,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":15126156
}
},
"network":{
"tcp":{
"active_opens":1513869,
"passive_opens":224206,
"curr_estab":52,
"in_segs":1929140868,
"out_segs":2005198310,
"retrans_segs":175635,
"estab_resets":37,
"attempt_fails":669,
"in_errs":3,
"out_rsts":88
}
},
"fs":{
"timestamp":1404880773912,
"total":{
"total_in_bytes":158534184960,
"free_in_bytes":148425240576,
"available_in_bytes":140372176896,
"disk_reads":1993,
"disk_writes":16068284,
"disk_io_op":16070277,
"disk_read_size_in_bytes":7445504,
"disk_write_size_in_bytes":650206560256,
"disk_io_size_in_bytes":650214005760
},
"data":[
{
"path":"/mnt/esdata/ab_elastic/nodes/0",
"mount":"/mnt/esdata",
"dev":"/dev/xvdb",
"total_in_bytes":158534184960,
"free_in_bytes":148425240576,
"available_in_bytes":140372176896,
"disk_reads":1993,
"disk_writes":16068284,
"disk_io_op":16070277,
"disk_read_size_in_bytes":7445504,
"disk_write_size_in_bytes":650206560256,
"disk_io_size_in_bytes":650214005760
}
]
},
"transport":{
"server_open":26,
"rx_count":214082686,
"rx_size_in_bytes":148941344505,
"tx_count":209856841,
"tx_size_in_bytes":90356130749
},
"http":{
"current_open":0,
"total_opened":0
},
"fielddata_breaker":{
"maximum_size_in_bytes":25714019532,
"maximum_size":"23.9gb",
"estimated_size_in_bytes":265401821,
"estimated_size":"253.1mb",
"overhead":1.03
}
},
"erc5l7dQai3mKBIX6sw":{
"timestamp":1404880733754,
"name":"Eddie Brock",
"transport_address":"removed",
"host":"removed",
"ip":[
"removed",
"NONE"
],
"indices":{
"docs":{
"count":17793781,
"deleted":529675
},
"store":{
"size_in_bytes":9969074949,
"throttle_time_in_millis":9235478
},
"indexing":{
"index_total":3531955,
"index_time_in_millis":2783791,
"index_current":0,
"delete_total":0,
"delete_time_in_millis":0,
"delete_current":0
},
"get":{
"total":2947020,
"time_in_millis":217006,
"exists_total":2946589,
"exists_time_in_millis":216977,
"missing_total":431,
"missing_time_in_millis":29,
"current":0
},
"search":{
"open_contexts":0,
"query_total":155416200,
"query_time_in_millis":1076023052,
"query_current":0,
"fetch_total":4101442,
"fetch_time_in_millis":2337321,
"fetch_current":0
},
"merges":{
"current":0,
"current_docs":0,
"current_size_in_bytes":0,
"total":563,
"total_time_in_millis":89211,
"total_docs":1099389,
"total_size_in_bytes":470013054
},
"refresh":{
"total":15123966,
"total_time_in_millis":63368992
},
"flush":{
"total":2335,
"total_time_in_millis":414790
},
"warmer":{
"current":0,
"total":2690030,
"total_time_in_millis":7339293
},
"filter_cache":{
"memory_size_in_bytes":695354935,
"evictions":1377260
},
"id_cache":{
"memory_size_in_bytes":131817497
},
"fielddata":{
"memory_size_in_bytes":137907189,
"evictions":0
},
"percolate":{
"total":0,
"time_in_millis":0,
"current":0,
"memory_size_in_bytes":-1,
"memory_size":"-1b",
"queries":0
},
"completion":{
"size_in_bytes":0
},
"segments":{
"count":282,
"memory_in_bytes":217468955
},
"translog":{
"operations":86,
"size_in_bytes":0
}
},
"os":{
"timestamp":1404880733801,
"uptime_in_millis":923467,
"load_average":[
3.71,
4.57,
4.46
],
"cpu":{
"sys":0,
"user":46,
"idle":52,
"usage":46,
"stolen":0
},
"mem":{
"free_in_bytes":4438659072,
"used_in_bytes":59720933376,
"free_percent":32,
"used_percent":67,
"actual_free_in_bytes":21124698112,
"actual_used_in_bytes":43034894336
},
"swap":{
"used_in_bytes":0,
"free_in_bytes":0
}
},
"process":{
"timestamp":1404880733801,
"open_file_descriptors":301,
"cpu":{
"percent":377,
"sys_in_millis":26524340,
"user_in_millis":947578840,
"total_in_millis":974103180
},
"mem":{
"resident_in_bytes":33945767936,
"share_in_bytes":692912128,
"total_virtual_in_bytes":47473528832
}
},
"jvm":{
"timestamp":1404880733802,
"uptime_in_millis":310301938,
"mem":{
"heap_used_in_bytes":10113402832,
"heap_used_percent":31,
"heap_committed_in_bytes":32142524416,
"heap_max_in_bytes":32142524416,
"non_heap_used_in_bytes":71762640,
"non_heap_committed_in_bytes":106397696,
"pools":{
"young":{
"used_in_bytes":218720808,
"max_in_bytes":558432256,
"peak_used_in_bytes":558432256,
"peak_max_in_bytes":558432256
},
"survivor":{
"used_in_bytes":9630920,
"max_in_bytes":69730304,
"peak_used_in_bytes":69730304,
"peak_max_in_bytes":69730304
},
"old":{
"used_in_bytes":9885051104,
"max_in_bytes":31514361856,
"peak_used_in_bytes":23868903928,
"peak_max_in_bytes":31514361856
}
}
},
"threads":{
"count":121,
"peak_count":144
},
"gc":{
"collectors":{
"young":{
"collection_count":209658,
"collection_time_in_millis":12089132
},
"old":{
"collection_count":11,
"collection_time_in_millis":6628
}
}
},
"buffer_pools":{
"direct":{
"count":162,
"used_in_bytes":195159665,
"total_capacity_in_bytes":195159665
},
"mapped":{
"count":1292,
"used_in_bytes":9897821921,
"total_capacity_in_bytes":9897821921
}
}
},
"thread_pool":{
"generic":{
"threads":1,
"queue":0,
"active":0,
"rejected":0,
"largest":23,
"completed":67214
},
"index":{
"threads":8,

...

--
You received this message because you are subscribed to the Google Groups "elasticsearch" group.
To unsubscribe from this group and stop receiving emails from it, send an email to elasticsearch+unsubscribe@googlegroups.com.
To view this discussion on the web visit https://groups.google.com/d/msgid/elasticsearch/904664f8-2d70-4983-8d92-38064989d781%40googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[webish]

After changing to Oracle Java it seemed the problem was resolved. CPU on
both nodes dropped to 5% or less during the same state; however, in the
last 24 hrs history is repeating itself.

CPU on one node is 75% and on the other is 50%....

On Wednesday, July 9, 2014 3:47:12 AM UTC-4, webish wrote:

Ahh, ok. Sorry.

NodeA:
nodeA> curl -XGET 'http://localhost:9200/_nodes/hot_threads' · GitHub

NodeB:
nodeB> curl -XGET 'http://localhost:9200/_nodes/hot_threads' · GitHub

On Wednesday, July 9, 2014 3:36:53 AM UTC-4, Mark Walkom wrote:

The output of hot threads is only from the one node, ie requesting it
isn't at a cluster level, if you can add the other node it'd help.

Regards,
Mark Walkom

Infrastructure Engineer
Campaign Monitor
email: ma...@campaignmonitor.com
web: www.campaignmonitor.com

On 9 July 2014 17:32, webish gre...@yoursports.com wrote:

Definitely.

Here are all the captures thus far in gists.

CLUSTER:
curl -XGET 'http://localhost:9200/_nodes/stats'
curl -XGET 'http://localhost:9200/_nodes/stats' · GitHub

curl -XGET 'http://localhost:9200/'
curl -XGET 'http://localhost:9200/' · GitHub

THREADS:
curl -XGET 'http://localhost:9200/_nodes/hot_threads'
nodeA> curl -XGET 'http://localhost:9200/_nodes/hot_threads' · GitHub

curl localhost:9200/_cat/thread_pool
curl localhost:9200/_cat/thread_pool · GitHub

On Wednesday, July 9, 2014 3:11:25 AM UTC-4, Ivan Brusic wrote:

Yes, please use gist/pastebin in the future.

Try using the hot threads API [1] to see if there are any threads that are
truly busy. If your system is truly idle, your thread pools should be
almost empty [2]. Your output above has only one management thread in use,
which could be simply the thread serving up the output.

[1] Elasticsearch Platform — Find real-time answers at scale | Elastic
reference/current/cluster-nodes-hot-threads.html
[2] Elasticsearch Platform — Find real-time answers at scale | Elastic
reference/current/cat-thread-pool.html

Cheers,

Ivan

On Tue, Jul 8, 2014 at 10:54 PM, webish gre...@yoursports.com wrote:

Thanks Mark. My mistake.

I can try switching to Oracle Java. There is no TTL. I've used Marvel
for development testing. Perhaps I can install the plugin...

On Wednesday, July 9, 2014 1:35:20 AM UTC-4, Mark Walkom wrote:

It's better to pop long output like that into a gist/pastebin, it makes it
easier to read the thread.
It's also worth installing a monitoring plugin like ElasticHQ or marvel,
as they provide graphical insight into what is happening and will
extrapolate some of the raw figures out.

If you can change to Oracle java you will get better performance. Are you
using TTL?

Regards,
Mark Walkom

Infrastructure Engineer
Campaign Monitor
email: ma...@campaignmonitor.com
web: www.campaignmonitor.com

On 9 July 2014 14:54, webish gre...@yoursports.com wrote:

I'm seeing very high CPU utilization in an idle or close to idle state. I
haven't seen it use less than 40% on each node over the course of days.
I'm not certain how long it has been like this but could be weeks.

Any help resolving this would be greatly appreciated!

Details:

There are 2 nodes in the cluster.

AWS instances are:
vCPU ECUMemory (GiB) Instance Storage (GB) Linux/UNIX Usage

r3.2xlarge 82661 1 x 160 SSD

curl -XGET 'http://localhost:9200/'
{
"status" : 200,
"name" : "Eddie Brock",
"version" : {
"number" : "1.1.0",
"build_hash" : "2181e113dea80b4a9e31e58e9686658a2d46e363",
"build_timestamp" : "2014-03-25T15:59:51Z",
"build_snapshot" : false,
"lucene_version" : "4.7"
},
"tagline" : "You Know, for Search"
}

% java -version
java version "1.7.0_55"
OpenJDK Runtime Environment (IcedTea 2.4.7) (7u55-2.4.7-1ubuntu1~0.12.04.2
)
OpenJDK 64-Bit Server VM (build 24.51-b03, mixed mode)

curl -XGET 'http://localhost:9200/_nodes/stats'
{
"cluster_name":"ab_elastic",
"nodes":{
"kpHPVkBCyjOkRaSpS5Q":{
"timestamp":1404880773859,
"name":"Milan",
"transport_address":"removed",
"host":"removed",
"ip":[
"removed",
"NONE"
],
"indices":{
"docs":{
"count":17793781,
"deleted":466537
},
"store":{
"size_in_bytes":9902722189,
"throttle_time_in_millis":8777423
},
"indexing":{
"index_total":3548638,
"index_time_in_millis":2610694,
"index_current":1,
"delete_total":0,
"delete_time_in_millis":0,
"delete_current":0
},
"get":{
"total":0,
"time_in_millis":0,
"exists_total":0,
"exists_time_in_millis":0,
"missing_total":0,
"missing_time_in_millis":0,
"current":0
},
"search":{
"open_contexts":0,
"query_total":155361213,
"query_time_in_millis":817989500,
"query_current":0,
"fetch_total":4119309,
"fetch_time_in_millis":1967525,
"fetch_current":0
},
"merges":{
"current":0,
"current_docs":0,
"current_size_in_bytes":0,
"total":564,
"total_time_in_millis":76065,
"total_docs":1093579,
"total_size_in_bytes":470304093
},
"refresh":{
"total":15117234,
"total_time_in_millis":50581020
},
"flush":{
"total":2336,
"total_time_in_millis":99980
},
"warmer":{
"current":0,
"total":2719640,
"total_time_in_millis":5980882
},
"filter_cache":{
"memory_size_in_bytes":722232396,
"evictions":1179740
},
"id_cache":{
"memory_size_in_bytes":131672871
},
"fielddata":{
"memory_size_in_bytes":133728950,
"evictions":0
},
"percolate":{
"total":0,
"time_in_millis":0,
"current":0,
"memory_size_in_bytes":-1,
"memory_size":"-1b",
"queries":0
},
"completion":{
"size_in_bytes":0
},
"segments":{
"count":278,
"memory_in_bytes":220799236
},
"translog":{
"operations":68,
"size_in_bytes":0
}
},
"os":{
"timestamp":1404880773910,
"uptime_in_millis":915746,
"load_average":[
2.68,
3.17,
3.35
],
"cpu":{
"sys":0,
"user":38,
"idle":61,
"usage":38,
"stolen":0
},
"mem":{
"free_in_bytes":12301606912,
"used_in_bytes":51857985536,
"free_percent":35,
"used_percent":64,
"actual_free_in_bytes":23005929472,
"actual_used_in_bytes":41153662976
},
"swap":{
"used_in_bytes":0,
"free_in_bytes":0
}
},
"process":{
"timestamp":1404880773911,
"open_file_descriptors":300,
"cpu":{
"percent":313,
"sys_in_millis":14021810,
"user_in_millis":794249870,
"total_in_millis":808271680
},
"mem":{
"resident_in_bytes":32342851584,
"share_in_bytes":712302592,
"total_virtual_in_bytes":47406919680
}
},
"jvm":{
"timestamp":1404880773911,
"uptime_in_millis":309661546,
"mem":{
"heap_used_in_bytes":18409783280,
"heap_used_percent":57,
"heap_committed_in_bytes":32142524416,
"heap_max_in_bytes":32142524416,
"non_heap_used_in_bytes":65012920,
"non_heap_committed_in_bytes":98009088,
"pools":{
"young":{
"used_in_bytes":538302112,
"max_in_bytes":558432256,
"peak_used_in_bytes":558432256,
"peak_max_in_bytes":558432256
},
"survivor":{
"used_in_bytes":11403704,
"max_in_bytes":69730304,
"peak_used_in_bytes":69730304,
"peak_max_in_bytes":69730304
},
"old":{
"used_in_bytes":17860077464,
"max_in_bytes":31514361856,
"peak_used_in_bytes":23694417088,
"peak_max_in_bytes":31514361856
}
}
},
"threads":{
"count":117,
"peak_count":133
},
"gc":{
"collectors":{
"young":{
"collection_count":200644,
"collection_time_in_millis":10870850
},
"old":{
"collection_count":9,
"collection_time_in_millis":2987
}
}
},
"buffer_pools":{
"direct":{
"count":154,
"used_in_bytes":78176363,
"total_capacity_in_bytes":78176363
},
"mapped":{
"count":1293,
"used_in_bytes":9831615457,
"total_capacity_in_bytes":9831615457
}
}
},
"thread_pool":{
"generic":{
"threads":1,
"queue":0,
"active":0,
"rejected":0,
"largest":14,
"completed":56826
},
"index":{
"threads":8,
"queue":0,
"active":0,
"rejected":0,
"largest":8,
"completed":3607948
},
"get":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"snapshot":{
"threads":4,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":95697
},
"merge":{
"threads":2,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":282257
},
"suggest":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"bulk":{
"threads":8,
"queue":0,
"active":0,
"rejected":0,
"largest":8,
"completed":1081
},
"optimize":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"warmer":{
"threads":2,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":5556773
},
"flush":{
"threads":1,
"queue":0,
"active":0,
"rejected":0,
"largest":2,
"completed":2336
},
"search":{
"threads":24,
"queue":0,
"active":0,
"rejected":0,
"largest":24,
"completed":160139043
},
"percolate":{
"threads":0,
"queue":0,
"active":0,
"rejected":0,
"largest":0,
"completed":0
},
"management":{
"threads":1,
"queue":0,
"active":1,
"rejected":0,
"largest":5,
"completed":1064
},
"refresh":{
"threads":4,
"queue":0,
"active":0,
"rejected":0,
"largest":4,
"completed":15126156
}
},
"network":{
"tcp":{
"active_opens":1513869,

...

--
You received this message because you are subscribed to the Google Groups "elasticsearch" group.
To unsubscribe from this group and stop receiving emails from it, send an email to elasticsearch+unsubscribe@googlegroups.com.
To view this discussion on the web visit https://groups.google.com/d/msgid/elasticsearch/712110a5-a554-46c6-a9e5-4eba8c27c2a5%40googlegroups.com.
For more options, visit https://groups.google.com/d/optout.