We are facing severe performance issue with ES 6.2.3 in comparison to ES 2.3.4. Not sure, what we are missing. We were planning to shift to ES6 and have setup a new 6.2.3 cluster with 20 m5d.4xlarge data, 9 m5.4xlarge client & 5 c5.xlarge master nodes. Indexes from current ES2 cluster were copied and delta updates pipeline setup is inplace. We moved 5% of prod search traffic to this new ES6 cluster and found continuous spikes in p95 latency graphs. Also, the latencies are very high in comparison to ES2.
ES6 client and data nodes have 26GB of max jvm allocation.
BTW, ES 2.3.4 has 60 data c5.4xlarge and 9 master c5.large nodes.
ES 6.2.3 /_cluster/stats:
{
"_nodes" : {
"total" : 34,
"successful" : 34,
"failed" : 0
},
"cluster_name" : "se_cluster",
"timestamp" : 1539674434541,
"status" : "green",
"indices" : {
"count" : 5,
"shards" : {
"total" : 85,
"primaries" : 30,
"replication" : 1.8333333333333333,
"index" : {
"shards" : {
"min" : 10,
"max" : 30,
"avg" : 17.0
},
"primaries" : {
"min" : 5,
"max" : 10,
"avg" : 6.0
},
"replication" : {
"min" : 1.0,
"max" : 4.0,
"avg" : 1.8
}
}
},
"docs" : {
"count" : 112875644,
"deleted" : 51467647
},
"store" : {
"size_in_bytes" : 281022181476
},
"fielddata" : {
"memory_size_in_bytes" : 24608071400,
"evictions" : 0
},
"query_cache" : {
"memory_size_in_bytes" : 3484131424,
"total_count" : 902027508,
"hit_count" : 714578426,
"miss_count" : 187449082,
"cache_size" : 100056,
"cache_count" : 29192312,
"evictions" : 29092256
},
"completion" : {
"size_in_bytes" : 0
},
"segments" : {
"count" : 1188,
"memory_in_bytes" : 703497198,
"terms_memory_in_bytes" : 528131307,
"stored_fields_memory_in_bytes" : 107156920,
"term_vectors_memory_in_bytes" : 0,
"norms_memory_in_bytes" : 9819136,
"points_memory_in_bytes" : 32723979,
"doc_values_memory_in_bytes" : 25665856,
"index_writer_memory_in_bytes" : 1347092314,
"version_map_memory_in_bytes" : 13714760,
"fixed_bit_set_memory_in_bytes" : 62525048,
"max_unsafe_auto_id_timestamp" : -1,
"file_sizes" : { }
}
},
"nodes" : {
"count" : {
"total" : 34,
"data" : 20,
"coordinating_only" : 9,
"master" : 5,
"ingest" : 20
},
"versions" : [
"6.2.3"
],
"os" : {
"available_processors" : 484,
"allocated_processors" : 484,
"names" : [
{
"name" : "Linux",
"count" : 34
}
],
"mem" : {
"total_in_bytes" : 1954498506752,
"free_in_bytes" : 690253582336,
"used_in_bytes" : 1264244924416,
"free_percent" : 35,
"used_percent" : 65
}
},
"process" : {
"cpu" : {
"percent" : 48
},
"open_file_descriptors" : {
"min" : 927,
"max" : 1229,
"avg" : 1148
}
},
"jvm" : {
"max_uptime_in_millis" : 1367385507,
"versions" : [
{
"version" : "1.8.0_131",
"vm_name" : "Java HotSpot(TM) 64-Bit Server VM",
"vm_version" : "25.131-b11",
"vm_vendor" : "Oracle Corporation",
"count" : 29
},
{
"version" : "1.8.0_181",
"vm_name" : "Java HotSpot(TM) 64-Bit Server VM",
"vm_version" : "25.181-b13",
"vm_vendor" : "Oracle Corporation",
"count" : 5
}
],
"mem" : {
"heap_used_in_bytes" : 318745528344,
"heap_max_in_bytes" : 827613904896
},
"threads" : 5068
},
"fs" : {
"total_in_bytes" : 7380778016768,
"free_in_bytes" : 7022425059328,
"available_in_bytes" : 6660544962560
},
"plugins" : [
{
"name" : "repository-s3",
"version" : "6.2.3",
"description" : "The S3 repository plugin adds S3 repositories",
"classname" : "org.elasticsearch.repositories.s3.S3RepositoryPlugin",
"extended_plugins" : [ ],
"has_native_controller" : false,
"requires_keystore" : false
}
],
"network_types" : {
"transport_types" : {
"netty4" : 34
},
"http_types" : {
"netty4" : 34
}
}
}
}
ES6 sample data:
{
"took": 75,
"timed_out": false,
"_shards": {
"total": 10,
"successful": 10,
"skipped": 0,
"failed": 0
},
"hits": {
"total": 43986609,
"max_score": 1,
"hits": [
{
"_index": "catalog",
"_type": "dd",
"_id": "1718708443456",
"_score": 1,
"_source": {
"id": 1718708443456,
"name": "New Women Long sheet",
"bd": "Default",
"bid": 43973023,
"sku": "XYI443-4",
"v_id": 212,
"cid": 742366,
"promo_text": null,
"url_key": "nwdwdw",
"tag": null,
"pvc": 2,
"oc": 0,
"p_updated_at": "2018-09-08T18:30:00.000Z",
"description": " some text",
"price": 603,
"mrp": 1270,
"is_in_stock": 1,
"search_weight": 0.05542687223333736,
"brw": 0,
"cwt": 2,
"cname": "LA",
"cfp": " Women->LS-> LA",
"child_product_names": [
"some name"
],
"child_product_ids": [
11111111
],
"child_instock_product_ids": [
22222222
],
"cats": [
555555,
52404444
],
"child_merchants": [
777777
],
"authorisation_level": [
3
],
"meta_keyword": [
],
"attributes_list": [
"Black"
],
"attributes": [
],
"cccs": [
0
],
"created_at": "2018-04-10T06:56:52.000Z",
"index_created_at": "2018-10-04T01:31:50.547Z",
"variants": [
{
"id": 11111111,
"merchant_id": 77777,
"is_in_stock": 1,
"price": 603
}
],
"filter_color": [
"Black"
],
"filter_sz": [
"Free"
]
}
}
]
}
}
Let me know if you need any more details.
