Elasticsearch First Response is very slow

Elastic Stack Elasticsearch
1

I have 7 days of data in 3 node cluster. 1 shard per index with one segment. I am planning to have data for 90 days later.
I try to view data for last 90 days(only 7 days of data really available) using date picker in kibana.
i start with clearing the cache.
Initial load take 70 seconds and then when i use the same time range took time goes to 4ms.
when change the time range again first call take over 1 min and goes below 1 second when i do the same search again.

How can I improve this?

Other info:
Es, kibana - 6.3.0
3 nodes with 4core 15gb with 7gb assigned to heap
index size is 2.2gb and has one replica

9 visualizations in the dashboard

2

What kind of server/hardware is this deployed on? How many visualisations does your dashboard have?

3

Google cloud
n1-standard-4 (4 vCPUs, 15 GB memory)

Disk
400GB per node
Standard persistent disk

CentOS

9 visualizations in the dashboard

4

So you have 14 shards in the cluster and only around 15GB of data? What is the output of the cluster stats API? Are you indexing at the same time you are querying? Do you have anything else running on these nodes?

5

I don't do any indexing at the moment. Nothing else is running on this cluster.
Now I have 18 shards as I added 3 indexes in the mean time.

cluster stats API Output

{
"_nodes": {
"total": 3,
"successful": 3,
"failed": 0
},
"cluster_name": "nuwans-elk-cluster1",
"timestamp": 1547537312699,
"status": "green",
"indices": {
"count": 24,
"shards": {
"total": 48,
"primaries": 24,
"replication": 1,
"index": {
"shards": {
"min": 2,
"max": 2,
"avg": 2
},
"primaries": {
"min": 1,
"max": 1,
"avg": 1
},
"replication": {
"min": 1,
"max": 1,
"avg": 1
}
}
},
"docs": {
"count": 490350031,
"deleted": 26139547
},
"store": {
"size": "43.7gb",
"size_in_bytes": 46980963876
},
"fielddata": {
"memory_size": "57.7kb",
"memory_size_in_bytes": 59104,
"evictions": 0
},
"query_cache": {
"memory_size": "0b",
"memory_size_in_bytes": 0,
"total_count": 0,
"hit_count": 0,
"miss_count": 0,
"cache_size": 0,
"cache_count": 0,
"evictions": 0
},
"completion": {
"size": "0b",
"size_in_bytes": 0
},
"segments": {
"count": 442,
"memory": "324.4mb",
"memory_in_bytes": 340159639,
"terms_memory": "278.6mb",
"terms_memory_in_bytes": 292200192,
"stored_fields_memory": "29.4mb",
"stored_fields_memory_in_bytes": 30931216,
"term_vectors_memory": "0b",
"term_vectors_memory_in_bytes": 0,
"norms_memory": "4.8kb",
"norms_memory_in_bytes": 4992,
"points_memory": "14.2mb",
"points_memory_in_bytes": 14893247,
"doc_values_memory": "2mb",
"doc_values_memory_in_bytes": 2129992,
"index_writer_memory": "0b",
"index_writer_memory_in_bytes": 0,
"version_map_memory": "0b",
"version_map_memory_in_bytes": 0,
"fixed_bit_set": "0b",
"fixed_bit_set_memory_in_bytes": 0,
"max_unsafe_auto_id_timestamp": 1547510407937,
"file_sizes": {}
}
},
"nodes": {
"count": {
"total": 3,
"data": 3,
"coordinating_only": 0,
"master": 3,
"ingest": 3
},
"versions": [
"6.3.0"
],
"os": {
"available_processors": 12,
"allocated_processors": 12,
"names": [
{
"name": "Linux",
"count": 3
}
],
"mem": {
"total": "43.5gb",
"total_in_bytes": 46799904768,
"free": "1.1gb",
"free_in_bytes": 1190825984,
"used": "42.4gb",
"used_in_bytes": 45609078784,
"free_percent": 3,
"used_percent": 97
}
},
"process": {
"cpu": {
"percent": 17
},
"open_file_descriptors": {
"min": 344,
"max": 366,
"avg": 351
}
},
"jvm": {
"max_uptime": "9d",
"max_uptime_in_millis": 781731013,
"versions": [
{
"version": "1.8.0_191",
"vm_name": "OpenJDK 64-Bit Server VM",
"vm_version": "25.191-b12",
"vm_vendor": "Oracle Corporation",
"count": 3
}
],
"mem": {
"heap_used": "9.8gb",
"heap_used_in_bytes": 10625182904,
"heap_max": "20.9gb",
"heap_max_in_bytes": 22443982848
},
"threads": 243
},
"fs": {
"total": "1.1tb",
"total_in_bytes": 1288439058432,
"free": "1.1tb",
"free_in_bytes": 1223634808832,
"available": "1.1tb",
"available_in_bytes": 1223634808832
},
"plugins": ,
"network_types": {
"transport_types": {
"security4": 3
},
"http_types": {
"security4": 3
}
}
}
}

6

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.