I ran this test again and I think these results from the hot nodes are only for the autocomplete suggestion queries.
gistfile1.txt
"@timestamp","host.name","elasticsearch.slowlog.took",message
"Sep 18, 2019 @ 00:43:55.256",server04,"1.2s","{""type"": ""index_search_slowlog"", ""timestamp"": ""2019-09-17T19:43:55,256-0500"", ""level"": ""TRACE"", ""component"": ""i.s.s.query"", ""cluster.name"": ""swlogstash"", ""node.name"": ""server04"", ""cluster.uuid"": ""NxY3dzLATVuFgDsd_plg5Q"", ""node.id"": ""MBJ-Z1X8Tp2XrMnqBPLEMg"", ""message"": ""[indexnameprefix-3.0-2019.09.13-000043][0] took[1.2s], took_millis[1232], total_hits[10000+ hits], types[], stats[], search_type[QUERY_THEN_FETCH], total_shards[209], source[{\""size\"":0,\""timeout\"":\""1000ms\"",\""terminate_after\"":100000,\""query\"":{\""match_all\"":{\""boost\"":1.0}},\""aggregations\"":{\""suggestions\"":{\""terms\"":{\""field\"":\""customer.name\"",\""size\"":10,\""shard_size\"":10,\""min_doc_count\"":1,\""shard_min_doc_count\"":0,\""show_term_doc_count_error\"":false,\""execution_hint\"":\""map\"",\""order\"":[{\""_count\"":\""desc\""},{\""_key\"":\""asc\""}],\""include\"":\""kqlcustomer02.*\""}}}}], id[], "" }"
"Sep 18, 2019 @ 00:43:55.255",server03,"1.2s","{""type"": ""index_search_slowlog"", ""timestamp"": ""2019-09-17T19:43:55,255-0500"", ""level"": ""TRACE"", ""component"": ""i.s.s.query"", ""cluster.name"": ""swlogstash"", ""node.name"": ""server03"", ""cluster.uuid"": ""NxY3dzLATVuFgDsd_plg5Q"", ""node.id"": ""76WG1dQZR1a3K5i1nQm2-g"", ""message"": ""[indexnameprefix-3.0-2019.09.16-000046][0] took[1.2s], took_millis[1231], total_hits[10000+ hits], types[], stats[], search_type[QUERY_THEN_FETCH], total_shards[209], source[{\""size\"":0,\""timeout\"":\""1000ms\"",\""terminate_after\"":100000,\""query\"":{\""match_all\"":{\""boost\"":1.0}},\""aggregations\"":{\""suggestions\"":{\""terms\"":{\""field\"":\""customer.name\"",\""size\"":10,\""shard_size\"":10,\""min_doc_count\"":1,\""shard_min_doc_count\"":0,\""show_term_doc_count_error\"":false,\""execution_hint\"":\""map\"",\""order\"":[{\""_count\"":\""desc\""},{\""_key\"":\""asc\""}],\""include\"":\""kqlcustomer02.*\""}}}}], id[], "" }"
"Sep 18, 2019 @ 00:43:55.248",server04,"1.2s","{""type"": ""index_search_slowlog"", ""timestamp"": ""2019-09-17T19:43:55,248-0500"", ""level"": ""TRACE"", ""component"": ""i.s.s.query"", ""cluster.name"": ""swlogstash"", ""node.name"": ""server04"", ""cluster.uuid"": ""NxY3dzLATVuFgDsd_plg5Q"", ""node.id"": ""MBJ-Z1X8Tp2XrMnqBPLEMg"", ""message"": ""[indexnameprefix-3.0-2019.09.13-000042][0] took[1.2s], took_millis[1224], total_hits[10000+ hits], types[], stats[], search_type[QUERY_THEN_FETCH], total_shards[209], source[{\""size\"":0,\""timeout\"":\""1000ms\"",\""terminate_after\"":100000,\""query\"":{\""match_all\"":{\""boost\"":1.0}},\""aggregations\"":{\""suggestions\"":{\""terms\"":{\""field\"":\""customer.name\"",\""size\"":10,\""shard_size\"":10,\""min_doc_count\"":1,\""shard_min_doc_count\"":0,\""show_term_doc_count_error\"":false,\""execution_hint\"":\""map\"",\""order\"":[{\""_count\"":\""desc\""},{\""_key\"":\""asc\""}],\""include\"":\""kqlcustomer02.*\""}}}}], id[], "" }"
"Sep 18, 2019 @ 00:43:55.234",server03,"1.2s","{""type"": ""index_search_slowlog"", ""timestamp"": ""2019-09-17T19:43:55,234-0500"", ""level"": ""TRACE"", ""component"": ""i.s.s.query"", ""cluster.name"": ""swlogstash"", ""node.name"": ""server03"", ""cluster.uuid"": ""NxY3dzLATVuFgDsd_plg5Q"", ""node.id"": ""76WG1dQZR1a3K5i1nQm2-g"", ""message"": ""[indexnameprefix-3.0-2019.09.14-000044][0] took[1.2s], took_millis[1201], total_hits[10000+ hits], types[], stats[], search_type[QUERY_THEN_FETCH], total_shards[209], source[{\""size\"":0,\""timeout\"":\""1000ms\"",\""terminate_after\"":100000,\""query\"":{\""match_all\"":{\""boost\"":1.0}},\""aggregations\"":{\""suggestions\"":{\""terms\"":{\""field\"":\""customer.name\"",\""size\"":10,\""shard_size\"":10,\""min_doc_count\"":1,\""shard_min_doc_count\"":0,\""show_term_doc_count_error\"":false,\""execution_hint\"":\""map\"",\""order\"":[{\""_count\"":\""desc\""},{\""_key\"":\""asc\""}],\""include\"":\""kqlcustomer02.*\""}}}}], id[], "" }"
"Sep 18, 2019 @ 00:43:55.217",server04,"1.1s","{""type"": ""index_search_slowlog"", ""timestamp"": ""2019-09-17T19:43:55,217-0500"", ""level"": ""TRACE"", ""component"": ""i.s.s.query"", ""cluster.name"": ""swlogstash"", ""node.name"": ""server04"", ""cluster.uuid"": ""NxY3dzLATVuFgDsd_plg5Q"", ""node.id"": ""MBJ-Z1X8Tp2XrMnqBPLEMg"", ""message"": ""[indexnameprefix-3.0-2019.09.17-000048][0] took[1.1s], took_millis[1193], total_hits[10000+ hits], types[], stats[], search_type[QUERY_THEN_FETCH], total_shards[209], source[{\""size\"":0,\""timeout\"":\""1000ms\"",\""terminate_after\"":100000,\""query\"":{\""match_all\"":{\""boost\"":1.0}},\""aggregations\"":{\""suggestions\"":{\""terms\"":{\""field\"":\""customer.name\"",\""size\"":10,\""shard_size\"":10,\""min_doc_count\"":1,\""shard_min_doc_count\"":0,\""show_term_doc_count_error\"":false,\""execution_hint\"":\""map\"",\""order\"":[{\""_count\"":\""desc\""},{\""_key\"":\""asc\""}],\""include\"":\""kqlcustomer02.*\""}}}}], id[], "" }"
"Sep 18, 2019 @ 00:43:55.193",server05,"1.1s","{""type"": ""index_search_slowlog"", ""timestamp"": ""2019-09-17T19:43:55,193-0500"", ""level"": ""TRACE"", ""component"": ""i.s.s.query"", ""cluster.name"": ""swlogstash"", ""node.name"": ""server05"", ""cluster.uuid"": ""NxY3dzLATVuFgDsd_plg5Q"", ""node.id"": ""m_LHRVyCRz6GC0wqy-7YuQ"", ""message"": ""[indexnameprefix-3.0-2019.09.12-000041][0] took[1.1s], took_millis[1165], total_hits[10000+ hits], types[], stats[], search_type[QUERY_THEN_FETCH], total_shards[209], source[{\""size\"":0,\""timeout\"":\""1000ms\"",\""terminate_after\"":100000,\""query\"":{\""match_all\"":{\""boost\"":1.0}},\""aggregations\"":{\""suggestions\"":{\""terms\"":{\""field\"":\""customer.name\"",\""size\"":10,\""shard_size\"":10,\""min_doc_count\"":1,\""shard_min_doc_count\"":0,\""show_term_doc_count_error\"":false,\""execution_hint\"":\""map\"",\""order\"":[{\""_count\"":\""desc\""},{\""_key\"":\""asc\""}],\""include\"":\""kqlcustomer02.*\""}}}}], id[], "" }"
"Sep 18, 2019 @ 00:43:55.084",server03,1s,"{""type"": ""index_search_slowlog"", ""timestamp"": ""2019-09-17T19:43:55,084-0500"", ""level"": ""TRACE"", ""component"": ""i.s.s.query"", ""cluster.name"": ""swlogstash"", ""node.name"": ""server03"", ""cluster.uuid"": ""NxY3dzLATVuFgDsd_plg5Q"", ""node.id"": ""76WG1dQZR1a3K5i1nQm2-g"", ""message"": ""[indexnameprefix-3.0-2019.09.15-000045][0] took[1s], took_millis[1060], total_hits[10000+ hits], types[], stats[], search_type[QUERY_THEN_FETCH], total_shards[209], source[{\""size\"":0,\""timeout\"":\""1000ms\"",\""terminate_after\"":100000,\""query\"":{\""match_all\"":{\""boost\"":1.0}},\""aggregations\"":{\""suggestions\"":{\""terms\"":{\""field\"":\""customer.name\"",\""size\"":10,\""shard_size\"":10,\""min_doc_count\"":1,\""shard_min_doc_count\"":0,\""show_term_doc_count_error\"":false,\""execution_hint\"":\""map\"",\""order\"":[{\""_count\"":\""desc\""},{\""_key\"":\""asc\""}],\""include\"":\""kqlcustomer02.*\""}}}}], id[], "" }"
"Sep 18, 2019 @ 00:43:55.080",server03,1s,"{""type"": ""index_search_slowlog"", ""timestamp"": ""2019-09-17T19:43:55,080-0500"", ""level"": ""TRACE"", ""component"": ""i.s.s.query"", ""cluster.name"": ""swlogstash"", ""node.name"": ""server03"", ""cluster.uuid"": ""NxY3dzLATVuFgDsd_plg5Q"", ""node.id"": ""76WG1dQZR1a3K5i1nQm2-g"", ""message"": ""[indexnameprefix-3.0-2019.09.15-000038][0] took[1s], took_millis[1055], total_hits[10000+ hits], types[], stats[], search_type[QUERY_THEN_FETCH], total_shards[209], source[{\""size\"":0,\""timeout\"":\""1000ms\"",\""terminate_after\"":100000,\""query\"":{\""match_all\"":{\""boost\"":1.0}},\""aggregations\"":{\""suggestions\"":{\""terms\"":{\""field\"":\""customer.name\"",\""size\"":10,\""shard_size\"":10,\""min_doc_count\"":1,\""shard_min_doc_count\"":0,\""show_term_doc_count_error\"":false,\""execution_hint\"":\""map\"",\""order\"":[{\""_count\"":\""desc\""},{\""_key\"":\""asc\""}],\""include\"":\""kqlcustomer02.*\""}}}}], id[], "" }"
"Sep 18, 2019 @ 00:43:55.067",server03,1s,"{""type"": ""index_search_slowlog"", ""timestamp"": ""2019-09-17T19:43:55,067-0500"", ""level"": ""TRACE"", ""component"": ""i.s.s.query"", ""cluster.name"": ""swlogstash"", ""node.name"": ""server03"", ""cluster.uuid"": ""NxY3dzLATVuFgDsd_plg5Q"", ""node.id"": ""76WG1dQZR1a3K5i1nQm2-g"", ""message"": ""[indexnameprefix-3.0-2019.09.16-000039][0] took[1s], took_millis[1043], total_hits[10000+ hits], types[], stats[], search_type[QUERY_THEN_FETCH], total_shards[209], source[{\""size\"":0,\""timeout\"":\""1000ms\"",\""terminate_after\"":100000,\""query\"":{\""match_all\"":{\""boost\"":1.0}},\""aggregations\"":{\""suggestions\"":{\""terms\"":{\""field\"":\""customer.name\"",\""size\"":10,\""shard_size\"":10,\""min_doc_count\"":1,\""shard_min_doc_count\"":0,\""show_term_doc_count_error\"":false,\""execution_hint\"":\""map\"",\""order\"":[{\""_count\"":\""desc\""},{\""_key\"":\""asc\""}],\""include\"":\""kqlcustomer02.*\""}}}}], id[], "" }"show original
I'm unsure why the actual queries aren't showing up there, unless I'm reading that wrong. I'm using filebeat's ES module to send slowlog.json to a separate monitoring ES server and used this query: event.dataset:"elasticsearch.slowlog" and "kqlcustomer02" and host.name:(server01 or server02 or server03 or server04 or server05) where those hostnames are the hot nodes.
But then I went into my Kibana Advanced settings and set filterEditor:suggestValues to Off. And now all of my searches using KQL return lickety-split with no timeouts.
I don't suppose there is a way to limit those auto-suggest queries to the time range of the actual query? Otherwise, we will live without auto-suggest.
@Bargs , thank you very much for your time walking through this with me. It's greatly appreciated!