Hi All,
Elasticsearch: 7.13.2
I am wondering if anyone could help provide some input into an issue that I'm seeing.
I am using a fairly simple query:
Query is executed against filebeat-*
"query": {
"bool": {
"must": [],
"filter": [
{
"bool": {
"should": [
{
"bool": {
"should": [
{
"query_string": {
"fields": [
"url.path"
],
"query": "\\/_apps\\/<app_name>*"
}
}
],
"minimum_should_match": 1
}
},
{
"bool": {
"should": [
{
"match_phrase": {
"url.original.text": "/_apps/<app_name>"
}
}
],
"minimum_should_match": 1
}
}
],
"minimum_should_match": 1
}
},
{
"bool": {
"should": [
{
"bool": {
"should": [
{
"query_string": {
"fields": [
"url.path"
],
"query": "\\/_apps\\/<app_name>*"
}
}
],
"minimum_should_match": 1
}
},
{
"bool": {
"should": [
{
"match_phrase": {
"url.original.text": "/_apps/<app_name>"
}
}
],
"minimum_should_match": 1
}
}
],
"minimum_should_match": 1
}
},
{
"bool": {
"minimum_should_match": 1,
"should": [
{
"match_phrase": {
"host.name": "<host_1>"
}
},
{
"match_phrase": {
"host.name": "<host_2>"
}
},
{
"match_phrase": {
"host.name": "<host_3>"
}
},
{
"match_phrase": {
"host.name": "<host_4>"
}
}
]
}
},
{
"match_phrase": {
"event.module": "iis"
}
},
{
"range": {
"@timestamp": {
"gte": "2021-06-22T12:36:38.778Z",
"lte": "2021-06-22T12:51:38.778Z",
"format": "strict_date_optional_time"
}
}
}
],
"should": [],
"must_not": []
}
}
}
Response:
{
"took": 114809,
"timed_out": false,
"_shards": {
"total": 48,
"successful": 48,
"skipped": 46,
"failed": 0
},
"hits": {
"total": 18588,
"max_score": null,
"hits": []
},
"aggregations": {
"b8a65013-8337-436c-b9ca-e50c817a2c3b": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [
{
"key": "<ip_address>",
"doc_count": 18588
}
]
}
}
}
Based on the amount of docs (19k) and there only being 1 return key/bucket, I wouldn't expect this type of query to take this long.
If I run a similar query, but change out source.ip with destination.ip I get a much better response time:
"query": {
"bool": {
"must": [],
"filter": [
{
"bool": {
"should": [
{
"bool": {
"should": [
{
"query_string": {
"fields": [
"url.path"
],
"query": "\\/_apps\\/<app_name>*"
}
}
],
"minimum_should_match": 1
}
},
{
"bool": {
"should": [
{
"match_phrase": {
"url.original.text": "/_apps/<app_name>"
}
}
],
"minimum_should_match": 1
}
}
],
"minimum_should_match": 1
}
},
{
"bool": {
"should": [
{
"bool": {
"should": [
{
"query_string": {
"fields": [
"url.path"
],
"query": "\\/_apps\\/<app_name>*"
}
}
],
"minimum_should_match": 1
}
},
{
"bool": {
"should": [
{
"match_phrase": {
"url.original.text": "/_apps/<app_name>"
}
}
],
"minimum_should_match": 1
}
}
],
"minimum_should_match": 1
}
},
{
"bool": {
"minimum_should_match": 1,
"should": [
{
"match_phrase": {
"host.name": "<host_1>"
}
},
{
"match_phrase": {
"host.name": "<host_2>"
}
},
{
"match_phrase": {
"host.name": "<host_3>"
}
},
{
"match_phrase": {
"host.name": "<host_4>"
}
}
]
}
},
{
"match_phrase": {
"event.module": "iis"
}
},
{
"range": {
"@timestamp": {
"gte": "2021-06-22T12:36:38.778Z",
"lte": "2021-06-22T12:51:38.778Z",
"format": "strict_date_optional_time"
}
}
}
],
"should": [],
"must_not": []
}
}
The response contains the same amount of records, but took significantly less time:
{
"took": 8386,
"timed_out": false,
"_shards": {
"total": 48,
"successful": 48,
"skipped": 46,
"failed": 0
},
"hits": {
"total": 18588,
"max_score": null,
"hits": []
},
"aggregations": {
"b8a65013-8337-436c-b9ca-e50c817a2c3b": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [
{
"key": "<ip_address_1>",
"doc_count": 4724
},
{
"key": "<ip_address_2>",
"doc_count": 4669
},
{
"key": "<ip_address_3>",
"doc_count": 4602
},
{
"key": "<ip_address_4>",
"doc_count": 4593
}
]
}
}
}