Hey,
So I have a monitor in kibana that queries my elasticsearch to find specific errors and to raise an alert if they are found. The issue I am having though is that when the elasticsearch cluster is under high load the query gets a timeout and this timeout sets off the alert for the error. What I am wondering is if there is any way to force it so that the alert is only raised when the occurrence of a timeout is false. The query that I am currently using is this:
{
"size": 500,
"query": {
"bool": {
"must": [
{
"match_all": {
"boost": 1
}
},
{
"match_phrase": {
"message": {
"query": "SQLSTATE[HY000] [2002] Connection refused",
"slop": 0,
"boost": 1
}
}
},
{
"range": {
"@timestamp": {
"from": "now-60s",
"to": "now",
"include_lower": true,
"include_upper": true,
"format": "epoch_millis",
"boost": 1
}
}
}
],
"adjust_pure_negative": true,
"boost": 1
}
},
"version": true,
"_source": {
"includes": [],
"excludes": []
},
"stored_fields": "*",
"docvalue_fields": [
"@timestamp",
"git_date"
],
"script_fields": {},
"sort": [
{
"@timestamp": {
"order": "desc",
"unmapped_type": "boolean"
}
}
],
"aggregations": {
"2": {
"date_histogram": {
"field": "@timestamp",
"time_zone": "Europe/London",
"interval": "1s",
"offset": 0,
"order": {
"_key": "asc"
},
"keyed": false,
"min_doc_count": 1
}
}
},
"highlight": {
"pre_tags": [
"@kibana-highlighted-field@"
],
"post_tags": [
"@/kibana-highlighted-field@"
],
"fragment_size": 2147483647,
"fields": {
"*": {}
}
}
}