AWS ELB Error Rates and Alerts

[alex2020 (Alex)]

Hello,

I have a requirement to define error rates from an AWS ELB. I setup filebeats and used the aws module using the elb fileset and that is working well. I have the data visible in Kabana now.

I then proceed to attempt and define a query that is a count of !(http status code 200) / all. I was not able to do this in Visualizations using a TSVB. Maybe there is a way I did not think of? So I used Timelion. Here is the query:

    .es(index=filebeat-*,
        timefield='@timestamp',
        metric='count:http.response.status_code',
        q='fileset.name: "elb" AND !(http.response.status_code:>=200 AND http.response.status_code:<300)')
    .divide(
    .es(index=filebeat-*,
        timefield='@timestamp',
        metric='count:http.response.status_code',
        q='fileset.name: "elb"')
    )
    .multiply(100)
    .label('Error Rate (%)')

So two questions:

1. Is it possible to alert on this using watcher?
2. If not, is it possible to rewrite this in a format watcher can accept?

[alex2020 (Alex)]

I was able to resolve this myself. Posting watcher json for anyone who cares:

{
   "trigger":{
      "schedule":{
         "interval":"5m"
      }
   },
   "input":{
      "chain":{
         "inputs":[
            {
               "first":{
                  "search":{
                     "extract":[
                        "hits.total"
                     ],
                     "request":{
                        "indices":[
                           "filebeat-*"
                        ],
                        "body":{
                           "query":{
                              "bool":{
                                 "must":[
                                    {
                                       "query_string":{
                                          "query": "fileset.name: \"elb\" +- (http.response.status_code >=200 AND http.response.status_code < 300)"
                                       }
                                    },
                                    {
                                       "range":{
                                          "@timestamp":{
                                             "lte":"now-5m",
                                             "gte":"now-10m"
                                          }
                                       }
                                    }
                                 ]
                              }
                           }
                        }
                     }
                  }
               }
            },
            {
               "second":{
                  "search":{
                     "extract":[
                        "hits.total"
                     ],
                     "request":{
                        "indices":[
                           "filebeat-*"
                        ],
                        "body":{
                           "query":{
                              "bool":{
                                 "must":[
                                    {
                                       "range":{
                                          "@timestamp":{
                                             "lte":"now-5m",
                                             "gte":"now-10m"
                                          }
                                       }
                                    },
                                    {
                                       "query_string":{
                                          "query": "fileset.name: \"elb\""
                                       }
                                    }
                                 ]
                              }
                           }
                        }
                     }
                  }
               }
            }
         ]
      }
   },
   "condition":{
      "script":{
         "source":"return (ctx.payload.first.hits.total / ctx.payload.second.hits.total) > 0.1"
      }
   },
   "actions":{
      "my-logging-action":{
         "logging":{
            "text":"There are {{ ctx.payload.first.hits.total }} / {{ ctx.payload.second.hits.total }} documents in your index. Threshold is 10%."
         }
      }
   }
}

[Rahul_Kumar4 (Rahul Kumar)]

Interesting that you did not have specify the date_format in the timestamp range in your watcher json.

[alex2020 (Alex)]

I cannot figure out how to edit my prior post but the JSON was slightly wrong. Here is the final working version:

    {
      "trigger": {
        "schedule": {
          "interval": "5m"
        }
      },
      "input": {
        "chain": {
          "inputs": [
            {
              "first": {
                "search": {
                  "request": {
                    "search_type": "query_then_fetch",
                    "indices": [
                      "filebeat-*"
                    ],
                    "rest_total_hits_as_int": true,
                    "body": {
                      "query": {
                        "bool": {
                          "must": [
                            {
                              "query_string": {
                                "query": "fileset.name: \"elb\""
                              }
                            },
                            {
                              "range": {
                                "@timestamp": {
                                  "lte": "now-5m",
                                  "gte": "now-10m"
                                }
                              }
                            }
                          ],
                          "must_not": {
                            "range": {
                              "http.response.status_code": {
                                "gte": 200,
                                "lte": 300
                              }
                            }
                          }
                        }
                      }
                    }
                  },
                  "extract": [
                    "hits.total"
                  ]
                }
              }
            },
            {
              "second": {
                "search": {
                  "request": {
                    "search_type": "query_then_fetch",
                    "indices": [
                      "filebeat-*"
                    ],
                    "rest_total_hits_as_int": true,
                    "body": {
                      "query": {
                        "bool": {
                          "must": [
                            {
                              "range": {
                                "@timestamp": {
                                  "lte": "now-5m",
                                  "gte": "now-10m"
                                }
                              }
                            },
                            {
                              "query_string": {
                                "query": "fileset.name: \"elb\""
                              }
                            }
                          ]
                        }
                      }
                    }
                  },
                  "extract": [
                    "hits.total"
                  ]
                }
              }
            }
          ]
        }
      },
      "condition": {
        "script": {
          "source": "return (ctx.payload.first.hits.total / ctx.payload.second.hits.total) > 0.1",
          "lang": "painless"
        }
      },
      "actions": {
        "logging-action": {
          "logging": {
            "level": "info",
            "text": "There are {{ ctx.payload.first.hits.total }} / {{ ctx.payload.second.hits.total }} documents in your index. Threshold is 10%."
          }
        },
        "email-action": {
          "email": {
            "profile": "standard",
            "to": [
              "username@example.org"
            ],
            "subject": "Watcher Notification",
            "body": {
              "text": "{{ctx.payload.hits.total}} error logs found"
            }
          }
        }
      }
    }