Watcher with Heartbeat and monitor status up

Elastic Stack Elasticsearch
1

Hi Team

We've configured Heartbeat to monitor services and got setup Watcher to monitor heartbeat monitor.status to send alert on a Slack channel. Following is the configuration:

{
  "trigger": {
    "schedule": {
      "interval": "1m"
    }
  },
  "input": {
    "search": {
      "request": {
        "search_type": "query_then_fetch",
        "indices": [
          "heartbeat-*"
        ],
        "rest_total_hits_as_int": true,
        "body": {
          "size": 0,
          "query": {
            "bool": {
              "must": [
                {
                  "term": {
                    "monitor.status": {
                      "value": "down"
                    }
                  }
                }
              ],
              "filter": [
                {
                  "range": {
                    "@timestamp": {
                      "from": "now-1m"
                    }
                  }
                }
              ]
            }
          },
          "aggregations": {
            "by_monitors": {
              "terms": {
                "field": "monitor.name",
                "size": 10,
                "min_doc_count": 1
              }
            }
          }
        }
      }
    }
  },
  "condition": {
    "compare": {
      "ctx.payload.hits.total": {
        "gt": 1
      }
    }
  },
  "actions": {
    "notify-slack": {
      "throttle_period_in_millis": 300000,
      "slack": {
        "message": {
          "from": "Heartbeat",
          "to": [
            "#ops"
          ],
          "text": "Some systems are not replying.",
          "dynamic_attachments": {
            "list_path": "ctx.payload.aggregations.by_monitors.buckets",
            "attachment_template": {
              "color": "warning",
              "title": "{{key}}",
              "text": "Amount of events: {{doc_count}}"
            }
          }
        }
      }
    }
  }
}

This works fine and sends required alert when service is down. What we need is, that the watcher should send an alert when service is up again. How would that be done ?
If we check monitor.status:up, then it will keep on sending alert. So how to manage that the watcher sends only one alert after service is up again. Any examples or suggestions would be much appreciated.

2

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.