Advance Watcher Script

Hi All,

I am implementing the uptime alerts for the heartbeat index and my requirement is as below.

I want to send an alert to the corresponding monitor mail id from the same index with trigger time =2m when monitor.status"="down", "filter range": last 15 m, agg_by_monitor,Size=10.

Action >> mail need to be sent to corresponding monitor mail id: {{fields.INFY_EA_CostCenterMailid}} and each document is having own mail id in the heartbeat index.

body >>Status is down for {{monitor.name}} , {{url.domain}} for the location {{observer.geo.name.}} >>this message is to be sent to a mail id corresponding to each monitors.

Please suggest the way to get the "email to" and "body" parts in the results and also suggest how to make iterations over results because I need to send an alert to the monitor mail id of the corresponding monitor if the monitor status is down.

Advance watcher script :

{
  "trigger": {
    "schedule": {
      "interval": "2m"
    }
  },
  "input": {
    "search": {
      "request": {
        "search_type": "query_then_fetch",
        "indices": [
          "heartbeat*"
        ],
        "rest_total_hits_as_int": true,
        "body": {
          "size": 0,
          "query": {
            "bool": {
              "must": [
                {
                  "term": {
                    "monitor.status": {
                      "value": "down"
                    }
                  }
                }
              ],
              "filter": [
                {
                  "range": {
                    "@timestamp": {
                      "from": "now-15m"
                    }
                  }
                }
              ]
            }
          },
          "aggregations": {
            "by_monitors_CenterMailid": {
              "terms": {
                "field": "monitor.name",
                "size": 10,
                "min_doc_count": 1
              }
            }
          },
          "aggs": {
            "top_hits": {
              "top_hits": {
                "size": 10,
                "_source": {
                  "includes": [
                    "observer.geo.name",
                    "fields.INFY_EA_CostCenterMailid",
                    "url.full"
                  ]
                }
              }
            }
          }
        }
      }
    }
  },
  "condition": {
    "compare": {
      "ctx.payload.hits.total": {
        "gt": 1
      }
    }
  },
  "actions": {
    "email_action": {
      "email": {
        "profile": "outlook",
        "from": "Heartbeat@info.com",
        "to": [
          "{{ctx.payload.aggregations.top_hits.hits.hits._source.fields.INFY_EA_CostCenterMailid}}"
        ],
        "subject": "Encountered {{ctx.payload.hits.total}} errors",
        "body": {
          "text": "{{ctx.payload.aggregations.top_hits.hits.hits._source.url.full}} is down for the location {{ctx.payload.aggregations.top_hits.hits.hits.observer.geo.name}} "
        }
      }
    }
  }
}

Simulation results(parts of the results) :

{
  "watch_id": "_inlined_",
  "node": "TD28dC9ZTQqdRnzbx0uvsw",
  "state": "executed",
  "user": "elastic",
  "status": {
    "state": {
      "active": true,
      "timestamp": "2021-09-22T07:52:38.646Z"
    },
    "last_checked": "2021-09-22T07:52:38.646Z",
    "last_met_condition": "2021-09-22T07:52:38.646Z",
    "actions": {
      "email_action": {
        "ack": {
          "timestamp": "2021-09-22T07:52:38.646Z",
          "state": "ackable"
        },
        "last_execution": {
          "timestamp": "2021-09-22T07:52:38.646Z",
          "successful": true
        },
        "last_successful_execution": {
          "timestamp": "2021-09-22T07:52:38.646Z",
          "successful": true
        }
      }
    },
    "execution_state": "executed",
    "version": -1
  },
  "trigger_event": {
    "type": "manual",
    "triggered_time": "2021-09-22T07:52:38.646Z",
    "manual": {
      "schedule": {
        "scheduled_time": "2021-09-22T07:52:38.646Z"
      }
    }
  },
  "input": {
    "search": {
      "request": {
        "search_type": "query_then_fetch",
        "indices": [
          "heartbeat*"
        ],
        "rest_total_hits_as_int": true,
        "body": {
          "size": 0,
          "query": {
            "bool": {
              "must": [
                {
                  "term": {
                    "monitor.status": {
                      "value": "down"
                    }
                  }
                }
              ],
              "filter": [
                {
                  "range": {
                    "@timestamp": {
                      "from": "now-15m"
                    }
                  }
                }
              ]
            }
          },
          "aggregations": {
            "by_monitors_CenterMailid": {
              "terms": {
                "field": "monitor.name",
                "size": 10,
                "min_doc_count": 1
              }
            }
          },
          "aggs": {
            "top_hits": {
              "top_hits": {
                "size": 10,
                "_source": {
                  "includes": [
                    "observer.geo.name",
                    "fields.INFY_EA_CostCenterMailid",
                    "url.full"
                  ]
                }
              }
            }
          }
        }
      }
    }
  },
  "condition": {
    "compare": {
      "ctx.payload.hits.total": {
        "gt": 1
      }
    }
  },
  "metadata": {
    "name": "uptime watcher",
    "xpack": {
      "type": "json"
    }
  },
  "result": {
    "execution_time": "2021-09-22T07:52:38.646Z",
    "execution_duration": 9,
    "input": {
      "type": "search",
      "status": "success",
      "payload": {
        "_shards": {
          "total": 34,
          "failed": 0,
          "successful": 34,
          "skipped": 0
        },
        "hits": {
          "hits": [],
          "total": 557,
          "max_score": null
        },
        "took": 8,
        "timed_out": false,
        "aggregations": {
          "top_hits": {
            "hits": {
              "hits": [
                {
                  "_ignored": [
                    "http.response.body.content"
                  ],
                  "_index": "heartbeat-7.9.1-2021.09.22",
                  "_type": "_doc",
                  "_source": {
                    "observer": {
                      "geo": {
                        "name": "OnPrem-Chennai"
                      }
                    },
                    "fields": {
                      "INFY_EA_CostCenterMailid": "xyz@infosys.com"
                    },
                    "url": {
                      "full": "xxxxxxxxxxxxxxxxxxxxxx"
                    }
                  },
                  "_id": "uWBzDHwBbh4zm7j4jecp",
                  "_score": 2.0666566
                },
                {
                  "_index": "heartbeat-7.9.1-2021.09.22",
                  "_type": "_doc",
                  "_source": {
                    "observer": {
                      "geo": {
                        "name": "Polycloud-Dublin"
                      }
                    },
                    "fields": {
                      "INFY_EA_CostCenterMailid": "xyz@infosys.com"
                    },
                    "url": {
                      "full": "xxxxxxxxxxxxxxxxxxxxxx"
                    }
                  },
                  "_id": "XGBzDHwBbh4zm7j4kOky",
                  "_score": 2.0666566
                },
                {
                  "_index": "heartbeat-7.9.1-2021.09.22",
                  "_type": "_doc",
                  "_source": {
                    "observer": {
                      "geo": {
                        "name": "Polycloud-Mumbai"
                      }
                    },
                    "fields": {
                      "INFY_EA_CostCenterMailid": "xysss@infosys.com"
                    },
                    "url": {
                      "full": "icmp://cowin.com"
                    }
                  },
                  "_id": "lGBzDHwBbh4zm7j4le2G",
                  "_score": 2.0666566
                },
                {
                  "_index": "heartbeat-7.9.1-2021.09.22",
                  "_type": "_doc",
                  "_source": {
                    "observer": {
                      "geo": {
                        "name": "Polycloud-Mumbai"
                      }
                    },
                    "fields": {
                      "INFY_EA_CostCenterMailid": "mns@infosys.com"
                    },
                    "url": {
                      "full": "icmp://cowin.com"
                    }
                  },
                  "_id": "eGF0DHwBbh4zm7j4CkPB",
                  "_score": 2.0666566
                },
                {
                  "_index": "heartbeat-7.9.1-2021.09.22",
                  "_type": "_doc",
                  "_source": {
                    "observer": {
                      "geo": {
                        "name": "Polycloud-Mumbai"
                      }
                    },
                    "fields": {
                      "INFY_EA_CostCenterMailid": "indiamail@infosys.com"
                    },
                    "url": {
                      "full": "icmp://cowin.com"
                    }
                  },
                  "_id": "TmFzDHwBbh4zm7j45CZz",
                  "_score": 2.0666566
                }
              ],
              "total": 557,
              "max_score": 2.0666566
            }
          }
        }
      },
      "search": {
        "request": {
          "search_type": "query_then_fetch",
          "indices": [
            "heartbeat*"
          ],
          "rest_total_hits_as_int": true,
          "body": {
            "size": 0,
            "query": {
              "bool": {
                "must": [
                  {
                    "term": {
                      "monitor.status": {
                        "value": "down"
                      }
                    }
                  }
                ],
                "filter": [
                  {
                    "range": {
                      "@timestamp": {
                        "from": "now-15m"
                      }
                    }
                  }
                ]
              }
            },
            "aggregations": {
              "by_monitors_CenterMailid": {
                "terms": {
                  "field": "monitor.name",
                  "size": 10,
                  "min_doc_count": 1
                }
              }
            },
            "aggs": {
              "top_hits": {
                "top_hits": {
                  "size": 10,
                  "_source": {
                    "includes": [
                      "observer.geo.name",
                      "fields.INFY_EA_CostCenterMailid",
                      "url.full"
                    ]
                  }
                }
              }
            }
          }
        }
      }
    },
    "condition": {
      "type": "compare",
      "status": "success",
      "met": true,
      "compare": {
        "resolved_values": {
          "ctx.payload.hits.total": 557
        }
      }
    },
    "actions": [
      {
        "id": "email_action",
        "type": "email",
        "status": "simulated",
        "email": {
          "message": {
            "id": "email_action__inlined__eff44d77-13ca-4d4f-ba9a-f69aa374cba2-2021-09-22T07:52:38.646340031Z_37",
            "from": "Heartbeat@info.com",
            "sent_date": "2021-09-22T07:52:38.655952136Z",
            "to": [],
            "subject": "Encountered 557 errors",
            "body": {
              "text": " is down for the location  "
            }
          }
        }
      }
    ]
  },
  "messages": []
}