ELK Watcher Alarms

Hey, I am trying to create alarms in elk watcher with the following parameters like 5 datapoints in 20 min for value > 10 but i cannot see anything apart from average median, etc. How can we achieve this type pf alarm config in elk watcher.

You will need to create an "advanced watch" via Kibana , or use the PUT watch API.

If your 5 data points can be achieved with a single _search query and/or aggregation, you should look at the search input with a compare condition. There is a full example here

If you need to pull your 5 data points from multiple queries or aggregations you may need to use the chained input and script condition to handle the complexity.

Thanks @jakelandis i will go through the links and try them out and will let you know in case of any issues.

I have formulated this query for an index, now i can get the count using ctx.payload.hits right?

{
"query": {
            "bool": {
              "filter": [
                {
                  "range": {
                    "@timestamp": {
                      "gte": "now-3h"
                    }
                  }
                },
                {
                  "match": {
                    "metricname": "Api_success_latency" 
                  }
                },
                {
                  "range": {
                    "metricvalue": {
                      "gte": 17
                    }
                  }
                }
              ]
            }
          }
}

now i can get the count using ctx.payload.hits right?

Correct, if all you need is the hits, then you can put that in a compare condition .

In this context what will be the difference between hits and count?

This is an alarm i created but it is showing condition failed skipping runtime -
{
"trigger": {
"schedule": {
"interval": "1m"
}
},
"input": {
"search": {
"request": {
"search_type": "query_then_fetch",
"indices": [
"prod-queueworker-service-*"
],
"types": ,
"body": {
"size": 0,
"query": {
"bool": {
"filter": {
"range": {
"@timestamp": {
"gte": "{{ctx.trigger.scheduled_time}}||-5m",
"lte": "{{ctx.trigger.scheduled_time}}",
"format": "strict_date_optional_time||epoch_millis"
}
}
}
}
},
"aggs": {
"metricAgg": {
"avg": {
"field": "metric_lqs-available-messages_QueueName_vae-plan-dead-letter-queue"
}
}
}
}
}
}
},
"condition": {
"always": {}
},
"actions": {
"pagerduty": {
"throttle_period_in_millis": 1800000,
"condition": {
"script": {
"source": "if (ctx.payload.aggregations.metricAgg.value > params.threshold) { return true; } return false;",
"lang": "painless",
"params": {
"threshold": 1
}
}
},
"webhook": {
"scheme": "https",
"host": "events.pagerduty.com",
"port": 443,
"method": "post",
"path": "/generic/2010-04-15/create_event.json",
"params": {},
"headers": {
"Content-type": "application/json"
},
"body": "{"service_key": "abcd","incident_key": "vaedlq","event_type": "trigger","description": "Breaching threshold for LAZADA for VAE general DLQ"}"
}
},
"pagerduty-resolve": {
"condition": {
"script": {
"source": "if (ctx.payload.aggregations.metricAgg.value < params.threshold) { return true; } return false;",
"lang": "painless",
"params": {
"threshold": 1
}
}
},
"webhook": {
"scheme": "https",
"host": "events.pagerduty.com",
"port": 443,
"method": "post",
"path": "/generic/2010-04-15/create_event.json",
"params": {},
"headers": {
"Content-type": "application/json"
},
"body": "{"service_key": "abcd","incident_key": "vaedlq","event_type": "resolve","description": "Issue resolved"}"
}
}
}
}

output - ---------------------------------------------------------------------

{
  "watch_id": "lqs-available-messages_QueueName_taxy-general-dead-letter-queue",
  "node": "Ud7KVIhsTfe44UC0q4540w",
  "state": "executed",
  "user": "elastic",
  "status": {
    "state": {
      "active": true,
      "timestamp": "2020-01-13T10:38:46.225Z"
    },
    "last_checked": "2020-01-14T09:43:36.205Z",
    "last_met_condition": "2020-01-14T09:43:36.205Z",
    "actions": {
      "pagerduty-resolve": {
        "ack": {
          "timestamp": "2020-01-13T10:38:46.225Z",
          "state": "awaits_successful_execution"
        }
      },
      "pagerduty": {
        "ack": {
          "timestamp": "2020-01-13T10:39:36.137Z",
          "state": "ackable"
        },
        "last_execution": {
          "timestamp": "2020-01-13T10:39:36.137Z",
          "successful": true
        },
        "last_successful_execution": {
          "timestamp": "2020-01-13T10:39:36.137Z",
          "successful": true
        },
        "last_throttle": {
          "timestamp": "2020-01-13T11:08:36.303Z",
          "reason": "throttling interval is set to [30m] but time elapsed since last execution is [29m]"
        }
      }
    },
    "execution_state": "executed",
    "version": -1
  },
  "trigger_event": {
    "type": "schedule",
    "triggered_time": "2020-01-14T09:43:36.205Z",
    "schedule": {
      "scheduled_time": "2020-01-14T09:43:35.980Z"
    }
  },
  "input": {
    "search": {
      "request": {
        "search_type": "query_then_fetch",
        "indices": [
          "prod-queueworker-service-*"
        ],
        "types": [],
        "body": {
          "size": 0,
          "query": {
            "bool": {
              "filter": {
                "range": {
                  "@timestamp": {
                    "gte": "{{ctx.trigger.scheduled_time}}||-5m",
                    "lte": "{{ctx.trigger.scheduled_time}}",
                    "format": "strict_date_optional_time||epoch_millis"
                  }
                }
              }
            }
          },
          "aggs": {
            "metricAgg": {
              "avg": {
                "field": "metric_lqs-available-messages_QueueName_taxy-general-dead-letter-queue"
              }
            }
          }
        }
      }
    }
  },
  "condition": {
    "always": {}
  },
  "metadata": {
    "name": "lqs-available-messages_QueueName_taxy-general-dead-letter-queue",
    "xpack": {
      "type": "json"
    }
  },
  "result": {
    "execution_time": "2020-01-14T09:43:36.205Z",
    "execution_duration": 15,
    "input": {
      "type": "search",
      "status": "success",
      "payload": {
        "_shards": {
          "total": 10,
          "failed": 0,
          "successful": 10,
          "skipped": 0
        },
        "hits": {
          "hits": [],
          "total": 225188,
          "max_score": 0
        },
        "took": 11,
        "timed_out": false,
        "aggregations": {
          "metricAgg": {
            "value": null
          }
        }
      },
      "search": {
        "request": {
          "search_type": "query_then_fetch",
          "indices": [
            "prod-queueworker-service-*"
          ],
          "types": [],
          "body": {
            "size": 0,
            "query": {
              "bool": {
                "filter": {
                  "range": {
                    "@timestamp": {
                      "gte": "2020-01-14T09:43:35.980Z||-5m",
                      "lte": "2020-01-14T09:43:35.980Z",
                      "format": "strict_date_optional_time||epoch_millis"
                    }
                  }
                }
              }
            },
            "aggs": {
              "metricAgg": {
                "avg": {
                  "field": "metric_lqs-available-messages_QueueName_taxy-general-dead-letter-queue"
                }
              }
            }
          }
        }
      }
    },
    "condition": {
      "type": "always",
      "status": "success",
      "met": true
    },
    "actions": [
      {
        "id": "pagerduty-resolve",
        "type": "webhook",
        "status": "condition_failed",
        "reason": "condition failed. skipping: runtime error"
      },
      {
        "id": "pagerduty",
        "type": "webhook",
        "status": "condition_failed",
        "reason": "condition failed. skipping: runtime error"
      }
    ]
  },
  "messages": []
}