ELK Watcher Input condition question

Currently, i am trying to write a condition where in if that metricname (condition should be must)in the logs is not encountered in last 1m then perform an action. But it seems to fetching metricnames based on words and not exact match. please ignore the body as it has some confidential data which i cannot add.

       {
          "trigger": {
            "schedule": {
              "interval": "1m"
            }
          },
          "input": {
            "search": {
              "request": {
                "search_type": "query_then_fetch",
                "indices": [
                  "prod-lqs-*"
                ],
                "types": [],
                "body": {
                  "query": {
                    "bool": {
                      "must": [
                        {
                          "query_string": {
                            "query": "lqs-available-messages_Namespace_aliyun-lazada-prod_QueueName_dispatcher-general-dead-letter-queue",
                            "default_field": "metricname"
                          }
                        },
                        {
                          "range": {
                            "@timestamp": {
                              "gte": "{{ctx.trigger.scheduled_time}}||-1m",
                              "lte": "{{ctx.trigger.scheduled_time}}",
                              "format": "strict_date_optional_time||epoch_millis"
                            }
                          }
                        }
                      ]
                    }
                  },
                  "_source": [
                    "message"
                  ],
                  "sort": [
                    {
                      "@timestamp": {
                        "order": "desc"
                      }
                    }
                  ]
                }
              }
            }
          },
          "condition": {
            "always": {}
          },
          "actions": {
            "pagerduty-resolve": {
              "condition": {
                "script": {
                  "source": "if (ctx.payload.hits.total < params.threshold) { return true; } return false;",
                  "lang": "painless",
                  "params": {
                    "threshold": 5
                  }
                }
              },
              "webhook": {
                "scheme": "https",
                "host": "events.pagerduty.com",
                "port": 443,
                "method": "post",
                "path": "/generic/2010-04-15/create_event.json",
                "params": {},
                "headers": {
                  "Content-type": "application/json"
                },
                "body": "{\"service_key\": \"abcd\",\"incident_key\": \"abcd\",\"event_type\": \"resolve\",\"description\": \"Issue resolved for abcd through API\"}"
              }
            }
          }
        }```
------------------------------------------------Result----------------------------------------------------
And following is the search output because of which it is not running as per my expectations - 
   ``` "result": {
        "execution_time": "2020-05-06T06:24:10.207Z",
        "execution_duration": 110,
        "input": {
          "type": "search",
          "status": "success",
          "payload": {
            "_shards": {
              "total": 110,
              "failed": 0,
              "successful": 110,
              "skipped": 0
            },
            "hits": {
              "hits": [
                {
                  "_index": "prod-lqs-service-2020.05.06",
                  "_type": "doc",
                  "_source": {
                    "message": "2020-05-06 06:24:08.694 [system-pool-3-thread-879] metric lqs-in-transit-messages_Namespace_aliyun-prod_QueueName_system-periodic-queue int32 1 count"
                  },
                  "_id": "-82n6HEBFyKZdczg530d",
                  "sort": [
                    1588746248997
                  ],
                  "_score": null
                },
                {
                  "_index": "prod-lqs-service-2020.05.06",
                  "_type": "doc",
                  "_source": {
                    "message": "2020-05-06 06:24:08.381 [system-pool-3-thread-876] metric lqs-in-transit-messages_Namespace_aliyun-prod_QueueName_system-periodic-queue int32 1 count"
                  },
                  "_id": "bc2n6HEBFyKZdczg534g",
                  "sort": [
                    1588746248995
                  ],
                  "_score": null
                },
                {
                  "_index": "prod-lqs-service-2020.05.06",
                  "_type": "doc",
                  "_source": {
                    "message": "2020-05-06 06:24:08.588 [system-pool-3-thread-878] metric lqs-available-messages_Namespace_aliyun-lazada-prod_QueueName_aliyun-lazada-prod int32 101 count"
                  },
                  "_id": "fM2n6HEBFyKZdczg534g",
                  "sort": [
                    1588746248995
                  ],
                  "_score": null
                },
                {
                  "_index": "prod-lqs-service-2020.05.06",
                  "_type": "doc",
                  "_source": {
                    "message": "2020-05-06 06:24:08.379 [system-pool-3-thread-876] metric lqs-available-messages_Namespace_aliyun-lazada-prod_QueueName_taxy-task-periodic-dead-letter-queue int32 3 count"
                  },
                  "_id": "3M2n6HEBFyKZdczg534m",
                  "sort": [
                    1588746248995
                  ],
                  "_score": null
                },
                {
                  "_index": "prod-lqs-service-2020.05.06",
                  "_type": "doc",
                  "_source": {
                    "message": "2020-05-06 06:24:08.500 [system-pool-3-thread-877] metric lqs-available-messages_Namespace_aliyun-lazada-prod_QueueName_taxy-user-periodic int32 9 count"
                  },
                  "_id": "5c2n6HEBFyKZdczg534m",
                  "sort": [
                    1588746248995
                  ],
                  "_score": null
                },
                {
                  "_index": "prod-lqs-service-2020.05.06",
                  "_type": "doc",
                  "_source": {
                    "message": "2020-05-06 06:24:08.588 [system-pool-3-thread-878] metric lqs-available-messages_Namespace_aliyun-lazada-prod_QueueName_taxy-task-periodic-dead-letter-queue int32 3 count"
                  },
                  "_id": "8c2n6HEBFyKZdczg530d",
                  "sort": [
                    1588746248995
                  ],
                  "_score": null
                },
                {
                  "_index": "prod-lqs-service-2020.05.06",
                  "_type": "doc",
                  "_source": {
                    "message": "2020-05-06 06:24:08.169 [system-pool-3-thread-874] metric lqs-available-messages_Namespace_aliyun-lazada-prod_QueueName_taxy-user-periodic int32 9 count"
                  },
                  "_id": "ys2n6HEBFyKZdczg534m",
                  "sort": [
                    1588746248994
                  ],
                  "_score": null
                },
                {
                  "_index": "prod-lqs-service-2020.05.06",
                  "_type": "doc",
                  "_source": {
                    "message": "2020-05-06 06:24:08.169 [system-pool-3-thread-874] metric lqs-available-messages_Namespace_aliyun-lazada-prod_QueueName_taxy-user-periodic-dead-letter-queue int32 5 count"
                  },
                  "_id": "n0en6HEBlSmJMJWq57Qn",
                  "sort": [
                    1588746248994
                  ],
                  "_score": null
                },
                {
                  "_index": "prod-lqs-service-2020.05.06",
                  "_type": "doc",
                  "_source": {
                    "message": "2020-05-06 06:24:08.274 [system-pool-3-thread-875] metric lqs-available-messages_Namespace_aliyun-lazada-prod_QueueName_taxy-user-periodic-dead-letter-queue int32 5 count"
                  },
                  "_id": "1M2n6HEBFyKZdczg530d",
                  "sort": [
                    1588746248994
                  ],
                  "_score": null
                },
                {
                  "_index": "prod-lqs-service-2020.05.06",
                  "_type": "doc",
                  "_source": {
                    "message": "2020-05-06 06:24:08.065 [system-pool-3-thread-873] metric lqs-available-messages_Namespace_aliyun-lazada-prod_QueueName_taxy-task-periodic-dead-letter-queue int32 3 count"
                  },
                  "_id": "k0en6HEBlSmJMJWq57Qn",
                  "sort": [
                    1588746248993
                  ],
                  "_score": null
                }
              ],
              "total": 7501,
              "max_score": null

There are 2 suggestions I would like to share:

  • Elasticsearch supports the Pager duty action, there is no need to use webhook
  • What is the mapping of the field metricname in all the indices prod-lqs-*? Maybe you should search on metricname.keyword (if exists)

solution 2 worked for me, Thanks a lot

1 Like

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.