Webhook watcher action to elastic no_http_response_exception

[nitzan.karni]

Keep on getting no_http_response_exception on my watcher's webhook action.
ctx.payload.body is the body of an update by query request. I create it in the transform section.

the watcher:

{
.
.
.
"actions": {
  "update_alerts": {
     "webhook": {
        "method": "POST",
        "host": "<hostname>",
        "port": "9200",
        "path": "<index-pattern>*/_update_by_query",
        "body": "{{#toJson}}ctx.payload.body{{/toJson}}",
        "auth": {
           "basic": {
             "username": "<username>",
             "password": "<password>"
           }
        }
     }
  }
}
}

the response I get for this specific action:

{
  "id": "update_alerts",
  "type": "webhook",
  "status": "failure",
  "error": {
    "root_cause": [ 
       {
         "type": "no_http_response_exception",
         "reason": "<hostname> failed to respond"
       }
     ],
     "type": "no_http_response_exception",
     "reason": "<hostname> failed to respond"
  }
}

[spinscale (Alexander Reelsen)]

hey,

please share the full watch as well as the output of the Execute Watch API, as this eases debugging and allows for easier understanding. Please take care of hiding any possible credentials in those responses. Thanks!

--Alex

[nitzan.karni]

watcher:

PUT _watcher/watch/_execute

{ "watch": {

 

    "trigger": {

      "schedule": {

        "interval": "30m"

      }

    },

    "input": {

      "search": {

        "request": {

          "indices": [

            "shob-alerts*"

          ],

          "body": {

            "query": {

              "bool": {

                "filter": {

                  "range": {

                    "time": {

                      "gte": "now-1w"

                    }

                  }

                }

              }

            },

            "aggs": {

              "monitors": {

                "terms": {

                  "field": "monitor.keyword",

                  "size": 100

                },

                "aggs": {

                  "time": {

                    "max": {

                      "field": "time"

                    }

                  },

                  "interval": {

                    "terms": {

                      "field": "interval.keyword",

                      "size": 1

                    }

                  }

                }

              }

            }

          }

        }

      }

    },

    "transform": {

      "script": {

        "source":

        """

        Map m = [:];

        List l = new ArrayList();

        l = ctx.payload.aggregations.monitors.buckets.stream().filter( watcher ->

          watcher.time.value <

            (ctx.trigger.scheduled_time.toInstant().toEpochMilli() -

            (Integer.parseInt(watcher.interval.buckets.0.key.substring(0, watcher.interval.buckets.0.key.length()-1)) * 60 * 1000)))

            .map( w -> { return [ w.key ]}).collect(Collectors.toList());

        m = ['script': ['source': "ctx._source.severity = 'clearVeryMuch'", 'lang': "painless"], 'query': ['bool': ['filter': [['terms': [ 'monitor.keyword' : l[0]]], ['range': [ 'time': ['gte': "now-10h"]]]]]]];

        return ['body': l, 'm': m ];

        """,

        "lang": "painless"

      }

    },

    "condition": {

      "script": {

        "source":

        """

        ctx.payload.aggregations.monitors.buckets.stream().filter( watcher ->

          watcher.time.value >

            (ctx.trigger.scheduled_time.toInstant().toEpochMilli() -

            (Integer.parseInt(watcher.interval.buckets.0.key.substring(0, watcher.interval.buckets.0.key.length()-1)) * 60 * 1000))   

        ).count() > 0

        """,

        "lang": "painless"

      }

    },

    "actions": {

       "clear_alerts": {

         "webhook": {

           "method": "POST",

           "host": "prodlnxelkm03.idb.local",

           "port": 9200,

           "auth": {

            "basic":{

               "username": "xxxxxxx",

               "password": "xxxxxxx"

            }

           },

           "path": "shob-alerts*/_update_by_query",

           "body": "{{#toJson}}ctx.payload.m{{/toJson}}"

         }

       },

       "my-logging-action": {

         "logging": {

           "text": "{{#toJson}}ctx.payload.m{{/toJson}}"

         }

       }

    }

  }

}

[nitzan.karni]

Response:

"result": {

"execution_time" : "2019-06-18T12:23:13.872Z",

      "execution_duration" : 14,

      "input" : {

        "type" : "search",

        "status" : "success",

        "payload" : {

          "_shards" : {

            "total" : 9,

            "failed" : 0,

            "successful" : 9,

            "skipped" : 0

          },

          "hits" : {

            "hits": {…}

            "total" : 1167,

            "max_score" : 0.0

          },

          "took" : 8,

          "timed_out" : false,

          "aggregations" : {

            "monitors" : {

              "doc_count_error_upper_bound" : 0,

              "sum_other_doc_count" : 0,

              "buckets" : [

                {

                  "doc_count" : 545,

                  "interval" : {

                    "doc_count_error_upper_bound" : 0,

                    "sum_other_doc_count" : 0,

                    "buckets" : [

                      {

                        "doc_count" : 530,

                        "key" : "8m"

                      }

                    ]

                  },

                  "time" : {

                    "value_as_string" : "2019-06-16T08:08:05.215Z",

                    "value" : 1.560672485215E12

                  },

                  "key" : "host1.memory-90.0%-8m-8"

                },

                {

                  "doc_count" : 341,

                  "interval" : {

                    "doc_count_error_upper_bound" : 0,

                    "sum_other_doc_count" : 0,

                    "buckets" : [

                      {

                        "doc_count" : 334,

                        "key" : "15m"

                      }

                    ]

                  },

                  "time" : {

                    "value_as_string" : "2019-06-16T20:45:05.215Z",

                    "value" : 1.560717905215E12

                  },

                  "key" : "host1.filesystem-90.0%-15m-1"

                },

                {

                  "doc_count" : 53,

                  "interval" : {

                    "doc_count_error_upper_bound" : 0,

                    "sum_other_doc_count" : 0,

                    "buckets" : [

                      {

                        "doc_count" : 53,

                        "key" : "8m"

                      }

                    ]

                  },

                  "time" : {

                    "value_as_string" : "2019-06-18T12:17:31.012Z",

                    "value" : 1.560860251012E12

                  },

                  "key" : "host2.memory-90.0%-8m-8"

                },

                {

                  "doc_count" : 25,

                  "interval" : {

                    "doc_count_error_upper_bound" : 0,

                    "sum_other_doc_count" : 0,

                    "buckets" : [

                      {

                        "doc_count" : 25,

                        "key" : "8m"

                      }

                    ]

                  },

                  "time" : {

                    "value_as_string" : "2019-06-18T00:53:06.196Z",

                    "value" : 1.560819186196E12

                  },

                  "key" : "host3.cpu-90.0%-8m-5"

                }

              ]

            }

          }

        },

        "search" : {

          "request" : {

            "search_type" : "query_then_fetch",

            "indices" : [

              "shob-alerts*"

            ],

            "rest_total_hits_as_int" : true,

            "body" : {

              "query" : {

                "bool" : {

                  "filter" : {

                    "range" : {

                      "time" : {

                        "gte" : "now-1w"

                      }

                    }

                  }

                }

              },

              "aggs" : {

                "monitors" : {

                  "terms" : {

                    "field" : "monitor.keyword",

                    "size" : 100

                  },

                  "aggs" : {

                    "time" : {

                      "max" : {

                        "field" : "time"

                      }

                    },

                    "interval" : {

                      "terms" : {

                        "field" : "interval.keyword",

                        "size" : 1

                      }

                    }

                  }

                }

              }

            }

          }

        }

      },

      "condition" : {

        "type" : "script",

        "status" : "success",

        "met" : true

      },

      "transform" : {

        "type" : "script",

        "status" : "success",

        "payload" : {

          "body" : [

            [

              "devlnxsystem01.memory-90.0%-8m-8"

            ],

            [

              "devlnxsystem01.filesystem-90.0%-15m-1"

            ],

            [

              "DC1HOUNIMSUMP01.cpu-90.0%-8m-5"

            ]

          ],

          "m" : {

            "query" : {

              "bool" : {

                "filter" : [

                  {

                    "terms" : {

                      "monitor.keyword" : [

                        "devlnxsystem01.memory-90.0%-8m-8"

                      ]

                    }

                  },

                  {

                    "range" : {

                      "time" : {

                        "gte" : "now-10h"

                      }

                    }

                  }

                ]

              }

            },

            "script" : {

              "source" : "ctx._source.severity = 'clearVeryMuch'",

              "lang" : "painless"

            }

          }

        }

      },

      "actions" : [

        {

          "id" : "my-logging-action",

          "type" : "logging",

          "status" : "success",

          "logging" : {

            "logged_text" : """{"query":{"bool":{"filter":[{"terms":{"monitor.keyword":["host1.memory-90.0%-8m-8"]}},{"range":{"time":{"gte":"now-10h"}}}]}},"script":{"source":"ctx._source.severity = 'clearVeryMuch'","lang":"painless"}}"""

          }

        },

        {

          "id" : "clear_alerts",

          "type" : "webhook",

          "status" : "failure",

          "error" : {

            "root_cause" : [

              {

                "type" : "no_http_response_exception",

                "reason" : "host:9200 failed to respond"

              }

            ],

            "type" : "no_http_response_exception",

            "reason" : "host:9200 failed to respond"

          }

        }

      ]

    },

    "messages" : [ ]

  }

}