How to show hostname in watcher text

sarahpark · April 16, 2020, 3:30am

Hi, I want to show hostname in watcher text.
I made the threshold alert, Kibana UI.
Its purpose is alerting the hostname, that using disk size over 80%.

I want to make the text like "Disk Usage Warning !! IS-SC-001 is used disk over 80%."
I figure out the only one, that {{ctx.metadata.name}} is watcher name, 'Disk Usage Warning'.

I cannot find the parameter for hostname.

This is the part of execute watcher API.

    "input" : {
      "search" : {
        "request" : {
          "search_type" : "query_then_fetch",
          "indices" : [
            "metricbeat-7.3*"
          ],
          "rest_total_hits_as_int" : true,
          "body" : {
            "size" : 0,
            "query" : {
              "bool" : {
                "filter" : {
                  "range" : {
                    "@timestamp" : {
                      "gte" : "{{ctx.trigger.scheduled_time}}||-10m",
                      "lte" : "{{ctx.trigger.scheduled_time}}",
                      "format" : "strict_date_optional_time||epoch_millis"
                    }
                  }
                }
              }
            },
            "aggs" : {
              "bucketAgg" : {
                "terms" : {
                  "field" : "host.name.keyword",
                  "size" : "40",
                  "order" : {
                    "metricAgg" : "asc"
                  }
                },
                "aggs" : {
                  "metricAgg" : {
                    "avg" : {
                      "field" : "system.filesystem.used.pct"
                    }
                  }
                }
              }
            }
          }
        }
      }
    },
    "condition" : {
      "script" : {
        "source" : "ArrayList arr = ctx.payload.aggregations.bucketAgg.buckets; for (int i = 0; i < arr.length; i++) { if (arr[i]['metricAgg'].value >= params.threshold) { return true; } } return false;",
        "lang" : "painless",
        "params" : {
          "threshold" : 0.8
        }
      }
    },
    "metadata" : {
      "name" : "Disk Usage TEST",
      "watcherui" : {
        "trigger_interval_unit" : "m",
        "agg_type" : "avg",
        "time_field" : "@timestamp",
        "trigger_interval_size" : 10,
        "term_size" : "40",
        "time_window_unit" : "m",
        "threshold_comparator" : ">=",
        "term_field" : "host.name.keyword",
        "index" : [
          "metricbeat-7.3*"
        ],
        "time_window_size" : 10,
        "threshold" : 0.8,
        "agg_field" : "system.filesystem.used.pct"
      },
      "xpack" : {
        "type" : "threshold"
      }
    },
    "result" : {
      "execution_time" : "2020-04-16T02:24:03.774Z",
      "execution_duration" : 1575,
      "input" : {
        "type" : "search",
        "status" : "success",
        "payload" : {
          "_shards" : {
            "total" : 1,
            "failed" : 0,
            "successful" : 1,
            "skipped" : 0
          },
          "hits" : {
            "hits" : [ ],
            "total" : 10000,
            "max_score" : null
          },
          "took" : 1573,
          "timed_out" : false,
          "aggregations" : {
            "bucketAgg" : {
              "doc_count_error_upper_bound" : 0,
              "sum_other_doc_count" : 0,
              "buckets" : [
                {
                  "doc_count" : 8320,
                  "metricAgg" : {
                    "value" : 0.08243333175778389
                  },
                  "key" : "IS-SC-052"
                },
                {
                  "doc_count" : 5016,
                  "metricAgg" : {
                    "value" : 0.12124999985098839
                  },
                  "key" : "IS-SC-025"
                },
                {
                  "doc_count" : 8863,
                  "metricAgg" : {
                    "value" : 0.13904142784220833
                  },
                  "key" : "IS-SC-027"
                },
                {
                  "doc_count" : 8661,
                  "metricAgg" : {
                    "value" : 0.14300000667572021
                  },
                  "key" : "IS-SC-030"
                },
                {
                  "doc_count" : 5737,
                  "metricAgg" : {
                    "value" : 0.1590999960899353
                  },
                  "key" : "IS-SC-005"
                },
                {
                  "doc_count" : 9399,
                  "metricAgg" : {
                    "value" : 0.15913333122928938
                  },
                  "key" : "IS-SC-033"
                },
                {
                  "doc_count" : 10821,
                  "metricAgg" : {
                    "value" : 0.1721000038087368
                  },
                  "key" : "IS-SC-007"
                },
                {
                  "doc_count" : 2698,
                  "metricAgg" : {
                    "value" : 0.17494999803602695
                  },
                  "key" : "IS-SC-021"
                },
                {
                  "doc_count" : 2041,
                  "metricAgg" : {
                    "value" : 0.1915999948978424
                  },
                  "key" : "IS-VSC-055"
                },
                {
                  "doc_count" : 5095,
                  "metricAgg" : {
                    "value" : 0.2037999927997589
                  },
                  "key" : "IS-VSC-054"
                },
                {
                  "doc_count" : 4728,
                  "metricAgg" : {
                    "value" : 0.21940000355243683
                  },
                  "key" : "IS-VSC-053"
                },
                {
                  "doc_count" : 3258,
                  "metricAgg" : {
                    "value" : 0.22503500059247017
                  },
                  "key" : "IS-SC-W013"
                },
                {
                  "doc_count" : 5047,
                  "metricAgg" : {
                    "value" : 0.23309999704360962
                  },
                  "key" : "IS-VSC-052"
                },
                {
                  "doc_count" : 2237,
                  "metricAgg" : {
                    "value" : 0.23430000245571136
                  },
                  "key" : "IS-VSC-056"
                },
                {
                  "doc_count" : 10361,
                  "metricAgg" : {
                    "value" : 0.23972285794360296
                  },
                  "key" : "IS-SC-W014"
                },
                {
                  "doc_count" : 10594,
                  "metricAgg" : {
                    "value" : 0.2402785707797323
                  },
                  "key" : "IS-SC-W012"
                },
                {
                  "doc_count" : 5755,
                  "metricAgg" : {
                    "value" : 0.2653000056743622
                  },
                  "key" : "IS-VSC-051"
                },
                {
                  "doc_count" : 2017,
                  "metricAgg" : {
                    "value" : 0.2888999983668327
                  },
                  "key" : "IS-VSC-W023"
                },
                {
                  "doc_count" : 7299,
                  "metricAgg" : {
                    "value" : 0.29521905311516355
                  },
                  "key" : "IS-SC-015"
                },
                {
                  "doc_count" : 9896,
                  "metricAgg" : {
                    "value" : 0.29759999985496205
                  },
                  "key" : "IS-SC-034"
                }, 
                {
                  "doc_count" : 10341,
                  "metricAgg" : {
                    "value" : 0.31350428121430535
                  },
                  "key" : "IS-SC-W011"
                },
                {
                  "doc_count" : 1980,
                  "metricAgg" : {
                    "value" : 0.32471000850200654
                  },
                  "key" : "IS-VSC-W022"
                },
                {
                  "doc_count" : 11081,
                  "metricAgg" : {
                    "value" : 0.3354800004111374
                  },
                  "key" : "IS-SC-016"
                },
                {
                  "doc_count" : 5805,
                  "metricAgg" : {
                    "value" : 0.34869999686876935
                  },
                  "key" : "IS-SC-032"
                },
                {
                  "doc_count" : 4662,
                  "metricAgg" : {
                    "value" : 0.35280001163482666
                  },
                  "key" : "IS-SC-018"
                },
                {
                  "doc_count" : 6577,
                  "metricAgg" : {
                    "value" : 0.36875381374931226
                  },
                  "key" : "IS-SC-037"
                },
                {
                  "doc_count" : 6391,
                  "metricAgg" : {
                    "value" : 0.4001999869942665
                  },
                  "key" : "IS-SC-022"
                },
                {
                  "doc_count" : 6464,
                  "metricAgg" : {
                    "value" : 0.4035000006357829
                  },
                  "key" : "IS-SC-026"
                },
                {
                  "doc_count" : 4733,
                  "metricAgg" : {
                    "value" : 0.45899999141693115
                  },
                  "key" : "IS-VSC-W054"
                },
                {
                  "doc_count" : 6154,
                  "metricAgg" : {
                    "value" : 0.5841390521753402
                  },
                  "key" : "IS_SC_035"
                },
                {
                  "doc_count" : 5463,
                  "metricAgg" : {
                    "value" : 0.6119000017642975
                  },
                  "key" : "IS-VSC-W052"
                },
                {
                  "doc_count" : 5627,
                  "metricAgg" : {
                    "value" : 0.6254000067710876
                  },
                  "key" : "IS-VSC-W051"
                },
                {
                  "doc_count" : 5375,
                  "metricAgg" : {
                    "value" : 0.6842000037431717
                  },
                  "key" : "IS-VSC-W053"
                },
                {
                  "doc_count" : 6008,
                  "metricAgg" : {
                    "value" : 0.7868185843740191
                  },
                  "key" : "IS-SC-003"
                },
                {
                  "doc_count" : 2390,
                  "metricAgg" : {
                    "value" : 0.8041399856408437
                  },
                  "key" : "IS-VSC-050"
                }
              ]
            }
          }
        },
        "search" : {
          "request" : {
            "search_type" : "query_then_fetch",
            "indices" : [
              "metricbeat-7.3*"
            ],
            "rest_total_hits_as_int" : true,
            "body" : {
              "size" : 0,
              "query" : {
                "bool" : {
                  "filter" : {
                    "range" : {
                      "@timestamp" : {
                        "gte" : "2020-04-16T02:24:03.77487Z||-10m",
                        "lte" : "2020-04-16T02:24:03.77487Z",
                        "format" : "strict_date_optional_time||epoch_millis"
                      }
                    }
                  }
                }
              },
              "aggs" : {
                "bucketAgg" : {
                  "terms" : {
                    "field" : "host.name.keyword",
                    "size" : "40",
                    "order" : {
                      "metricAgg" : "asc"
                    }
                  },
                  "aggs" : {
                    "metricAgg" : {
                      "avg" : {
                        "field" : "system.filesystem.used.pct"
                      }
                    }
                  }
                }
              }
            }
          }
        }
      },
      "condition" : {
        "type" : "script",
        "status" : "success",
        "met" : true
      },
      "transform" : {
        "type" : "script",
        "status" : "success",
        "payload" : {
          "results" : [
            {
              "value" : 0.8041399856408437,
              "key" : "IS-VSC-050"
            }
          ]
        }
      },
      "actions" : [
        {
          "id" : "logging_1",
          "type" : "logging",
          "status" : "success",
          "logging" : {
            "logged_text" : "Disk Usage Warning !!  is used disk over 80%. "
          }
        }
      ]
    },
    "messages" : [ ]
  }
}

monfera · April 20, 2020, 3:47pm

Hi Sarah, let me check with the team

monfera · April 20, 2020, 5:54pm

Sarah, please check out this thread, ie. consider creating a log action with {{ctx}} to get it all emitted to the log.

sarahpark · April 20, 2020, 11:57pm

Hi monfera, Thanks for help.
I solved it.
'{{#ctx.payload.results}} {{key}} : {{value}}, {{/ctx.payload.results}} ' is worked.

cjcenizal · April 21, 2020, 12:09am

Hi Sarah, can you share your watch configuration with us? You can do this by searching for the watch in Dev Tools > Console:

GET .watches/_doc/<id>

Look for a script transform that will look something like this:

    "transform" : {
      "script" : {
        "source" : "HashMap result = new HashMap(); result.result = ctx.payload.hits.total; return result;",
        "lang" : "painless",
        "params" : {
          "threshold" : 0
        }
      }
    },

Normally, you could access the documents and fields in your search result like this:

 {{ctx.metadata.name}} !!  {{ctx.payload.hits.hits.0._source. hostname}}  is used disk over 80%

Unfortunately, the above transform will overwrite payload so that it only has a single property (result) and all of the search result data will be lost. I've filed https://github.com/elastic/kibana/issues/64022 to track this problem.

In the meantime, you can recreate the watch in Dev Tools Console by doing the following:

Use the GET .watches/_doc/<id> request to retrieve the watch configuration.
Insert the object defined under the _source key into the request below, but leave out the transform configuration.
Lastly, replace the metadata object with the one below. You don't want to use the metadata originally defined, because that identifies it as a threshold watch. The metadata object below defines it as an advanced watch.

PUT _watcher/watch/g
{
  <Put the trigger, input, condition, and actions in here. Leave out transform.>
  "metadata": {
    "xpack": {
      "type": "json"
    }
  }
}

After you submit this request you should be able to edit the watch in Watcher, and you can change the action to interpolate values from the search results, which will look something like this:

    "actions" : {
      "logging_1" : {
        "logging" : {
          "level" : "info",
          "text" : "{{ctx.metadata.name}} !!  {{ctx.payload.hits.hits.0._source. hostname}}  is used disk over 80%"
        }
      }
    },

Hope this helps!

sarahpark · April 21, 2020, 12:32am

Hi cjcnizal, Thanks for help.

I found the parameter of hostname and percentage, I did so many test at Dev Tool on Kibana UI.
I used command " POST _watcher/watch/(watcher ID)/_execute ", and I checked text results.
It was very helpful.

This is the configuration after solved the problem.

{
  "trigger": {
    "schedule": {
      "interval": "1h"
    }
  },
  "input": {
    "search": {
      "request": {
        "search_type": "query_then_fetch",
        "indices": [
          "metricbeat-7.3*"
        ],
        "rest_total_hits_as_int": true,
        "body": {
          "size": 0,
          "query": {
            "bool": {
              "filter": [
                {
                  "range": {
                    "@timestamp": {
                      "gte": "{{ctx.trigger.scheduled_time}}||-10m",
                      "lte": "{{ctx.trigger.scheduled_time}}",
                      "format": "strict_date_optional_time||epoch_millis"
                    }
                  }
                },
                {
                  "term": {
                    "system.filesystem.type": "fixed"
                  }
                }
              ]
            }
          },
          "aggs": {
            "bucketAgg": {
              "terms": {
                "field": "agent.hostname.keyword",
                "size": "40",
                "order": {
                  "metricAgg": "asc"
                }
              },
              "aggs": {
                "metricAgg": {
                  "max": {
                    "field": "system.filesystem.used.pct"
                  }
                }
              }
            }
          }
        }
      }
    }
  },
  "condition": {
    "script": {
      "source": "ArrayList arr = ctx.payload.aggregations.bucketAgg.buckets; for (int i = 0; i < arr.length; i++) { if (arr[i]['metricAgg'].value >= params.threshold) { return true; } } return false;",
      "lang": "painless",
      "params": {
        "threshold": 0.9
      }
    }
  },
  "actions": {
    "logging_1": {
      "logging": {
        "level": "info",
        "text": "{{ctx.metadata.name}} !! {{#ctx.payload.results}} {{key}} : {{value}}\\n {{/ctx.payload.results}}"
      }
    },
    "slack_1": {
      "slack": {
        "message": {
          "text": "{{ctx.metadata.name}}\n{{#ctx.payload.results}} {{key}} : {{value}}, {{/ctx.payload.results}}"
        }
      }
    }
  },
  "transform": {
    "script": {
      "source": "HashMap result = new HashMap(); ArrayList arr = ctx.payload.aggregations.bucketAgg.buckets; ArrayList filteredHits = new ArrayList(); for (int i = 0; i < arr.length; i++) { HashMap filteredHit = new HashMap(); filteredHit.key = arr[i].key; filteredHit.value = arr[i]['metricAgg'].value; if (filteredHit.value >= params.threshold) { filteredHits.add(filteredHit); } } result.results = filteredHits; return result;",
      "lang": "painless",
      "params": {
        "threshold": 0.9
      }
    }
  }
}

system · May 19, 2020, 12:32am

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.