Watcher for missing log event

Elastic Stack Elasticsearch
1

Hi, I'm trying to write a watcher for two specific hosts, for when we do not receive a log entry for them. I found a previous post looking for similar, which linked to this example. I've been trying to modify this to look for specific hostnames, with "MOL" in the name.

However it fails to trigger with error "failed to execute watch input". Modified code below.

{
  "metadata": {
    "window_period": "1h",
    "last_period":"8m"
  },
  "trigger": {
    "schedule": {
      "interval": "3m"
    }
  },
  "input": {
    "search": {
      "request": {
        "indices": [
          "perf-metrics-*"
        ],
        "types": [],
        "body": {
          "query": {
            "@timestamp": {
                "gte": "now-{{ctx.metadata.window_period}}"
              }
          },
          "aggs": {
            "periods": {
              "filters": {
                "filters": {
                  "history": {
                    "range": {
                      "@timestamp": {
                        "gte": "now-{{ctx.metadata.window_period}}"
                      }
                    }
                  },
                  "last_period": {
                    "range": {
                      "@timestamp": {
                        "gte": "now-{{ctx.metadata.last_period}}"
                      }
                    }
                  },
                  "match": {
                    "host": "*MOL*"
                  }
                }
              },
              "aggs": {
                "hosts": {
                  "terms": {
                    "field": "host",
                    "size": 10000
                  }
                }
              }
            }
          },
          "size": 0
        }
      }
    }
  },
  "condition": {
    "script": {
      "id": "condition"
    }
  },
  "actions": {
    "script": {
	    "lang": "painless",
	    "source": "return ctx.payload.aggregations.periods.buckets.history.hosts.buckets.size() > ctx.payload.aggregations.periods.buckets.last_period.hosts.buckets.size();"
	  }

}
2

please provide the output of the Execute Watch API to aid with further debugging. Thanks.

3

So I found that issue was due to missing "range" in the query, fixed that and get this issue now.

{
  "watch_id": "Logstash_watcher",
  "node": "k3GzQ4XpT_q-VpDWytcURQ",
  "state": "failed",
  "user": "kanderson",
  "status": {
    "state": {
      "active": true,
      "timestamp": "2019-06-10T13:49:10.977Z"
    },
    "actions": {
      "log": {
        "ack": {
          "timestamp": "2019-06-10T13:49:10.977Z",
          "state": "awaits_successful_execution"
        }
      }
    },
    "execution_state": "failed",
    "version": -1
  },
  "trigger_event": {
    "type": "schedule",
    "triggered_time": "2019-06-10T13:52:11.390Z",
    "schedule": {
      "scheduled_time": "2019-06-10T13:52:10.978Z"
    }
  },
  "input": {
    "search": {
      "request": {
        "search_type": "query_then_fetch",
        "indices": [
          "perf-metrics-*"
        ],
        "types": [],
        "body": {
          "query": {
            "range": {
              "@timestamp": {
                "gte": "now-{{ctx.metadata.window_period}}"
              }
            }
          },
          "aggs": {
            "periods": {
              "filters": {
                "filters": {
                  "history": {
                    "range": {
                      "@timestamp": {
                        "gte": "now-{{ctx.metadata.window_period}}"
                      }
                    }
                  },
                  "last_period": {
                    "range": {
                      "@timestamp": {
                        "gte": "now-{{ctx.metadata.last_period}}"
                      }
                    }
                  },
                  "match": {
                    "hostname": "*MOL*"
                  }
                }
              },
              "aggs": {
                "hosts": {
                  "terms": {
                    "field": "host",
                    "size": 10000
                  }
                }
              }
            }
          },
          "size": 0
        }
      }
    }
  },
  "condition": {
    "script": {
      "source": "return ctx.payload.aggregations.periods.buckets.history.hosts.buckets.size() > ctx.payload.aggregations.periods.buckets.last_period.hosts.buckets.size();",
      "lang": "painless"
    }
  },
  "metadata": {
    "last_period": "8m",
    "window_period": "1h",
    "name": "Logstash",
    "xpack": {
      "type": "json"
    }
  },
  "result": {
    "execution_time": "2019-06-10T13:52:11.390Z",
    "execution_duration": 0,
    "input": {
      "type": "search",
      "status": "failure",
      "error": {
        "root_cause": [
          {
            "type": "parsing_exception",
            "reason": "[hostname] query malformed, no start_object after query name",
            "line": 1,
            "col": 221
          }
        ],
        "type": "parsing_exception",
        "reason": "[hostname] query malformed, no start_object after query name",
        "line": 1,
        "col": 221
      },
      "search": {
        "request": {
          "search_type": "query_then_fetch",
          "indices": [
            "perf-metrics-*"
          ],
          "types": [],
          "body": {
            "query": {
              "range": {
                "@timestamp": {
                  "gte": "now-1h"
                }
              }
            },
            "aggs": {
              "periods": {
                "filters": {
                  "filters": {
                    "history": {
                      "range": {
                        "@timestamp": {
                          "gte": "now-1h"
                        }
                      }
                    },
                    "last_period": {
                      "range": {
                        "@timestamp": {
                          "gte": "now-8m"
                        }
                      }
                    },
                    "match": {
                      "hostname": "*MOL*"
                    }
                  }
                },
                "aggs": {
                  "hosts": {
                    "terms": {
                      "field": "host",
                      "size": 10000
                    }
                  }
                }
              }
            },
            "size": 0
          }
        }
      }
    },
    "actions": []
  },
  "messages": [
    "failed to execute watch input"
  ]
}
4

before you put the query into a watch, please put it into a regular search request and check if it executes properly. it seems your filters aggregation is still off.

5

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.