Muli query watcher

Elastic Stack Elasticsearch
1

I am trying to build a watcher which alerts me when the following occurs:

  1. Event A (based on three tags and field result="InvalidPassword" for username=y) occurs three times.
  2. Event A is followed by event B (based on three tags and field result="BlockedUser" for username=y) occurs once.

I have the following but this does not seem to work

{
  "trigger": {
    "schedule": {
      "interval": "10s"
    }
  },
  "input": {
    "chain":{
      "inputs": [
        {
          "InvalidPassword": {
            "search": {
              "request": {
                "body": {
                  "size": 0,
                  "query": {
                    "bool": {
                      "must": [
                        {
                          "range": {
                            "@timestamp": {
                              "gte": "now-2m"
                            }
                          }
                        },
                        {
                          "match_phrase": {
                            "tags": "login"
                          }
                        },
                        {
                          "match_phrase": {
                            "result": "InvalidPassword"
                          }
                        }
                      ]
                    }
                  },
                  "aggs": {
                    "byUser": {
                      "terms": {
                        "field": "username"
                      }
                    }
                  }
                },
                "indices": [
                  "filebeat-*"
                ]
              }
            }
          }
        },
        {
          "BlockedUser": {
            "search": {
              "request": {
                "body": {
                  "size": 0,
                  "query": {
                    "bool": {
                      "must": [
                        {
                          "range": {
                            "@timestamp": {
                              "gte": "now-2m"
                            }
                          }
                        },
                        {
                          "match_phrase": {
                            "tags": "login"
                          }
                        },
                        {
                          "match_phrase": {
                            "result": "BlockedUser"
                          }
                        }
                      ]
                    }
                  },
                  "aggs": {
                    "byUser": {
                      "terms": {
                        "field": "username"
                      }
                    }
                  }
                },
                "indices": [
                  "filebeat-*"
                ]
              }
            }
          }
        }
      ]
    }
  },
  "condition": {
    "script": {
      "source": "return ctx.payload.InvalidPassword.username == ctx.payload.BlockedUser.username"
    }
  },
  "actions": {
    //my action
  }
}
2

To troubleshoot, I would suggest to output the payload to the log file. I suspect that the you need to find the username ctx.payload.InvalidPassword.aggregations.byUser.username.buckets[0].key , or something like that (and may need to compare arrays). Logging the full payload should help see exactly how to reference the values in the script compare.

  "actions": {
    "mylog": {
      "logging": {
        "text": "The payload is: {{ctx.payload}} "
      }
...
3

Hi @jakelandis , thanks for the hint.

Did help me a step forward, but now I don't know what to make of the results.
(I included my full watcher code below this time)

The response from a log event:

The payload is: {metadata=null, watch_id=soc_test, payload={BlockedUser=
{_shards={total=0, failed=0, successful=0, skipped=0}, hits={hits=[], total=0, 
max_score=0.0}, took=0, timed_out=false}, InvalidPassword={_shards={total=0, 
failed=0, successful=0, skipped=0}, hits={hits=[], total=0, max_score=0.0}, 
took=0, timed_out=false}}, 
id=soc_test_3c1dddfa-1187-4563-9a22-896b9defcc59-2019-12-28T12:00:22.969733Z, 
trigger={triggered_time=2019-12-28T12:00:22.969Z, 
scheduled_time=2019-12-28T12:00:22.738Z}, vars={}, 
execution_time=2019-12-28T12:00:22.969733Z}

However when I use the BlockedUser query directly on elasticsearch the response is:

{
    "took": 1129,
    "timed_out": false,
    "_shards": {
        "total": 2950,
        "successful": 2950,
        "skipped": 2911,
        "failed": 0
    },
    "hits": {
        "total": {
            "value": 2,
            "relation": "eq"
        },
        "max_score": null,
        "hits": []
    }
}

Watcher

{
  "trigger": {
    "schedule": {
      "interval": "30s"
    }
  },
  "input": {
    "chain": {
      "inputs": [
        {
          "InvalidPassword": {
            "search": {
              "request": {
                "search_type": "query_then_fetch",
                "indices": [
                  "filebeat-*"
                ],
                "rest_total_hits_as_int": true,
                "body": {
                  "size": 6,
                  "query": {
                    "bool": {
                      "must": [
                        {
                          "range": {
                            "@timestamp": {
                              "gte": "now-8m"
                            }
                          }
                        },
                        {
                          "match_phrase": {
                            "tags": "login"
                          }
                        },
                        {
                          "match_phrase": {
                            "tags": "outsystems"
                          }
                        },
                        {
                          "match_phrase": {
                            "envcode": "fac"
                          }
                        },
                        {
                          "match_phrase": {
                            "application": "outsystems"
                          }
                        },
                        {
                          "match_phrase": {
                            "result": "InvalidPassword"
                          }
                        }
                      ]
                    }
                  }
                }
              }
            }
          }
        },
        {
          "BlockedUser": {
            "search": {
              "request": {
                "search_type": "query_then_fetch",
                "indices": [
                  "filebeat-*"
                ],
                "rest_total_hits_as_int": true,
                "body": {
                  "size": 6,
                  "query": {
                    "bool": {
                      "must": [
                        {
                          "range": {
                            "@timestamp": {
                              "gte": "now-8m"
                            }
                          }
                        },
                        {
                          "match_phrase": {
                            "tags": "login"
                          }
                        },
                        {
                          "match_phrase": {
                            "tags": "outsystems"
                          }
                        },
                        {
                          "match_phrase": {
                            "envcode": "fac"
                          }
                        },
                        {
                          "match_phrase": {
                            "application": "outsystems"
                          }
                        },
                        {
                          "match_phrase": {
                            "result": "BlockedUser"
                          }
                        }
                      ]
                    }
                  }
                }
              }
            }
          }
        }
      ]
    }
  },
  "condition": {
    "always": {}
  },
  "actions": {
    "mylog": {
      "logging": {
        "text": "The payload is: {{ctx}}"
      }
    }
  }
}

Elasticsearch wget

wget --quiet \
  --method GET \
  --header 'Content-Type: application/json' \
  --header 'Authorization: Basic <auth>' \
  --header 'User-Agent: PostmanRuntime/7.20.1' \
  --header 'Accept: */*' \
  --header 'Cache-Control: no-cache' \
  --header 'Host: <elasticsearchhost>:9200' \
  --header 'Accept-Encoding: gzip, deflate' \
  --header 'Content-Length: 1013' \
  --header 'Connection: keep-alive' \
  --header 'cache-control: no-cache' \
  --body-data '{\n    "size": 0,\n    "query": {\n        "bool": {\n            "must": [\n                {\n                    "range": {\n                        "@timestamp": {\n                            "gte": "now-8m"\n                        }\n                    }\n                },\n                {\n                    "match_phrase": {\n                        "tags": "login"\n                    }\n                },\n                {\n                    "match_phrase": {\n                        "tags": "outsystems"\n                    }\n                },\n                {\n                    "match_phrase": {\n                        "envcode": "fac"\n                    }\n                },\n                {\n                    "match_phrase": {\n                        "application": "outsystems"\n                    }\n                },\n                {\n                    "match_phrase": {\n                        "result": "BlockedUser"\n                    }\n                }\n            ]\n        }\n    }\n}' \
  --output-document \
  - http://<elasticsearchhost>:9200/filebeat-%2A/_search
4

It looks like your query is not returning any results.

I would suggest to pull the search request out of the Watch and ensure that it is returning what you expect. The query syntax is the same, so it should be mostly copy/paste to a normal _search request.

5

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.