Watcher querying data within indices from the day before

Hi there, another question from another ES noob. So I've managed to set up a Watch to alert on any successful logins, however, when the Watch executes, it appears to be querying data from the day before, even when I've included datemath in the "indices" section of "input" within the Watch JSON script. Please see below:

POST _watcher/watch/successful_ssh_login_2
{
  "trigger": {
  "schedule": {
    "interval": "1m"
  }
},
"input": {
  "search": {
    "request": {
      "indices": [
        "<filebeat-{now}>",
        "<filebeat-{now/d}>"
        ],
        "body": {
          "size": 0,
          "query": {
            "bool": {
              "filter": [
                {
                  "range": {
                    "@timestamp": {
                      "from": "now-1h",
                      "to": "now"
                    }
                  }
                },
                {
                  "term": {
                    "event.type": "authentication_success"
                  }
                }
              ]
            }
          }
        }
    }
  }
},
  "condition": {
    "compare": {
      "ctx.payload.hits.total": {
        "gte": 0
      }
    }
  },
  "actions": {
    "log": {
      "logging": {
        "text": "WARNING: {{ctx.payload.hits.total}} Successful SSH Login(s)!!!__2"
      }
    }
  }
}

This is the output that I get from Dev Tools when I execute the Watch myself (Time and dat being: 16:26 31/03/2020):

{
  "_id" : "successful_ssh_login_2_dc0b2e50-0ac9-4da5-9d31-d9836531bbda-2020-03-30T18:16:51.368152Z",
  "watch_record" : {
    "watch_id" : "successful_ssh_login_2",
    "node" : "npDAvvM1RtiYaNo4on4Ktg",
    "state" : "executed",
    "status" : {
      "state" : {
        "active" : true,
        "timestamp" : "2020-03-30T17:46:02.946Z"
      },
      "last_checked" : "2020-03-30T18:16:51.368Z",
      "last_met_condition" : "2020-03-30T18:16:51.368Z",
      "actions" : {
        "log" : {
          "ack" : {
            "timestamp" : "2020-03-30T17:46:35.867Z",
            "state" : "ackable"
          },
          "last_execution" : {
            "timestamp" : "2020-03-30T18:16:51.368Z",
            "successful" : true
          },
          "last_successful_execution" : {
            "timestamp" : "2020-03-30T18:16:51.368Z",
            "successful" : true
          }
        }
      },
      "execution_state" : "executed",
      "version" : 181
    },
    "trigger_event" : {
      "type" : "manual",
      "triggered_time" : "2020-03-30T18:16:51.368Z",
      "manual" : {
        "schedule" : {
          "scheduled_time" : "2020-03-30T18:16:51.368Z"
        }
      }
    },
    "input" : {
      "search" : {
        "request" : {
          "search_type" : "query_then_fetch",
          "indices" : [
            "<filebeat-{now}>",
            "<filebeat-{now/d}>"
          ],
          "rest_total_hits_as_int" : true,
          "body" : {
            "size" : 0,
            "query" : {
              "bool" : {
                "filter" : [
                  {
                    "range" : {
                      "@timestamp" : {
                        "from" : "now-1h",
                        "to" : "now"
                      }
                    }
                  },
                  {
                    "term" : {
                      "event.type" : "authentication_success"
                    }
                  }
                ]
              }
            }
          }
        }
      }
    },
    "condition" : {
      "compare" : {
        "ctx.payload.hits.total" : {
          "gte" : 0
        }
      }
    },
    "result" : {
      "execution_time" : "2020-03-30T18:16:51.368Z",
      "execution_duration" : 0,
      "input" : {
        "type" : "search",
        "status" : "success",
        "payload" : {
          "_shards" : {
            "total" : 0,
            "failed" : 0,
            "successful" : 0,
            "skipped" : 0
          },
          "hits" : {
            "hits" : [ ],
            "total" : 0,
            "max_score" : 0.0
          },
          "took" : 0,
          "timed_out" : false
        },
        "search" : {
          "request" : {
            "search_type" : "query_then_fetch",
            "indices" : [
              "<filebeat-{now}>",
              "<filebeat-{now/d}>"
            ],
            "rest_total_hits_as_int" : true,
            "body" : {
              "size" : 0,
              "query" : {
                "bool" : {
                  "filter" : [
                    {
                      "range" : {
                        "@timestamp" : {
                          "from" : "now-1h",
                          "to" : "now"
                        }
                      }
                    },
                    {
                      "term" : {
                        "event.type" : "authentication_success"
                      }
                    }
                  ]
                }
              }
            }
          }
        }
      },
      "condition" : {
        "type" : "compare",
        "status" : "success",
        "met" : true,
        "compare" : {
          "resolved_values" : {
            "ctx.payload.hits.total" : 0
          }
        }
      },
      "actions" : [
        {
          "id" : "log",
          "type" : "logging",
          "status" : "success",
          "logging" : {
            "logged_text" : "WARNING: 0 Successful SSH Login(s)!!!__2"
          }
        }
      ]
    },
    "messages" : [ ]
  }
}

Just wondering if anyone knows why this occurs and how remedy this so the Watcher only queries a more recent timeframe?
Have scoured online to find an answer but still unsure of how to go about correcting this. Any and all help would be much appreciated. Cheers.

So just to add to this issue, I've looked at the indices that filebeat creates daily, which are:
filebeat-7.6.1-2020.03.30
filebeat-7.6.1-2020.03.31
filebeat-7.6.1-2020.04.01

When breaking down my query and querying these individual indices over the ES API I'm only recieving data from filebeat-7.6.1-2020.03.30, even though I do have data within the others that relate to successful SSH logins as I am able to view these within Kibana.

Query used:

{
  "query": {
    "bool": {
      "must": {"term": {"event.type": "authentication_success"}},
      "filter": {"range": {"@timestamp":{"from": "now-1d", "to": "now"}}}
  }
}
}

And this has been queried with:
GET filebeat-*/_search
GET filebeat-7.6.1-2020.03.30/_search
GET filebeat-7.6.1-2020.03.31/_search
GET filebeat-7.6.1-2020.03.31/_search

The index from the 30th being the only index to return anything.

Any ideas as to why this would be happening?

Judging from the execution date in the watch

"execution_time" : "2020-03-30T18:16:51.368Z",

Is it possible, that the data you index lacks the information of the timezone and thus is not stored in UTC or converted the wrong way?

Elasticsearch stores all dates as UTC...

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.