Watcher doesn't work (empty result)

Elastic Stack Elasticsearch
1

Hi all,

I have a watcher in Kibana that doesn't work (condition is if ctx.payload.hits.total == 0, then send notification via slack). BUT:

If you check the Execution Output of Watcher you can see that there are no totals, hits, shards etc (so as i understand watcher see nothing, or smth is blocking it).

But if i go to Simulate this watcher i will have results and Watcher will not send mail/slack notification whatever i use in actions - as stats a really presented (checked in discovery, visualizations etc)

Can someone advice what is wrong with Watcher, cause it doesn't work, should find stats and not notify me.

Really thanks in advance!

When it worked by itself:

  "metadata": {
    "xpack": {
      "type": "json"
    }
  },
  "result": {
    "execution_time": "2018-06-18T12:21:46.677Z",
    "execution_duration": 82,
    "input": {
      "type": "search",
      "status": "success",
      "payload": {
        "_shards": {
          "total": 0,
          "failed": 0,
          "successful": 0,
          "skipped": 0
        },
        "hits": {
          "hits": [],
          "total": 0,
          "max_score": 0
        },
        "took": 2,
        "timed_out": false
      },
      "search": {
        "request": {
          "search_type": "query_then_fetch",
          "indices": [
            "name"
          ],
          "types": [
            "events"

    "condition": {
      "type": "script",
      "status": "success",
      "met": true
    },
    "actions": [
      {
        "id": "notify-slack",
        "type": "slack",
        "status": "success",
        "slack": {
          "account": "name_acc",
          "sent_messages": [
            {
              "status": "success",
              "to": "#name",
              "message": {
                "from": "Kibana - Name",
                "text": " ",
                "attachments": [
                  {
                    "title": "NO results",
                    "text": "There were no results within the last 30 min"
                  }
                ]
              }
            }
          ]
        }
      }
    ]
  },
  "messages": []
}
2

When i simulate this wathcer, I have:

{
  "watch_id": "_inlined_",
  "node": "ScQM65WcSAydJhw7gzOxUA",
  "state": "execution_not_needed",
  "status": {
    "state": {
      "active": true,
      "timestamp": "2018-06-18T13:38:17.744Z"
    },
    "last_checked": "2018-06-18T13:38:17.745Z",
    "actions": {
      "notify-slack": {
        "ack": {
          "timestamp": "2018-06-18T13:38:17.744Z",
          "state": "awaits_successful_execution"
        }
      }
    },
    "execution_state": "execution_not_needed",
    "version": -1
  },
  "trigger_event": {
    "type": "manual",
    "triggered_time": "2018-06-18T13:38:17.745Z",
    "manual": {
      "schedule": {
        "scheduled_time": "2018-06-18T13:38:17.745Z"
      }
    }
  },
  "input": {
    "search": {
      "request": {
        "search_type": "query_then_fetch",
        "indices": [
          "name*"
        ],
        "types": [
          "events"
        ],
        "body": {
          "query": {
            "bool": {
              "must": [
                {
                  "match_all": {}
                },
                {
                  "bool": {
                    "minimum_should_match": 1,
                    "should": [
                      {
                        "match_phrase": {
                          "name": "name1"
                        }
                      },
                      {
                        "match_phrase": {
                          "name": "name2"
                        }
                      }
                    ]
                  }
                },
                {
                  "match_phrase": {
                    "mtd": {
                      "query": "name"
                    }
                  }
                },
                {
                  "match_phrase": {
                    "name": {
                      "query": 1
                    }
                  }
                },
                {
                  "range": {
                    "@timestamp": {
                      "gte": "now-30m",
                      "lte": "now"
                    }
                  }
                }
              ]
            }
          },
          "size": 0
        }
      }
    }
  },
  "condition": {
    "script": {
      "source": "if (ctx.payload.hits.total == 0) return true;else return false; ",
      "lang": "painless"
    }
  },
  "metadata": {
    "xpack": {
      "type": "json"
    }
  },
  "result": {
    "execution_time": "2018-06-18T13:38:17.745Z",
    "execution_duration": 991,
    "input": {
      "type": "search",
      "status": "success",
      "payload": {
        "_shards": {
          "total": 992,
          "failed": 0,
          "successful": 992,
          "skipped": 654
        },
        "hits": {
          "hits": [],
          "total": 692,
          "max_score": 0
        },
        "took": 980,
        "timed_out": false
      },
      "search": {
        "request": {
          "search_type": "query_then_fetch",
          "indices": [
            "name*"
          ],
          "types": [
            "events"
          ],
          "body": {
            "query": {
              "bool": {
                "must": [
                  {
                    "match_all": {}
                  },
                  {
                    "bool": {
                      "minimum_should_match": 1,
                      "should": [
                        {
                          "match_phrase": {
                            "name": "name1"
                          }
                        },
                        {
                          "match_phrase": {
                            "name": "name2"
                          }
                        }
                      ]
                    }
                  },
                  {
                    "match_phrase": {
                      "mtd": {
                        "query": "name"
                      }
                    }
                  },
                  {
                    "match_phrase": {
                      "name": {
                        "query": 1
                      }
                    }
                  },
                  {
                    "range": {
                      "@timestamp": {
                        "gte": "now-30m",
                        "lte": "now"
                      }
                    }
                  }
                ]
              }
            },
            "size": 0
          }
        }
      }
    },
    "condition": {
      "type": "script",
      "status": "success",
      "met": false
    },
    "actions": []
  },
  "messages": []
}
3

Hey guys, really need your help/advice on this, at least where can be the problem or where to check.

Thanks

4

Add this in your query

size: "50"

Like this :

 "body": {
      "size":50,
          "query": {
            "bool": {
              "must": [
                {
                  "match_all
5

i think you should try executing your watch in the dev tools environment : like this :slight_smile:

POST _xpack/watcher/watch/_execute
{
  "watch": {
    "trigger": {
      "schedule": {
        "interval": "30m"
      }
    },
    "input": {
      "search": {
        "request": {
          "body": {
            "size": 0,
            "query": {
              "match_all": {}
            }
          },
          "indices": [
            "*"
          ]
        }
      }
    },
    "condition": {
      "compare": {
        "ctx.payload.hits.total": {
          "gte": 10
        }
      }
    },
    "actions": {
      "my-logging-action": {
        "logging": {
          "text": "There are {{ctx.payload.hits.total}} documents in your index. Threshold is 10."
        }
      }
    }
  }
}
6

Thank you Guillaume,

Size was already mentioned in my query "size" : 0, i tried 50 but the same result ( AFAIK size 0 means take all docs, 50 only 50, no?)

Then i go to dev tools (already treid) it gives the same result as I simulate watch - so i see totals shards... seems ok

But why watcher doesn't find when it works by itself, i dunno (

7

We can see that you have 692 hits

 "hits": {
          "hits": [],
          "total": 692,
          "max_score": 0
        },

And just below your script do this
"if (ctx.payload.hits.total == 0) return true;else return false; "
So logically the condition "met": false because your ctx.payload.hits.total = 692

So you have no actions triggered .

8

Yes, it is correct - but these 692 hits total are presented only when I simulate watch (or in Dev tool) manually.
When watcher runs by itself it doesnt find anything - and so start to alert me but shoudnt cause stats are presented(

9

:thinking:
Both watchers you use are not querying the same index .
When it works by itself , you have this

 "indices": [
            "name"]

and when you simulated you have this .

 "indices": [
            "geocomply-nj*"
          ],
10

Well, this is the same, i just wanna not to mention real names, and all where name or name1 name2 are the same thing are hided :slight_smile:
So basically it is all the same in query, i just go to Watchers -> Edit -> Simulate and have results (so no alerts no need) but when i go to execution output of ran watcher (history) that watcher generated by itself - no hits no results and alert was generated.

11

In don't really understand why watcher has not the same behavior while running .

Maybe you should try to execute your watcher in the dev tools with the syntax shown above , and if it works . Implement it with

POST _xpack/watcher/watch/thenameofyourwatch
{
    "trigger": {
      "schedule": {
        "interval": "30m"
      }
    },
    "input": {
      "search": {
        "request": {
          "body": {
            "size": 0,
            "query": {
              "match_all": {}
            }
          },
          "indices": [
            "*"
          ]
        }
      }
    },
    "condition": {
      "compare": {
        "ctx.payload.hits.total": {
          "gte": 10
        }
      }
    },
    "actions": {
      "my-logging-action": {
        "logging": {
          "text": "There are {{ctx.payload.hits.total}} documents in your index. Threshold is 10."
        }
      }
    }
}

Don't forget to remove this
"watch": { }

And then compare both results .

12

If you store the watch, you can also call it with POST _xpack/watcher/watch/thenameofyourwatch/_execute to make sure you dont execute another watch than the one you stored.

That history entry, which is returned then should show, that there is a hit count - feel free to paste it here.

@elastock thanks so much for jumping in here and helping as well (as well as in the other posts), much appreciated!

1 Like
13

Hello Alexander,

So i tried a lot of times in Dev tools.

Now I tried this : POST _xpack/watcher/watch/thenameofyourwatch/_execute

I have this result (so I see hits), but watcher by itself don't show those results (totals 0 as well hits 0), :frowning:

{
  "_id": "NAMEOF THE WATCHER_10670f19-c92e-45b9-9af1-7042152a083d-2018-06-22T14:16:43.757Z",
  "watch_record": {
    "watch_id": "NAMEOF THE WATCHER",
    "node": "ScQM65WcSAydJhw7gzOxUA",
    "state": "execution_not_needed",
    "status": {
      "state": {
        "active": true,
        "timestamp": "2018-06-22T14:16:39.629Z"
      },
      "last_checked": "2018-06-22T14:16:43.757Z",
      "actions": {
        "notify-slack": {
          "ack": {
            "timestamp": "2018-06-21T09:07:42.912Z",
            "state": "awaits_successful_execution"
          }
        }
      },
      "execution_state": "execution_not_needed",
      "version": 341
    },
    "trigger_event": {
      "type": "manual",
      "triggered_time": "2018-06-22T14:16:43.757Z",
      "manual": {
        "schedule": {
          "scheduled_time": "2018-06-22T14:16:43.757Z"
        }
      }
    },
    "input": {
      "search": {
        "request": {
          "search_type": "query_then_fetch",
          "indices": [
            "name_of_index"
          ],
          "types": [
            "events"
          ],
          "body": {
            "query": {
              "bool": {
                "must": [
                  {
                    "match_all": {}
                  },
                  {
                    "bool": {
                      "minimum_should_match": 1,
                      "should": [
                        {
                          "match_phrase": {
                            "field_name": "name1"
                          }
                        },
                        {
                          "match_phrase": {
                            "field_name": "name2"
                          }
                        }
                      ]
                    }
                  },
                  {
                    "match_phrase": {
                      "mtd": {
                        "query": "name"
                      }
                    }
                  },
                  {
                    "match_phrase": {
                      "field_name2": {
                        "query": 1
                      }
                    }
                  },
                  {
                    "range": {
                      "@timestamp": {
                        "gte": "now-30m",
                        "lte": "now"
                      }
                    }
                  }
                ]
              }
            },
            "size": 0
          }
        }
      }
    },
    "condition": {
      "script": {
        "source": "if (ctx.payload.hits.total == 0) return true;else return false; ",
        "lang": "painless"
      }
    },
    "metadata": {
      "xpack": {
        "type": "json"
      }
    },
    "result": {
      "execution_time": "2018-06-22T14:16:43.757Z",
      "execution_duration": 984,
      "input": {
        "type": "search",
        "status": "success",
        "payload": {
          "_shards": {
            "total": 992,
            "failed": 0,
            "successful": 992,
            "skipped": 654
          },
          "hits": {
            "hits": [],
            "total": 30,
            "max_score": 0
          },
          "took": 972,
          "timed_out": false
        },
        "search": {
          "request": {
            "search_type": "query_then_fetch",
            "indices": [
              "name_of_index"
            ],
            "types": [
              "events"
            ],
            "body": {
              "query": {
                "bool": {
                  "must": [
                    {
                      "match_all": {}
                    },
                    {
                      "bool": {
                        "minimum_should_match": 1,
                        "should": [
                          {
                            "match_phrase": {
                              "field_name": "name1"
                            }
                          },
                          {
                            "match_phrase": {
                              "solution": "name2"
                            }
                          }
                        ]
                      }
                    },
                    {
                      "match_phrase": {
                        "mtd": {
                          "query": "name"
                        }
                      }
                    },
                    {
                      "match_phrase": {
                        "field_name2": {
                          "query": 1
                        }
                      }
                    },
                    {
                      "range": {
                        "@timestamp": {
                          "gte": "now-30m",
                          "lte": "now"
                        }
                      }
                    }
                  ]
                }
              },
              "size": 0
            }
          }
        }
      },
      "condition": {
        "type": "script",
        "status": "success",
        "met": false
      },
      "actions": []
    },
    "messages": []
  }
}
14

Check your condition, I think it is reversed to what it should be

You return true, if there are no hits, false otherwise. So in this case there are hits but you return false, stopping the watch execution.

Did you mean:

return ctx.payload.hits.total > 0

--Alex

15

So I want to notify me when there is no transactions, so that's why i use

source": "if (ctx.payload.hits.total == 0) return true;else return false; ",

But this watcher doesn't see results (but as you see in Dev tool it sees transactions) and notifies me every triggered time...

16

but the total hits are 30, so the condition is not met as expected then?

17

you also specified size: 0, which excludes all results

18

Well,

Sorry i will try to explain the problem: I have this watcher to notify me if( for the last 30mins) there is no transactions. I saved this watcher, and thought that should be ok. But then i started to receive every 30 mins that there is no transaction in system - then i go to dashboards/visualizations and see that transactions are presented. Then i check both in Dev tool and in Simulate mode and have those results as on dashboard/Visualization ... but watcher by itself doesn't see them. That is the problem , that automatically watcher doesnt work.

"hits": { "hits": , "total": 30, "max_score": 0 },

So as hits are not equal 0, no need to send me notify, it is clear.

Size 0 - i thought it means took all docs, size 50 took only 50 docs (i take this from visualization in request tab), mb i am wrong, can you share a link to proper useful literature to read and learn, please

19

According to your last description and your last history entry, watcher works as expected. There we transactions (aka search results in this case), thus the condition was not met and no email was sent.

Can you share a watch history that triggered an action, because the condition was not met?

20

Hey Alex,

This is from output history of Wathcer (field_name name 1,2 - I changed real names). As i told, by itself watcher doesnt see anything (shards total skiped 0) but when i take this watcher to Dev tools/Simulate - I will have totals and hits - so notification will not send to me (which is ok)...but watcher by itself in needed time notify me that 0 transactions ...(

 {
  "watch_id": "NAMEOF THE WATCHER",
  "node": "pwiX8UuHQrew6Z-cDYenMQ",
  "state": "executed",
  "status": {
    "state": {
      "active": true,
      "timestamp": "2018-06-21T09:05:39.768Z"
    },
    "last_checked": "2018-06-21T09:07:40.527Z",
    "last_met_condition": "2018-06-21T09:07:40.527Z",
    "actions": {
      "notify-slack": {
        "ack": {
          "timestamp": "2018-06-21T09:06:39.962Z",
          "state": "ackable"
        },
        "last_execution": {
          "timestamp": "2018-06-21T09:07:40.527Z",
          "successful": true
        },
        "last_successful_execution": {
          "timestamp": "2018-06-21T09:07:40.527Z",
          "successful": true
        }
      }
    },
    "execution_state": "executed",
    "version": -1
  },
  "trigger_event": {
    "type": "schedule",
    "triggered_time": "2018-06-21T09:07:40.527Z",
    "schedule": {
      "scheduled_time": "2018-06-21T09:07:40.060Z"
    }
  },
  "input": {
    "search": {
      "request": {
        "search_type": "query_then_fetch",
        "indices": [
          "index_name*"
        ],
        "types": [
          "events"
        ],
        "body": {
          "query": {
            "bool": {
              "must": [
                {
                  "match_all": {}
                },
                {
                  "bool": {
                    "minimum_should_match": 1,
                    "should": [
                      {
                        "match_phrase": {
                          "field_name": "name1"
                        }
                      },
                      {
                        "match_phrase": {
                          "field_name": "name2"
                        }
                      }
                    ]
                  }
                },
                {
                  "match_phrase": {
                    "mtd": {
                      "query": "name"
                    }
                  }
                },
                {
                  "match_phrase": {
                    "field_name2": {
                      "query": 1
                    }
                  }
                },
                {
                  "range": {
                    "@timestamp": {
                      "gte": "now-30m",
                      "lte": "now"
                    }
                  }
                }
              ]
            }
          },
          "size": 0
        }
      }
    }
  },
  "condition": {
    "script": {
      "source": "if (ctx.payload.hits.total == 0) return true;else return false; ",
      "lang": "painless"
    }
  },
  "metadata": {
    "xpack": {
      "type": "json"
    }
  },
  "result": {
    "execution_time": "2018-06-21T09:07:40.527Z",
    "execution_duration": 73,
    "input": {
      "type": "search",
      "status": "success",
      "payload": {
        "_shards": {
          "total": 0,
          "failed": 0,
          "successful": 0,
          "skipped": 0
        },
        "hits": {
          "hits": [],
          "total": 0,
          "max_score": 0
        },
        "took": 3,
        "timed_out": false
      },
      "search": {
        "request": {
          "search_type": "query_then_fetch",
          "indices": [
            "index_name*"
          ],
          "types": [
            "events"
          ],
          "body": {
            "query": {
              "bool": {
                "must": [
                  {
                    "match_all": {}
                  },
                  {
                    "bool": {
                      "minimum_should_match": 1,
                      "should": [
                        {
                          "match_phrase": {
                            "field_name": "name1"
                          }
                        },
                        {
                          "match_phrase": {
                            "field_name": "name2"
                          }
                        }
                      ]
                    }
                  },
                  {
                    "match_phrase": {
                      "mtd": {
                        "query": "name"
                      }
                    }
                  },
                  {
                    "match_phrase": {
                      "field_name2": {
                        "query": 1
                      }
                    }
                  },
                  {
                    "range": {
                      "@timestamp": {
                        "gte": "now-30m",
                        "lte": "now"
                      }
                    }
                  }
                ]
              }
            },
            "size": 0
          }
        }
      }
    },
    "condition": {
      "type": "script",
      "status": "success",
      "met": true
    },
    "actions": [
      {
        "id": "notify-slack",
        "type": "slack",
        "status": "success",
        "slack": {
          "account": "elk_test",
          "sent_messages": [
            {
              "status": "success",
              "to": "#fortests",
              "message": {
                "from": "Kibana",
                "text": " ",
                "attachments": [
                  {
                    "title": "NO pass transactions",
                    "text": "there were not pass transactions within the last 30 min"
                  }
                ]
              }
            }
          ]
        }
      }
    ]
  },
  "messages": []
}