Watcher on ECE - Comparing an Aggs result to a threshold

Hi, I'm starting with ECE/Elastic in general, and now I'm trying to create a watcher to send an e-mail based on a query that aggregates me a metric coming from APM.
This watcher runs but never is triggered.

{
  "trigger": {
    "schedule": {
      "interval": "1m"
    }
  },
  "input": {
    "search": {
      "request": {
        "search_type": "query_then_fetch",
        "indices": [
          "apm-*"
        ],
        "rest_total_hits_as_int": true,
        "body": {
          "size": 0,
          "query": {
            "bool": {
              "must": [
                {
                  "match": {
                    "service.name": "MY-SERVER"
                  }
                }
              ],
              "filter": {
                "range": {
                  "@timestamp": {
                    "gte": "now-1h"
                  }
                }
              }
            }
          },
          "aggs": {
            "jvm.thread.count": {
              "max": {
                "field": "jvm.thread.count"
              }
            }
          }
        }
      }
    }
  },
  "condition": {
    "compare": {
      "ctx.payload.aggregations.jvm.thread.count.value": {
        "gte": 5
      }
    }
  },
  "actions": {
    "my-logging-action": {
      "logging": {
        "level": "info",
        "text": "There are {{jvm.thread.count}} threads. Threshold is 5."
      }
    },
    "send_email": {
      "email": {
        "profile": "standard",
        "to": [
          "my-email"
        ],
        "subject": "Alerta Watch - {{service.name}}",
        "body": {
          "text": "Quantidade de threads ativas é: {{jvm.thread.count}}. Threshold é 5."
        }
      }
    }
  }
}

In the Watcher execution history I got:

{
  "watch_id": "4cd11254-acb1-4c79-849e-6f9984ee91a4",
  "node": "MXBCDigARAaX1kdhaSAebQ",
  "state": "execution_not_needed",
  "user": "elastic",
  "status": {
    "state": {
      "active": true,
      "timestamp": "2019-10-10T17:36:21.275Z"
    },
    "last_checked": "2019-10-10T17:56:09.887Z",
    "actions": {
      "my-logging-action": {
        "ack": {
          "timestamp": "2019-10-10T17:36:21.275Z",
          "state": "awaits_successful_execution"
        }
      },
      "send_email": {
        "ack": {
          "timestamp": "2019-10-10T17:36:21.275Z",
          "state": "awaits_successful_execution"
        }
      }
    },
    "execution_state": "execution_not_needed",
    "version": -1
  },
  "trigger_event": {
    "type": "schedule",
    "triggered_time": "2019-10-10T17:56:09.887Z",
    "schedule": {
      "scheduled_time": "2019-10-10T17:56:09.546Z"
    }
  },
  "input": {
    "search": {
      "request": {
        "search_type": "query_then_fetch",
        "indices": [
          "apm-*"
        ],
        "rest_total_hits_as_int": true,
        "body": {
          "size": 0,
          "query": {
            "bool": {
              "must": [
                {
                  "match": {
                    "service.name": "MY-SERVER"
                  }
                }
              ],
              "filter": {
                "range": {
                  "@timestamp": {
                    "gte": "now-1h"
                  }
                }
              }
            }
          },
          "aggs": {
            "jvm.thread.count": {
              "max": {
                "field": "jvm.thread.count"
              }
            }
          }
        }
      }
    }
  },
  "condition": {
    "compare": {
      "ctx.payload.aggregations.jvm.thread.count.value": {
        "gte": 5
      }
    }
  },
  "metadata": {
    "name": "Alert JVM Thread Count",
    "xpack": {
      "type": "json"
    }
  },
  "result": {
    "execution_time": "2019-10-10T17:56:09.887Z",
    "execution_duration": 109,
    "input": {
      "type": "search",
      "status": "success",
      "payload": {
        "_shards": {
          "total": 165,
          "failed": 0,
          "successful": 165,
          "skipped": 161
        },
        "hits": {
          "hits": [],
          "total": 3838,
          "max_score": null
        },
        "took": 108,
        "timed_out": false,
        "aggregations": {
          "jvm.thread.count": {
            "value": 49
          }
        }
      },
      "search": {
        "request": {
          "search_type": "query_then_fetch",
          "indices": [
            "apm-*"
          ],
          "rest_total_hits_as_int": true,
          "body": {
            "size": 0,
            "query": {
              "bool": {
                "must": [
                  {
                    "match": {
                      "service.name": "MY-SERVER"
                    }
                  }
                ],
                "filter": {
                  "range": {
                    "@timestamp": {
                      "gte": "now-1h"
                    }
                  }
                }
              }
            },
            "aggs": {
              "jvm.thread.count": {
                "max": {
                  "field": "jvm.thread.count"
                }
              }
            }
          }
        }
      }
    },
    "condition": {
      "type": "compare",
      "status": "success",
      "met": false,
      "compare": {
        "resolved_values": {
          "ctx.payload.aggregations.jvm.thread.count.value": null
        }
      }
    },
    "actions": []
  },
  "messages": []
}

I tested the query on my elastic and I have results:

{
  "took" : 114,
  "timed_out" : false,
  "_shards" : {
    "total" : 165,
    "successful" : 165,
    "skipped" : 161,
    "failed" : 0
  },
  "hits" : {
    "total" : {
      "value" : 3730,
      "relation" : "eq"
    },
    "max_score" : null,
    "hits" : [ ]
  },
  "aggregations" : {
    "jvm.thread.count" : {
      "value" : 39.0
    }
  }
}

Can someone give me a hint on what's wrong?

Thanks!! :+1:

Hey,

I think the dots in the fieldname are confusing for the mustache parser, as the mustache parser now thinks there is a JSON structure like jvm : { thread : { count : .. in your JSON and thus returns null. Can you try a fieldname without any dots and see if that is working?

--Alex

1 Like

Hi @spinscale, thanks for your reply!
I'll change that name and use "_" instead of dots.
Thanks!! :+1:

Hi @spinscale thanks for your suggestion! It worked!
Well I'll take this as a best practice, not use dots on the Aggs name! :+1: