Watcher returns null values

Elastic Stack Elasticsearch
1

Hello!
I'd appreciate for any help, can't figure it by myself for a long time.
I have a quite simple watcher, like this:

{
  "trigger": {
"schedule": {
  "interval": "5m"
}
  },
  "input": {
"search": {
  "request": {
    "search_type": "query_then_fetch",
    "indices": [
      "metricbeat-*"
    ],
    "rest_total_hits_as_int": true,
    "body": {
      "size": 0,
      "query": {
        "bool": {
          "must": [
            {
              "exists": {
                "field": "system.load.5"
              }
            },
            {
              "range": {
                "@timestamp": {
                "gte": "{{ctx.trigger.scheduled_time}}||-2m",
                "lte": "{{ctx.trigger.scheduled_time}}",
                "format": "strict_date_optional_time||epoch_millis"
              }
              }
            }
          ]
        }
      },
      "aggs": {
        "host": {
          "terms": {
            "field": "agent.id",
            "size": 10,
            "order": {
              "load_5": "desc"
            }
          },
          "aggs": {
            "load_1": {
              "max": {
                "field": "system.load.1"
              }
            },
            "load_5": {
              "max": {
                "field": "system.load.5"
              }
            },
            "load_15": {
              "max": {
                "field": "system.load.15"
              }
            },
            "total": {
              "max": {
                "field": "system.cpu.total.pct"
              }
            }
          }
        }
      }
    }
  }
}
  },
  "condition": {
"script": "return ctx.payload.aggregations.host.buckets.0.load_5.value > 1 || ctx.payload.aggregations.host.buckets.0.load_15.value > 1 "
  },
  "actions": {
"slack_alert_1": {
  "condition": {
    "script": "return ctx.payload.aggregations.host.buckets.0.load_5.value > 1 || ctx.payload.aggregations.host.buckets.0.load_15.value > 1 "
  },
  "slack": {
    "message": {
      "from": "Watcher. Server -> {{ctx.payload.aggregations.host.buckets.0.key}}",
      "attachments": [
        {
          "color": "warning",
          "title": "High CPU Usage Alert!",
          "text": "INFO \n LOAD 1 - {{ctx.payload.aggregations.host.buckets.0.load_1.value}} | LOAD 5 - {{ctx.payload.aggregations.host.buckets.0.load_5.value}} | LOAD 15 - {{ctx.payload.aggregations.host.buckets.0.load_15.value}} \n CPU Usage - {{ctx.payload.aggregations.host.buckets.0.total.value}}%"
        }
      ]
    }
  }
}
  }
}

The return output is:

"result" : {
  "execution_time" : "2020-01-15T07:16:10.664Z",
  "execution_duration" : 22,
  "input" : {
    "type" : "search",
    "status" : "success",
    "payload" : {
      "_shards" : {
        "total" : 1,
        "failed" : 0,
        "successful" : 1,
        "skipped" : 0
      },
      "hits" : {
        "hits" : [ ],
        "total" : 115,
        "max_score" : null
      },
      "took" : 21,
      "timed_out" : false,
      "aggregations" : {
        "host" : {
          "doc_count_error_upper_bound" : 0,
          "sum_other_doc_count" : 0,
          "buckets" : [
            {
              "doc_count" : 11,
              "total" : {
                "value" : null
              },
              "load_5" : {
                "value" : 0.71
              },
              "load_15" : {
                "value" : 0.81
              },
              "load_1" : {
                "value" : 1.23
              },
              "key" : "516e37ef-b19e-4663-a326-070558a62bb8"
            },
            {
              "doc_count" : 11,
              "total" : {
                "value" : null
              },
              "load_5" : {
                "value" : 0.2
              },
              "load_15" : {
                "value" : 0.12
              },
              "load_1" : {
                "value" : 0.64
              },
              "key" : "890367ad-8016-4e0e-baa0-9aa4a6373bb3"
            },
            {
              "doc_count" : 12,
              "total" : {
                "value" : null
              },
              "load_5" : {
                "value" : 0.17
              },
              "load_15" : {
                "value" : 0.21
              },
              "load_1" : {
                "value" : 0.13
              },
              "key" : "e4f739ee-b7c8-4632-a137-39dca8f2a975"
            },
            {
              "doc_count" : 12,
              "total" : {
                "value" : null
              },
              "load_5" : {
                "value" : 0.11
              },
              "load_15" : {
                "value" : 0.12
              },
              "load_1" : {
                "value" : 0.12
              },
              "key" : "f48d09b0-a7d9-4250-aa8a-25b47603e2f7"
            },
            {
              "doc_count" : 11,
              "total" : {
                "value" : null
              },
              "load_5" : {
                "value" : 0.1
              },
              "load_15" : {
                "value" : 0.06
              },
              "load_1" : {
                "value" : 0.2
              },
              "key" : "9dc84233-8336-4c28-9569-fe59a285800b"
            },
            {
              "doc_count" : 11,
              "total" : {
                "value" : null
              },
              "load_5" : {
                "value" : 0.09
              },
              "load_15" : {
                "value" : 0.02
              },
              "load_1" : {
                "value" : 0.25
              },
              "key" : "f1f8bb43-a32d-42e0-9471-116aeeec20c7"
            },
            {
              "doc_count" : 12,
              "total" : {
                "value" : null
              },
              "load_5" : {
                "value" : 0.08
              },
              "load_15" : {
                "value" : 0.14
              },
              "load_1" : {
                "value" : 0.22
              },
              "key" : "44a2094e-219e-4730-84d8-6ab0ee5c5351"
            },
            {
              "doc_count" : 12,
              "total" : {
                "value" : null
              },
              "load_5" : {
                "value" : 0.08
              },
              "load_15" : {
                "value" : 0.06
              },
              "load_1" : {
                "value" : 0.2
              },
              "key" : "e60625e8-d6f4-4a89-bd23-49e9dde473ba"
            },
            {
              "doc_count" : 12,
              "total" : {
                "value" : null
              },
              "load_5" : {
                "value" : 0.03
              },
              "load_15" : {
                "value" : 0.05
              },
              "load_1" : {
                "value" : 0.0
              },
              "key" : "15c6246b-1b3c-42fb-96f2-afccc66f549d"
            },
            {
              "doc_count" : 11,
              "total" : {
                "value" : null
              },
              "load_5" : {
                "value" : 0.02
              },
              "load_15" : {
                "value" : 0.05
              },
              "load_1" : {
                "value" : 0.0
              },
              "key" : "e3ef14e4-a748-4867-a19c-e5f6f7477db0"
            }
          ]
        }
      }
    },

I can't understand, why it returns "null" values of "total" (system.cpu.total.pct).

"total" : {
                "value" : null
              },

But when I look at same metric at "Metrics Explorer" I can see values and grafhs.

image
image1643×386 71.2 KB

I use Elastic Cloud with Kibana 7.5.1 and Metricbeat 7.5.1 on hosts via docker.

2

Hey,

you can remove the size: 0 to verify that the documents that do have a system.load.5 field also have a system.cpu.total.pct - or maybe those are two different documents?

--Alex

3

Oh... Your advice made me think it was in this code.

"exists": {
                "field": "system.load.5"
              }

Did not notice this in the docs, that this may cause some filtering of gathered metrics.
Now I think it works fine

Thank you for help :wink:

4

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.