Alert for disk space on server with multiple disks

Elastic Stack Elasticsearch
1

Hello Community,

I working to get an alert for diskspace over 75% on server which have multiple drives (A, B, C, D). I having difficulty to in compare condition with multiple drives in a bucket. I hope below snippet of code and results will give some idea
{
"trigger": {
"schedule": {
"interval": "1m"
}
},
"input": {
"search": {
"request": {
"search_type": "query_then_fetch",
"indices": [
"metricbeat-*"
],
"types": [],
"body": {
"size": 0,
"query": {
"bool": {
"must": [
{
"exists": {
"field": "system.filesystem.used.pct"
}
},
{
"range": {
"@timestamp": {
"gte": "now-15m",
"lte": "now",
"format": "epoch_millis"
}
}
}
]
}
},
"aggs": {
"host": {
"terms": {
"field": "beat.hostname",
"size": 20,
"order": {
"pct": "desc"
}
},
"aggs": {
"pct": {
"max": {
"field": "system.filesystem.used.pct",
"script": {
"source": "doc['system.filesystem.used.pct'].value *100",
"lang": "painless"
}
}
},
"mpoint": {
"terms": {
"field": "system.filesystem.mount_point",
"size": 5,
"order": {
"pct": "desc"
}
},
"aggs": {
"pct": {
"max": {
"field": "system.filesystem.used.pct"
}
}
}
}
}
}
}
}
}
}
},
"condition": {
"script": {
"source": "ArrayList arr = ctx.payload.aggregations.bucketAgg.buckets; for (int i = 0; i < arr.length; i++) { if (arr[i]['pct'].value > params.threshold) { return true; } } return false;",
"lang": "painless",
"params": {
"threshold": 0.75
}
}
},
"actions": {
"email_1": {
"email": {
"profile": "standard",
"priority": "high",
"to": [
"test@test.com"
],
"subject": "High FS Usage on {{ctx.payload.aggregations.host.buckets.0.key}}",
"body": {
"html": "{{ctx.payload.aggregations.host.buckets.0.mpoint.buckets.0.key}} reached {{ctx.payload.aggregations.host.buckets.0.pct.value}}%"
}
}
}
}
}

2

And the results are ,

`{

"watch_id": "inlined",
"node": "Bd-EWXLQTbKMG-oCTDvGhQ",
"state": "executed",
"status": {
"state": {
"active": true,
"timestamp": "2018-06-28T20:38:31.505Z"
},
"last_checked": "2018-06-28T20:38:31.505Z",
"last_met_condition": "2018-06-28T20:38:31.505Z",
"actions": {
"email_1": {
"ack": {
"timestamp": "2018-06-28T20:38:31.505Z",
"state": "ackable"
},
"last_execution": {
"timestamp": "2018-06-28T20:38:31.505Z",
"successful": true
},
"last_successful_execution": {
"timestamp": "2018-06-28T20:38:31.505Z",
"successful": true
}
}
},
"execution_state": "executed",
"version": -1
},
"trigger_event": {
"type": "manual",
"triggered_time": "2018-06-28T20:38:31.505Z",
"manual": {
"schedule": {
"scheduled_time": "2018-06-28T20:38:31.505Z"
}
}
},
"input": {
"search": {
"request": {
"search_type": "query_then_fetch",
"indices": [
"metricbeat-*"
],
"types": [],
"body": {
"size": 0,
"query": {
"bool": {
"must": [
{
"exists": {
"field": "system.filesystem.used.pct"
}
},
{
"range": {
"@timestamp": {
"gte": "now-15m",
"lte": "now",
"format": "epoch_millis"
}
}
}
]
}
},
"aggs": {
"host": {
"terms": {
"field": "beat.hostname",
"size": 20,
"order": {
"pct": "desc"
}
},
"aggs": {
"pct": {
"max": {
"field": "system.filesystem.used.pct",
"script": {
"source": "doc['system.filesystem.used.pct'].value *100",
"lang": "painless"
}
}
},
"mpoint": {
"terms": {
"field": "system.filesystem.mount_point",
"size": 5,
"order": {
"pct": "desc"
}
},
"aggs": {
"pct": {
"max": {
"field": "system.filesystem.used.pct"
}
}
}
}
}
}
}
}
}
}
},
"condition": {
"compare": {
"ctx.payload.aggregations.host.buckets.0.pct.value": {
"gt": 75
}
}
},
"metadata": {
"xpack": {
"type": "json"
}
},
"result": {
"execution_time": "2018-06-28T20:38:31.505Z",
"execution_duration": 165,
"input": {
"type": "search",
"status": "success",
"payload": {
"_shards": {
"total": 186,
"failed": 0,
"successful": 186,
"skipped": 180
},
"hits": {
"hits": [],
"total": 644,
"max_score": 0
},
"took": 163,
"timed_out": false,
"aggregations": {
"host": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [
{
"pct": {
"value": 90.4
},
"doc_count": 55,
"mpoint": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [
{
"pct": {
"value": 0.904
},
"doc_count": 11,
"key": "/opt/data2"
},
{
"pct": {
"value": 0.88
},
"doc_count": 11,
"key": "/opt/data"
},
{
"pct": {
"value": 0.709
},
"doc_count": 11,
"key": "/opt/data3"
},
{
"pct": {
"value": 0.618
},
"doc_count": 11,
"key": "/"
},
{
"pct": {
"value": 0.225
},
"doc_count": 11,
"key": "/opt/data4"
}
]
},
"key": "Server-1"
},
{
"pct": {
"value": 73.10000000000001
},
"doc_count": 15,
"mpoint": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [
{
"pct": {
"value": 0.7310000000000001
},
"doc_count": 15,
"key": "A:\"
}
]
},
"key": "server - 2"
}
]
}
}
},

3

Continuation of results

  `   "search": {
    
"request": {
      "search_type": "query_then_fetch",
      "indices": [
        "metricbeat-*"
      ],
      "types": [],
      "body": {
        "size": 0,
        "query": {
          "bool": {
            "must": [
              {
                "exists": {
                  "field": "system.filesystem.used.pct"
                }
              },
              {
                "range": {
                  "@timestamp": {
                    "gte": "now-15m",
                    "lte": "now",
                    "format": "epoch_millis"
                  }
                }
              }
            ]
          }
        },
        "aggs": {
          "host": {
            "terms": {
              "field": "beat.hostname",
              "size": 20,
              "order": {
                "pct": "desc"
              }
            },
            "aggs": {
              "pct": {
                "max": {
                  "field": "system.filesystem.used.pct",
                  "script": {
                    "source": "doc['system.filesystem.used.pct'].value *100",
                    "lang": "painless"
                  }
                }
              },
              "mpoint": {
                "terms": {
                  "field": "system.filesystem.mount_point",
                  "size": 5,
                  "order": {
                    "pct": "desc"
                  }
                },
                "aggs": {
                  "pct": {
                    "max": {
                      "field": "system.filesystem.used.pct"
                    }
                  }
                }
              }
            }
          }
        }
      }
    }
  }
},
"condition": {
  "type": "compare",
  "status": "success",
  "met": true,
  "compare": {
    "resolved_values": {
      "ctx.payload.aggregations.host.buckets.0.pct.value": 90.4
    }
  }
},
"actions": [
  {
    "id": "email_1",
    "type": "email",
    "status": "simulated",
    "email": {
      "message": {
        "id": "_inlined__f13ff6e7-56aa-426d-beb8-c69a328b70e6-2018-06-28T20:38:31.505Z",
        "priority": "high",
        "sent_date": "2018-06-28T20:38:31.670Z",
        "to": [
          "rpendela@creditshop.com"
        ],
        "subject": "Disk space on <b>A:\\</b> reached <b>90.4%</b> on server "server-1",
        "body": {
          "text": "test"
        }
      }
    }
  }
]

},
"messages": []
}`

I want to see the results as

Disk space on disk A:\ reached to 90.4% on server "server-1"
Disk space on disk B:\ reached to 88.0% on server "server-1"
Disk space on disk A:\ reached to 73.0% on server "server-2"
Disk space on disk C:\ reached to 85.8% on server "server-3"`

4

please format all your messages properly using markdown, those snippets above are super hard to read and reduce the chances of getting help.

try this as a starting point, this also shows how to keep debugging things better using the execute watch API and also to reduce the feedback loop how much sense your watch makes. See this blog post for more information

POST _xpack/watcher/watch/_execute
{
  "alternative_input": {
    "_shards": {
      "total": 186,
      "failed": 0,
      "successful": 186,
      "skipped": 180
    },
    "hits": {
      "hits": [],
      "total": 644,
      "max_score": 0
    },
    "took": 163,
    "timed_out": false,
    "aggregations": {
      "host": {
        "doc_count_error_upper_bound": 0,
        "sum_other_doc_count": 0,
        "buckets": [
          {
            "pct": {
              "value": 90.4
            },
            "doc_count": 55,
            "mpoint": {
              "doc_count_error_upper_bound": 0,
              "sum_other_doc_count": 0,
              "buckets": [
                {
                  "pct": {
                    "value": 0.904
                  },
                  "doc_count": 11,
                  "key": "/opt/data2"
                },
                {
                  "pct": {
                    "value": 0.88
                  },
                  "doc_count": 11,
                  "key": "/opt/data"
                },
                {
                  "pct": {
                    "value": 0.709
                  },
                  "doc_count": 11,
                  "key": "/opt/data3"
                },
                {
                  "pct": {
                    "value": 0.618
                  },
                  "doc_count": 11,
                  "key": "/"
                },
                {
                  "pct": {
                    "value": 0.225
                  },
                  "doc_count": 11,
                  "key": "/opt/data4"
                }
              ]
            },
            "key": "Server-1"
          }
        ]
      }
    }
  },
  "watch": {
    "trigger": {
      "schedule": {
        "interval": "10h"
      }
    },
    "input": {
      "simple": {
        "foo": "bar"
      }
    },
    "actions": {
      "logme": {
        "transform" : {
          "script" : "def hosts = []; ctx.payload.aggregations.host.buckets.stream().forEach(b -> b.mpoint.buckets.stream().forEach(b2 -> hosts.add(['host':b.key, 'mountpoint':b2.key, 'size': b2.pct.value*100.0]))) ; return ['hosts': hosts]"
        },
        "logging": {
          "text": "{{#ctx.payload.hosts}}{{mountpoint}} on {{host}} has {{size}}{{/ctx.payload.hosts}}\n"
        }
      }
    }
  }
}