Alert for disk space on server with multiple disks

Elastic Stack Elasticsearch
1

Hello Community,

I working to get an alert for diskspace over 75% on server which have multiple drives (A, B, C, D). I having difficulty to in compare condition with multiple drives in a bucket. I hope below snippet of code and results will give some idea
{
"trigger": {
"schedule": {
"interval": "1m"
}
},
"input": {
"search": {
"request": {
"search_type": "query_then_fetch",
"indices": [
"metricbeat-*"
],
"types": [],
"body": {
"size": 0,
"query": {
"bool": {
"must": [
{
"exists": {
"field": "system.filesystem.used.pct"
}
},
{
"range": {
"@timestamp": {
"gte": "now-15m",
"lte": "now",
"format": "epoch_millis"
}
}
}
]
}
},
"aggs": {
"host": {
"terms": {
"field": "beat.hostname",
"size": 20,
"order": {
"pct": "desc"
}
},
"aggs": {
"pct": {
"max": {
"field": "system.filesystem.used.pct",
"script": {
"source": "doc['system.filesystem.used.pct'].value *100",
"lang": "painless"
}
}
},
"mpoint": {
"terms": {
"field": "system.filesystem.mount_point",
"size": 5,
"order": {
"pct": "desc"
}
},
"aggs": {
"pct": {
"max": {
"field": "system.filesystem.used.pct"
}
}
}
}
}
}
}
}
}
}
},
"condition": {
"script": {
"source": "ArrayList arr = ctx.payload.aggregations.bucketAgg.buckets; for (int i = 0; i < arr.length; i++) { if (arr[i]['pct'].value > params.threshold) { return true; } } return false;",
"lang": "painless",
"params": {
"threshold": 0.75
}
}
},
"actions": {
"email_1": {
"email": {
"profile": "standard",
"priority": "high",
"to": [
"test@test.com"
],
"subject": "High FS Usage on {{ctx.payload.aggregations.host.buckets.0.key}}",
"body": {
"html": "{{ctx.payload.aggregations.host.buckets.0.mpoint.buckets.0.key}} reached {{ctx.payload.aggregations.host.buckets.0.pct.value}}%"
}
}
}
}
}

2

And the results are ,

`{

"watch_id": "inlined",
"node": "Bd-EWXLQTbKMG-oCTDvGhQ",
"state": "executed",
"status": {
"state": {
"active": true,
"timestamp": "2018-06-28T20:38:31.505Z"
},
"last_checked": "2018-06-28T20:38:31.505Z",
"last_met_condition": "2018-06-28T20:38:31.505Z",
"actions": {
"email_1": {
"ack": {
"timestamp": "2018-06-28T20:38:31.505Z",
"state": "ackable"
},
"last_execution": {
"timestamp": "2018-06-28T20:38:31.505Z",
"successful": true
},
"last_successful_execution": {
"timestamp": "2018-06-28T20:38:31.505Z",
"successful": true
}
}
},
"execution_state": "executed",
"version": -1
},
"trigger_event": {
"type": "manual",
"triggered_time": "2018-06-28T20:38:31.505Z",
"manual": {
"schedule": {
"scheduled_time": "2018-06-28T20:38:31.505Z"
}
}
},
"input": {
"search": {
"request": {
"search_type": "query_then_fetch",
"indices": [
"metricbeat-*"
],
"types": [],
"body": {
"size": 0,
"query": {
"bool": {
"must": [
{
"exists": {
"field": "system.filesystem.used.pct"
}
},
{
"range": {
"@timestamp": {
"gte": "now-15m",
"lte": "now",
"format": "epoch_millis"
}
}
}
]
}
},
"aggs": {
"host": {
"terms": {
"field": "beat.hostname",
"size": 20,
"order": {
"pct": "desc"
}
},
"aggs": {
"pct": {
"max": {
"field": "system.filesystem.used.pct",
"script": {
"source": "doc['system.filesystem.used.pct'].value *100",
"lang": "painless"
}
}
},
"mpoint": {
"terms": {
"field": "system.filesystem.mount_point",
"size": 5,
"order": {
"pct": "desc"
}
},
"aggs": {
"pct": {
"max": {
"field": "system.filesystem.used.pct"
}
}
}
}
}
}
}
}
}
}
},
"condition": {
"compare": {
"ctx.payload.aggregations.host.buckets.0.pct.value": {
"gt": 75
}
}
},
"metadata": {
"xpack": {
"type": "json"
}
},
"result": {
"execution_time": "2018-06-28T20:38:31.505Z",
"execution_duration": 165,
"input": {
"type": "search",
"status": "success",
"payload": {
"_shards": {
"total": 186,
"failed": 0,
"successful": 186,
"skipped": 180
},
"hits": {
"hits": [],
"total": 644,
"max_score": 0
},
"took": 163,
"timed_out": false,
"aggregations": {
"host": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [
{
"pct": {
"value": 90.4
},
"doc_count": 55,
"mpoint": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [
{
"pct": {
"value": 0.904
},
"doc_count": 11,
"key": "/opt/data2"
},
{
"pct": {
"value": 0.88
},
"doc_count": 11,
"key": "/opt/data"
},
{
"pct": {
"value": 0.709
},
"doc_count": 11,
"key": "/opt/data3"
},
{
"pct": {
"value": 0.618
},
"doc_count": 11,
"key": "/"
},
{
"pct": {
"value": 0.225
},
"doc_count": 11,
"key": "/opt/data4"
}
]
},
"key": "Server-1"
},
{
"pct": {
"value": 73.10000000000001
},
"doc_count": 15,
"mpoint": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [
{
"pct": {
"value": 0.7310000000000001
},
"doc_count": 15,
"key": "A:\"
}
]
},
"key": "server - 2"
}
]
}
}
},

3

Continuation of results

  `   "search": {
    
"request": {
      "search_type": "query_then_fetch",
      "indices": [
        "metricbeat-*"
      ],
      "types": [],
      "body": {
        "size": 0,
        "query": {
          "bool": {
            "must": [
              {
                "exists": {
                  "field": "system.filesystem.used.pct"
                }
              },
              {
                "range": {
                  "@timestamp": {
                    "gte": "now-15m",
                    "lte": "now",
                    "format": "epoch_millis"
                  }
                }
              }
            ]
          }
        },
        "aggs": {
          "host": {
            "terms": {
              "field": "beat.hostname",
              "size": 20,
              "order": {
                "pct": "desc"
              }
            },
            "aggs": {
              "pct": {
                "max": {
                  "field": "system.filesystem.used.pct",
                  "script": {
                    "source": "doc['system.filesystem.used.pct'].value *100",
                    "lang": "painless"
                  }
                }
              },
              "mpoint": {
                "terms": {
                  "field": "system.filesystem.mount_point",
                  "size": 5,
                  "order": {
                    "pct": "desc"
                  }
                },
                "aggs": {
                  "pct": {
                    "max": {
                      "field": "system.filesystem.used.pct"
                    }
                  }
                }
              }
            }
          }
        }
      }
    }
  }
},
"condition": {
  "type": "compare",
  "status": "success",
  "met": true,
  "compare": {
    "resolved_values": {
      "ctx.payload.aggregations.host.buckets.0.pct.value": 90.4
    }
  }
},
"actions": [
  {
    "id": "email_1",
    "type": "email",
    "status": "simulated",
    "email": {
      "message": {
        "id": "_inlined__f13ff6e7-56aa-426d-beb8-c69a328b70e6-2018-06-28T20:38:31.505Z",
        "priority": "high",
        "sent_date": "2018-06-28T20:38:31.670Z",
        "to": [
          "rpendela@creditshop.com"
        ],
        "subject": "Disk space on <b>A:\\</b> reached <b>90.4%</b> on server "server-1",
        "body": {
          "text": "test"
        }
      }
    }
  }
]

},
"messages": []
}`

I want to see the results as

Disk space on disk A:\ reached to 90.4% on server "server-1"
Disk space on disk B:\ reached to 88.0% on server "server-1"
Disk space on disk A:\ reached to 73.0% on server "server-2"
Disk space on disk C:\ reached to 85.8% on server "server-3"`

4

please format all your messages properly using markdown, those snippets above are super hard to read and reduce the chances of getting help.

try this as a starting point, this also shows how to keep debugging things better using the execute watch API and also to reduce the feedback loop how much sense your watch makes. See this blog post for more information

POST _xpack/watcher/watch/_execute
{
  "alternative_input": {
    "_shards": {
      "total": 186,
      "failed": 0,
      "successful": 186,
      "skipped": 180
    },
    "hits": {
      "hits": [],
      "total": 644,
      "max_score": 0
    },
    "took": 163,
    "timed_out": false,
    "aggregations": {
      "host": {
        "doc_count_error_upper_bound": 0,
        "sum_other_doc_count": 0,
        "buckets": [
          {
            "pct": {
              "value": 90.4
            },
            "doc_count": 55,
            "mpoint": {
              "doc_count_error_upper_bound": 0,
              "sum_other_doc_count": 0,
              "buckets": [
                {
                  "pct": {
                    "value": 0.904
                  },
                  "doc_count": 11,
                  "key": "/opt/data2"
                },
                {
                  "pct": {
                    "value": 0.88
                  },
                  "doc_count": 11,
                  "key": "/opt/data"
                },
                {
                  "pct": {
                    "value": 0.709
                  },
                  "doc_count": 11,
                  "key": "/opt/data3"
                },
                {
                  "pct": {
                    "value": 0.618
                  },
                  "doc_count": 11,
                  "key": "/"
                },
                {
                  "pct": {
                    "value": 0.225
                  },
                  "doc_count": 11,
                  "key": "/opt/data4"
                }
              ]
            },
            "key": "Server-1"
          }
        ]
      }
    }
  },
  "watch": {
    "trigger": {
      "schedule": {
        "interval": "10h"
      }
    },
    "input": {
      "simple": {
        "foo": "bar"
      }
    },
    "actions": {
      "logme": {
        "transform" : {
          "script" : "def hosts = []; ctx.payload.aggregations.host.buckets.stream().forEach(b -> b.mpoint.buckets.stream().forEach(b2 -> hosts.add(['host':b.key, 'mountpoint':b2.key, 'size': b2.pct.value*100.0]))) ; return ['hosts': hosts]"
        },
        "logging": {
          "text": "{{#ctx.payload.hosts}}{{mountpoint}} on {{host}} has {{size}}{{/ctx.payload.hosts}}\n"
        }
      }
    }
  }
}
5

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.