# WATCHER SCRIPTS

**URL:** <https://discuss.elastic.co/t/watcher-scripts/373850>\
**Category:** Elasticsearch\
**Tags:** painless\
**Created:** [January 29, 2025, 4:43pm UTC](https://discuss.elastic.co/t/watcher-scripts/373850 "2025-01-29T16:43:22Z")\
**Posts on this page:** 1\
**Page:** 1

<div class="post-metadata">

**Author:** ![PRASHANT\_MEHTA](https://sea2.discourse-cdn.com/elastic/user_avatar/discuss.elastic.co/prashant_mehta/32/101764_2.png) [@PRASHANT\_MEHTA](https://discuss.elastic.co/u/PRASHANT_MEHTA)\
**Post date:** [January 29, 2025, 4:43pm UTC](https://discuss.elastic.co/t/watcher-scripts/373850/1 "2025-01-29T16:43:22Z")

</div>

```auto
TOMCAT ALERT COUNT

PUT _watcher/watch/tomcat_error_count_alerts
{
  "trigger": {
    "schedule": {
      "interval": "1h"
    }
  },
  "input": {
    "search": {
      "request": {
        "search_type": "query_then_fetch",
        "indices": [
          "mis-monitoring-webserver-*"
        ],
        "rest_total_hits_as_int": true,
        "body": {
          "aggs": {
            "host": {
              "terms": {
                "field": "tomcat.host_name.keyword",
                "order": {
                  "_key": "desc"
                },
                "size": 10000
              },
              "aggs": {
                "application_name": {
                  "terms": {
                    "field": "tomcat.application_name",
                    "order": {
                      "_key": "desc"
                    },
                    "size": 10000
                  },
                  "aggs": {
                    "port": {
                      "terms": {
                        "field": "tomcat.port",
                        "order": {
                          "_key": "desc"
                        },
                        "size": 10000
                      },
                      "aggs": {
                        "error_count": {
                          "top_hits": {
                            "_source": [
                              "tomcat.error_count"
                            ],
                            "size": 1,
                            "sort": [
                              {
                                "@timestamp": {
                                  "order": "desc"
                                }
                              }
                            ]
                          }
                        }
                      }
                    }
                  }
                }
              }
            }
          },
          "size": 0,
          "query": {
            "bool": {
              "must": [],
              "filter": [
                {
                  "bool": {
                    "should": [
                      {
                        "term": {
                          "tomcat.error_status": {
                            "value": "Critical"
                          }
                        }
                      }
                    ],
                    "minimum_should_match": 1
                  }
                },
                {
                  "range": {
                    "@timestamp": {
                      "gte": "now-15m",
                      "lte": "now"
                    }
                  }
                }
              ]
            }
          }
        }
      }
    }
  },
  "condition": {
    "compare": {
      "ctx.payload.hits.total": {
        "gt": 0
      }
    }
  },
  "actions": {
    "send_email": {
      "email": {
        "profile": "standard",
        "to": [
          "prashant.mehta@dell.com"
        ],
        "subject": "Tomcat Critical Error Count Status",
        "body": {
          "html": "\n<html>\n <body>\n <h2>Tomcat Critical Error Count:</h2>\n {{#ctx.payload.aggregations.host.buckets}}\n <p><b>Summary:</b> Tomcat is critical for error count on server: {{key}}</p>\n <p><b>Date and Time:</b> {{ctx.trigger.scheduled_time}}</p>\n <p><b>Description:</b> Tomcat error count for below mentioned port and application, Please investigate further details on server.</p>\n <ul>\n {{#application_name.buckets}}\n <li><b>Application:</b> {{key}}\n <ul>\n {{#port.buckets}}\n <li><b>Port:</b> {{key}}, <b>Error Count:</b> {{error_count.hits.hits.0._source.tomcat.error_count}}</li>\n {{/port.buckets}}\n </ul>\n </li>\n {{/application_name.buckets}}\n </ul>\n <p><b>Issued By:</b> CIS Monitoring System</p>\n <hr />\n {{/ctx.payload.aggregations.host.buckets}}\n </body>\n</html>\n"
        }
      }
    }
  },
  "metadata": {
    "xpack": {
      "type": "json"
    },
    "name": "TOMCAT ERROR COUNT ALERTS"
  }
}

```

```auto
TOMCAT ERROR COUNT ALERTS

PUT _watcher/watch/tomcat_error_count_alerts
{
  "trigger": {
    "schedule": {
      "interval": "1h"
    }
  },
  "input": {
    "search": {
      "request": {
        "search_type": "query_then_fetch",
        "indices": [
          "mis-monitoring-webserver-*"
        ],
        "rest_total_hits_as_int": true,
        "body": {
          "aggs": {
            "host": {
              "terms": {
                "field": "tomcat.host_name.keyword",
                "order": {
                  "_key": "desc"
                },
                "size": 10000
              },
              "aggs": {
                "application_name": {
                  "terms": {
                    "field": "tomcat.application_name",
                    "order": {
                      "_key": "desc"
                    },
                    "size": 10000
                  },
                  "aggs": {
                    "port": {
                      "terms": {
                        "field": "tomcat.port",
                        "order": {
                          "_key": "desc"
                        },
                        "size": 10000
                      },
                      "aggs": {
                        "error_count": {
                          "top_hits": {
                            "_source": [
                              "tomcat.error_count"
                            ],
                            "size": 1,
                            "sort": [
                              {
                                "@timestamp": {
                                  "order": "desc"
                                }
                              }
                            ]
                          }
                        }
                      }
                    }
                  }
                }
              }
            }
          },
          "size": 0,
          "query": {
            "bool": {
              "must": [],
              "filter": [
                {
                  "bool": {
                    "should": [
                      {
                        "term": {
                          "tomcat.error_status": {
                            "value": "Critical"
                          }
                        }
                      }
                    ],
                    "minimum_should_match": 1
                  }
                },
                {
                  "range": {
                    "@timestamp": {
                      "gte": "now-15m",
                      "lte": "now"
                    }
                  }
                }
              ]
            }
          }
        }
      }
    }
  },
  "condition": {
    "compare": {
      "ctx.payload.hits.total": {
        "gt": 0
      }
    }
  },
  "actions": {
    "send_email": {
      "email": {
        "profile": "standard",
        "to": [
          "prashant.mehta@dell.com"
        ],
        "subject": "Tomcat Critical Error Count Status",
        "body": {
          "html": "\n<html>\n <body>\n <h2>Tomcat Critical Error Count:</h2>\n {{#ctx.payload.aggregations.host.buckets}}\n <p><b>Summary:</b> Tomcat is critical for error count on server: {{key}}</p>\n <p><b>Date and Time:</b> {{ctx.trigger.scheduled_time}}</p>\n <p><b>Description:</b> Tomcat error count for below mentioned port and application, Please investigate further details on server.</p>\n <ul>\n {{#application_name.buckets}}\n <li><b>Application:</b> {{key}}\n <ul>\n {{#port.buckets}}\n <li><b>Port:</b> {{key}}, <b>Error Count:</b> {{error_count.hits.hits.0._source.tomcat.error_count}}</li>\n {{/port.buckets}}\n </ul>\n </li>\n {{/application_name.buckets}}\n </ul>\n <p><b>Issued By:</b> CIS Monitoring System</p>\n <hr />\n {{/ctx.payload.aggregations.host.buckets}}\n </body>\n</html>\n"
        }
      }
    }
  },
  "metadata": {
    "xpack": {
      "type": "json"
    },
    "name": "TOMCAT ERROR COUNT ALERTS"
  }
}

```

```auto
MEMORY USAGE ALERTS

PUT _watcher/watch/memory_usage_alerts
{
  "trigger": {
    "schedule": {
      "interval": "1h"
    }
  },
  "input": {
    "search": {
      "request": {
        "search_type": "query_then_fetch",
        "indices": [
          "cis-metric"
        ],
        "rest_total_hits_as_int": true,
        "body": {
          "aggs": {
            "host": {
              "terms": {
                "field": "host.name.keyword",
                "size": "1000",
                "order": {
                  "memory_used_pct": "desc"
                }
              },
              "aggs": {
                "memory_used_pct": {
                  "avg": {
                    "field": "system.memory.actual.used.pct"
                  }
                }
              }
            }
          },
          "size": 0,
          "query": {
            "bool": {
              "must": [],
              "filter": [
                {
                  "range": {
                    "@timestamp": {
                      "gte": "now-30m",
                      "lte": "now"
                    }
                  }
                }
              ],
              "should": [],
              "must_not": []
            }
          }
        }
      }
    }
  },
  "condition": {
    "script": {
      "source": "ArrayList arr = ctx.payload.aggregations.host.buckets; for (int i = 0; i < arr.length; i++) { if (arr[i]['memory_used_pct'].value > params.threshold) {return true; }} return false;",
      "lang": "painless",
      "params": {
        "threshold": 0.85
      }
    }
  },
  "actions": {
    "email_1": {
      "email": {
        "profile": "standard",
        "to": ["REPLACE_NOTIFICATION_ADDRESS"],
        "subject": "System Memory Critical Alert",
        "body": {
          "html": "{{#ctx.payload._value}}\n<p><b>Summary:</b> {{summary}}</p>\n<p><b>Date and Time:</b> {{date_and_time}}</p>\n<p><b>Memory Usage:</b> {{memory_usage}}</p>\n<p><b>Description:</b> {{description}}</p>\n<p><b>Issued By:</b> {{issued_by}}</p>\n<hr />\n{{/ctx.payload._value}}\n\n "
        }
      }
    }
  },
  "transform": {
    "script": {
      "source": "\n ArrayList arr = ctx.payload.aggregations.host.buckets;\n ArrayList results = new ArrayList();\n \n for (int i = 0; i < arr.length; i++) {\n double memoryUsedPct = arr[i]['memory_used_pct'].value * 100.00;\n if (memoryUsedPct > params.threshold) {\n def serverAlert = [\n 'summary': \"System memory critical alert on server \" + arr[i].key,\n 'date_and_time': ctx.trigger.scheduled_time,\n 'memory_usage': memoryUsedPct,\n 'description': \"The system memory usage is reached to \" + memoryUsedPct + \"%. Hence, the server performance will be downgraded. Please investigate further details on the server.\",\n 'issued_by': \"CIS Monitoring System\"\n ];\n results.add(serverAlert);\n }\n }\n return results;\n ",
      "lang": "painless",
      "params": {
        "threshold": 85
      }
    }
  },
  "metadata": {
    "xpack": {
      "type": "json"
    },
    "name": "MEMORY USAGE ABOVE 85%"
  }
}

```

```auto
DISK SPACE ALERTS

PUT _watcher/watch/disk_space_alerts
{
  "trigger": {
    "schedule": {
      "interval": "1h"
    }
  },
  "input": {
    "search": {
      "request": {
        "search_type": "query_then_fetch",
        "indices": [
          "cis-metric"
        ],
        "rest_total_hits_as_int": true,
        "body": {
          "aggs": {
            "host": {
              "terms": {
                "field": "host.name.keyword",
                "order": {
                  "_key": "desc"
                },
                "size": 10000
              },
              "aggs": {
                "mount_point": {
                  "terms": {
                    "field": "system.filesystem.mount_point.keyword",
                    "order": {
                      "_key": "desc"
                    },
                    "size": 10000
                  },
                  "aggs": {
                    "disk_usage_pct": {
                      "max": {
                        "field": "system.filesystem.used.pct"
                      }
                    }
                  }
                }
              }
            }
          },
          "size": 0,
          "query": {
            "bool": {
              "must": [],
              "filter": [
                {
                  "range": {
                    "@timestamp": {
                      "gte": "now-30m",
                      "lte": "now"
                    }
                  }
                }
              ],
              "should": [],
              "must_not": []
            }
          }
        }
      }
    }
  },
  "condition": {
    "script": {
      "source": "ArrayList arr = ctx.payload.aggregations.host.buckets; for (int i = 0; i < arr.length; i++) { ArrayList arr2 = arr[i].mount_point.buckets;for (int j = 0; j < arr2.length; j++){if (arr2[j]['disk_usage_pct'].value > params.threshold) { return true; }}} return false;",
      "lang": "painless",
      "params": {
        "threshold": 0.85
      }
    }
  },
  "actions": {
    "email_1": {
      "email": {
        "profile": "standard",
        "to": ["REPLACE_NOTIFICATION_ADDRESS"],
        "subject": "System Disk Space Alert",
        "body": {
          "html": "{{#ctx.payload._value}}\n<p><b>Summary:</b> {{summary}}</p>\n<p><b>Date and Time:</b> {{date_and_time}}</p>\n<p><b>Mount Point:</b> {{mount_point}}</p>\n<p><b>Description:</b> The mountpoint usage is reached to {{disk_usage_pct}}%. Hence, the server performance will be downgraded. Please investigate further details in the server.</p>\n<p><b>Issued By:</b> CIS Monitoring System</p>\n<hr />\n{{/ctx.payload._value}}"
        }
      }
    }
  },
  "transform": {
    "script": {
      "source": "\n ArrayList arr = ctx.payload.aggregations.host.buckets;\n ArrayList results = new ArrayList();\n \n for (int i = 0; i < arr.length; i++) {\n ArrayList arr2 = arr[i].mount_point.buckets;\n for (int j = 0; j < arr2.length; j++) {\n double diskUsagePct = arr2[j]['disk_usage_pct'].value * 100.00;\n if (diskUsagePct > params.threshold) {\n def serverAlert = [\n 'summary': \"Disk space critical alert on server \" + arr[i].key,\n 'date_and_time': ctx.trigger.scheduled_time,\n 'mount_point': arr2[j].key,\n 'disk_usage_pct': diskUsagePct,\n 'description': \"The mountpoint usage is reached to \" + diskUsagePct + \"%. Hence, the server performance will be downgraded. Please investigate further details in the server.\",\n 'issued_by': \"CIS Monitoring System\"\n ];\n results.add(serverAlert);\n }\n }\n }\n \n return results;\n ",
      "lang": "painless",
      "params": {
        "threshold": 85
      }
    }
  },
  "metadata": {
    "xpack": {
      "type": "json"
    },
    "name": "DISK SPACE ABOVE 85%"
  }
}

```

```auto
CPU UTILIZATION
PUT _watcher/watch/cpu_utilization_alerts
{
  "trigger": {
    "schedule": {
      "interval": "1h"
    }
  },
  "input": {
    "search": {
      "request": {
        "search_type": "query_then_fetch",
        "indices": [
          "cis-metric"
        ],
        "rest_total_hits_as_int": true,
        "body": {
          "aggs": {
            "host": {
              "terms": {
                "field": "host.name.keyword",
                "size": 1000,
                "order": {
                  "cpu_usage_pct": "desc"
                }
              },
              "aggs": {
                "cpu_usage_pct": {
                  "avg": {
                    "field": "host.cpu.usage"
                  }
                }
              }
            }
          },
          "size": 0,
          "query": {
            "bool": {
              "filter": [
                {
                  "range": {
                    "@timestamp": {
                      "gte": "now-30m",
                      "lte": "now"
                    }
                  }
                }
              ]
            }
          }
        }
      }
    }
  },
  "condition": {
    "script": {
      "source": "\n ArrayList arr = ctx.payload.aggregations.host.buckets;\n for (int i = 0; i < arr.length; i++) {\n if (arr[i]['cpu_usage_pct'].value > params.threshold) {\n return true;\n }\n }\n return false;\n ",
      "lang": "painless",
      "params": {
        "threshold": 0.85
      }
    }
  },
  "actions": {
    "email_1": {
      "email": {
        "profile": "standard",
        "to": ["REPLACE_NOTIFICATION_ADDRESS"],
        "subject": "CPU Usage Critical Alert",
        "body": {
          "html": "\n {{#ctx.payload._value}}\n <p><b>Summary:</b> {{summary}}</p>\n <p><b>Date and Time:</b> {{date_and_time}}</p>\n <p><b>CPU Usage:</b> {{cpu_usage}}%</p>\n <p><b>Description:</b> {{description}}</p>\n <p><b>Issued By:</b> {{issued_by}}</p>\n <hr />\n {{/ctx.payload._value}}\n "
        }
      }
    }
  },
  "transform": {
    "script": {
      "source": "\n ArrayList arr = ctx.payload.aggregations.host.buckets;\n ArrayList results = new ArrayList();\n \n for (int i = 0; i < arr.length; i++) {\n double cpuUsagePct = arr[i]['cpu_usage_pct'].value * 100.00;\n if (cpuUsagePct > params.threshold) {\n def serverAlert = [\n 'summary': \"CPU usage critical alert on server \" + arr[i].key,\n 'date_and_time': ctx.trigger.scheduled_time,\n 'cpu_usage': cpuUsagePct,\n 'description': \"The CPU usage is reached to \" + cpuUsagePct + \"%. Hence, the server performance will be downgraded. Please investigate further details on the server.\",\n 'issued_by': \"CIS Monitoring System\"\n ];\n results.add(serverAlert);\n }\n }\n \n return results;\n ",
      "lang": "painless",
      "params": {
        "threshold": 85
      }
    }
  },
  "metadata": {
    "xpack": {
      "type": "json"
    },
    "name": "CPU USAGE ABOVE 85%"
  }
}

```