{
"trigger": {
"schedule": {
"interval": "30m"
}
},
"input": {
"chain": {
"inputs": [
{
"first": {
"search": {
"request": {
"search_type": "query_then_fetch",
"indices": [
"my_logs"
],
"rest_total_hits_as_int": true,
"body": {
"query": {
"bool": {
"filter": [
{
"range": {
"@timestamp": {
"from": "now-1h",
"to": "now"
}
}
}
]
}
},
"aggs": {
"inst_name": {
"terms": {
"field": "inst_name.keyword",
"size": 10000
},
"aggs": {
"messages": {
"filters": {
"filters": {
"failure": {
"match": {
"status": "0"
},
"siteError": {
"match": {
"error_code.keyword": "SITE"
}
}
}
}
}
}
}
}
}
}
}
}
}
},
{
"second": {
"search": {
"request": {
"search_type": "query_then_fetch",
"indices": [
"my_logs"
],
"rest_total_hits_as_int": true,
"body": {
"query": {
"bool": {
"filter": [
{
"range": {
"@timestamp": {
"from": "now-30m",
"to": "now"
}
}
}
]
}
},
"aggs": {
"inst_name": {
"terms": {
"field": "inst_name.keyword",
"size": 10000
},
"aggs": {
"messages": {
"filters": {
"filters": {
"failure": {
"match": {
"status": "0"
}
},
"siteError": {
"match": {
"error_code.keyword": "SITE"
}
}
}
}
}
}
}
}
}
}
}
}
}
]
}
},
"condition": {
"script": {
"source": """
def message = "";
for (first_org_item in ctx.payload.first.aggregations.inst_name.buckets) {
def first_org_name = first_org_item.key;
def first_total = first_org_item.doc_count;
def first_failure_count = first_org_item.messages.buckets.failure.doc_count;
def first_fail_per = Math.round( first_failure_count * 100 / first_total);
def first_siterr_count = first_org_item.messages.buckets.siteError.doc_count;
def first_site_per = Math.round( first_siterr_count * 100 / first_total);
for (sec_org_item in ctx.payload.second.aggregations.inst_name.buckets) {
def sec_org_name = sec_org_item.key;
def sec_total = sec_org_item.doc_count;
def sec_failure_count = sec_org_item.messages.buckets.failure.doc_count;
def sec_fail_per = Math.round( sec_failure_count * 100 / sec_total);
def sec_siterr_count = sec_org_item.messages.buckets.siteError.doc_count;
def sec_site_per = Math.round( sec_siterr_count * 100 / sec_total);
if (first_org_name == sec_org_name) {
if (sec_fail_per > first_fail_per ) {
def diff = sec_fail_per - first_fail_per;
message += "Error [Alert]:" + first_org_name +" Institution Overall failure percentage increased by " + diff +"% in last 30mins" +(String)(char)0x0a;
}
if (sec_site_per > first_site_per) {
def s_diff = sec_site_per - first_site_per;
message += "Error [Alert]:" + first_org_name +" Institution Site failure percentage increased by " + s_diff +"% in last 30mins" +(String)(char)0x0a;
}
}
}
}
if(message.equals("")){
return false;
}
else{
ctx.payload["result"] = message;
return true;
}
""",
"lang": "painless"
}
},
"actions": {
"slack_webhook": {
"webhook": {
"scheme": "https",
"host": "hooks.slack.com",
"port": 443,
"method": "post",
"path": "/services/T1LHPRL20/B02NASMCLU8/bKM7Pd54WctoZf2uRi4oCWlN",
"params": {
"watch_id": "{{ctx.watch_id}}"
},
"headers": {
"Content-Type": "application/json"
},
"body": """__json__::{"text": "{{ctx.payload.result}}"}"""
}
}
}
}
Error is
"actions": []
},
"exception": {
"type": "script_exception",
"reason": "runtime error",
"script_stack": [
"for (first_org_item in ctx.payload.first.aggregations.inst_name.buckets) {\n def ",
" ^---- HERE"
],
"script": " ...",
"lang": "painless",
"position": {
"offset": 88,
"start": 35,
"end": 126
},
"caused_by": {
"type": "null_pointer_exception",
"reason": null,
"stack_trace": "java.lang.NullPointerException\n\tat org.elasticsearch.painless.DefBootstrap$PIC.fallback(DefBootstrap.java:209)\n\tat org.elasticsearch.painless.PainlessScript$Script.execute( ...:89)\n\tat org.elasticsearch.xpack.watcher.condition.ScriptCondition.doExecute(ScriptCondition.java:60)\n\tat org.elasticsearch.xpack.watcher.condition.ScriptCondition.execute(ScriptCondition.java:55)\n\tat org.elasticsearch.xpack.watcher.execution.ExecutionService.executeInner(ExecutionService.java:512)\n\tat org.elasticsearch.xpack.watcher.execution.ExecutionService.execute(ExecutionService.java:319)\n\tat org.elasticsearch.xpack.watcher.transport.actions.execute.TransportExecuteWatchAction$1.doRun(TransportExecuteWatchAction.java:158)\n\tat org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)\n\tat org.elasticsearch.xpack.watcher.execution.ExecutionService$WatchExecutionTask.run(ExecutionService.java:626)\n\tat org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)\n\tat java.lang.Thread.run(Thread.java:748)\n"
},
"stack_trace": "ScriptException[runtime error]; nested: NullPointerException;\n\tat org.elasticsearch.painless.PainlessScript.convertToScriptException(PainlessScript.java:96)\n\tat org.elasticsearch.painless.PainlessScript$Script.execute( ...:1)\n\tat org.elasticsearch.xpack.watcher.condition.ScriptCondition.doExecute(ScriptCondition.java:60)\n\tat org.elasticsearch.xpack.watcher.condition.ScriptCondition.execute(ScriptCondition.java:55)\n\tat org.elasticsearch.xpack.watcher.execution.ExecutionService.executeInner(ExecutionService.java:512)\n\tat org.elasticsearch.xpack.watcher.execution.ExecutionService.execute(ExecutionService.java:319)\n\tat org.elasticsearch.xpack.watcher.transport.actions.execute.TransportExecuteWatchAction$1.doRun(TransportExecuteWatchAction.java:158)\n\tat org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)\n\tat org.elasticsearch.xpack.watcher.execution.ExecutionService$WatchExecutionTask.run(ExecutionService.java:626)\n\tat org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)\n\tat java.lang.Thread.run(Thread.java:748)\nCaused by: java.lang.NullPointerException\n\tat org.elasticsearch.painless.DefBootstrap$PIC.fallback(DefBootstrap.java:209)\n\tat org.elasticsearch.painless.PainlessScript$Script.execute( ...:89)\n\t... 11 more\n"
}