Hi! I have created a webhook to test out altering using some of the new machine learning features and run into some issues when attempting to send test alerts from the Kibana UI.
Currently I am not concerned with alerting on anything in particular and just want to ensure that I can alert on anything as a test of the functionality.
When attempting to test the webhook the request fails with the root cause of: socket_timeout_exception
When attempting to test an alert via email the request fails with the following exceptions:
messaging_exception
send_failed_exception
s_m_t_p_address_failed_exception
The webhook is meant to send a notification to a Microsoft Teams Channel and the email is meant to go to a zendesk support email account.
I know that my request to the webhook is formatted correctly because I have sent the request manually using CURL from my own machnie, which succeeded.
My elasticsearch instance is a free-tier cloud instance.
I am currently unsure as to what is causing these issue for me and do not know how to remediate them. My current hypothesis is that there may be some network misconfiguration or similar issue preventing the cluster from communicating to the destinations that I wish to send request to.
Any help in resolving this would be greatly appreciated. Please let me know if you require more information to assist me.
My watcher configuration is listed below. I have replaced some parts of my config with placeholders for the purpose of posting here.
{
"trigger": {
"schedule": {
"interval": "1m"
}
},
"input": {
"search": {
"request": {
"search_type": "query_then_fetch",
"indices": [
".ml-anomalies-rare_process_by_host_windows_ecs"
],
"rest_total_hits_as_int": true,
"body": {
"size": 0,
"query": {
"bool": {
"filter": {
"range": {
"timestamp": {
"gte": "{{ctx.trigger.scheduled_time}}||-5d",
"lte": "{{ctx.trigger.scheduled_time}}",
"format": "strict_date_optional_time||epoch_millis"
}
}
}
}
}
}
}
}
},
"condition": {
"script": {
"source": "if (ctx.payload.hits.total > params.threshold) { return true; } return false;",
"lang": "painless",
"params": {
"threshold": 100
}
}
},
"transform": {
"script": {
"source": "HashMap result = new HashMap(); result.result = ctx.payload.hits.total; return result;",
"lang": "painless",
"params": {
"threshold": 100
}
}
},
"actions": {
"webhook_1": {
"webhook": {
"scheme": "https",
"host": "outlook.office.com",
"port": 443,
"method": "post",
"path": "/webhook/<web_hook_uuid>",
"params": {},
"headers": {},
"body": """{
"@type": "MessageCard",
"summary": "Elasticsearch Alert",
"themeColor": "0078D7",
"sections": [
{
"activityTitle": "**Elasticsearch Alert**",
"activityText": "Watch {{ctx.metadata.name}} has exceeded the threshold"
}
]
}"""
}
},
"email_1": {
"email": {
"profile": "standard",
"to": [
"support@<company_name>.zendesk.com"
],
"subject": "Watch [{{ctx.metadata.name}}] has exceeded the threshold",
"body": {
"text": "Elasticsearch Alert\nWatch [{{ctx.metadata.name}}] has exceeded the threshold"
}
}
}
},
"metadata": {
"name": "rare_process_by_host_windows_alert",
"watcherui": {
"trigger_interval_unit": "m",
"agg_type": "count",
"time_field": "timestamp",
"trigger_interval_size": 1,
"term_size": 5,
"time_window_unit": "d",
"threshold_comparator": ">",
"term_field": null,
"index": [
".ml-anomalies-rare_process_by_host_windows_ecs"
],
"time_window_size": 5,
"threshold": 100,
"agg_field": null
},
"xpack": {
"type": "threshold"
}
}
}