Hello Team ,
We are using ELK enterprise license 7.17 with 5 data node and 3 master nodes.
We are getting continous alerts from one of the watcher (system watch)
Could someone help to resolve this or how can we deactivate the watcher .
where we cannot deactivate / edit any setting.
The execution status is Log state not found in Firing status .
Attached the output of the API
GET _watcher/watch/8KRfo_1OSs2ooNEJgb-bsA_elasticsearch_cluster_status
{
"found" : true,
"_id" : "8KRfo_1OSs2ooNEJgb-bsA_elasticsearch_cluster_status",
"_version" : 1018253,
"_seq_no" : 6188047,
"_primary_term" : 234,
"status" : {
"state" : {
"active" : true,
"timestamp" : "2022-01-24T12:35:07.482Z"
},
"last_checked" : "2023-03-16T09:37:14.555Z",
"last_met_condition" : "2023-03-16T09:37:14.555Z",
"actions" : {
"send_email_to_admin" : {
"ack" : {
"timestamp" : "2022-01-24T12:35:07.482Z",
"state" : "awaits_successful_execution"
}
},
"add_to_alerts_index" : {
"ack" : {
"timestamp" : "2022-01-24T13:04:19.897Z",
"state" : "awaits_successful_execution"
},
"last_execution" : {
"timestamp" : "2022-01-24T12:56:45.266Z",
"successful" : true
},
"last_successful_execution" : {
"timestamp" : "2022-01-24T12:56:45.266Z",
"successful" : true
}
},
"log_state_not_found" : {
"ack" : {
"timestamp" : "2022-01-24T13:04:19.812Z",
"state" : "ackable"
},
"last_execution" : {
"timestamp" : "2023-03-16T09:37:14.555Z",
"successful" : true
},
"last_successful_execution" : {
"timestamp" : "2023-03-16T09:37:14.555Z",
"successful" : true
},
"last_throttle" : {
"timestamp" : "2022-01-24T16:02:12.912Z",
"reason" : "throttling interval is set to [5s] but time elapsed since last execution is [4.9s]"
}
}
},
"execution_state" : "executed",
"version" : 1018253
},
"watch" : {
"trigger" : {
"schedule" : {
"interval" : "1m"
}
},
"input" : {
"chain" : {
"inputs" : [
{
"check" : {
"search" : {
"request" : {
"search_type" : "query_then_fetch",
"indices" : [
".monitoring-es-*"
],
"rest_total_hits_as_int" : true,
"body" : {
"size" : 1,
"sort" : [
{
"timestamp" : {
"order" : "desc"
}
}
],
"_source" : [
"cluster_state.status"
],
"query" : {
"bool" : {
"filter" : [
{
"term" : {
"cluster_uuid" : "{{ctx.metadata.xpack.cluster_uuid}}"
}
},
{
"term" : {
"type" : "cluster_stats"
}
},
{
"range" : {
"timestamp" : {
"gte" : "now-2m"
}
}
}
]
}
}
}
}
}
}
},
{
"alert" : {
"search" : {
"request" : {
"search_type" : "query_then_fetch",
"indices" : [
".monitoring-alerts-7"
],
"rest_total_hits_as_int" : true,
"body" : {
"size" : 1,
"terminate_after" : 1,
"query" : {
"bool" : {
"filter" : {
"term" : {
"_id" : "{{ctx.watch_id}}"
}
}
}
},
"sort" : [
{
"timestamp" : {
"order" : "desc"
}
}
]
}
}
}
}
},
{
"kibana_settings" : {
"search" : {
"request" : {
"search_type" : "query_then_fetch",
"indices" : [
".monitoring-kibana-7-*"
],
"rest_total_hits_as_int" : true,
"body" : {
"size" : 1,
"query" : {
"bool" : {
"filter" : {
"term" : {
"type" : "kibana_settings"
}
}
}
},
"sort" : [
{
"timestamp" : {
"order" : "desc"
}
}
]
}
}
}
}
}
]
}
},
"condition" : {
"script" : {
"source" : "ctx.vars.fails_check = ctx.payload.check.hits.total != 0 && ctx.payload.check.hits.hits[0]._source.cluster_state.status != 'green';ctx.vars.not_resolved = ctx.payload.alert.hits.total == 1 && ctx.payload.alert.hits.hits[0]._source.resolved_timestamp == null;return ctx.vars.fails_check || ctx.vars.not_resolved",
"lang" : "painless"
}
},
"transform" : {
"script" : {
"source" : "ctx.vars.email_recipient = (ctx.payload.kibana_settings.hits.total > 0 && ctx.payload.kibana_settings.hits.hits[0]._source.kibana_settings.xpack != null) ? ctx.payload.kibana_settings.hits.hits[0]._source.kibana_settings.xpack.default_admin_email : null;ctx.vars.is_new = ctx.vars.fails_check && !ctx.vars.not_resolved;ctx.vars.is_resolved = !ctx.vars.fails_check && ctx.vars.not_resolved;ctx.vars.found_state = ctx.payload.check.hits.total != 0;def state = ctx.vars.found_state ? ctx.payload.check.hits.hits[0]._source.cluster_state.status : 'unknown';if (ctx.vars.not_resolved){ctx.payload = ctx.payload.alert.hits.hits[0]._source;if (ctx.vars.fails_check == false) {ctx.payload.resolved_timestamp = ctx.execution_time;}} else {ctx.payload = ['timestamp': ctx.execution_time, 'metadata': ctx.metadata.xpack];}if (ctx.vars.fails_check) {ctx.payload.prefix = 'Elasticsearch cluster status is ' + state + '.';if (state == 'red') {ctx.payload.message = 'Allocate missing primary shards and replica shards.';ctx.payload.metadata.severity = 2100;} else {ctx.payload.message = 'Allocate missing replica shards.';ctx.payload.metadata.severity = 1100;}}ctx.vars.state = state.toUpperCase();ctx.payload.update_timestamp = ctx.execution_time;return ctx.payload;",
"lang" : "painless"
}
},
"actions" : {
"log_state_not_found" : {
"condition" : {
"script" : {
"source" : "!ctx.vars.found_state",
"lang" : "painless"
}
},
"logging" : {
"level" : "debug",
"text" : "Watch [{{ctx.metadata.xpack.watch}}] could not determine cluster state for cluster [{{ctx.metadata.xpack.cluster_uuid}}]. This likely means the cluster has not sent any monitoring data recently."
}
},
"add_to_alerts_index" : {
"condition" : {
"script" : {
"source" : "ctx.vars.found_state",
"lang" : "painless"
}
},
"index" : {
"index" : ".monitoring-alerts-7",
"doc_id" : "8KRfo_1OSs2ooNEJgb-bsA_elasticsearch_cluster_status"
}
},
"send_email_to_admin" : {
"condition" : {
"script" : {
"source" : "return ctx.vars.email_recipient != null && ctx.vars.found_state && (ctx.vars.is_new || ctx.vars.is_resolved)",
"lang" : "painless"
}
},
"email" : {
"profile" : "standard",
"from" : "X-Pack Admin <{{ctx.vars.email_recipient}}>",
"to" : [
"X-Pack Admin <{{ctx.vars.email_recipient}}>"
],
"subject" : "[{{#ctx.vars.is_new}}NEW{{/ctx.vars.is_new}}{{#ctx.vars.is_resolved}}RESOLVED{{/ctx.vars.is_resolved}}] {{ctx.metadata.name}} [{{ctx.vars.state}}]",
"body" : {
"text" : "{{#ctx.vars.is_resolved}}This cluster alert has been resolved: {{/ctx.vars.is_resolved}}{{ctx.payload.prefix}} {{ctx.payload.message}}"
}
}
}
},
"metadata" : {
"name" : "X-Pack Monitoring: Cluster Status (8KRfo_1OSs2ooNEJgb-bsA)",
"xpack" : {
"severity" : 2100,
"cluster_uuid" : "8KRfo_1OSs2ooNEJgb-bsA",
"version_created" : "7050099",
"watch" : "elasticsearch_cluster_status",
"link" : "elasticsearch/indices",
"alert_index" : ".monitoring-alerts-7",
"type" : "monitoring"
}
}
}
}