Hi
I have a cluster with 4nodes of data and 1node for kibana. about 5 days ago one of the built-in watcher (Nodes Changed) run most of the times.
{
"watch_id": "kVfChQJ8TWWjr5Mrbw_nEg_elasticsearch_nodes",
"node": "YU8RQoTRT1SC4ebMkjWg",
"state": "executed",
"user": "_xpack",
"status": {
"state": {
"active": true,
"timestamp": "2021-03-14T11:42:10"
},
"last_checked": "2021-05-08T11:15:11",
"last_met_condition": "2021-05-08T11:15:11",
"actions": {
"send_email_to_admin": {
"ack": {
"timestamp": "2021-05-08T11:15:11",
"state": "ackable"
},
"last_execution": {
"timestamp": "2021-05-08T11:15:11",
"successful": true
},
"last_successful_execution": {
"timestamp": "2021-05-08T11:15:11",
"successful": true
}
},
"add_to_alerts_index": {
"ack": {
"timestamp": "2021-05-08T11:15:11",
"state": "ackable"
},
"last_execution": {
"timestamp": "2021-05-08T11:15:11",
"successful": true
},
"last_successful_execution": {
"timestamp": "2021-05-08T11:15:11",
"successful": true
}
}
},
"execution_state": "executed",
"version": -1
},
"trigger_event": {
"type": "schedule",
"triggered_time": "2021-05-08T11:15:11.",
"schedule": {
"scheduled_time": "2021-05-08T11:15:10"
}
},
"input": {
"chain": {
"inputs": [
{
"check": {
"search": {
"request": {
"search_type": "query_then_fetch",
"indices": [
".monitoring-es-*"
],
"rest_total_hits_as_int": true,
"body": {
"size": 2,
"sort": [
{
"timestamp": {
"order": "desc"
}
}
],
"_source": [
"cluster_state.nodes_hash",
"cluster_state.nodes.*.name",
"cluster_state.nodes.*.ephemeral_id"
],
"collapse": {
"field": "cluster_state.nodes_hash"
},
"query": {
"bool": {
"filter": [
{
"term": {
"cluster_uuid": "kVfChQJ8TWWjr5bw_nEg"
}
},
{
"term": {
"type": "cluster_stats"
}
},
{
"range": {
"timestamp": {
"gte": "now-2d"
}
}
}
]
}
}
}
}
}
}
},
{
"alert": {
"search": {
"request": {
"search_type": "query_then_fetch",
"indices": [
".monitoring-alerts-7"
],
"rest_total_hits_as_int": true,
"body": {
"size": 1,
"_source": [
"nodes.hash"
],
"query": {
"bool": {
"filter": [
{
"term": {
"metadata.cluster_uuid": "kVfChQJ8TWjr5Mrbw_nEg"
}
},
{
"term": {
"metadata.watch": "elasticsearch_nodes"
}
}
]
}
},
"sort": [
{
"timestamp": {
"order": "desc"
}
}
]
}
}
}
}
},
{
"kibana_settings": {
"search": {
"request": {
"search_type": "query_then_fetch",
"indices": [
".monitoring-kibana-7-*"
],
"rest_total_hits_as_int": true,
"body": {
"size": 1,
"query": {
"bool": {
"filter": {
"term": {
"type": "kibana_settings"
}
}
}
},
"sort": [
{
"timestamp": {
"order": "desc"
}
}
]
}
}
}
}
}
]
}
},
"condition": {
"script": {
"source": "ctx.payload.check.hits.hits.length > 1 && (ctx.payload.alert.hits.total == 0 || ctx.payload.alert.hits.hits[0]._source.nodes.hash != ctx.payload.check.hits.hits[0]._source.cluster_state.nodes_hash)",
"lang": "painless"
}
},
"metadata": {
"name": "X-Pack Monitoring: Nodes Changed (kVfChQJ8TWWjr5Mrbw_nEg)",
"xpack": {
"severity": 1999,
"cluster_uuid": "kVfChQJ8TWW5Mrbw_nEg",
"version_created": "7050099",
"watch": "elasticsearch_nodes",
"link": "elasticsearch/nodes",
"alert_index": ".monitoring-alerts-7",
"type": "monitoring"
}
},
"result": {
"execution_time": "2021-05-08T11:15:11",
"execution_duration": 8526,
"input": {
"type": "chain",
"status": "success",
"payload": {
"alert": {
"_shards": {
"total": 1,
"failed": 0,
"successful": 1,
"skipped": 0
},
"hits": {
"hits": [
{
"_index": ".monitoring-alerts-7",
"_type": "_doc",
"_source": {
"nodes": {
"hash": -226668508
}
},
"_id": "4RisS3kBUvTA5fBQmGTB",
"sort": [
1620472271379
],
"_score": null
}
],
"total": 10000,
"max_score": null
},
"took": 7,
"timed_out": false
},
"kibana_settings": {
"_shards": {
"total": 7,
"failed": 0,
"successful": 7,
"skipped": 0
},
"hits": {
"hits": [
{
"_index": ".monitoring-kibana-7-mb-2021.05.08",
"_type": "_doc",
"_source": {
"interval_ms": 10000,
"agent": {
"hostname": "elk-lnx1",
"id": "3a2d68b3-19e5-4ffd-a5969481278df",
"type": "metricbeat",
"ephemeral_id": "935b3d77-7f1d-46b9-9a00-01c76006f",
"version": "7.6.2"
},
"@timestamp": "2021-05-08T11:15:13",
"ecs": {
"version": "1.4.0"
},
"cluster_uuid": "kVfChQJ8TWWjrrbw_nEg",
"service": {
"address": "elk-lnx1:5601",
"type": "kibana"
},
"kibana_settings": {
"kibana": {
"transport_address": "0.0.0.0:5601",
"name": "kibana",
"host": "0.0.0.0",
"index": ".kibana",
"version": "7.6.2",
"uuid": "fbcdffc8-0ef8-412f-9706-d1ff9197c669",
"snapshot": false,
"status": "green"
},
"xpack": {
"default_admin_email": "my_mail"
}
},
"host": {
"hostname": "elk-lnx1",
"os": {
"kernel": "3.10.0-1062.1.2.el7.x86_64",
"codename": "Core",
"name": "CentOS Linux",
"family": "redhat",
"version": "7 (Core)",
"platform": "centos"
},
"containerized": false,
"name": "elk-lnx1",
"id": "4866f81736374490920cba3ce7d33247",
"architecture": "x86_64"
},
"metricset": {
"period": 10000,
"name": "stats"
},
"event": {
"duration": 229333470,
"module": "kibana",
"dataset": "kibana.stats"
},
"type": "kibana_settings",
"timestamp": "2021-05-08T11:15:13.243Z"
},
"_id": "fxiwS3kBUvTA5fBQR3st",
"sort": [
1620472513243
],
"_score": null
}
],
"total": 10000,
"max_score": null
},
"took": 279,
"timed_out": false
},
"check": {
"_shards": {
"total": 14,
"failed": 0,
"successful": 14,
"skipped": 0
},
"hits": {
"hits": [
{
"_index": ".monitoring-es-7-mb-2021.05.08",
"_type": "_doc",
"_source": {
"cluster_state": {
"nodes_hash": -846563061,
"nodes": {
"fnGT0sQgSeqUAWiLXbSe1Q": {
"name": "elk-nginx1",
"ephemeral_id": "Nh67C3TaQTquLE2FWHQXvQ"
},
"weetJo22TOuqO1191f2SKw": {
"name": "elk-lnx2",
"ephemeral_id": "6QX_2x9NQ0SGhaPdlKF59Q"
},
"ujUegzAiSyKDPFzjUbEhgQ": {
"name": "elk-lnx102",
"ephemeral_id": "zshwFN04Sw6LxyhUgUBFlg"
},
"YU8RQoTRT1SC4eA9bMkjWg": {
"name": "elk-lnx1",
"ephemeral_id": "zWQ3LvIDQMKMg1kWiLfYbw"
},
"oORrByqaQ7i2JAdquk1pjg": {
"name": "elk-lnx101",
"ephemeral_id": "GqB6ywN0Tz6tNaCPhgWJLQ"
},
"KsW8IE8uS-6NNejn5PmJzg": {
"name": "elk-nginx2",
"ephemeral_id": "8YBOgAo9TUi1WkVyRkjN3A"
}
}
}
},