Watcher X-Pack Monitoring: Nodes Changed

Hi
I have a cluster with 4nodes of data and 1node for kibana. about 5 days ago one of the built-in watcher (Nodes Changed) run most of the times.

{
  "watch_id": "kVfChQJ8TWWjr5Mrbw_nEg_elasticsearch_nodes",
  "node": "YU8RQoTRT1SC4ebMkjWg",
  "state": "executed",
  "user": "_xpack",
  "status": {
    "state": {
      "active": true,
      "timestamp": "2021-03-14T11:42:10"
    },
    "last_checked": "2021-05-08T11:15:11",
    "last_met_condition": "2021-05-08T11:15:11",
    "actions": {
      "send_email_to_admin": {
        "ack": {
          "timestamp": "2021-05-08T11:15:11",
          "state": "ackable"
        },
        "last_execution": {
          "timestamp": "2021-05-08T11:15:11",
          "successful": true
        },
        "last_successful_execution": {
          "timestamp": "2021-05-08T11:15:11",
          "successful": true
        }
      },
      "add_to_alerts_index": {
        "ack": {
          "timestamp": "2021-05-08T11:15:11",
          "state": "ackable"
        },
        "last_execution": {
          "timestamp": "2021-05-08T11:15:11",
          "successful": true
        },
        "last_successful_execution": {
          "timestamp": "2021-05-08T11:15:11",
          "successful": true
        }
      }
    },
    "execution_state": "executed",
    "version": -1
  },
  "trigger_event": {
    "type": "schedule",
    "triggered_time": "2021-05-08T11:15:11.",
    "schedule": {
      "scheduled_time": "2021-05-08T11:15:10"
    }
  },
  "input": {
    "chain": {
      "inputs": [
        {
          "check": {
            "search": {
              "request": {
                "search_type": "query_then_fetch",
                "indices": [
                  ".monitoring-es-*"
                ],
                "rest_total_hits_as_int": true,
                "body": {
                  "size": 2,
                  "sort": [
                    {
                      "timestamp": {
                        "order": "desc"
                      }
                    }
                  ],
                  "_source": [
                    "cluster_state.nodes_hash",
                    "cluster_state.nodes.*.name",
                    "cluster_state.nodes.*.ephemeral_id"
                  ],
                  "collapse": {
                    "field": "cluster_state.nodes_hash"
                  },
                  "query": {
                    "bool": {
                      "filter": [
                        {
                          "term": {
                            "cluster_uuid": "kVfChQJ8TWWjr5bw_nEg"
                          }
                        },
                        {
                          "term": {
                            "type": "cluster_stats"
                          }
                        },
                        {
                          "range": {
                            "timestamp": {
                              "gte": "now-2d"
                            }
                          }
                        }
                      ]
                    }
                  }
                }
              }
            }
          }
        },
        {
          "alert": {
            "search": {
              "request": {
                "search_type": "query_then_fetch",
                "indices": [
                  ".monitoring-alerts-7"
                ],
                "rest_total_hits_as_int": true,
                "body": {
                  "size": 1,
                  "_source": [
                    "nodes.hash"
                  ],
                  "query": {
                    "bool": {
                      "filter": [
                        {
                          "term": {
                            "metadata.cluster_uuid": "kVfChQJ8TWjr5Mrbw_nEg"
                          }
                        },
                        {
                          "term": {
                            "metadata.watch": "elasticsearch_nodes"
                          }
                        }
                      ]
                    }
                  },
                  "sort": [
                    {
                      "timestamp": {
                        "order": "desc"
                      }
                    }
                  ]
                }
              }
            }
          }
        },
        {
          "kibana_settings": {
            "search": {
              "request": {
                "search_type": "query_then_fetch",
                "indices": [
                  ".monitoring-kibana-7-*"
                ],
                "rest_total_hits_as_int": true,
                "body": {
                  "size": 1,
                  "query": {
                    "bool": {
                      "filter": {
                        "term": {
                          "type": "kibana_settings"
                        }
                      }
                    }
                  },
                  "sort": [
                    {
                      "timestamp": {
                        "order": "desc"
                      }
                    }
                  ]
                }
              }
            }
          }
        }
      ]
    }
  },
  "condition": {
    "script": {
      "source": "ctx.payload.check.hits.hits.length > 1 && (ctx.payload.alert.hits.total == 0 || ctx.payload.alert.hits.hits[0]._source.nodes.hash != ctx.payload.check.hits.hits[0]._source.cluster_state.nodes_hash)",
      "lang": "painless"
    }
  },
  "metadata": {
    "name": "X-Pack Monitoring: Nodes Changed (kVfChQJ8TWWjr5Mrbw_nEg)",
    "xpack": {
      "severity": 1999,
      "cluster_uuid": "kVfChQJ8TWW5Mrbw_nEg",
      "version_created": "7050099",
      "watch": "elasticsearch_nodes",
      "link": "elasticsearch/nodes",
      "alert_index": ".monitoring-alerts-7",
      "type": "monitoring"
    }
  },
  "result": {
    "execution_time": "2021-05-08T11:15:11",
    "execution_duration": 8526,
    "input": {
      "type": "chain",
      "status": "success",
      "payload": {
        "alert": {
          "_shards": {
            "total": 1,
            "failed": 0,
            "successful": 1,
            "skipped": 0
          },
          "hits": {
            "hits": [
              {
                "_index": ".monitoring-alerts-7",
                "_type": "_doc",
                "_source": {
                  "nodes": {
                    "hash": -226668508
                  }
                },
                "_id": "4RisS3kBUvTA5fBQmGTB",
                "sort": [
                  1620472271379
                ],
                "_score": null
              }
            ],
            "total": 10000,
            "max_score": null
          },
          "took": 7,
          "timed_out": false
        },
        "kibana_settings": {
          "_shards": {
            "total": 7,
            "failed": 0,
            "successful": 7,
            "skipped": 0
          },
          "hits": {
            "hits": [
              {
                "_index": ".monitoring-kibana-7-mb-2021.05.08",
                "_type": "_doc",
                "_source": {
                  "interval_ms": 10000,
                  "agent": {
                    "hostname": "elk-lnx1",
                    "id": "3a2d68b3-19e5-4ffd-a5969481278df",
                    "type": "metricbeat",
                    "ephemeral_id": "935b3d77-7f1d-46b9-9a00-01c76006f",
                    "version": "7.6.2"
                  },
                  "@timestamp": "2021-05-08T11:15:13",
                  "ecs": {
                    "version": "1.4.0"
                  },
                  "cluster_uuid": "kVfChQJ8TWWjrrbw_nEg",
                  "service": {
                    "address": "elk-lnx1:5601",
                    "type": "kibana"
                  },
                  "kibana_settings": {
                    "kibana": {
                      "transport_address": "0.0.0.0:5601",
                      "name": "kibana",
                      "host": "0.0.0.0",
                      "index": ".kibana",
                      "version": "7.6.2",
                      "uuid": "fbcdffc8-0ef8-412f-9706-d1ff9197c669",
                      "snapshot": false,
                      "status": "green"
                    },
                    "xpack": {
                      "default_admin_email": "my_mail"
                    }
                  },
                  "host": {
                    "hostname": "elk-lnx1",
                    "os": {
                      "kernel": "3.10.0-1062.1.2.el7.x86_64",
                      "codename": "Core",
                      "name": "CentOS Linux",
                      "family": "redhat",
                      "version": "7 (Core)",
                      "platform": "centos"
                    },
                    "containerized": false,
                    "name": "elk-lnx1",
                    "id": "4866f81736374490920cba3ce7d33247",
                    "architecture": "x86_64"
                  },
                  "metricset": {
                    "period": 10000,
                    "name": "stats"
                  },
                  "event": {
                    "duration": 229333470,
                    "module": "kibana",
                    "dataset": "kibana.stats"
                  },
                  "type": "kibana_settings",
                  "timestamp": "2021-05-08T11:15:13.243Z"
                },
                "_id": "fxiwS3kBUvTA5fBQR3st",
                "sort": [
                  1620472513243
                ],
                "_score": null
              }
            ],
            "total": 10000,
            "max_score": null
          },
          "took": 279,
          "timed_out": false
        },
        "check": {
          "_shards": {
            "total": 14,
            "failed": 0,
            "successful": 14,
            "skipped": 0
          },
          "hits": {
            "hits": [
              {
                "_index": ".monitoring-es-7-mb-2021.05.08",
                "_type": "_doc",
                "_source": {
                  "cluster_state": {
                    "nodes_hash": -846563061,
                    "nodes": {
                      "fnGT0sQgSeqUAWiLXbSe1Q": {
                        "name": "elk-nginx1",
                        "ephemeral_id": "Nh67C3TaQTquLE2FWHQXvQ"
                      },
                      "weetJo22TOuqO1191f2SKw": {
                        "name": "elk-lnx2",
                        "ephemeral_id": "6QX_2x9NQ0SGhaPdlKF59Q"
                      },
                      "ujUegzAiSyKDPFzjUbEhgQ": {
                        "name": "elk-lnx102",
                        "ephemeral_id": "zshwFN04Sw6LxyhUgUBFlg"
                      },
                      "YU8RQoTRT1SC4eA9bMkjWg": {
                        "name": "elk-lnx1",
                        "ephemeral_id": "zWQ3LvIDQMKMg1kWiLfYbw"
                      },
                      "oORrByqaQ7i2JAdquk1pjg": {
                        "name": "elk-lnx101",
                        "ephemeral_id": "GqB6ywN0Tz6tNaCPhgWJLQ"
                      },
                      "KsW8IE8uS-6NNejn5PmJzg": {
                        "name": "elk-nginx2",
                        "ephemeral_id": "8YBOgAo9TUi1WkVyRkjN3A"
                      }
                    }
                  }
                },

Continue of previous watcher log:

                "_id": "nBiwS3kBUvTA5fBQIXqn",
                "sort": [
                  1620472503250
                ],
                "_score": null,
                "fields": {
                  "cluster_state.nodes_hash": [
                    -846563061
                  ]
                }
              },
              {
                "_index": ".monitoring-es-7-2021.05.08",
                "_type": "_doc",
                "_source": {
                  "cluster_state": {
                    "nodes_hash": -226668508,
                    "nodes": {
                      "weetJo22TOuqO11912SKw": {
                        "name": "elk-lnx2";
                        "ephemeral_id": "6QX_2xQ0SGhaPdlKF59Q"
                      },
                      "fnGT0sQgSeqUiLXbSe1Q": {
                        "name": "elk-nginx1",
                        "ephemeral_id": "Nh3TaQTquLE2FWHQXvQ"
                      },
                      "ujUegzAiSyPFzjUbEhgQ": {
                        "name": "elk-lnx102",
                        "ephemeral_id": "zshwFNSw6LxyhUgUBFlg"
                      },
                      "YU8RQoTRT1SC4eA9kjWg": {
                        "name": "elk-lnx1",
                        "ephemeral_id": "zWQ3LDQMKMg1kWiLfYbw"
                      },
                      "oORrByqaQ7i2JAdquk1pjg": {
                        "name": "elk-lnx101",
                        "ephemeral_id": "GqB6y0Tz6tNaCPhgWJLQ"
                      },
                      "KsW8IE8uS-6NNe5PmJzg": {
                        "name": "elk-nginx2",
                        "ephemeral_id": "8YBOo9TUi1WkVyRkjN3A"
                      }
                    }
                  }
                },
                "_id": "szewS3kBtcyV7r5O52",
                "sort": [
                  1620472497873
                ],
                "_score": null,
                "fields": {
                  "cluster_state.nodes_hash": [
                    -226668508
                  ]
                }
              }
            ],
            "total": 10000,
            "max_score": null
          },
          "took": 8189,
          "timed_out": false
        }
      },
      "chain": {
        "check": {
          "type": "search",
          "status": "success",
          "payload": {
            "_shards": {
              "total": 14,
              "failed": 0,
              "successful": 14,
              "skipped": 0
            },
            "hits": {
              "hits": [
                {
                  "_index": ".monitoring-es-7-mb-2021.05.08",
                  "_type": "_doc",
                  "_source": {
                    "cluster_state": {
                      "nodes_hash": -846563061,
                      "nodes": {
                        "fnGT0sQgSeqWiLXbSe1Q": {
                          "name": "elk-nginx1",
                          "ephemeral_id": "Nh67C3TTquLE2FWHQXvQ"
                        },
                        "weetJo22TOuqO119f2SKw": {
                          "name": "elk-lnx2",
                          "ephemeral_id": "6QX_2NQ0SGhaPdlKF59Q"
                        },
                        "ujUegzAiSyKDPFzjbEhgQ": {
                          "name": "elk-lnx102",
                          "ephemeral_id": "zshwFN04SwxyhUgUBFlg"
                        },
                        "YU8RQoTRT1SC4eA9bMkjWg": {
                          "name": "elk-lnx1",
                          "ephemeral_id": "zWQ3LvIMKMg1kWiLfYbw"
                        },
                        "oORrByqaQ7i2JAdquk1pjg": {
                          "name": "elk-lnx101",
                          "ephemeral_id": "GqB6ywNz6tNaCPhgWJLQ"
                        },
                        "KsW8IE8uS-6NNejn5PmJzg": {
                          "name": "elk-nginx2",
                          "ephemeral_id": "8YBOgAo9i1WkVyRkjN3A"
                        }
                      }
                    }
                  },
                  "_id": "nBiwS3kBUvTA5fQIXqn",
                  "sort": [
                    1620472503250
                  ],
                  "_score": null,
                  "fields": {
                    "cluster_state.nodes_hash": [
                      -846563061
                    ]
                  }
                },
    {
                  "_index": ".monitoring-es-7-2021.05.08",
                  "_type": "_doc",
                  "_source": {
                    "cluster_state": {
                      "nodes_hash": -226668508,
                      "nodes": {
                        "weetJo22TOuqO1191f2SKw": {
                          "name": "elk-lnx2",
                          "ephemeral_id": "6QX_2x9NQ0SaPdlKF59Q"
                        },
                        "fnGT0sQgSeqUAWiLXbSe1Q": {
                          "name": "elk-nginx1",
                          "ephemeral_id": "Nh67C3TaQTquLFWHQXvQ"
                        },
                        "ujUegzAiSyKDPFzjUbEhgQ": {
                          "name": "elk-lnx102",
                          "ephemeral_id": "zshwFN04Sw6yhUgUBFlg"
                        },
                        "YU8RQoTRT1SC4eA9bMkjWg": {
                          "name": "elk-lnx1",
                          "ephemeral_id": "zWQ3LvIDQKMg1kWiLfYbw"
                        },
                        "oORrByqaQ7i2JAdquk1pjg": {
                          "name": "elk-lnx101",
                          "ephemeral_id": "GqB6ywN0Tz6tNaCgWJLQ"
                        },
                        "KsW8IE8uS-6NNejn5PmJzg": {
                          "name": "elk-nginx2",
                          "ephemeral_id": "8YBOgAo9TUi1kVyRkjN3A"
                        }
                      }
                    }
                  },
                  "_id": "szewS3kBtcyV7r5sCO52",
                  "sort": [
                    1620472497873
                  ],
                  "_score": null,
                  "fields": {
                    "cluster_state.nodes_hash": [
                      -226668508
                    ]
                  }
                }
              ],
              "total": 10000,
              "max_score": null
            },
            "took": 8189,
            "timed_out": false
          },
          "search": {
            "request": {
              "search_type": "query_then_fetch",
              "indices": [
                ".monitoring-es-*"
              ],
              "rest_total_hits_as_int": true,
              "body": {
                "size": 2,
                "sort": [
                  {
                    "timestamp": {
                      "order": "desc"
                    }
                  }
                ],
                "_source": [
                  "cluster_state.nodes_hash",
                  "cluster_state.nodes.*.name",
                  "cluster_state.nodes.*.ephemeral_id"
                ],
                "collapse": {
                  "field": "cluster_state.nodes_hash"
                },
                "query": {
                  "bool": {
                    "filter": [
                      {
                        "term": {
                          "cluster_uuid": "kVfChQJ8TWWjr5Mrbw_nEg"
                        }
                      },
                      {
                        "term": {
                          "type": "cluster_stats"
                        }
                      },
                      {
                        "range": {
                          "timestamp": {
                            "gte": "now-2d"
                          }
                        }
                      }
                    ]
                  }
                }
              }
            }
          }
        },
        "alert": {
          "type": "search",
          "status": "success",
          "payload": {
            "_shards": {
              "total": 1,
              "failed": 0,
              "successful": 1,
              "skipped": 0
            },
            "hits": {
              "hits": [
                {
                  "_index": ".monitoring-alerts-7",
                  "_type": "_doc",
                  "_source": {
                    "nodes": {
                      "hash": -226668508
                    }
                  },
                  "_id": "4RisS3kBUvTA5fBQmGTB",
                  "sort": [
                    1620472271379
                  ],
                  "_score": null
                }
              ],
              "total": 10000,
              "max_score": null
            },
            "took": 7,
            "timed_out": false
          },
          "search": {
            "request": {
              "search_type": "query_then_fetch",
              "indices": [
                ".monitoring-alerts-7"
              ],
              "rest_total_hits_as_int": true,
              "body": {
                "size": 1,
                "_source": [
                  "nodes.hash"
                ],
                "query": {
                  "bool": {
                    "filter": [
                      {
                        "term": {
                          "metadata.cluster_uuid": "kVfChQJ8TWWjr5Mrbw_nEg"
                        }
                      },
                      {
                        "term": {
                          "metadata.watch": "elasticsearch_nodes"
                        }
                      }
                    ]
                  }
                },
                "sort": [
                  {
                    "timestamp": {
                      "order": "desc"
                    }
                  }
                ]
              }
            }
          }
        },
        "kibana_settings": {
          "type": "search",
          "status": "success",
          "payload": {
            "_shards": {
              "total": 7,
              "failed": 0,
              "successful": 7,
              "skipped": 0
            },

Continue of previous watcher log:

            "hits": {
              "hits": [
                {
                  "_index": ".monitoring-kibana-7-mb-2021.05.08",
                  "_type": "_doc",
                  "_source": {
                    "interval_ms": 10000,
                    "agent": {
                      "hostname": "elk-lnx1",
                      "id": "3a2d68b3-19e5-4ffd-acf9-5969481278df",
                      "type": "metricbeat",
                      "ephemeral_id": "935b3d77-7f1d-46b9-9a00-82201c766f",
                      "version": "7.6.2"
                    },
                    "@timestamp": "2021-05-08T11:15:13",
                    "ecs": {
                      "version": "1.4.0"
                    },
                    "cluster_uuid": "kVfChQJ8TWWjr5Mrb_nEg",
                    "service": {
                      "address": "elk-lnx15601",
                      "type": "kibana"
                    },
                    "kibana_settings": {
                      "kibana": {
                        "transport_address": "0.0.0.0:5601",
                        "name": "kibana",
                        "host": "0.0.0.0",
                        "index": ".kibana",
                        "version": "7.6.2",
                        "uuid": "fbcdffc8-0ef8-412f-9706-d1ff9197c",
                        "snapshot": false,
                        "status": "green"
                      },
                      "xpack": {
                        "default_admin_email": "my_mail"
                      }
                    },
                    "host": {
                      "hostname": "elk-lnx1",
                      "os": {
                        "kernel": "3.10.0-1062.1.2.el7.x86_64",
                        "codename": "Core",
                        "name": "CentOS Linux",
                        "family": "redhat",
                        "version": "7 (Core)",
                        "platform": "centos"
                      },
                      "containerized": false,
                      "name": "elk-lnx1",
                      "id": "4866f81736374490920cba3ce7d33247",
                      "architecture": "x86_64"
                    },
                    "metricset": {
                      "period": 10000,
                      "name": "stats"
                    },
                    "event": {
                      "duration": 229333470,
                      "module": "kibana",
                      "dataset": "kibana.stats"
                    },
                    "type": "kibana_settings",
                    "timestamp": "2021-05-08T11:15:13"
                  },
                  "_id": "fxiwS3kBUvTAfBQR3st",
                  "sort": [
                    1620472513243
                  ],
                  "_score": null
                }
              ],
              "total": 10000,
              "max_score": null
            },
            "took": 279,
            "timed_out": false
          },
          "search": {
            "request": {
              "search_type": "query_then_fetch",
              "indices": [
                ".monitoring-kibana-7-*"
              ],
              "rest_total_hits_as_int": true,
              "body": {
                "size": 1,
                "query": {
                  "bool": {
                    "filter": {
                      "term": {
                        "type": "kibana_settings"
                      }
                    }
                  }
                },
                "sort": [
                  {
                    "timestamp": {
                      "order": "desc"
                    }
                  }
                ]
              }
            }
          }
        }
      }
    },
    "condition": {
      "type": "script",
      "status": "success",
      "met": true
    },
    "transform": {
      "type": "script",
      "status": "success",
      "payload": {
        "metadata": {
          "severity": 1999,
          "cluster_uuid": "kVfChQJ8TWWjr5Mrbw_nEg",
          "version_created": "7050099",
          "watch": "elasticsearch_nodes",
          "link": "elasticsearch/nodes",
          "alert_index": ".monitoring-alerts-7",
          "type": "monitoring"
        },
        "nodes": {
          "removed": {},
          "added": {},
          "restarted": {},
          "hash": -846563061
        },
        "updated_timestamp": "2021-05-08T11:15:11",
        "prefix": "Elasticsearch cluster nodes have changed!",
        "message": "",
        "resolved_timestamp": "2021-05-08T11:15:11",
        "timestamp": "2021-05-08T11:15:11"
      }
    },
    "actions": [
      {
        "id": "send_email_to_admin",
        "type": "email",
        "status": "success",
        "condition": {
          "type": "script",
          "status": "success",
          "met": true
        },
        "email": {
          "account": "exchange_account",
          "message": {
            "id": "send_email_to_admin_kVfChQJ8TWWjr5Mrbw_nEg_elasticsearch_nodes_6fb85189-67d3-4017-9c40-4ff2514daf09-2021-05-08T11:15:11",
            "from": "X-Pack Admin <my_mail>",
            "sent_date": "2021-05-08T11:15:19",
            "to": [
              "X-Pack Admin <my_mail>"
            ],
            "subject": "X-Pack Monitoring: Nodes Changed (kVfChQJ8TWWjr5Mrbw_nEg)",
            "body": {
              "text": "Elasticsearch cluster nodes have changed! "
            }
          }
        }
      },
      {
        "id": "add_to_alerts_index",
        "type": "index",
        "status": "success",
        "index": {
          "response": {
            "created": true,
            "result": "created",
            "id": "lRiwS3kBUvTA5fBQW3to",
            "version": 1,
            "type": "_doc",
            "index": ".monitoring-alerts-7"
          }
        }
      }
    ]
  },
  "messages": []

Where is the problem?
This is the log of one my nodes:

[2021-05-03T15:06:35,477][INFO ][o.e.c.s.ClusterApplierService] [elk-lnx102] master node changed {previous [], current [{elk-lnx2}{weetJo22TOuqO1191f2SKw}{6QX_2x9NQ0SGhaPdlKF59Q}{172.22.33.186}{172.22.33.186:9300}{dilm}{ml.machine_memory=8200634368, ml.max_open_jobs=20, xpack.installed=true}]}, term: 1817, version: 1297263, reason: ApplyCommitRequest{term=1817, version=1297263, sourceNode={elk-lnx2}{weetJo22TOuqO1191f2SKw}{6QX_2x9NQ0SGhaPdlKF59Q}{172.22.33.186}{172.22.33.186:9300}{dilm}{ml.machine_memory=8200634368, ml.max_open_jobs=20, xpack.installed=true}}
[2021-05-03T15:06:42,514][WARN ][o.e.c.a.s.ShardStateAction] [elk-lnx102] unexpected failure while sending request [internal:cluster/shard/failure] to [{elk-lnx2}{weetJo22TOuqO1191f2SKw}{6QX_2x9NQ0SGhaPdlKF59Q}{172.22.33.186}{172.22.33.186:9300}{dilm}{ml.machine_memory=8200634368, ml.max_open_jobs=20, xpack.installed=true}] for shard entry [shard id [[log-oss-webservice-2021][0]], allocation id [-9mdPWgZSo2Si2glJVtlqg], primary term [29], message [failed to perform indices:data/write/bulk[s] on replica [log-oss-webservice-2021][0], node[YU8RQoTRT1SC4eA9bMkjWg], [R], s[STARTED], a[id=-9mdPWgZSo2Si2glJVtlqg]], failure [NodeDisconnectedException[[elk-lnx1][172.22.33.187:9300][indices:data/write/bulk[s][r]] disconnected]], markAsStale [true]]
org.elasticsearch.transport.RemoteTransportException: [elk-lnx2][172.22.33.186:9300][internal:cluster/shard/failure]
Caused by: org.elasticsearch.cluster.action.shard.ShardStateAction$NoLongerPrimaryShardException: primary term [29] did not match current primary term [30]
        at org.elasticsearch.cluster.action.shard.ShardStateAction$ShardFailedClusterStateTaskExecutor.execute(ShardStateAction.java:365) ~[elasticsearch-7.6.2.jar:7.6.2]
        at org.elasticsearch.cluster.service.MasterService.executeTasks(MasterService.java:702) ~[elasticsearch-7.6.2.jar:7.6.2]
        at org.elasticsearch.cluster.service.MasterService.calculateTaskOutputs(MasterService.java:324) ~[elasticsearch-7.6.2.jar:7.6.2]
        at org.elasticsearch.cluster.service.MasterService.runTasks(MasterService.java:219) ~[elasticsearch-7.6.2.jar:7.6.2]
        at org.elasticsearch.cluster.service.MasterService.access$000(MasterService.java:73) ~[elasticsearch-7.6.2.jar:7.6.2]
        at org.elasticsearch.cluster.service.MasterService$Batcher.run(MasterService.java:151) ~[elasticsearch-7.6.2.jar:7.6.2]
        at org.elasticsearch.cluster.service.TaskBatcher.runIfNotProcessed(TaskBatcher.java:150) ~[elasticsearch-7.6.2.jar:7.6.2]
        at org.elasticsearch.cluster.service.TaskBatcher$BatchedTask.run(TaskBatcher.java:188) ~[elasticsearch-7.6.2.jar:7.6.2]
        at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:633) ~[elasticsearch-7.6.2.jar:7.6.2]
        at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:252) ~[elasticsearch-7.6.2.jar:7.6.2]
        at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:215) ~[elasticsearch-7.6.2.jar:7.6.2]
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]
        at java.lang.Thread.run(Thread.java:830) [?:?]
[2021-05-03T15:06:42,522][WARN ][o.e.c.a.s.ShardStateAction] [elk-lnx102] unexpected failure while sending request [internal:cluster/shard/failure] to [{elk-lnx2}{weetJo22TOuqO1191f2SKw}{6QX_2x9NQ0SGhaPdlKF59Q}{172.22.33.186}{172.22.33.186:9300}{dilm}{ml.machine_memory=8200634368, ml.max_open_jobs=20, xpack.installed=true}] for shard entry [shard id [[.monitoring-es-7-2021.05.03][0]], allocation id [tyJlusBOR1y_2wplFyvFWw], primary term [1], message [failed to perform indices:data/write/bulk[s] on replica [.monitoring-es-7-2021.05.03][0], node[YU8RQoTRT1SC4eA9bMkjWg], [R], s[STARTED], a[id=tyJlusBOR1y_2wplFyvFWw]], failure [NodeDisconnectedException[[elk-lnx1][172.22.33.187:9300][indices:data/write/bulk[s][r]] disconnected]], markAsStale [true]]
org.elasticsearch.transport.RemoteTransportException: [elk-lnx2][172.22.33.186:9300][internal:cluster/shard/failure]
Caused by: org.elasticsearch.cluster.action.shard.ShardStateAction$NoLongerPrimaryShardException: primary term [1] did not match current primary term [2]
        at org.elasticsearch.cluster.action.shard.ShardStateAction$ShardFailedClusterStateTaskExecutor.execute(ShardStateAction.java:365) ~[elasticsearch-7.6.2.jar:7.6.2]
        at org.elasticsearch.cluster.service.MasterService.executeTasks(MasterService.java:702) ~[elasticsearch-7.6.2.jar:7.6.2]
        at org.elasticsearch.cluster.service.MasterService.calculateTaskOutputs(MasterService.java:324) ~[elasticsearch-7.6.2.jar:7.6.2]
        at org.elasticsearch.cluster.service.MasterService.runTasks(MasterService.java:219) ~[elasticsearch-7.6.2.jar:7.6.2]
        at org.elasticsearch.cluster.service.MasterService.access$000(MasterService.java:73) ~[elasticsearch-7.6.2.jar:7.6.2]
        at org.elasticsearch.cluster.service.MasterService$Batcher.run(MasterService.java:151) ~[elasticsearch-7.6.2.jar:7.6.2]
        at org.elasticsearch.cluster.service.TaskBatcher.runIfNotProcessed(TaskBatcher.java:150) ~[elasticsearch-7.6.2.jar:7.6.2]
        at org.elasticsearch.cluster.service.TaskBatcher$BatchedTask.run(TaskBatcher.java:188) ~[elasticsearch-7.6.2.jar:7.6.2]
        at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:633) ~[elasticsearch-7.6.2.jar:7.6.2]
        at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:252) ~[elasticsearch-7.6.2.jar:7.6.2]
        at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:215) ~[elasticsearch-7.6.2.jar:7.6.2]
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]
        at java.lang.Thread.run(Thread.java:830) [?:?]
[2021-05-03T15:06:42,522][WARN ][o.e.c.a.s.ShardStateAction] [elk-lnx102] unexpected failure while sending request [internal:cluster/shard/failure] to [{elk-lnx2}{weetJo22TOuqO1191f2SKw}{6QX_2x9NQ0SGhaPdlKF59Q}{172.22.33.186}{172.22.33.186:9300}{dilm}{ml.machine_memory=8200634368, ml.max_open_jobs=20, xpack.installed=true}] for shard entry [shard id [[log-referral-2021][0]], allocation id [an3uzhcwTbaA0DgVf6bR5A], primary term [33], message [failed to perform indices:data/write/bulk[s] on replica [log-referral-2021][0], node[weetJo22TOuqO1191f2SKw], [R], s[STARTED], a[id=an3uzhcwTbaA0DgVf6bR5A]], failure [NodeDisconnectedException[[elk-lnx2][172.22.33.186:9300][indices:data/write/bulk[s][r]] disconnected]], markAsStale [true]]
org.elasticsearch.transport.RemoteTransportException: [elk-lnx2][172.22.33.186:9300][internal:cluster/shard/failure]
Caused by: org.elasticsearch.cluster.action.shard.ShardStateAction$NoLongerPrimaryShardException: primary term [33] did not match current primary term [34]
        at org.elasticsearch.cluster.action.shard.ShardStateAction$ShardFailedClusterStateTaskExecutor.execute(ShardStateAction.java:365) ~[elasticsearch-7.6.2.jar:7.6.2]
        at org.elasticsearch.cluster.service.MasterService.executeTasks(MasterService.java:702) ~[elasticsearch-7.6.2.jar:7.6.2]
        at org.elasticsearch.cluster.service.MasterService.calculateTaskOutputs(MasterService.java:324) ~[elasticsearch-7.6.2.jar:7.6.2]
        at org.elasticsearch.cluster.service.MasterService.runTasks(MasterService.java:219) ~[elasticsearch-7.6.2.jar:7.6.2]
        at org.elasticsearch.cluster.service.MasterService.access$000(MasterService.java:73) ~[elasticsearch-7.6.2.jar:7.6.2]
        at org.elasticsearch.cluster.service.MasterService$Batcher.run(MasterService.java:151) ~[elasticsearch-7.6.2.jar:7.6.2]
        at org.elasticsearch.cluster.service.TaskBatcher.runIfNotProcessed(TaskBatcher.java:150) ~[elasticsearch-7.6.2.jar:7.6.2]
        at org.elasticsearch.cluster.service.TaskBatcher$BatchedTask.run(TaskBatcher.java:188) ~[elasticsearch-7.6.2.jar:7.6.2]
        at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:633) ~[elasticsearch-7.6.2.jar:7.6.2]
        at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:252) ~[elasticsearch-7.6.2.jar:7.6.2]
        at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:215) ~[elasticsearch-7.6.2.jar:7.6.2]
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]
        at java.lang.Thread.run(Thread.java:830) [?:?]
[2021-05-03T15:07:06,823][WARN ][o.e.i.s.IndexShard       ] [elk-lnx102] [log-2020.05.02][0] failed to turn off translog retention
org.apache.lucene.store.AlreadyClosedException: engine is closed
        at org.elasticsearch.index.shard.IndexShard.getEngine(IndexShard.java:2528) ~[elasticsearch-7.6.2.jar:7.6.2]
        at org.elasticsearch.index.shard.IndexShard.trimTranslog(IndexShard.java:1106) ~[elasticsearch-7.6.2.jar:7.6.2]
        at org.elasticsearch.index.shard.IndexShard$3.doRun(IndexShard.java:1944) [elasticsearch-7.6.2.jar:7.6.2]
        at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692) [elasticsearch-7.6.2.jar:7.6.2]
        at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.6.2.jar:7.6.2]
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]
        at java.lang.Thread.run(Thread.java:830) [?:?]

It's not clear to me what you are asking here. Are you saying that you have been getting lots of alerts that nodes in the cluster have changed? Or something else?

Thank you for your response
Yes exactly, I didn't changed anything but it`s about 5days this watcher (Node Changed) runs most the time

We'd need to see the logs from the other nodes please.

[2021-05-09T14:26:17,704][INFO ][o.e.c.s.ClusterApplierService] [elk-lnx101] removed {{elk-nginx1}{fnGT0sQgSeqUAWiLXbSe1Q}{Nh67C3TaQTquLE2FWHQXvQ}{172.22.33.189}{172.22.33.189:9300}{il}{ml.machine_memory=1927270400, ml.max_open_jobs=20, xpack.installed=true}}, term: 1818, version: 1299331, reason: ApplyCommitRequest{term=1818, version=1299331, sourceNode={elk-lnx1}{YU8RQoTRT1SC4eA9bMkjWg}{zWQ3LvIDQMKMg1kWiLfYbw}{172.22.33.187}{172.22.33.187:9300}{dilm}{ml.machine_memory=8200634368, ml.max_open_jobs=20, xpack.installed=true}}
[2021-05-09T14:26:17,725][DEBUG][o.e.a.a.c.n.i.TransportNodesInfoAction] [elk-lnx101] failed to execute on node [fnGT0sQgSeqUAWiLXbSe1Q]
org.elasticsearch.transport.NodeDisconnectedException: [elk-nginx1][172.22.33.189:9300][cluster:monitor/nodes/info[n]] disconnected
[2021-05-09T14:27:08,538][INFO ][o.e.c.s.ClusterApplierService] [elk-lnx101] added {{elk-nginx1}{fnGT0sQgSeqUAWiLXbSe1Q}{Nh67C3TaQTquLE2FWHQXvQ}{172.22.33.189}{172.22.33.189:9300}{il}{ml.machine_memory=1927270400, ml.max_open_jobs=20, xpack.installed=true}}, term: 1818, version: 1299335, reason: ApplyCommitRequest{term=1818, version=1299335, sourceNode={elk-lnx1}{YU8RQoTRT1SC4eA9bMkjWg}{zWQ3LvIDQMKMg1kWiLfYbw}{172.22.33.187}{172.22.33.187:9300}{dilm}{ml.machine_memory=8200634368, ml.max_open_jobs=20, xpack.installed=true}}
[2021-05-10T04:30:01,826][INFO ][o.e.c.m.MetaDataCreateIndexService] [elk-lnx1] [.watcher-history-10-2021.05.10] creating index, cause [auto(bulk api)], templates [.watch-history-10], shards [1]/[0], mappings [_doc]
[2021-05-10T04:30:01,926][INFO ][o.e.c.r.a.AllocationService] [elk-lnx1] updating number_of_replicas to [1] for indices [.watcher-history-10-2021.05.10]
[2021-05-10T04:30:06,931][INFO ][o.e.c.m.MetaDataCreateIndexService] [elk-lnx1] [.monitoring-es-7-2021.05.10] creating index, cause [auto(bulk api)], templates [.monitoring-es], shards [1]/[0], mappings [_doc]
[2021-05-10T04:30:06,944][INFO ][o.e.c.r.a.AllocationService] [elk-lnx1] updating number_of_replicas to [1] for indices [.monitoring-es-7-2021.05.10]
[2021-05-10T04:30:07,289][INFO ][o.e.c.m.MetaDataCreateIndexService] [elk-lnx1] [.monitoring-logstash-7-mb-2021.05.10] creating index, cause [auto(bulk api)], templates [.monitoring-logstash], shards [1]/[0], mappings [_doc]
[2021-05-10T04:30:07,301][INFO ][o.e.c.r.a.AllocationService] [elk-lnx1] updating number_of_replicas to [1] for indices [.monitoring-logstash-7-mb-2021.05.10]
[2021-05-10T04:30:08,094][INFO ][o.e.c.m.MetaDataMappingService] [elk-lnx1] [.watcher-history-10-2021.05.10/QP3yC6fLT-W1PE6zlxkC8Q] update_mapping [_doc]
[2021-05-10T04:30:08,410][INFO ][o.e.x.i.IndexLifecycleTransition] [elk-lnx1] moving index [.watcher-history-10-2021.05.10] from [null] to [{"phase":"new","action":"complete","name":"complete"}] in policy [watch-history-ilm-policy]
[2021-05-10T04:30:08,659][INFO ][o.e.x.i.IndexLifecycleTransition] [elk-lnx1] moving index [.watcher-history-10-2021.05.03] from [{"phase":"new","action":"complete","name":"complete"}] to [{"phase":"delete","action":"delete","name":"wait-for-shard-history-leases"}] in policy [watch-history-ilm-policy]
[2021-05-10T04:30:09,139][INFO ][o.e.c.m.MetaDataCreateIndexService] [elk-lnx1] [.monitoring-es-7-mb-2021.05.10] creating index, cause [auto(bulk api)], templates [.monitoring-es], shards [1]/[0], mappings [_doc]
[2021-05-10T04:30:09,148][INFO ][o.e.c.r.a.AllocationService] [elk-lnx1] updating number_of_replicas to [1] for indices [.monitoring-es-7-mb-2021.05.10]
[2021-05-10T04:30:09,558][INFO ][o.e.c.m.MetaDataCreateIndexService] [elk-lnx1] [.monitoring-kibana-7-mb-2021.05.10] creating index, cause [auto(bulk api)], templates [.monitoring-kibana], shards [1]/[0], mappings [_doc]
[2021-05-10T04:30:09,564][INFO ][o.e.c.r.a.AllocationService] [elk-lnx1] updating number_of_replicas to [1] for indices [.monitoring-kibana-7-mb-2021.05.10]
[2021-05-10T04:30:11,178][INFO ][o.e.c.r.a.AllocationService] [elk-lnx1] Cluster health status changed from [YELLOW] to [GREEN] (reason: [shards started [[.monitoring-kibana-7-mb-2021.05.10][0]]]).
[2021-05-10T04:30:15,481][INFO ][o.e.c.m.MetaDataMappingService] [elk-lnx1] [.watcher-history-10-2021.05.10/QP3yC6fLT-W1PE6zlxkC8Q] update_mapping [_doc]
[2021-05-10T04:30:15,638][INFO ][o.e.c.m.MetaDataMappingService] [elk-lnx1] [.watcher-history-10-2021.05.10/QP3yC6fLT-W1PE6zlxkC8Q] update_mapping [_doc]
[2021-05-10T04:31:21,314][INFO ][o.e.c.m.MetaDataMappingService] [elk-lnx1] [.watcher-history-10-2021.05.10/QP3yC6fLT-W1PE6zlxkC8Q] update_mapping [_doc]
[2021-05-10T04:36:19,532][INFO ][o.e.x.i.IndexLifecycleTransition] [elk-lnx1] moving index [.watcher-history-10-2021.05.03] from [{"phase":"delete","action":"delete","name":"wait-for-shard-history-leases"}] to [{"phase":"delete","action":"delete","name":"delete"}] in policy [watch-history-ilm-policy]
[2021-05-10T04:36:22,219][INFO ][o.e.c.m.MetaDataDeleteIndexService] [elk-lnx1] [.watcher-history-10-2021.05.03/Goq0aCsQTkW4nq0UZwpqSA] deleting index
[2021-05-10T05:30:00,001][INFO ][o.e.x.m.e.l.LocalExporter] [elk-lnx1] cleaning up [4] old indices
[2021-05-10T05:30:00,042][INFO ][o.e.c.m.MetaDataDeleteIndexService] [elk-lnx1] [.monitoring-es-7-2021.05.03/R6XFUHDkTB6ag2JCINdx4A] deleting index
[2021-05-10T05:30:00,042][INFO ][o.e.c.m.MetaDataDeleteIndexService] [elk-lnx1] [.monitoring-kibana-7-mb-2021.05.03/HBmHfAGnSoSWASyB0w1upg] deleting index
[2021-05-10T05:30:00,042][INFO ][o.e.c.m.MetaDataDeleteIndexService] [elk-lnx1] [.monitoring-logstash-7-mb-2021.05.03/o-AXVXwRRUOqDxlQM7EJJQ] deleting index
[2021-05-10T05:30:00,042][INFO ][o.e.c.m.MetaDataDeleteIndexService] [elk-lnx1] [.monitoring-es-7-mb-2021.05.03/25n2jBhESWSrU2sfPeuVUQ] deleting index
[2021-05-10T06:00:00,013][INFO ][o.e.x.s.SnapshotRetentionTask] [elk-lnx1] starting SLM retention snapshot cleanup task

What about logs from that node?

Thank you so much for helping me
This is elk-nginx1 log:

[2021-05-12T08:33:58,189][WARN ][o.e.m.j.JvmGcMonitorService] [elk-nginx1] [gc][12298074] overhead, spent [1s] collecting in the last [1.7s]
[2021-05-12T08:48:30,799][WARN ][o.e.m.j.JvmGcMonitorService] [elk-nginx1] [gc][young][12298945][41995] duration [1s], collections [1]/[1.9s], total [1s]/[5.8h], memory [753.6mb]->[488.9mb]/[990.7mb], all_pools {[young] [266.1mb]->[2.4mb]/[266.2mb]}{[survivor] [138.5kb]->[56.8kb]/[33.2mb]}{[old] [487.3mb]->[487.3mb]/[691.2mb]}
[2021-05-12T08:48:31,007][WARN ][o.e.m.j.JvmGcMonitorService] [elk-nginx1] [gc][12298945] overhead, spent [1s] collecting in the last [1.9s]
[2021-05-12T08:58:12,768][WARN ][o.e.m.j.JvmGcMonitorService] [elk-nginx1] [gc][young][12299525][41997] duration [1.4s], collections [1]/[2.2s], total [1.4s]/[5.8h], memory [753.2mb]->[489.9mb]/[990.7mb], all_pools {[young] [265.7mb]->[2.5mb]/[266.2mb]}{[survivor] [145.7kb]->[48.8kb]/[33.2mb]}{[old] [487.3mb]->[487.3mb]/[691.2mb]}
[2021-05-12T08:58:12,960][WARN ][o.e.m.j.JvmGcMonitorService] [elk-nginx1] [gc][12299525] overhead, spent [1.4s] collecting in the last [2.2s]
[2021-05-12T09:07:52,130][INFO ][o.e.m.j.JvmGcMonitorService] [elk-nginx1] [gc][young][12300103][41999] duration [847ms], collections [1]/[1.8s], total [847ms]/[5.8h], memory [753.1mb]->[490.2mb]/[990.7mb], all_pools {[young] [265.6mb]->[2.8mb]/[266.2mb]}{[survivor] [137kb]->[52.7kb]/[33.2mb]}{[old] [487.3mb]->[487.3mb]/[691.2mb]}
[2021-05-12T09:07:52,132][INFO ][o.e.m.j.JvmGcMonitorService] [elk-nginx1] [gc][12300103] overhead, spent [847ms] collecting in the last [1.8s]
[2021-05-12T09:17:32,748][WARN ][o.e.m.j.JvmGcMonitorService] [elk-nginx1] [gc][young][12300682][42001] duration [1.3s], collections [1]/[2.1s], total [1.3s]/[5.8h], memory [751.7mb]->[487.4mb]/[990.7mb], all_pools {[young] [264.2mb]->[2.2mb]/[266.2mb]}{[survivor] [70.1kb]->[68.8kb]/[33.2mb]}{[old] [487.3mb]->[487.3mb]/[691.2mb]}
[2021-05-12T09:17:32,975][WARN ][o.e.m.j.JvmGcMonitorService] [elk-nginx1] [gc][12300682] overhead, spent [1.3s] collecting in the last [2.1s]
[2021-05-12T09:27:14,888][WARN ][o.e.m.j.JvmGcMonitorService] [elk-nginx1] [gc][young][12301263][42003] duration [1.1s], collections [1]/[1.5s], total [1.1s]/[5.8h], memory [753.6mb]->[489.8mb]/[990.7mb], all_pools {[young] [266.2mb]->[2.4mb]/[266.2mb]}{[survivor] [69kb]->[52.5kb]/[33.2mb]}{[old] [487.3mb]->[487.3mb]/[691.2mb]}
[2021-05-12T09:27:15,148][WARN ][o.e.m.j.JvmGcMonitorService] [elk-nginx1] [gc][12301263] overhead, spent [1.1s] collecting in the last [1.5s]
[2021-05-12T09:46:37,719][WARN ][o.e.m.j.JvmGcMonitorService] [elk-nginx1] [gc][young][12302423][42007] duration [1.2s], collections [1]/[1s], total [1.2s]/[5.8h], memory [752.7mb]->[753.6mb]/[990.7mb], all_pools {[young] [265.3mb]->[2.3mb]/[266.2mb]}{[survivor] [54.3kb]->[55.7kb]/[33.2mb]}{[old] [487.3mb]->[487.3mb]/[691.2mb]}
[2021-05-12T09:46:37,972][WARN ][o.e.m.j.JvmGcMonitorService] [elk-nginx1] [gc][12302423] overhead, spent [1.2s] collecting in the last [1s]
[2021-05-12T09:56:19,436][WARN ][o.e.m.j.JvmGcMonitorService] [elk-nginx1] [gc][young][12303003][42009] duration [1.2s], collections [1]/[2s], total [1.2s]/[5.8h], memory [753.5mb]->[491mb]/[990.7mb], all_pools {[young] [266mb]->[3.7mb]/[266.2mb]}{[survivor] [79.5kb]->[63kb]/[33.2mb]}{[old] [487.3mb]->[487.3mb]/[691.2mb]}
[2021-05-12T09:56:19,564][WARN ][o.e.m.j.JvmGcMonitorService] [elk-nginx1] [gc][12303003] overhead, spent [1.2s] collecting in the last [2s]
[2021-05-12T10:05:59,342][INFO ][o.e.m.j.JvmGcMonitorService] [elk-nginx1] [gc][young][12303582][42011] duration [935ms], collections [1]/[1.2s], total [935ms]/[5.8h], memory [752mb]->[488.8mb]/[990.7mb], all_pools {[young] [264.6mb]->[3.1mb]/[266.2mb]}{[survivor] [74.2kb]->[95.9kb]/[33.2mb]}{[old] [487.3mb]->[487.3mb]/[691.2mb]}
[2021-05-12T10:05:59,501][WARN ][o.e.m.j.JvmGcMonitorService] [elk-nginx1] [gc][12303582] overhead, spent [935ms] collecting in the last [1.2s]
[2021-05-12T10:15:40,614][INFO ][o.e.m.j.JvmGcMonitorService] [elk-nginx1] [gc][12304162] overhead, spent [689ms] collecting in the last [1.7s]
[2021-05-12T10:30:15,071][WARN ][o.e.m.j.JvmGcMonitorService] [elk-nginx1] [gc][young][12305034][42016] duration [1.9s], collections [1]/[2.7s], total [1.9s]/[5.8h], memory [753.2mb]->[489.5mb]/[990.7mb], all_pools {[young] [265.8mb]->[2.8mb]/[266.2mb]}{[survivor] [68kb]->[61.5kb]/[33.2mb]}{[old] [487.3mb]->[487.3mb]/[691.2mb]}
[2021-05-12T10:30:15,320][WARN ][o.e.m.j.JvmGcMonitorService] [elk-nginx1] [gc][12305034] overhead, spent [1.9s] collecting in the last [2.7s]
[2021-05-12T10:39:56,391][INFO ][o.e.m.j.JvmGcMonitorService] [elk-nginx1] [gc][young][12305614][42018] duration [879ms], collections [1]/[1.6s], total [879ms]/[5.8h], memory [753.6mb]->[489.3mb]/[990.7mb], all_pools {[young] [266.2mb]->[2.7mb]/[266.2mb]}{[survivor] [72.5kb]->[47.5kb]/[33.2mb]}{[old] [487.3mb]->[487.3mb]/[691.2mb]}
[2021-05-12T10:39:56,548][WARN ][o.e.m.j.JvmGcMonitorService] [elk-nginx1] [gc][12305614] overhead, spent [879ms] collecting in the last [1.6s]
[2021-05-12T10:49:39,965][WARN ][o.e.m.j.JvmGcMonitorService] [elk-nginx1] [gc][young][12306196][42020] duration [1.2s], collections [1]/[1.8s], total [1.2s]/[5.8h], memory [752.1mb]->[488.7mb]/[990.7mb], all_pools {[young] [264.6mb]->[2.4mb]/[266.2mb]}{[survivor] [65.3kb]->[60.1kb]/[33.2mb]}{[old] [487.3mb]->[487.3mb]/[691.2mb]}
[2021-05-12T10:49:40,198][WARN ][o.e.m.j.JvmGcMonitorService] [elk-nginx1] [gc][12306196] overhead, spent [1.2s] collecting in the last [1.8s]
[2021-05-12T10:59:20,773][WARN ][o.e.m.j.JvmGcMonitorService] [elk-nginx1] [gc][young][12306775][42022] duration [1.1s], collections [1]/[2s], total [1.1s]/[5.8h], memory [751.8mb]->[487.4mb]/[990.7mb], all_pools {[young] [264.4mb]->[2.3mb]/[266.2mb]}{[survivor] [69.2kb]->[50kb]/[33.2mb]}{[old] [487.3mb]->[487.3mb]/[691.2mb]}
[2021-05-12T10:59:21,037][WARN ][o.e.m.j.JvmGcMonitorService] [elk-nginx1] [gc][12306775] overhead, spent [1.1s] collecting in the last [2s]
[2021-05-12T11:09:01,119][WARN ][o.e.m.j.JvmGcMonitorService] [elk-nginx1] [gc][young][12307354][42024] duration [1.4s], collections [1]/[1.6s], total [1.4s]/[5.8h], memory [753mb]->[489.4mb]/[990.7mb], all_pools {[young] [265.6mb]->[2mb]/[266.2mb]}{[survivor] [62.6kb]->[57.7kb]/[33.2mb]}{[old] [487.3mb]->[487.3mb]/[691.2mb]}
[2021-05-12T11:09:01,325][WARN ][o.e.m.j.JvmGcMonitorService] [elk-nginx1] [gc][12307354] overhead, spent [1.4s] collecting in the last [1.6s]
[2021-05-12T11:18:42,745][WARN ][o.e.m.j.JvmGcMonitorService] [elk-nginx1] [gc][young][12307934][42026] duration [1.1s], collections [1]/[1.9s], total [1.1s]/[5.8h], memory [752.1mb]->[489.6mb]/[990.7mb], all_pools {[young] [264.6mb]->[3.5mb]/[266.2mb]}{[survivor] [120.8kb]->[116.7kb]/[33.2mb]}{[old] [487.3mb]->[487.3mb]/[691.2mb]}
[2021-05-12T11:18:42,914][WARN ][o.e.m.j.JvmGcMonitorService] [elk-nginx1] [gc][12307934] overhead, spent [1.1s] collecting in the last [1.9s]
[2021-05-12T11:28:22,865][WARN ][o.e.m.j.JvmGcMonitorService] [elk-nginx1] [gc][young][12308513][42028] duration [1s], collections [1]/[1.5s], total [1s]/[5.8h], memory [752.2mb]->[488.1mb]/[990.7mb], all_pools {[young] [264.8mb]->[1.7mb]/[266.2mb]}{[survivor] [70.2kb]->[83.6kb]/[33.2mb]}{[old] [487.3mb]->[487.3mb]/[691.2mb]}
[2021-05-12T11:28:23,036][WARN ][o.e.m.j.JvmGcMonitorService] [elk-nginx1] [gc][12308513] overhead, spent [1s] collecting in the last [1.5s]
[2021-05-12T11:38:03,488][WARN ][o.e.m.j.JvmGcMonitorService] [elk-nginx1] [gc][young][12309092][42030] duration [1.2s], collections [1]/[2s], total [1.2s]/[5.8h], memory [750.5mb]->[488.8mb]/[990.7mb], all_pools {[young] [263mb]->[2.4mb]/[266.2mb]}{[survivor] [111.2kb]->[94.2kb]/[33.2mb]}{[old] [487.3mb]->[487.3mb]/[691.2mb]}
[2021-05-12T11:38:03,669][WARN ][o.e.m.j.JvmGcMonitorService] [elk-nginx1] [gc][12309092] overhead, spent [1.2s] collecting in the last [2s]
[2021-05-12T11:47:45,360][WARN ][o.e.m.j.JvmGcMonitorService] [elk-nginx1] [gc][young][12309672][42032] duration [1.4s], collections [1]/[2.1s], total [1.4s]/[5.8h], memory [753.5mb]->[489.1mb]/[990.7mb], all_pools {[young] [266.1mb]->[1.7mb]/[266.2mb]}{[survivor] [100.2kb]->[62.7kb]/[33.2mb]}{[old] [487.3mb]->[487.3mb]/[691.2mb]}
[2021-05-12T11:47:45,685][WARN ][o.e.m.j.JvmGcMonitorService] [elk-nginx1] [gc][12309672] overhead, spent [1.4s] collecting in the last [2.1s]

Can you share the output from the _cluster/stats?pretty&human API please?

Yes, sure:

{
  "_nodes" : {
    "total" : 6,
    "successful" : 6,
    "failed" : 0
  },
  "cluster_name" : "LogSummerize",
  "cluster_uuid" : "kVfChQJ8TWWjr5Mrbw_nEg",
  "timestamp" : 1621056068749,
  "status" : "green",
  "indices" : {
    "count" : 636,
    "shards" : {
      "total" : 1272,
      "primaries" : 636,
      "replication" : 1.0,
      "index" : {
        "shards" : {
          "min" : 2,
          "max" : 2,
          "avg" : 2.0
        },
        "primaries" : {
          "min" : 1,
          "max" : 1,
          "avg" : 1.0
        },
        "replication" : {
          "min" : 1.0,
          "max" : 1.0,
          "avg" : 1.0
        }
      }
    },
    "docs" : {
      "count" : 185687318,
      "deleted" : 4311706
    },
    "store" : {
      "size" : "183.6gb",
      "size_in_bytes" : 197177918145
    },
    "fielddata" : {
      "memory_size" : "239kb",
      "memory_size_in_bytes" : 244816,
      "evictions" : 0
    },
    "query_cache" : {
      "memory_size" : "71.2mb",
      "memory_size_in_bytes" : 74691470,
      "total_count" : 44044863,
      "hit_count" : 2584622,
      "miss_count" : 41460241,
      "cache_size" : 33781,
      "cache_count" : 96191,
      "evictions" : 62410
    },
    "completion" : {
      "size" : "0b",
      "size_in_bytes" : 0
    },
    "segments" : {
      "count" : 8842,
      "memory" : "627.1mb",
      "memory_in_bytes" : 657579612,
      "terms_memory" : "464.7mb",
      "terms_memory_in_bytes" : 487306820,
      "stored_fields_memory" : "62.6mb",
      "stored_fields_memory_in_bytes" : 65680144,
      "term_vectors_memory" : "0b",
      "term_vectors_memory_in_bytes" : 0,
      "norms_memory" : "44.5mb",
      "norms_memory_in_bytes" : 46689472,
      "points_memory" : "0b",
      "points_memory_in_bytes" : 0,
      "doc_values_memory" : "55.2mb",
      "doc_values_memory_in_bytes" : 57903176,
      "index_writer_memory" : "119.1mb",
      "index_writer_memory_in_bytes" : 124937376,
      "version_map_memory" : "48.1mb",
      "version_map_memory_in_bytes" : 50460479,
      "fixed_bit_set" : "18.9mb",
      "fixed_bit_set_memory_in_bytes" : 19849336,
      "max_unsafe_auto_id_timestamp" : 1784125209148,
      "file_sizes" : { }
    }
  },
  "nodes" : {
    "count" : {
      "total" : 6,
      "coordinating_only" : 0,
      "data" : 4,
      "ingest" : 6,
      "master" : 4,
      "ml" : 6,
      "voting_only" : 0
    },
    "versions" : [
      "7.6.2"
    ],
    "os" : {
      "available_processors" : 40,
      "allocated_processors" : 40,
      "names" : [
        {
          "name" : "Linux",
          "count" : 6
        }
      ],
      "pretty_names" : [
        {
          "pretty_name" : "CentOS Linux 7 (Core)",
          "count" : 6
        }
      ],
      "mem" : {
        "total" : "34.1gb",
        "total_in_bytes" : 36656943104,
        "free" : "715.6mb",
        "free_in_bytes" : 750460928,
        "used" : "33.4gb",
        "used_in_bytes" : 35906482176,
        "free_percent" : 2,
        "used_percent" : 98
      }
    },
    "process" : {
      "cpu" : {
        "percent" : 5
      },
      "open_file_descriptors" : {
        "min" : 338,
        "max" : 3949,
        "avg" : 2428
      }
    },
    "jvm" : {
      "max_uptime" : "95.8d",
      "max_uptime_in_millis" : 8282674536,
      "versions" : [
        {
          "version" : "13.0.2",
          "vm_name" : "OpenJDK 64-Bit Server VM",
          "vm_version" : "13.0.2+8",
          "vm_vendor" : "AdoptOpenJDK",
          "bundled_jdk" : true,
          "using_bundled_jdk" : true,
          "count" : 6
        }
      ],
      "mem" : {
        "heap_used" : "9.8gb",
        "heap_used_in_bytes" : 10592475968,
        "heap_max" : "17.6gb",
        "heap_max_in_bytes" : 18978701312
      },
      "threads" : 1015
    },
    "fs" : {
      "total" : "1012.5gb",
      "total_in_bytes" : 1087174082560,
      "free" : "814.6gb",
      "free_in_bytes" : 874739716096,
      "available" : "814.6gb",
      "available_in_bytes" : 874739716096
    },
    "plugins" : [ ],
    "network_types" : {
      "transport_types" : {
        "security4" : 6
      },
      "http_types" : {
        "security4" : 6
      }
    },
    "discovery_types" : {
      "zen" : 6
    },
    "packaging_types" : [
      {
        "flavor" : "default",
        "type" : "rpm",
        "count" : 6
      }
    ],
    "ingest" : {
      "number_of_pipelines" : 5,
      "processor_stats" : {
        "geoip" : {
          "count" : 0,
          "failed" : 0,
          "current" : 0,
          "time" : "0s",
          "time_in_millis" : 0
        },
        "gsub" : {
          "count" : 0,
          "failed" : 0,
          "current" : 0,
          "time" : "0s",
          "time_in_millis" : 0
        },
        "pipeline" : {
          "count" : 0,
          "failed" : 0,
          "current" : 0,
          "time" : "0s",
          "time_in_millis" : 0
        },
        "script" : {
          "count" : 0,
          "failed" : 0,
          "current" : 0,
          "time" : "0s",
          "time_in_millis" : 0
        },
        "user_agent" : {
          "count" : 0,
          "failed" : 0,
          "current" : 0,
          "time" : "0s",
          "time_in_millis" : 0
        }
      }
    }
  }
}

Thank you so much

I'd suggest upgrading, 7.12 is the latest minor version and you will find improvements there.
It might help some of the GC you are seeing on the elk-nginx1 node.

I`ll consider it
Thank you so much for helping me

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.