Error: elastic agent error failed to publish events: temporary bulk send failure

Elastic Stack Elasticsearch
1

hello I have a cluster of 3 ES nodes version 8.7.1
I am managing the agents using the fleet
I had installed Elasticagent on a Windows VM to ingest the fortigate logs and it was successfully
until one day logs ingestion stopped and the below error started rising in the elastic agent event logs
note that FortiGate is sending 20 GB of events per day

{"log.level":"error","@timestamp":"2024-03-28T09:36:24.659+0200","message":"failed to publish events: temporary bulk send failure","component":{"binary":"filebeat","dataset":"elastic_agent.filebeat","id":"udp-default","type":"udp"},"log":{"source":"udp-default"},"service.name":"filebeat","ecs.version":"1.6.0","log.logger":"publisher_pipeline_output","log.origin":{"file.line":176,"file.name":"pipeline/client_worker.go"},"ecs.version":"1.6.0"}
2

What is the full output of the cluster stats API?

3

i see no problem

{
  "_nodes": {
    "total": 3,
    "successful": 3,
    "failed": 0
  },
  "cluster_name": "co-cluster",
  "cluster_uuid": "yP6w8I5tQ5qEXuP9K9wBCw",
  "timestamp": 1711618392594,
  "status": "green",
  "indices": {
    "count": 57,
    "shards": {
      "total": 115,
      "primaries": 57,
      "replication": 1.0175438596491229,
      "index": {
        "shards": {
          "min": 2,
          "max": 3,
          "avg": 2.017543859649123
        },
        "primaries": {
          "min": 1,
          "max": 1,
          "avg": 1
        },
        "replication": {
          "min": 1,
          "max": 2,
          "avg": 1.0175438596491229
        }
      }
    },
    "docs": {
      "count": 214674445,
      "deleted": 783379
    },
    "store": {
      "size_in_bytes": 159955585845,
      "total_data_set_size_in_bytes": 159955585845,
      "reserved_in_bytes": 0
    },
    "fielddata": {
      "memory_size_in_bytes": 330816,
      "evictions": 0
    },
    "query_cache": {
      "memory_size_in_bytes": 7023887,
      "total_count": 65689768,
      "hit_count": 2408696,
      "miss_count": 63281072,
      "cache_size": 47,
      "cache_count": 291,
      "evictions": 244
    },
    "completion": {
      "size_in_bytes": 0
    },
    "segments": {
      "count": 775,
      "memory_in_bytes": 0,
      "terms_memory_in_bytes": 0,
      "stored_fields_memory_in_bytes": 0,
      "term_vectors_memory_in_bytes": 0,
      "norms_memory_in_bytes": 0,
      "points_memory_in_bytes": 0,
      "doc_values_memory_in_bytes": 0,
      "index_writer_memory_in_bytes": 57626244,
      "version_map_memory_in_bytes": 0,
      "fixed_bit_set_memory_in_bytes": 3168,
      "max_unsafe_auto_id_timestamp": 1711055972336,
      "file_sizes": {}
    },
    "mappings": {
      "total_field_count": 5873,
      "total_deduplicated_field_count": 5126,
      "total_deduplicated_mapping_size_in_bytes": 53634,
      "field_types": [
        {
          "name": "alias",
          "count": 1,
          "index_count": 1,
          "script_count": 0
        },
        {
          "name": "binary",
          "count": 4,
          "index_count": 4,
          "script_count": 0
        },
        {
          "name": "boolean",
          "count": 70,
          "index_count": 35,
          "script_count": 0
        },
        {
          "name": "byte",
          "count": 1,
          "index_count": 1,
          "script_count": 0
        },
        {
          "name": "constant_keyword",
          "count": 128,
          "index_count": 30,
          "script_count": 0
        },
        {
          "name": "date",
          "count": 116,
          "index_count": 38,
          "script_count": 0
        },
        {
          "name": "date_nanos",
          "count": 1,
          "index_count": 1,
          "script_count": 0
        },
        {
          "name": "date_range",
          "count": 1,
          "index_count": 1,
          "script_count": 0
        },
        {
          "name": "double",
          "count": 52,
          "index_count": 3,
          "script_count": 0
        },
        {
          "name": "double_range",
          "count": 1,
          "index_count": 1,
          "script_count": 0
        },
        {
          "name": "float",
          "count": 39,
          "index_count": 7,
          "script_count": 0
        },
        {
          "name": "float_range",
          "count": 1,
          "index_count": 1,
          "script_count": 0
        },
        {
          "name": "geo_point",
          "count": 10,
          "index_count": 8,
          "script_count": 0
        },
        {
          "name": "geo_shape",
          "count": 1,
          "index_count": 1,
          "script_count": 0
        },
        {
          "name": "half_float",
          "count": 1,
          "index_count": 1,
          "script_count": 0
        },
        {
          "name": "integer",
          "count": 6,
          "index_count": 4,
          "script_count": 0
        },
        {
          "name": "integer_range",
          "count": 1,
          "index_count": 1,
          "script_count": 0
        },
        {
          "name": "ip",
          "count": 79,
          "index_count": 31,
          "script_count": 0
        },
        {
          "name": "ip_range",
          "count": 1,
          "index_count": 1,
          "script_count": 0
        },
        {
          "name": "keyword",
          "count": 3046,
          "index_count": 40,
          "script_count": 0
        },
        {
          "name": "long",
          "count": 868,
          "index_count": 29,
          "script_count": 0
        },
        {
          "name": "long_range",
          "count": 1,
          "index_count": 1,
          "script_count": 0
        },
        {
          "name": "match_only_text",
          "count": 72,
          "index_count": 17,
          "script_count": 0
        },
        {
          "name": "nested",
          "count": 1,
          "index_count": 1,
          "script_count": 0
        },
        {
          "name": "object",
          "count": 1261,
          "index_count": 37,
          "script_count": 0
        },
        {
          "name": "scaled_float",
          "count": 42,
          "index_count": 5,
          "script_count": 0
        },
        {
          "name": "shape",
          "count": 1,
          "index_count": 1,
          "script_count": 0
        },
        {
          "name": "short",
          "count": 4,
          "index_count": 2,
          "script_count": 0
        },
        {
          "name": "text",
          "count": 55,
          "index_count": 31,
          "script_count": 0
        },
        {
          "name": "version",
          "count": 1,
          "index_count": 1,
          "script_count": 0
        },
        {
          "name": "wildcard",
          "count": 7,
          "index_count": 6,
          "script_count": 0
        }
      ],
      "runtime_field_types": []
    },
    "analysis": {
      "char_filter_types": [],
      "tokenizer_types": [],
      "filter_types": [],
      "analyzer_types": [],
      "built_in_char_filters": [],
      "built_in_tokenizers": [],
      "built_in_filters": [],
      "built_in_analyzers": []
    },
    "versions": [
      {
        "version": "8.7.1",
        "index_count": 57,
        "primary_shard_count": 57,
        "total_primary_bytes": 80121677604
      }
    ],
    "search": {
      "total": 855874,
      "queries": {
        "geo_bounding_box": 24,
        "bool": 711268,
        "prefix": 34,
        "match": 16933,
        "range": 325884,
        "nested": 34,
        "wildcard": 23,
        "match_phrase": 5850,
        "terms": 367485,
        "match_phrase_prefix": 49,
        "match_all": 17,
        "exists": 350702,
        "term": 418610,
        "simple_query_string": 48931
      },
      "sections": {
        "highlight": 39,
        "search_after": 35,
        "stored_fields": 750,
        "runtime_mappings": 28993,
        "query": 783932,
        "script_fields": 750,
        "pit": 1172,
        "_source": 45679,
        "terminate_after": 236,
        "fields": 31778,
        "collapse": 119,
        "aggs": 127126
      }
    }
  },
  "nodes": {
    "count": {
      "total": 3,
      "coordinating_only": 0,
      "data": 3,
      "data_cold": 3,
      "data_content": 3,
      "data_frozen": 3,
      "data_hot": 3,
      "data_warm": 3,
      "index": 0,
      "ingest": 3,
      "master": 3,
      "ml": 3,
      "remote_cluster_client": 3,
      "search": 0,
      "transform": 3,
      "voting_only": 0
    },
    "versions": [
      "8.7.1"
    ],
    "os": {
      "available_processors": 48,
      "allocated_processors": 48,
      "names": [
        {
          "name": "Linux",
          "count": 3
        }
      ],
      "pretty_names": [
        {
          "pretty_name": "Ubuntu 22.04.4 LTS",
          "count": 3
        }
      ],
      "architectures": [
        {
          "arch": "amd64",
          "count": 3
        }
      ],
      "mem": {
        "total_in_bytes": 98469335040,
        "adjusted_total_in_bytes": 98469335040,
        "free_in_bytes": 2925678592,
        "used_in_bytes": 95543656448,
        "free_percent": 3,
        "used_percent": 97
      }
    },
    "process": {
      "cpu": {
        "percent": 3
      },
      "open_file_descriptors": {
        "min": 629,
        "max": 891,
        "avg": 796
      }
    },
    "jvm": {
      "max_uptime_in_millis": 1456593835,
      "versions": [
        {
          "version": "20.0.1",
          "vm_name": "OpenJDK 64-Bit Server VM",
          "vm_version": "20.0.1+9-29",
          "vm_vendor": "Oracle Corporation",
          "bundled_jdk": true,
          "using_bundled_jdk": true,
          "count": 3
        }
      ],
      "mem": {
        "heap_used_in_bytes": 22030022448,
        "heap_max_in_bytes": 49249517568
      },
      "threads": 441
    },
    "fs": {
      "total_in_bytes": 315267784704,
      "free_in_bytes": 103001788416,
      "available_in_bytes": 86845329408
    },
    "plugins": [],
    "network_types": {
      "transport_types": {
        "security4": 3
      },
      "http_types": {
        "security4": 3
      }
    },
    "discovery_types": {
      "multi-node": 3
    },
    "packaging_types": [
      {
        "flavor": "default",
        "type": "deb",
        "count": 3
      }
    ],
    "ingest": {
      "number_of_pipelines": 70,
      "processor_stats": {
        "append": {
          "count": 2626420087,
          "failed": 0,
          "current": 0,
          "time_in_millis": 4816836
        },
        "convert": {
          "count": 7731577425,
          "failed": 486,
          "current": 0,
          "time_in_millis": 6535090
        },
        "date": {
          "count": 1065705485,
          "failed": 0,
          "current": 0,
          "time_in_millis": 19363700
        },
        "dissect": {
          "count": 2,
          "failed": 2,
          "current": 0,
          "time_in_millis": 146
        },
        "foreach": {
          "count": 0,
          "failed": 0,
          "current": 0,
          "time_in_millis": 0
        },
        "geoip": {
          "count": 2073893357,
          "failed": 0,
          "current": 0,
          "time_in_millis": 9202929
        },
        "grok": {
          "count": 351436783,
          "failed": 897,
          "current": 0,
          "time_in_millis": 5126628
        },
        "gsub": {
          "count": 2459544118,
          "failed": 0,
          "current": 0,
          "time_in_millis": 5656924
        },
        "kv": {
          "count": 0,
          "failed": 0,
          "current": 0,
          "time_in_millis": 0
        },
        "lowercase": {
          "count": 376037323,
          "failed": 0,
          "current": 0,
          "time_in_millis": 435094
        },
        "network_direction": {
          "count": 341625093,
          "failed": 0,
          "current": 0,
          "time_in_millis": 2912052
        },
        "pipeline": {
          "count": 714273752,
          "failed": 0,
          "current": 0,
          "time_in_millis": 22821682
        },
        "registered_domain": {
          "count": 9818198,
          "failed": 0,
          "current": 0,
          "time_in_millis": 11265
        },
        "remove": {
          "count": 1789833374,
          "failed": 3262,
          "current": 0,
          "time_in_millis": 4518945
        },
        "rename": {
          "count": 14741159634,
          "failed": 996952,
          "current": 0,
          "time_in_millis": 16036426
        },
        "script": {
          "count": 3467446881,
          "failed": 0,
          "current": 0,
          "time_in_millis": 24509980
        },
        "set": {
          "count": 3191798103,
          "failed": 42,
          "current": 0,
          "time_in_millis": 6776884
        },
        "set_security_user": {
          "count": 362869644,
          "failed": 0,
          "current": 0,
          "time_in_millis": 3653131
        },
        "split": {
          "count": 19636807,
          "failed": 0,
          "current": 0,
          "time_in_millis": 19921
        },
        "uppercase": {
          "count": 702863278,
          "failed": 0,
          "current": 0,
          "time_in_millis": 477065
        }
      }
    },
    "indexing_pressure": {
      "memory": {
        "current": {
          "combined_coordinating_and_primary_in_bytes": 0,
          "coordinating_in_bytes": 0,
          "primary_in_bytes": 0,
          "replica_in_bytes": 0,
          "all_in_bytes": 0
        },
        "total": {
          "combined_coordinating_and_primary_in_bytes": 0,
          "coordinating_in_bytes": 0,
          "primary_in_bytes": 0,
          "replica_in_bytes": 0,
          "all_in_bytes": 0,
          "coordinating_rejections": 0,
          "primary_rejections": 0,
          "replica_rejections": 0
        },
        "limit_in_bytes": 0
      }
    }
  }
}
4

Yes, that looks fine. Is there anything in the Elasticsearch logs?

5

nothing in the elasticsearch logs

but is it normal that the last record in elasticsearch.log is from 8/2/2024

6

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.