Fiebeat test output error (Java heap size)

Elastic Stack Beats
1

Hi All

Can anyone help one the below error ? the file beat fails the test and we unable to access the Wazuh's dashboard

[root@ELS01 conf]# filebeat test output
elasticsearch: http://1.1.1.1:9200...
parse url... OK
connection...
parse host... OK
dns lookup... OK
addresses: 1.1.1.1
dial up... OK
TLS... WARN secure connection disabled
talk to server... ERROR Connection marked as failed because the onConnect
callback failed: cannot retrieve the elasticsearch license from the /_xpack
endpoint, Filebeat requires the default distribution of Elasticsearch. Please
make the endpoint accessible to Filebeat so it can verify the license.: could
not retrieve the license information from the cluster: 429 Too Many Requests:
{"error":{"root_cause":
[{"type":"circuit_breaking_exception","reason":"[parent] Data too large, data
for [<http_request>] would be [1000976936/954.6mb], which is larger than the
limit of [986061209/940.3mb], real usage: [1000976936/954.6mb], new bytes
reserved: [0/0b], usages [request=0/0b, fielddata=0/0b,
in_flight_requests=0/0b,
accounting=144179448/137.5mb]","bytes_wanted":1000976936,"bytes_limit":9860612
09,"durability":"PERMANENT"}],"type":"circuit_breaking_exception","reason":"[p
arent] Data too large, data for [<http_request>] would be
[1000976936/954.6mb], which is larger than the limit of [986061209/940.3mb],
real usage: [1000976936/954.6mb], new bytes reserved: [0/0b], usages
[request=0/0b, fielddata=0/0b, in_flight_requests=0/0b,
accounting=144179448/137.5mb]","bytes_wanted":1000976936,"bytes_limit":9860612
09,"durability":"PERMANENT"},"status":429}

2

Welcome to our community! :smiley:

It looks like your Elasticsearch is overloaded. What is the output from the _cluster/stats?pretty&human API?

3

@warkolm Please see below per your request :
curl -XGET 'localhost:9200/_cluster/health?pretty'
{
"cluster_name" : "elasticsearch",
"status" : "green",
"timed_out" : false,
"number_of_nodes" : 1,
"number_of_data_nodes" : 1,
"active_primary_shards" : 462,
"active_shards" : 462,
"relocating_shards" : 0,
"initializing_shards" : 0,
"unassigned_shards" : 0,
"delayed_unassigned_shards" : 0,
"number_of_pending_tasks" : 0,
"number_of_in_flight_fetch" : 0,
"task_max_waiting_in_queue_millis" : 0,
"active_shards_percent_as_number" : 100.0
}
]0;root@XXX:~ [root@XXX~]# curl -XGET 'localhost:9200/_cluster/health?pretty' &' h' u' m' a' n'
{
"cluster_name" : "elasticsearch",
"status" : "green",
"timed_out" : false,
"number_of_nodes" : 1,
"number_of_data_nodes" : 1,
"active_primary_shards" : 462,
"active_shards" : 462,
"relocating_shards" : 0,
"initializing_shards" : 0,
"unassigned_shards" : 0,
"delayed_unassigned_shards" : 0,
"number_of_pending_tasks" : 0,
"number_of_in_flight_fetch" : 0,
"task_max_waiting_in_queue" : "0s",
"task_max_waiting_in_queue_millis" : 0,
"active_shards_percent" : "100.0%",
"active_shards_percent_as_number" : 100.0
}
]0;root@XXX:~ [root@XXX~]#

4

Please format your code/logs/config using the </> button, or markdown style back ticks. It helps to make things easy to read which helps us help you :slight_smile:

Also that is not the API that I asked for, it's _cluster/stats?pretty&human.

5 
{
  "_nodes" : {
    "total" : 1,
    "successful" : 1,
    "failed" : 0
  },
  "cluster_name" : "elasticsearch",
  "cluster_uuid" : "gJPxqzdeSqGMEhYL22a-HQ",
  "timestamp" : 1628150444964,
  "status" : "green",
  "indices" : {
    "count" : 191,
    "shards" : {
      "total" : 462,
      "primaries" : 462,
      "replication" : 0.0,
      "index" : {
        "shards" : {
          "min" : 1,
          "max" : 3,
          "avg" : 2.418848167539267
        },
        "primaries" : {
          "min" : 1,
          "max" : 3,
          "avg" : 2.418848167539267
        },
        "replication" : {
          "min" : 0.0,
          "max" : 0.0,
          "avg" : 0.0
        }
      }
    },
    "docs" : {
      "count" : 57398485,
      "deleted" : 10075
    },
    "store" : {
      "size" : "145.9gb",
      "size_in_bytes" : 156733409177
    },
    "fielddata" : {
      "memory_size" : "0b",
      "memory_size_in_bytes" : 0,
      "evictions" : 0
    },
    "query_cache" : {
      "memory_size" : "0b",
      "memory_size_in_bytes" : 0,
      "total_count" : 0,
      "hit_count" : 0,
      "miss_count" : 0,
      "cache_size" : 0,
      "cache_count" : 0,
      "evictions" : 0
    },
    "completion" : {
      "size" : "0b",
      "size_in_bytes" : 0
    },
    "segments" : {
      "count" : 4326,
      "memory" : "137.5mb",
      "memory_in_bytes" : 144179448,
      "terms_memory" : "104mb",
      "terms_memory_in_bytes" : 109059434,
      "stored_fields_memory" : "26.1mb",
      "stored_fields_memory_in_bytes" : 27389768,
      "term_vectors_memory" : "0b",
      "term_vectors_memory_in_bytes" : 0,
      "norms_memory" : "1.2mb",
      "norms_memory_in_bytes" : 1301312,
      "points_memory" : "1.4mb",
      "points_memory_in_bytes" : 1502798,
      "doc_values_memory" : "4.6mb",
      "doc_values_memory_in_bytes" : 4926136,
      "index_writer_memory" : "0b",
      "index_writer_memory_in_bytes" : 0,
      "version_map_memory" : "0b",
      "version_map_memory_in_bytes" : 0,
      "fixed_bit_set" : "528b",
      "fixed_bit_set_memory_in_bytes" : 528,
      "max_unsafe_auto_id_timestamp" : 1601563153739,
      "file_sizes" : { }
    }
  },
  "nodes" : {
    "count" : {
      "total" : 1,
      "coordinating_only" : 0,
      "data" : 1,
      "ingest" : 1,
      "master" : 1,
      "ml" : 1,
      "voting_only" : 0
    },
    "versions" : [
      "7.5.2"
    ],
    "os" : {
      "available_processors" : 12,
      "allocated_processors" : 12,
      "names" : [
        {
          "name" : "Linux",
          "count" : 1
        }
      ],
      "pretty_names" : [
        {
          "pretty_name" : "RHEL",
          "count" : 1
        }
      ],
      "mem" : {
        "total" : "62.7gb",
        "total_in_bytes" : 67386687488,
        "free" : "25.7gb",
        "free_in_bytes" : 27645804544,
        "used" : "37gb",
        "used_in_bytes" : 39740882944,
        "free_percent" : 41,
        "used_percent" : 59
      }
    },
    "process" : {
      "cpu" : {
        "percent" : 8
      },
      "open_file_descriptors" : {
        "min" : 9943,
        "max" : 9943,
        "avg" : 9943
      }
    },
    "jvm" : {
      "max_uptime" : "6.2d",
      "max_uptime_in_millis" : 540406620,
      "versions" : [
        {
          "version" : "13.0.1",
          "vm_name" : "OpenJDK 64-Bit Server VM",
          "vm_version" : "13.0.1+9",
          "vm_vendor" : "AdoptOpenJDK",
          "bundled_jdk" : true,
          "using_bundled_jdk" : true,
          "count" : 1
        }
      ],
      "mem" : {
        "heap_used" : "978.7mb",
        "heap_used_in_bytes" : 1026335768,
        "heap_max" : "989.8mb",
        "heap_max_in_bytes" : 1037959168
      },
      "threads" : 186
    },
    "fs" : {
      "total" : "215.4gb",
      "total_in_bytes" : 231380877312,
      "free" : "27.2gb",
      "free_in_bytes" : 29278244864,
      "available" : "27.2gb",
      "available_in_bytes" : 29278244864
    },
    "plugins" : [ ],
    "network_types" : {
      "transport_types" : {
        "security4" : 1
      },
      "http_types" : {
        "security4" : 1
      }
    },
    "discovery_types" : {
      "zen" : 1
    },
    "packaging_types" : [
      {
        "flavor" : "default",
        "type" : "rpm",
        "count" : 1
      }
    ]
  }
}
6

Hi @warkolm

Any Idea what might cause the issue ?

Thank you

7

What do you have the JVM heap set at?

Is it perhaps left at the default of 1GB?

On a 60GB host it should be set at 26 - 28GB

Also 7.5 pretty old, newer versions automatically set the proper JVM heap size

1 Like
8

Hi @stephenb

I managed to fix the filebeat issue by increasing the heapsize . however I "Cant" connect to management interface listening to port 5601.
_cluster/stats?pretty&human` new out put per below :

{
  "_nodes" : {
    "total" : 1,
    "successful" : 1,
    "failed" : 0
  },
  "cluster_name" : "elasticsearch",
  "cluster_uuid" : "gJPxqzdeSqGMEhYL22a-HQ",
  "timestamp" : 1629434471001,
  "status" : "green",
  "indices" : {
    "count" : 216,
    "shards" : {
      "total" : 525,
      "primaries" : 525,
      "replication" : 0.0,
      "index" : {
        "shards" : {
          "min" : 1,
          "max" : 3,
          "avg" : 2.4305555555555554
        },
        "primaries" : {
          "min" : 1,
          "max" : 3,
          "avg" : 2.4305555555555554
        },
        "replication" : {
          "min" : 0.0,
          "max" : 0.0,
          "avg" : 0.0
        }
      }
    },
    "docs" : {
      "count" : 59326820,
      "deleted" : 10420
    },
    "store" : {
      "size" : "149.4gb",
      "size_in_bytes" : 160490395427
    },
    "fielddata" : {
      "memory_size" : "0b",
      "memory_size_in_bytes" : 0,
      "evictions" : 0
    },
    "query_cache" : {
      "memory_size" : "0b",
      "memory_size_in_bytes" : 0,
      "total_count" : 0,
      "hit_count" : 0,
      "miss_count" : 0,
      "cache_size" : 0,
      "cache_count" : 0,
      "evictions" : 0
    },
    "completion" : {
      "size" : "0b",
      "size_in_bytes" : 0
    },
    "segments" : {
      "count" : 4780,
      "memory" : "145.2mb",
      "memory_in_bytes" : 152311951,
      "terms_memory" : "110.3mb",
      "terms_memory_in_bytes" : 115715159,
      "stored_fields_memory" : "26.9mb",
      "stored_fields_memory_in_bytes" : 28240488,
      "term_vectors_memory" : "0b",
      "term_vectors_memory_in_bytes" : 0,
      "norms_memory" : "1.4mb",
      "norms_memory_in_bytes" : 1494784,
      "points_memory" : "1.4mb",
      "points_memory_in_bytes" : 1551936,
      "doc_values_memory" : "5mb",
      "doc_values_memory_in_bytes" : 5309584,
      "index_writer_memory" : "0b",
      "index_writer_memory_in_bytes" : 0,
      "version_map_memory" : "0b",
      "version_map_memory_in_bytes" : 0,
      "fixed_bit_set" : "528b",
      "fixed_bit_set_memory_in_bytes" : 528,
      "max_unsafe_auto_id_timestamp" : 1628479262137,
      "file_sizes" : { }
    }
  },
  "nodes" : {
    "count" : {
      "total" : 1,
      "coordinating_only" : 0,
      "data" : 1,
      "ingest" : 1,
      "master" : 1,
      "ml" : 1,
      "voting_only" : 0
    },
    "versions" : [
      "7.5.2"
    ],
    "os" : {
      "available_processors" : 12,
      "allocated_processors" : 12,
      "names" : [
        {
          "name" : "Linux",
          "count" : 1
        }
      ],
      "pretty_names" : [
        {
          "pretty_name" : "RHEL",
          "count" : 1
        }
      ],
      "mem" : {
        "total" : "62.7gb",
        "total_in_bytes" : 67386687488,
        "free" : "9.7gb",
        "free_in_bytes" : 10433200128,
        "used" : "53gb",
        "used_in_bytes" : 56953487360,
        "free_percent" : 15,
        "used_percent" : 85
      }
    },
    "process" : {
      "cpu" : {
        "percent" : 0
      },
      "open_file_descriptors" : {
        "min" : 15185,
        "max" : 15185,
        "avg" : 15185
      }
    },
    "jvm" : {
      "max_uptime" : "13.4m",
      "max_uptime_in_millis" : 809013,
      "versions" : [
        {
          "version" : "13.0.1",
          "vm_name" : "OpenJDK 64-Bit Server VM",
          "vm_version" : "13.0.1+9",
          "vm_vendor" : "AdoptOpenJDK",
          "bundled_jdk" : true,
          "using_bundled_jdk" : true,
          "count" : 1
        }
      ],
      "mem" : {
        "heap_used" : "1.6gb",
        "heap_used_in_bytes" : 1756215688,
        "heap_max" : "25.9gb",
        "heap_max_in_bytes" : 27830059008
      },
      "threads" : 97
    },
    "fs" : {
      "total" : "215.4gb",
      "total_in_bytes" : 231380877312,
      "free" : "47.8gb",
      "free_in_bytes" : 51360935936,
      "available" : "47.8gb",
      "available_in_bytes" : 51360935936
    },
    "plugins" : [ ],
    "network_types" : {
      "transport_types" : {
        "security4" : 1
      },
      "http_types" : {
        "security4" : 1
      }
    },
    "discovery_types" : {
      "zen" : 1
    },
    "packaging_types" : [
      {
        "flavor" : "default",
        "type" : "rpm",
        "count" : 1
      }
    ]
  }
}

Netstat Status : :grinning:

[root@XXX ~]# netstat -a -n | grep tcp | grep 5601 tcp 0 0 127.0.0.1:5601 0.0.0.0:* LISTEN
[root@XXX ~]# netstat -ntlp
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name
tcp 0 0 0.0.0.0:55932 0.0.0.0:* LISTEN 2400/cvd
tcp 0 0 127.0.0.1:5601 0.0.0.0:* LISTEN 1204/node
tcp 0 0 0.0.0.0:49157 0.0.0.0:* LISTEN 2401/ClMgrS
tcp 0 0 0.0.0.0:1515 0.0.0.0:* LISTEN 2192/ossec-authd
tcp 0 0 127.0.0.1:51536 0.0.0.0:* LISTEN 2400/cvd
tcp 0 0 0.0.0.0:8400 0.0.0.0:* LISTEN 2400/cvd
tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 1619/sshd
tcp 0 0 127.0.0.1:25 0.0.0.0:* LISTEN 1973/master
tcp6 0 0 :::14942 :::* LISTEN 2330/splxhttpd
tcp6 0 0 :::14943 :::* LISTEN 2330/splxhttpd
tcp6 0 0 1.1.1.1:9200 :::* LISTEN 436/java
tcp6 0 0 :::80 :::* LISTEN 1620/httpd
tcp6 0 0 1.1.1.1:9300 :::* LISTEN 436/java
tcp6 0 0 :::22 :::* LISTEN 1619/sshd
tcp6 0 0 :::55000 :::* LISTEN 1629/node
tcp6 0 0 ::1:25 :::* LISTEN

9

There is no 'management interface" for Elasticsearch running on port 5601.

I Believe what you referring to is Kibana which is an entire separate app that you need to install and configure.

Kibana runs on Port 5601 and then connects to Elasticsearch. Kibana Is the management and data exploration interface.

See here

10

I just wanted to drop in and mention 7.5 is EOL, please upgrade :slight_smile:

11

Hi All

Filebeat issue resolved by increasing the JVM heapsize
Kibana issue resolved by disabling the Firewall-cmd module

Thank you for cool community support
Cheers

12

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.