Hello,
I hope my question finds you and your loved ones safe and healthy.
I am having errors trying to ingest system metrics (system module) from a single Ubuntu system running 22.04.1 using Metricbeat (version 7.17.7 and 8.4.3) via Logstash (7.17.7). The logs are ingested via a common beats pipeline running having the following configuration:
input {
beats {
port => 5074
#id => "beats_ingest"
}
}
output {
elasticsearch {
hosts => ["https://192.168.0.235:9200","https://192.168.0.236:9200"]
ssl => true
user => 'redacted'
password => 'redacted'
cacert => '/etc/logstash/elasticsearch-ca.pem'
ssl_certificate_verification => false
index => "%{[@metadata][beat]}-%{+YYYY.MM}"
ilm_enabled => auto
ilm_rollover_alias => "%{[@metadata][beat]}"
}
}
The error I get is:
Limit of total fields [1000] has been exceeded while adding new fields
I have applied the following setting from:
- What does "Limit of total fields [1000] in index has been exceeded" means in Elasticsearch - Stack Overflow
- Approaches to deal with "Limit of total fields [1000] in index has been exceeded"
I've applied the following:
PUT metricbeat/_settings
{
"index.mapping.total_fields.limit": 2000
}
and also
PUT metricbeat*/_settings
{
"index.mapping.total_fields.limit": 2000
}
However the error is persistent.
I am confused as to why Metricbeat running default configuration to capture system metrics is giving this error
Error log on logstash and it is odd that I see fields such as ios while the system is
[2022-10-27T23:20:39,929][WARN ][logstash.outputs.elasticsearch][beats_ingest][71731faf1263f2b407ad56129215210a4b6a97a861ce28f6c83821a199a44315] Could not index event to Elasticsearch. {:status=>400, :action=>["index", {:_id=>nil, :_index=>"metricbeat", :routing=>nil}, {"host"=>{"hostname"=>"inmum-x-pslp01", "mac"=>["00:50:56:ae:6c:e9"], "os"=>{"codename"=>"jammy", "version"=>"22.04.1 LTS (Jammy Jellyfish)", "type"=>"linux", "name"=>"Ubuntu", "kernel"=>"5.15.0-52-generic", "family"=>"debian", "platform"=>"ubuntu"}, "id"=>"9b066925c16c49b99a4aa4e287d27c85", "architecture"=>"x86_64", "ip"=>["192.168.0.50", "fe80::250:56ff:feae:6ce9"], "containerized"=>false, "name"=>"inmum-x-pslp01"}, "user"=>{"name"=>"root"}, "ecs"=>{"version"=>"8.0.0"}, "@timestamp"=>2022-10-27T23:20:38.803Z, "system"=>{"process"=>{"cgroup"=>{"path"=>"/system.slice/ElasticEndpoint.service", "io"=>{"stats"=>{"dm-1"=>{"discarded"=>{"bytes"=>0, "ios"=>0}, "read"=>{"bytes"=>78123008, "ios"=>1305}, "write"=>{"bytes"=>2068480, "ios"=>114}}, "loop7"=>{"discarded"=>{"bytes"=>0, "ios"=>0}, "read"=>{"bytes"=>7168, "ios"=>2}, "write"=>{"bytes"=>0, "ios"=>0}}, "loop14"=>{"discarded"=>{"bytes"=>0, "ios"=>0}, "read"=>{"bytes"=>4319232, "ios"=>44}, "write"=>{"bytes"=>0, "ios"=>0}}, "sda"=>{"discarded"=>{"bytes"=>0, "ios"=>0}, "read"=>{"bytes"=>78645760, "ios"=>2326}, "write"=>{"bytes"=>0, "ios"=>0}}, "dm-0"=>{"discarded"=>{"bytes"=>0, "ios"=>0}, "read"=>{"bytes"=>78123008, "ios"=>1305}, "write"=>{"bytes"=>2068480, "ios"=>114}}}, "path"=>"/system.slice/ElasticEndpoint.service", "pressure"=>{"some"=>{"total"=>255628, "300"=>{"pct"=>0}, "60"=>{"pct"=>0}, "10"=>{"pct"=>0}}, "full"=>{"total"=>252043, "300"=>{"pct"=>0}, "60"=>{"pct"=>0}, "10"=>{"pct"=>0}}}, "id"=>"ElasticEndpoint.service"}, "cpu"=>{"pressure"=>{"some"=>{"total"=>9621535, "300"=>{"pct"=>0.25}, "60"=>{"pct"=>0.03}, "10"=>{"pct"=>0.02}}, "full"=>{"total"=>9030204, "300"=>{"pct"=>0.21}, "60"=>{"pct"=>0.01}, "10"=>{"pct"=>0}}}, "path"=>"/system.slice/ElasticEndpoint.service", "stats"=>{"periods"=>0, "user"=>{"pct"=>0, "norm"=>{"pct"=>0}, "ns"=>10303865}, "system"=>{"pct"=>0, "norm"=>{"pct"=>0}, "ns"=>6652095}, "throttled"=>{"periods"=>0, "us"=>0}, "usage"=>{"pct"=>0, "norm"=>{"pct"=>0}, "ns"=>16955960}}, "id"=>"ElasticEndpoint.service"}, "id"=>"ElasticEndpoint.service", "memory"=>{"stats"=>{"page_faults"=>32150, "page_steal"=>0, "file_thp"=>{"bytes"=>0}, "inactive_anon"=>{"bytes"=>52658176}, "workingset_activate_anon"=>0, "swap_cached"=>{"bytes"=>0}, "page_refill"=>0, "slab_reclaimable"=>{"bytes"=>3350560}, "kernel_stack"=>{"bytes"=>376832}, "page_lazy_freed"=>0, "workingset_restore_anon"=>0, "file"=>{"bytes"=>43384832}, "slab"=>{"bytes"=>3579568}, "major_page_faults"=>72, "inactive_file"=>{"bytes"=>36892672}, "workingset_node_reclaim"=>0, "active_file"=>{"bytes"=>6492160}, "file_writeback"=>{"bytes"=>0}, "anon"=>{"bytes"=>52662272}, "workingset_restore_file"=>0, "page_lazy_free"=>0, "unevictable"=>{"bytes"=>0}, "workingset_refault_file"=>0, "file_mapped"=>{"bytes"=>8216576}, "active_anon"=>{"bytes"=>4096}, "workingset_refault_anon"=>0, "page_activate"=>1794, "anon_thp"=>{"bytes"=>0}, "page_scan"=>0, "page_deactivate"=>0, "shmem_thp"=>{"bytes"=>0}, "per_cpu"=>{"bytes"=>0}, "workingset_activate_file"=>0, "thp_fault_alloc"=>0, "shmem"=>{"bytes"=>0}, "file_dirty"=>{"bytes"=>16384}, "sock"=>{"bytes"=>0}, "page_tables"=>{"bytes"=>335872}, "htp_collapse_alloc"=>0, "slab_unreclaimable"=>{"bytes"=>229008}}, "mem"=>{"low"=>{"bytes"=>0}, "events"=>{"high"=>0, "max"=>0, "low"=>0, "oom"=>0, "oom_kill"=>0}, "usage"=>{"bytes"=>100360192}}, "path"=>"/system.slice/ElasticEndpoint.service", "memsw"=>{"events"=>{"high"=>0, "max"=>0, "fail"=>0}, "low"=>{"bytes"=>0}, "usage"=>{"bytes"=>0}}, "id"=>"ElasticEndpoint.service"}, "cgroups_version"=>2}, "cpu"=>{"total"=>{"pct"=>0.006, "norm"=>{"pct"=>0.006}, "value"=>16950}, "start_time"=>"2022-10-27T23:06:45.000Z"}, "memory"=>{"size"=>764297216, "share"=>12304384, "rss"=>{"bytes"=>63598592, "pct"=>0.0155}}, "state"=>"sleeping", "fd"=>{"open"=>121, "limit"=>{"soft"=>8192, "hard"=>8192}}, "cmdline"=>"/opt/Elastic/Endpoint/elastic-endpoint run"}}, "service"=>{"type"=>"system"}, "tags"=>["beats_input_raw_event"], "process"=>{"pid"=>932, "memory"=>{"pct"=>0.0155}, "executable"=>"/opt/Elastic/Endpoint/elastic-endpoint", "name"=>"elastic-endpoin", "state"=>"sleeping", "command_line"=>"/opt/Elastic/Endpoint/elastic-endpoint run", "pgid"=>932, "args"=>["/opt/Elastic/Endpoint/elastic-endpoint", "run"], "cpu"=>{"start_time"=>"2022-10-27T23:06:45.000Z", "pct"=>0.006}, "working_directory"=>"/", "parent"=>{"pid"=>1}}, "agent"=>{"version"=>"8.4.3", "type"=>"metricbeat", "name"=>"inmum-x-pslp01", "ephemeral_id"=>"1d78db85-6e26-4c81-9e37-de822c322216", "id"=>"81759243-2e2f-4bd8-bc61-83ff4b43b543"}, "@version"=>"1", "metricset"=>{"name"=>"process", "period"=>10000}, "event"=>{"dataset"=>"system.process", "duration"=>115778553, "module"=>"system"}}], :response=>{"index"=>{"_index"=>"metricbeat", "_id"=>"hHy_G4QBBPNtLXwjabQS", "status"=>400, "error"=>{"type"=>"mapper_parsing_exception", "reason"=>"failed to parse", "caused_by"=>{"type"=>"illegal_argument_exception", "reason"=>"Limit of total fields [1000] has been exceeded while adding new fields [2]"}}}}}