I am currently using kv filter Logstash for parsing the log lines such as following
type=LOGIN msg=audit(1539751621.172:6100294): pid=72964 uid=0 subj=system_u:system_r:crond_t:s0-s0:c0.c1023 old auid=4294967295 new auid=0 old ses=4294967295 new ses=2216742
I noticed an error in ES that "Limit of total fields [1000] in index [] has been exceeded" and when after increasing this limit via template I noticed strange mappings, such as below, being created for the index for these logs.
"""259Z","@metadata":{"beat":"filebeat","type":"doc","version":"6""": {
"properties": {
"4": {
"properties": {
"""2","topic":"log_audit"},"fields":{"document_type":"log_audit"},"beat":{"name":"server1""": {
"properties": {
"dc1": {
"properties": {
"server": {
"properties": {
"""com","hostname":"server1""": {
"properties": {
"dc1": {
"properties": {
"server": {
"properties": {
"""com","version":"6""": {
"properties": {
"4": {
"properties": {
"""2"},"host":{"name":"server1""": {
"properties": {
"dc1": {
"properties": {
"server": {
"properties": {
"""com"},"source":"audit""": {
"properties": {
"""log","offset":3146106,"message":"type""": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}
}
}
}
}
}
}
}
}
}
}
}
}
}
}
}
}
}
}
}
}
},
"""2","topic":"log_audit"},"message":"type""": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"""2","topic":"log_audit"},"offset":3145388,"message":"type""": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"""2","topic":"log_audit"},"offset":3145748,"message":"type""": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"""2","topic":"log_audit"},"source":"audit""": {
"properties": {
"""log","offset":3145943,"message":"type""": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}
}
}
},
I am using the below template for the index creation
input {
stdin {
id => "stdin_input"
}
}
filter {
kv { }
}
output {
elasticsearch {
hosts => "localhost:9200"
index => "log_audit-%{+YYYY.MM.dd}"
manage_template => true
template => "log_audit.json"
template_overwrite => true
template_name => "log_audit"
}
}
Please suggest what is going wrong in the set up
Thank you