Kv filter for misbehaving


(Hari Prasad) #1

I am currently using kv filter Logstash for parsing the log lines such as following

type=LOGIN msg=audit(1539751621.172:6100294): pid=72964 uid=0 subj=system_u:system_r:crond_t:s0-s0:c0.c1023 old auid=4294967295 new auid=0 old ses=4294967295 new ses=2216742

I noticed an error in ES that "Limit of total fields [1000] in index [] has been exceeded" and when after increasing this limit via template I noticed strange mappings, such as below, being created for the index for these logs.

"""259Z","@metadata":{"beat":"filebeat","type":"doc","version":"6""": {
                "properties": {
                  "4": {
                    "properties": {
                      """2","topic":"log_audit"},"fields":{"document_type":"log_audit"},"beat":{"name":"server1""": {
                        "properties": {
                          "dc1": {
                            "properties": {
                              "server": {
                                "properties": {
                                  """com","hostname":"server1""": {
                                    "properties": {
                                      "dc1": {
                                        "properties": {
                                          "server": {
                                            "properties": {
                                              """com","version":"6""": {
                                                "properties": {
                                                  "4": {
                                                    "properties": {
                                                      """2"},"host":{"name":"server1""": {
                                                        "properties": {
                                                          "dc1": {
                                                            "properties": {
                                                              "server": {
                                                                "properties": {
                                                                  """com"},"source":"audit""": {
                                                                    "properties": {
                                                                      """log","offset":3146106,"message":"type""": {
                                                                        "type": "text",
                                                                        "fields": {
                                                                          "keyword": {
                                                                            "type": "keyword",
                                                                            "ignore_above": 256
                                                                          }
                                                                        }
                                                                      }
                                                                    }
                                                                  }
                                                                }
                                                              }
                                                            }
                                                          }
                                                        }
                                                      }
                                                    }
                                                  }
                                                }
                                              }
                                            }
                                          }
                                        }
                                      }
                                    }
                                  }
                                }
                              }
                            }
                          }
                        }
                      },
                      """2","topic":"log_audit"},"message":"type""": {
                        "type": "text",
                        "fields": {
                          "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                          }
                        }
                      },
                      """2","topic":"log_audit"},"offset":3145388,"message":"type""": {
                        "type": "text",
                        "fields": {
                          "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                          }
                        }
                      },
                      """2","topic":"log_audit"},"offset":3145748,"message":"type""": {
                        "type": "text",
                        "fields": {
                          "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                          }
                        }
                      },
                      """2","topic":"log_audit"},"source":"audit""": {
                        "properties": {
                          """log","offset":3145943,"message":"type""": {
                            "type": "text",
                            "fields": {
                              "keyword": {
                                "type": "keyword",
                                "ignore_above": 256
                              }
                            }
                          }
                        }
                      }
                    }
                  }
                }
              },

I am using the below template for the index creation

input {
  stdin {
    id => "stdin_input"
  }
}
filter {
       kv { }
}
output {
elasticsearch {
       hosts => "localhost:9200"
       index => "log_audit-%{+YYYY.MM.dd}"
       manage_template => true
       template => "log_audit.json"
       template_overwrite => true
       template_name => "log_audit"
  }
}

Please suggest what is going wrong in the set up
Thank you


(system) #2

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.