Multiple Log Files With Different Output Fails


(Sparsh Singhal) #1

I am getting 3 log files from the FileBeat on 4 nodes. Only 1 log from single node is getting indexed in elasticsearch.

input {
  beats {
    port => 5044
    ssl => true
    ssl_certificate => "/etc/pki/tls/certs/logstash-forwarder.crt"
    ssl_key => "/etc/pki/tls/private/logstash-forwarder.key"
  }
}

filter {
    if [fields][log_type] == "namenode" {
    grok {
      match => {
      "message" => "%{DATESTAMP:log_timestamp} %{LOGLEVEL:log_level}  (?<log_action>\b\w+(\.\w+)?\b) (%{DATA:log_actor}) - %{GREEDYDATA:log_message}"
      }
      add_field => [ "received_from", "%{host}" ]
    }
    date {
      match => [ "log_timestamp", "yyyy-MM-dd HH:mm:ss,SSS"]
      target => "log_timestamp"
    }
  }
  else if [fields][log_type] == "namenode_audit" {
    grok {
      match => {
      "message" =>  "%{DATESTAMP:log_timestamp} %{LOGLEVEL:log_level} (?:\b\w+(\.\w+)?\b): allowed=(?<allowed>true|false)       ugi=%{DATA:user} (%{DATA:authentication_type})   ip=/%{IPV4:host_requested}      cmd=%{DATA:command}     src=%{DATA:source}      dst=%{DATA:destination}  perm=%{DATA:permission} proto=%{GREEDYDATA:protocol}"
      }
      add_field => [ "received_from", "%{host}" ]
    }
    date {
      match => [ "log_timestamp", "yyyy-MM-dd HH:mm:ss,SSS"]
      target => "log_timestamp"
    }
  }
  else if [fields][log_type] == "resource_manager" {
    grok {
      match => {
      "message" =>  "%{GREEDYDATA:data}"
      }
      add_field => [ "received_from", "%{host}" ]
    }
    date {
      match => [ "log_timestamp", "yyyy-MM-dd HH:mm:ss,SSS"]
      target => "log_timestamp"
    }
  }
}

output {
  elasticsearch {
    hosts => "localhost:9200"
    index => "%{[fields][log_type]}-%{[beat][hostname]}"
    manage_template => false
  }
}

(system) #2

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.