Identify filebeat input with tags and input type


(Nikhil Pawar) #1

HI, I am unsure of how to identify filebeat input both with tags and type

input {
    beats {
    client_inactivity_timeout => 86400
    port => 5044
    type => "log"
          }
      }
filter {
  if "vus" in [tags] {
    mutate {
      gsub => [
        "message", "\t", " ",
          "message", "\n", " "
            ]
          }
    grok {
      match => { "message" => "\[%{TIMESTAMP_ISO8601:timestamp_match}\]%{SPACE}%{WORD:level}%{SPACE}%{JAVACLASS:coid
key}%{SPACE}%{USER:ident}%{SPACE}%{GREEDYDATA:Url}"}
     add_field => { "[@metadata][es_index]" => "apps-vus" }
  }
} else if "download" in [tags] {
  mutate {
    gsub => [
      "message", "\t", " ",
      "message", "\n", " "
            ]
           }
    grok {
     match => { "message" => "\[%{TIMESTAMP_ISO8601:timestamp_match}\]%{SPACE}\:\|\:%{SPACE}%{WORD:level}%{SPACE}\:\
|\:%{SPACE}%{USERNAME:host_name}%{SPACE}\:\|\:%{SPACE}%{DATA:coidkey}%{SPACE}\:\|\:%{SPACE}%{GREEDYDATA:clientinfo}%
{SPACE}\:\|\:%{SPACE}(%{IP:clientip})?%{SPACE}\:\|\:%{SPACE}%{GREEDYDATA:Url}%{SPACE}\:\|\:%{SPACE}%{JAVACLASS:class
}%{SPACE}\:\|\:%{SPACE}%{USER:ident}%{SPACE}%{GREEDYDATA:msg}"}

   add_field => { "[@metadata][es_index]" => "apps-download" }
   remove_field => [ "@version","ident","host","beat.hostname" ]
   remove_tag => [ "beats_input_codec_plain_applied","beat.name" ]
   }
}else if "openofc" in [tags] {
  mutate {
    gsub => [
      "message", "\t", " ",
      "message", "\n", " "
            ]
           }
    grok {
     match => { "message" => "\[%{TIMESTAMP_ISO8601:timestamp_match}\]%{SPACE}%{WORD:level}%{SPACE}%{USERNAME:host_n
ame}%{SPACE}%{GREEDYDATA:msg}"}

   add_field => { "[@metadata][es_index]" => "apps-openofc" }
   remove_field => [ "@version","ident","host","beat.hostname" ]
   remove_tag => [ "beats_input_codec_plain_applied","beat.name" ]
 }
}else {
    mutate {
        add_field => { "[@metadata][es_index]" => "unknown" }
          }
      }
  }
 output {
  stdout { codec => rubydebug }
  if "_grokparsefailure" in [tags] {
    # write events that didn't match to a file
    file { "path" => "/tmp/grok_failures.txt" }
  }else {
     elasticsearch {
         hosts => ["https://search-iacapps-es-ggoluwfkrzo5ezjxqchkiva.us-east-1.es.amazonaws.com:443"]
         manage_template => false
         index => "%{[@metadata][es_index]}-%{+YYYY.MM.dd}"
         document_type => "log"
   }
  }
}

I am trying to add more

    input {
            tcp {
                ssl_enable => true
        	ssl_cert  =>  "/etc/logstash/cr/cylance.crt"
        	ssl_key =>  "/etc/logstash/cr/cylance.key"
                ssl_verify => false
                port => 5514
        #        ssl_key_passphrase => ""
                type => CylanceSyslog
            }
        }

with

filter {
    if [type] == "CylanceSyslog" {
        grok {
           match => { "message" => "Event Type: %{WORD:EventType}, Event Name: %{DATA:EventName}, %{GREEDYDATA:message2}" }
        }
        kv { 
            source => "message2"
            value_split => ":" 
            field_split => ","
            remove_char_key => " "
            remove_field => [ "message2" ]
        }
    }
}

and

output {
    if [type] == "CylanceSyslog" {

        elasticsearch {
            hosts => "localhost"
            index => "cylance-%{+YYYY.MM.dd}"
        }

    }
}

My first config works totally fine . Not sure how to merge with the current one

Can someone please help with this


(Magnus B├Ąck) #2

I suggest you set the type, fields, and tags on the Filebeat side rather than in Logstash.

But I don't really get your question. You want to run particular filters for events originating from Filebeat? Which filters? And what do such events look like? Use a stdout { codec => rubydebug } output to dump the raw events to the log.


(system) #3

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.