Multi Filebeats input and one output via logstash

Elastic Stack Logstash
1

Hi
Hi I have at the moment 2 vm's from where I want to send log files (centos 8 machines) to logstash. I have this input configuration:

# 01-inputs.conf
input {
  udp {
     port => 5140
  }
beats {
    port => 5044
}
}
filter {
  #Adjust to match the IP address of pfSense or OPNsense
  if [host] =~ /192\.168\.0\.XX/ {
    mutate {
      add_tag => ["pf", "Ready"]
    }
  }
  #To enable or ingest multiple pfSense or OPNsense instances uncomment the below section
  ##############################
  #if [host] =~ /172\.2\.22\.1/ {
  #  mutate {
  #    add_tag => ["pf-2", "Ready"]
  #  }
  #}
  ##############################
  if "pf" in [tags] {
    grok {
      # OPNsense - Enable/Disable the line below based on firewall platform
     # match => { "message" => "%{SYSLOGTIMESTAMP:pf_timestamp} %{SYSLOGHOST:pf_hostname} %{DATA:pf_program}(?:\[%{POSINT:pf_pid}\])?: %{GREEDYDATA:pf_message}" }
      # OPNsense
      # pfSense - Enable/Disable the line below based on firewall platform
        match => { "message" => "%{SYSLOGTIMESTAMP:pf_timestamp} %{DATA:pf_program}(?:\[%{POSINT:pf_pid}\])?: %{GREEDYDATA:pf_message}" }
      # pfSense
      add_field => [ "received_at", "%{@timestamp}" ]
      add_field => [ "received_from", "%{host}" ]
    }
    mutate {
      rename => { "[message]" => "[event][original]"}
      remove_tag => "Ready"
    }
   }
  if [host.hostname] == "SVGXXXX-XXXXX-XX.XXXXXX" {
    grok {
    match => [ "message" , "%{COMBINEDAPACHELOG}+%{GREEDYDATA:extra_fields}"]
    overwrite => [ "message" ]
  }
  mutate {
    convert => ["response", "integer"]
    convert => ["bytes", "integer"]
    convert => ["responsetime", "float"]
  }
  geoip {
    source => "clientip"
    target => "geoip"
    add_tag => [ "nginx-geoip" ]
  }
  date {
    match => [ "timestamp" , "dd/MMM/YYYY:HH:mm:ss Z" ]
    remove_field => [ "timestamp" ]
  }
    useragent {
    source => "agent"
  }
 }
}

the output:

output {
if [@metadata][beat] {
elasticsearch {
hosts => ["http://localhost:9200"]
manage_template => true
index => "%{[@metadata][beat]}-%{[@metadata][version]}"
}
} else {
elasticsearch {
hosts => ["http://localhost:9200"]
index => "pf-%{+YYYY.MM.dd}"
}
}
}

I don't understand why it doesn't work. I used to get it work from 1 machine but from the second no change. Probably in the input file something is wrong? I want to mutate differently if the logs ar comming from a different machine, that's why I have this "if [host.hostname] == "SVGXXXX-XXXXX-XX.XXXXXX" "

The Pfsense part is always working fine.
Any help is welcome.

2

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.