Help With Multiple If {} else if {} COnfiguration

Elastic Stack Logstash
1

I am seeking help with a Logstash configuration issue. Some of my syslog messages coming into my indexer are in JSON format and some are not. I am solving this using multiple if {} else if {} else {} statements after groking the syslog_message out of the event. (Is there a better way?)

My problem is that this works for some of my events (e.g. player events) but not others (e.g. sysstatf events). What am I doing wrong?

Here are two example input lines:

<190>Jan 28 19:00:32 host2 player-player[1034]: {\"player_message\":\"View finished\",\"player_extra\":{\"slot\":\"com.apps\",\"label\":\"https://s3.amazonaws.com/dev.jpg\"}}
<10>Jan 28 18:55:03 host2 sysstatf.py[14855]: {\"facter\": {\"system_uptime\": {\"seconds\": 617616, \"hours\": 171, \"uptime\": \"7 days\", \"days\": 7}}, \"sysstat\": {\"hosts\": [{\"sysname\": \"Linux\", \"statistics\": [{\"timestamp\": {\"date\": \"2016-01-28\", \"utc\": 1, \"interval\": 120, \"time\": \"18:54:01\"}, \"filesystems\": [{\"MBfsused\": 1483, \"%fsused\": 5.36, \"%Iused\": 4.98, \"Iused\": 89772, \"%ufsused\": 5.36, \"filesystem\": \"/dev/sbd1\", \"MBfsfree\": 26195, \"Ifree\": 1712468}, {\"MBfsused\": 14, \"%fsused\": 21.77, \"%Iused\": 0.0, \"Iused\": 0, \"%ufsused\": 21.77, \"filesystem\": \"/dev/sde1\", \"MBfsfree\": 50, \"Ifree\": 0}], \"cpu-load\": [{\"iowait\": 0.34, \"system\": 3.62, \"idle\": 88.61, \"user\": 7.43, \"steal\": 0.0, \"cpu\": \"all\", \"nice\": 0.0}], \"memory\": {\"memused-percent\": 33.68, \"cached\": 371060, \"memfree\": 1319568, \"inactive\": 172092, \"commit-percent\": 25.93, \"active\": 434424, \"commit\": 515972, \"memused\": 670028, \"buffers\": 157596, \"dirty\": 424}}], \"nodename\": \"host2\", \"file-date\": \"2016-01-28\", \"restarts\": [], \"number-of-cpus\": 4, \"machine\": \"amd64\", \"release\": \"3.4.0-comark\", \"file-utc-time\": \"00:00:01\"}], \"sysdata-version\": 2.19}}

Here is the logstash config:

input {
  redis {
    port => "6379"
    host => "127.0.0.1"
    key => "sourceevents"
    data_type => "list"
  }
}

filter {
  if [type] == "syslog" {
    grok {
      patterns_dir => "/etc/logstash/patterns"
      match => { "message" => "<%{POSINT:syslog_pri}>%{COMBOTIMESTAMP:syslog_timestamp} (?:%{SYSLOGHOST:syslog_hostname} )?%{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
      add_field => {
        "[@metadata][file_name]" => "source_unknown_syslog.log"
      }
    }
    cidr {
      add_tag => [ "index_server" ]
      address => [ "%{host}" ]
      network => "127.1.0.0/16"
    }
    cidr {
      add_tag => [ "index_network" ]
      address => [ "%{host}" ]
      network => "127.74.0.0/21"
      network => "127.75.0.0/21"
    }
    cidr {
      add_tag => [ "index_xray", "development" ]
      address => [ "%{host}" ]
      network => "127.16.13.0/24"
    }

    if "index_server" in [tags] {
      mutate {
        update => {
          "[@metadata][file_name]" => "source_server_syslog.log"
        }
      }
    } else if "index_network" in [tags] {
      mutate {
        update => {
          "[@metadata][file_name]" => "source_network_syslog.log"
        }
      }
    } else if "index_xray" in [tags] {
      if [syslog_program] =~ /player.*/ {
        json {
          add_tag => [ "player_event" ]
          source => [ "syslog_message" ]
        }
        mutate {
          update => {
            "[@metadata][file_name]" => "source_xray_player.log"
          }
        }
      } else if [syslog_program] == "sysstaf.py" {
        json {
          add_tag => [ "source_system_statistics", "metrics" ]
          source => [ "syslog_message"]
        }
        mutate {
          update => {
            "[@metadata][file_name]" => "source_xray_sysstat.log"
          }
        }
      } else {
        mutate {
          add_tag => "linux"
          update => {
            "[@metadata][file_name]" => "source_xray_linux.log"
          }
        }
      }
    }
  }
}

output {
  # Order is important: first match takes it
  if "_grokparsefailure" not in [tags] and "_grokparsefailure_sysloginput" not in [tags] {
    file {
      path => "/srv/logstash/%{[@metadata][file_name]}"
    }
  } else {
    file {
      path => "/srv/logstash/grokparsefailure.log"
    }
  }
}

I am hitting the character limit, so I will put the output in another message.

3

The problem is a typo: } else if [syslog_program] == "sysstaf.py" {. The real program is sysstatf.py.

So, maybe I should of asked, how do you test your configs against a known set of inputs?

The question about whether this is the best way to find the JSON hidden in some syslog messages still stands.

Thank you in advance.