Migrating Filter/Plugin/Pattern to Logstash 1.5

Elastic Stack Logstash
1

Hi,

I've just set up a new ELK-stack for monitoring of Cisco ASA Log-files. We have already one running with kibana 3 and logstash 1.4. Unfortunately, my filter is not working with the new version, since it contains "grep" and I can't find how to integrate this with the new plugin system.
My other issue are the patterns. As far as I remember, I had a patterns-folder in the installation path, but with the new installation there isn't one. How can I import my patterns?

Thanks in advance!

Here's my old filter that I want to adapt to logstash 1.5:

input {
file {
path => ["/var/log/fw/fw.log"]
sincedbpath => "/var/log/logstash"
startposition => "beginning"
type => "syslog"
tags => [ "netsyslog" ]
}
}

filter {
if [type] == "syslog" {
grok {
#strips timestamp and host off of the front of the syslog message leaving the raw message generated by the syslog client and saves it as "rawmessage"
patternsdir => "/opt/logstash/patterns"
match => [ "message", "%{CISCOTIMESTAMP:@timestamp} %{HOST:sysloghost} %{GREEDYDATA:rawmessage}" ]
}
}

classify network syslog logs as ASA or other log
if "netsyslog" in [tags] {
grep {
drop => false
match => [ "rawmessage", "%ASA-" ]
addtag => [ "asalog", "firewall" ]
}
grep {
drop => false
match => [ "rawmessage", "%PIX-" ]
addtag => [ "pixlog", "firewall" ]
}
if "firewall" not in [tags] {
mutate {
addtag => [ "genericlog" ]
}
}
}

if "asalog" in [tags] {
#parse ASA log
grok {
patternsdir => "/opt/logstash/patterns"
breakonmatch => false
match => [ "rawmessage", "%{CISCOFACSEVMNEM} %{WORD:Action} %{WORD:IPProtocol} src %{WORD:SourceZone}:%{IP:SourceAddress}\/%{POSINT:SourcePort} dst %{WORD:DestinationZone}:%{IP:DestinationAddress}\/%{POSINT:DestinationPort} by access-group \"%{NOTSPACE:rule}\"%{GREEDYDATA}",
"rawmessage", "%{CISCOFACSEVMNEM} %{WORD:Action} %{IPPROTOCOL:IPProtocol} src %{WORD:SourceZone}:%{IP:SourceAddress} dst %{WORD:DestinationZone}:%{IP:DestinationAddress} %{DATA:icmptypecode} by access-group \"%{WORD:Rule}\"%{GREEDYDATA}",
"rawmessage", "%{CISCOFACSEVMNEM} %{GREEDYDATA:description}" ]
}
mutate {
removefield => [ "message", "rawmessage" ]
addfield => [ "Application", "unknown" ]
lowercase => [ "Action" ]
}

} else {
#apply actions to logs that don't match any particular type of log
}

if "firewall" in [tags] and [SourceAddress] and [DestinationAddress] {
fingerprint {
concatenate_sources => true
method => "SHA1"
key => "logstash"
source => [ "SourceAddress", "DestinationAddress", "DestinationPort", "IPProtocol" ]
}
}
}

output {
elasticsearch {
protocol => "node"
nodename => "abc"
cluster => "abccluster"
host => "localhost"
}
}
2

A grep filter like this:

grep {
  drop => false
  match => [ "rawmessage", "%ASA-" ]
  addtag => [ "asalog", "firewall" ]
}

Can be rewritten like this:

if [rawmessage] =~ /%ASA-/ {
  mutate {
    add_tag => ["asalog", "firewall"]
  }
}
3

First of all, thanks for the reply, I have updated the filter and it seems to work somehow. Unfortunately I'm running now into other problems:

1.) The fw.log is read and indices are generated, but after some time logstash stops and throws this error:

{:timestamp=>"2015-06-11T11:19:15.354000+0200", :message=>"A plugin had an unrecoverable error. Will restart this plugin.\n Plugin: <LogStash::Inputs::File path=>["/var/log/fw/fw.log"], sincedb_path=>"/var/log/logstash", start_position=>"beginning", type=>"syslog", tags=>["netsyslog"], delimiter=>"\n">\n Error: Permission denied - /var/log/logstash.9824.23644.796693", :level=>:error}

How can I determine which plugin is causing the issue and resolve it?

2.) It seems that the data is not handled properly - instead of getting an output like this:

    {
  "_index": "logstash-2015.06.10",
  "_type": "syslog",
  "_id": "Uz6NB7p8TQytZjXdgaKb3w",
  "_score": null,
  "_source": {
    "@version": "1",
    "@timestamp": "2015-06-10T23:57:10.321Z",
    "type": "syslog",
    "tags": [
      "netsyslog",
      "asa_log",
      "firewall"
    ],
    "host": "rzklurambo.abc.local",
    "path": "/var/log/fw/fw.log",
    "syslog_host": "stw-fw-at-klu-sr3-201",
    "facility": "ASA",
    "severity": "4",
    "mnemonic": "106023",
    "Action": "deny",
    "IPProtocol": "udp",
    "SourceZone": "wired_guest",
    "SourceAddress": "192.168.31.13",
    "SourcePort": "137",
    "DestinationZone": "inside",
    "DestinationAddress": "172.16.0.20",
    "DestinationPort": "137",
    "rule": "acl_guest",
    "Application": "unknown",
    "fingerprint": "eb5aa6ddf3a695f1a4462b12f2f340ab1ebb7c83"
  }

I am getting an output like this:

{
  "_index": "logstash-2015.06.11",
  "_type": "syslog",
  "_id": "AU3h6pAKhbazHDFdRp8T",
  "_score": null,
  "_source": {
    "message": "Jun 11 11:19:14 10.210.15.4 %ASA-4-106023: Deny tcp src outside:194.100.100.5/59051 dst ABC:10.210.11.46/25 by access-group \"acl_outside\" [0x0, 0x0]",
    "@version": "1",
    "@timestamp": "2015-06-11T09:19:15.345Z",
    "type": "syslog",
    "tags": [
      "netsyslog",
      "generic_log"
    ],
    "host": "NM27",
    "path": "/var/log/fw/fw.log",
    "syslog_host": "10.210.15.4",
    "raw_message": "%ASA-4-106023: Deny tcp src outside:194.100.100.5/59051 dst ABC:10.210.11.46/25 by access-group \"acl_outside\" [0x0, 0x0]"
  },
  "fields": {
    "@timestamp": [
      1434014355345
    ]
  },
  "highlight": {
    "raw_message": [
      "%ASA-4-@kibana-highlighted-field@106023@/kibana-highlighted-field@: Deny tcp src outside:194.100.100.5/59051 dst ABC:10.210.11.46/25 by access-group \"acl_outside\" [0x0, 0x0]"
    ],
    "message": [
      "Jun 11 11:19:14 10.210.15.4 %ASA-4-@kibana-highlighted-field@106023@/kibana-highlighted-field@: Deny tcp src outside:194.100.100.5/59051 dst ABC:10.210.11.46/25 by access-group \"acl_outside\" [0x0, 0x0]"
    ]
  }

But normally, the filters should be the same...
Has anything changed regarding how filters are handled, or am I doing something wrong?

EDIT: Here's my changed filter - seems that it is not applying the tags asalog/pixlog correctly:

if "netsyslog" in [tags] {
 if [rawmessage] =~ /%PIX-/ {
  mutate {
    add_tag => ["pix_log", "firewall"]
  }
 }
 if [rawmessage] =~ /%ASA-/ {
  mutate {
    add_tag => ["asalog", "firewall"]
  }
 }
  if "firewall" not in [tags] {
    mutate {
      add_tag => [ "generic_log" ]
    }
  }
}

Any hint would be helpful. Thanks in advance!

4

I'm having the same issue.

My filter is similar to yours:

#classify network syslog logs as palo alto or ASA or other log
  if "netsyslog" in [tags] {
   if [rawmessage] =~ /%ASA-/ {
    mutate {
    add_tag => ["asalog", "firewall"]
    }
   }
   else if [rawmessage] =~ /,TRAFFIC,/ {
    mutate {
    add_tag => ["palo_alto_traffic", "firewall"]
    }
   }
    else if [rawmessage] =~ /THREAT/ {
    mutate {
    add_tag => ["palo_alto_threat", "firewall"]
    }
   }
    else if "firewall" not in [tags] {
     mutate {
       add_tag => [ "generic_log" ]
     }
    }
}

It seems to ignore the if [raw message] statements and just tag everything with generic_log. Any help would be appreciated!