How to create two kv filter

Elastic Stack Logstash
17

@lcguy

So the firewall are not sending JSON via UDP - just plain formatted strings. I also see that you got some help about your firewalls Fortigate here and Sidewinder here. Some of this advice I have retained and some I have done a little differently...

IMPORTANT!!!! Read my follow up notes post before diving in and trying the config below.

input {
  generator {
    lines => [
      '<189>date=2018-04-28 time=12:21:13 devname=FIREWALL1 devid=xxxxxxxx logid=0000000013 type=traffic subtype=forward level=notice vd=root srcip=x.x.x.x srcport=137 srcintf="VL_MGMT" dstip=x.x.x.x dstport=137 dstintf="port9" poluuid=xxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxx sessionid=1095303991 proto=17 action=deny policyid=498 dstcountry="Reserved" srccountry="Reserved" trandisp=noop service="HTTP Servcice" duration=0 sentbyte=0 rcvdbyte=0 sentpkt=0 appcat="unscanned" crscore=30 craction=131072 crlevel=high',
      '<45>Apr 28 12:21:14 FIREWALL2 auditd: date="2018-04-28 04:21:14 +0000",fac=f_kernel_ipfilter,area=a_general_area,type=t_nettraffic,pri=p_major,hostname=FIREWALL2.domain.local,event="session end",application=TCP-443,netsessid=b26845ae3f6b9,src_geo=XX,srcip=x.x.x.x,srcport=35478,srczone=VL_EXT,protocol=6,dstip=x.x.x.x,dstport=443,dstzone=VL_1,bytes_written_to_client=324,bytes_written_to_server=848,rule_name="TESTING RULE",cache_hit=0,start_time="2018-04-28 04:21:13 +0000"'
    ]
    count => 1
  }
}

filter {
  dissect {
    mapping => {
      message => "<%{priority}>%{rest}"
    }
  }
  if [rest] =~ /^\w{3}\s/ {
    dissect {
      mapping => {
        # using skip fields for month day and time because in kv 'date' has the UTC based timestamp
        rest => "%{?month} %{?day} %{?time} %{devname} %{process_name}: %{rest}"
      }
      add_tag => ["format2"]
    }
  }
  if "format2" in [tags] {
    kv {
      source => "[rest]"
      value_split => "="
      field_split => ","
      remove_field => [ "[message]", "[rest]" ]
    }
    mutate {
      rename => { "protocol" => "proto"}
      # !!!!! rename or remove any other fields here (not date though).
    }
    date {
      match => [ "date", "yyyy-MM-dd HH:mm:ss Z" ]
    }
  } else {
    # handle first format here
    kv {
      source => "[rest]"
      value_split => "="
      field_split => " "
      remove_field => [ "[message]", "[rest]" ]
    }
    mutate {
      # combine the date and time and add the timezone
      replace => { "date" => "%{date}T%{time}+0800" }
      # !!!!! rename or remove any other fields here (not date though).
      # add_field, remove_field etc always runs after any other mutate operations
      remove_field => [ "[time]" ]
    }
    date {
      match => [ "date", "ISO8601" ]
    }
  }
}

output {
  stdout {
    codec => rubydebug
  }
}

This gives...

{
            "vd" => "root",
      "sequence" => 0,
      "sentbyte" => "0",
       "subtype" => "forward",
       "srcintf" => "VL_MGMT",
      "policyid" => "498",
          "date" => "2018-04-28T12:21:13+0800",
         "level" => "notice",
    "srccountry" => "Reserved",
       "crscore" => "30",
         "devid" => "xxxxxxxx",
       "crlevel" => "high",
    "@timestamp" => 2018-04-28T04:21:13.000Z,
    "dstcountry" => "Reserved",
       "dstport" => "137",
      "@version" => "1",
       "sentpkt" => "0",
     "sessionid" => "1095303991",
        "appcat" => "unscanned",
       "devname" => "FIREWALL1",
         "srcip" => "x.x.x.x",
         "dstip" => "x.x.x.x",
       "poluuid" => "xxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxx",
      "trandisp" => "noop",
       "service" => "HTTP Servcice",
      "rcvdbyte" => "0",
       "dstintf" => "port9",
          "type" => "traffic",
        "action" => "deny",
      "priority" => "189",
          "host" => "Elastics-MacBook-Pro.local",
         "logid" => "0000000013",
       "srcport" => "137",
      "craction" => "131072",
      "duration" => "0",
         "proto" => "17"
}
{
                   "sequence" => 0,
                 "start_time" => "2018-04-28 04:21:13 +0000",
                       "date" => "2018-04-28 04:21:14 +0000",
                    "src_geo" => "XX",
                 "@timestamp" => 2018-04-28T04:21:14.000Z,
                   "hostname" => "FIREWALL2.domain.local",
                    "dstport" => "443",
                   "@version" => "1",
                       "area" => "a_general_area",
                  "rule_name" => "TESTING RULE",
                       "tags" => [
        [0] "format2"
    ],
                    "devname" => "FIREWALL2",
               "process_name" => "auditd",
                  "cache_hit" => "0",
                      "srcip" => "x.x.x.x",
                        "pri" => "p_major",
                  "netsessid" => "b26845ae3f6b9",
                      "dstip" => "x.x.x.x",
    "bytes_written_to_server" => "848",
                    "dstzone" => "VL_1",
                       "type" => "t_nettraffic",
                   "priority" => "45",
                    "srczone" => "VL_EXT",
                       "host" => "Elastics-MacBook-Pro.local",
                    "srcport" => "35478",
    "bytes_written_to_client" => "324",
                      "proto" => "6",
                      "event" => "session end",
                "application" => "TCP-443",
                        "fac" => "f_kernel_ipfilter"
}
1 Like