Can't dup NilClass event.get()

vicennt · May 17, 2018, 8:01am

Hi there!

First sorry for my English.

I have a problem when I try to get the value from grok field. I'm using the event.get but when I try to pass this value to a Time.parse() I always get `Can't dup NilClass error'.

ERROR:

[ERROR] 2018-05-17 09:51:32.407 [Ruby-0-Thread-10@[main]>worker0: /usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:385] ruby - Ruby exception occurred: can't dup NilClass

My grok pattern is this:

    grok {
      match => { "message" => "(?:\[%{DATA:apache.date_dayname} %{DATA:apache.date_month} %{DATA:apache.date_day} %{DATA:apache.hour} %{DATA:apache.date_year}\]) (?:\[%{DATA:apache.event}?:%{DATA:apache.event_result}\]) (?:\[%{DATA:apache.pid}\]) (?:\[client %{DATA:apache.ip}?:%{DATA:apache.port}\]) %{DATA:apache.errorcode}?: user %{DATA:apache.user}?: authentication %{DATA:apache.auth_result} for %{DATA:apache.urlaccess}?:" }
}

And my ruby code:

 ruby {
       code => "
         require 'time';
         hour = event.get('%{apache.hour}');
         arg = Time.parse(hour);
         ini = Time.parse('08:00:00');
         fin = Time.parse('17:00:00');
         if (arg.hour >= ini.hour) and (arg.hour <= fin.hour)
            event.set('office_hour','intime');
         end
     "
  }

Thanks!

Badger · May 17, 2018, 12:26pm

This should be

hour = event.get('apache.hour');

Using . in a field name will work most of the time, except for those times where it does not work. It is unsupported. If you want hour to be nested inside apache then change both references to be [apache][hour]

vicennt · May 23, 2018, 9:24am

I changed de grok pattern:

%{DATA:test}

And the event.get line:

hour = event.get('test');

But the error persist. I don't understant, because if I set this value to another field:

event.set('office_hour', hour)

The finally value into office_hour is correct.

magnusbaeck · May 27, 2018, 8:15pm

The error could be related to your excessive use of DATA patterns. Those are a performance killer anyway so I suggest you rewrite the grok expression to use more exact patterns. If it still doesn't work, post your updated configuration and an example event produced by Logstash (use a stdout { codec => rubydebug } output).

vicennt · May 28, 2018, 7:24am

Hi Magnus, thanks for your response.

Firstly I update the grok pattern to use more exact patterns:

(?:\[%{DAY:dayname} %{MONTH:month} %{DATA:day} %{TIME:hour} %{YEAR:year}\]) (?:\[%{WORD:event}?:%{WORD:resultevent}\]) (?:\[pid %{NUMBER:pid}\]) (?:\[client %{IPV4:ipdirection}?:%{NUMBER:port}\]) %{WORD:errorcode}?: user %{USERNAME:user}?: authentication %{WORD:authresult} for %{DATA:urlaccess}?:

My configuration:

input {
  beats {
     port => "5046"
  }
}


filter {
 grok {
      match => { "message" => "(?:\[%{DAY:dayname} %{MONTH:month} %{DATA:day} %{TIME:hour} %{YEAR:year}\]) (?:\[%{WORD:event}?:%{WORD:resultevent}\]) (?:\[pid %{NUMBER:pid}\]) (?:\[client %{IPV4:ipdirection}?:%{NUMBER:port}\]) %{WORD:errorcode}?: user %{USERNAME:user}?: authentication %{WORD:authresult} for %{DATA:urlaccess}?:"}
}


geoip {
    source => "ipdirection"
}


ruby {
    add_field => {"office_hour" => "outoftime"}
    code => "
      require 'time'
      h = event.get('hour');
      arg = Time.parse(h);
      inicio = Time.parse('12:00:00');
      final = Time.parse('15:00:00');
      if (arg.hour >= inicio.hour) and (arg.hour <= final.hour)
         event.set('office_hour','intime');
      end
    "
}

 if "_grokparsefailure" in [tags] {
    drop { }
 }
}

output {
  elasticsearch {
    hosts => ["192.168.126.6:9200"]
    index => "apacheaccesslog"
 }

 stdout {
    codec => rubydebug
 }
}

But the error persist, the output of the terminal is always:

[ERROR] 2018-05-28 09:15:46.835 [Ruby-0-Thread-10@[main]>worker0: /usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:385] ruby - Ruby exception occurred: can't dup NilClass
[ERROR] 2018-05-28 09:15:46.836 [Ruby-0-Thread-10@[main]>worker0: /usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:385] ruby - Ruby exception occurred: can't dup NilClass
[ERROR] 2018-05-28 09:15:46.847 [Ruby-0-Thread-10@[main]>worker0: /usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:385] ruby - Ruby exception occurred: can't dup NilClass
[ERROR] 2018-05-28 09:15:46.847 [Ruby-0-Thread-10@[main]>worker0: /usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:385] ruby - Ruby exception occurred: can't dup NilClass
[ERROR] 2018-05-28 09:15:46.847 [Ruby-0-Thread-10@[main]>worker0: /usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:385] ruby - Ruby exception occurred: can't dup NilClass
[ERROR] 2018-05-28 09:15:46.848 [Ruby-0-Thread-10@[main]>worker0: /usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:385] ruby - Ruby exception occurred: can't dup NilClass
[ERROR] 2018-05-28 09:15:46.848 [Ruby-0-Thread-10@[main]>worker0: /usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:385] ruby - Ruby exception occurred: can't dup NilClass
[ERROR] 2018-05-28 09:15:46.848 [Ruby-0-Thread-10@[main]>worker0: /usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:385] ruby - Ruby exception occurred: can't dup NilClass
[ERROR] 2018-05-28 09:15:46.848 [Ruby-0-Thread-10@[main]>worker0: /usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:385] ruby - Ruby exception occurred: can't dup NilClass
[ERROR] 2018-05-28 09:15:46.849 [Ruby-0-Thread-10@[main]>worker0: /usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:385] ruby - Ruby exception occurred: can't dup NilClass
[ERROR] 2018-05-28 09:15:46.849 [Ruby-0-Thread-10@[main]>worker0: /usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:385] ruby - Ruby exception occurred: can't dup NilClass
[ERROR] 2018-05-28 09:15:46.849 [Ruby-0-Thread-10@[main]>worker0: /usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:385] ruby - Ruby exception occurred: can't dup NilClass
[ERROR] 2018-05-28 09:15:46.849 [Ruby-0-Thread-10@[main]>worker0: /usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:385] ruby - Ruby exception occurred: can't dup NilClass
[ERROR] 2018-05-28 09:15:46.850 [Ruby-0-Thread-10@[main]>worker0: /usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:385] ruby - Ruby exception occurred: can't dup NilClass

magnusbaeck · May 28, 2018, 9:56am

Is the grok filter always successful, i.e. is the hour field always set?

vicennt · May 28, 2018, 10:10am

Yes. If I comment the ruby code, an example of event produced by Logstash is this:

{
           "hour" => "13:57:45.241518",
           "tags" => [
        [0] "beats_input_codec_plain_applied"
    ],
         "source" => "/var/log/httpd/error_log",
          "event" => "auth_basic",
     "authresult" => "failure",
           "port" => "54725",
    "ipdirection" => "62.190.56.0",
           "year" => "2018",
           "user" => "USER_LONDON",
         "offset" => 1220043,
      "urlaccess" => "\"/kibana\"",
          "month" => "May",
      "errorcode" => "AH01617",
    "resultevent" => "error",
           "beat" => {
            "name" => "formacionkibana",
        "hostname" => "formacionkibana",
         "version" => "6.2.4"
    },
     "@timestamp" => 2018-05-28T10:07:01.011Z,
        "dayname" => "Fri",
            "day" => "11",
       "@version" => "1",
           "host" => "formacionkibana",
        "message" => "[Fri May 11 13:57:45.241518 2018] [auth_basic:error] [pid 7014] [client 62.190.56.0:54725] AH01617: user USER_LONDON: authentication failure for \"/kibana\": Password Mismatch",
            "pid" => "7014",
     "prospector" => {
        "type" => "log"
    },
          "geoip" => {
                    "ip" => "62.190.56.0",
          "country_name" => "United Kingdom",
              "timezone" => "Europe/London",
              "location" => {
            "lon" => -0.1224,
            "lat" => 51.4964
        },
              "latitude" => 51.4964,
         "country_code2" => "GB",
        "continent_code" => "EU",
             "longitude" => -0.1224,
         "country_code3" => "GB"
    },
    "office_hour" => "outoftime"
}

magnusbaeck · May 28, 2018, 11:54am

Are you sure that's an event that didn't work with the ruby filter?

To narrow down the problem you can always comment out parts of the Ruby code.

vicennt · May 29, 2018, 6:38am

Hi,

Finally the problem is the code order. Because I don't remove the events that do not do maching before the ruby code. If I put the conditional to remove events with "_grokparsefailure" before the ruby code I don't get any error.

This is the finally pipeline configuration:

input {
  beats {
     port => "5046"
  }
}


filter {
 grok {
      match => { "message" => "(?:\[%{DAY:dayname} %{MONTH:month} %{DATA:day} %{TIME:hour} %{YEAR:year}\]) (?:\[%{WORD:event}?:%{WORD:resultevent}\]) (?:\[pid %{NUMBER:pid}\]) (?:\[client %{IPV4:ipdirection}?:%{NUMBER:port}\]) %{WORD:errorcode}?: user %{USERNAME:user}?: authentication %{WORD:authresult} for %{DATA:urlaccess}?:"}
}


geoip {
    source => "ipdirection"
}


 if "_grokparsefailure" in [tags] {
    drop { }
 }

if "_geoip_lookup_failure" in [tags] {
    drop {}
}

ruby {
    add_field => {"office_hour" => "outoftime"}
    code => "
      require 'time'
      h = event.get('hour');
      arg = Time.parse(h);
      inicio = Time.parse('12:00:00');
      final = Time.parse('15:00:00');
      if (arg.hour >= inicio.hour) and (arg.hour <= final.hour)
         event.set('office_hour','intime');
      end
    "
}

}
output {
  elasticsearch {
    hosts => ["192.168.126.6:9200"]
    index => "apacheaccesslog"
 }

 stdout {
    codec => rubydebug
 }
}

Thanks for all magnus and Badger!

system · June 26, 2018, 6:42am

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.