Why does my ruby filter seems to act randomly?

Elastic Stack Logstash
1

I have multiple timestamp fields that I want to transform in long (cf. my previous post).

Here is my logstash filter config :

input {
  jdbc { ... }
}
filter {
  ruby {
    path => "/etc/logstash/date_fields_convertor.rb"
    script_params => { "fields" => ['date1', 'date2', 'date3', 'date4', 'date5', 'date6'] }
  }
}
output {
  stdout {}
  elasticsearch { ... }
}

And the ruby script :

def register(prams)
   @field_list = params["fields"]
end

def filter(event)
  field = ""
  fields_mapping = {
    "date1" => "long1",
    "date2" => "long2",
    "date3" => "long3",
    "date4" => "long4",
    "date5" => "long5",
    "date6" => "long6",
  }
  while !@field_list.empty? do
    field = @field_list.pop
    unless event.get(field).nil?
      event.set(fields_mapping[field], (event.get(field).to_f.round(3)*1000).to_i)
    end
  end
  return [event]
end

It seems to work pretty much but depending on the case, I may have zero, one or more transformed fields in my resulting event !?!

{
    "date1": "2018-09-28T11:26:51.182Z",
    "date2": null,
    "long1": 1538134011182,
    "date3": "2018-07-30T00:00:00.000Z",
    "date4": "2018-09-27T20:13:30.500Z",
    "date5": "2018-07-26T00:00:00.000Z",
    "date6": "2018-09-28T11:26:50.703Z"
},
{
    "date1": "2018-09-28T11:02:21.041Z",
    "date2": null,
    "date3": "2018-07-30T00:00:00.000Z",
    "date4": "2018-09-27T20:13:30.500Z",
    "date5": "2018-07-26T00:00:00.000Z",
    "date6": "2018-09-28T11:02:20.403Z"
},
{
    "date1": "2018-09-28T10:31:06.544Z",
    "date2": null,
    "long5": 1532563200000,
    "date3": "2018-07-30T00:00:00.000Z",
    "date4": "2018-09-27T20:13:30.500Z",
    "date5": "2018-07-26T00:00:00.000Z",
    "date6": "2018-09-28T10:30:44.582Z"
},
{
    "date1": "2018-09-28T10:28:34.920Z",
    "date2": null,
    "date3": "2018-07-30T00:00:00.000Z",
    "date4": "2018-09-27T20:13:30.500Z",
    "date5": "2018-07-26T00:00:00.000Z",
    "date6": "2018-09-28T10:27:22.498Z"
},
{
    "date1": "2018-08-02T19:13:01.674Z",
    "date2": null,
    "date3": "2018-07-30T00:00:00.000Z",
    "date4": "2018-08-01T20:13:30.500Z",
    "date5": "2018-07-26T00:00:00.000Z",
    "date6": "2018-08-02T19:12:56.806Z"
},
{
    "date1": "2018-09-28T10:45:33.558Z",
    "date2": null,
    "date3": "2018-07-30T00:00:00.000Z",
    "date4": "2018-09-27T20:13:30.500Z",
    "date5": "2018-07-26T00:00:00.000Z",
    "date6": "2018-09-28T10:45:28.207Z"
},
{
    "date1": "2018-09-28T15:25:27.078Z",
    "date2": null,
    "date3": "2018-07-30T00:00:00.000Z",
    "date4": "2018-09-28T10:13:30.500Z",
    "date5": "2018-07-26T00:00:00.000Z",
    "date6": "2018-09-28T15:25:25.996Z"
},
{
    "date1": "2018-08-09T16:52:45.460Z",
    "date2": null,
    "date3": "2018-07-30T00:00:00.000Z",
    "date4": "2018-08-08T10:13:30.500Z",
    "date5": "2018-07-26T00:00:00.000Z",
    "date6": "2018-08-09T16:52:42.455Z"
},
{
    "date1": "2018-08-13T14:13:39.223Z",
    "long4": 1534148160500,
    "date2": null,
    "long3": 1535414400000,
    "date3": "2018-08-28T00:00:00.000Z",
    "date4": "2018-08-13T08:16:00.500Z",
    "long6": 1534169615486,
    "date5": "2018-08-24T00:00:00.000Z",
    "date6": "2018-08-13T14:13:35.486Z"
},
{
    "date1": "2018-09-28T10:47:14.323Z",
    "date2": null,
    "date3": "2018-07-30T00:00:00.000Z",
    "date4": "2018-09-27T20:13:30.500Z",
    "date5": "2018-07-26T00:00:00.000Z",
    "date6": "2018-09-28T10:46:39.604Z"
}
2

I got it, it comes from my ruby script. The way I get the values from my hash fields_mapping. I correct it this way :

$fields_mapping = {
  "date1" => "long1",
  "date2" => "long2",
  "date3" => "long3",
  "date4" => "long4",
  "date5" => "long5",
  "date6" => "long6",
}

def register(params)
  @field_list = params["fields"]
end

def filter(event)
  field = ""
  i = 0
  while i < @field_list.size do
    field = @field_list[i]
    unless event.get(field).nil?
      event.set($fields_mapping[field], (event.get(field).to_f.round(3)*1000).to_i)
    end
    i = i + 1
  end
  return [event]
end
3

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.