Processing Multiple Rows From JDBC Streaming

Elastic Stack Logstash
21

The full pipeline config that manipulates the data is here:

    if [question] {
      ruby {
        code => '
          data = event.get("question")
          if data.is_a? Array
            newData = []
            data.each { |h|
              q = h.delete("questionid")
              event.set("question#{q}", h)
            }
          end
        '
      }
      ruby {
              code => '
                  def EmptyField(object, name, event)
                      if object
                          if object.kind_of?(Hash) and object != {}
                              object.each { |k, v| EmptyField(v, "#{name}[#{k}]", event) }
                          elsif object.kind_of?(Array) and object != []
                              object.each_index { |i|
                                  EmptyField(object[i], "#{name}[#{i}]", event)
                              }
                          else
                              if object == "nil"
                                  event.remove(name)
                              end
                          end
                      end
                  end
      
                  event.to_hash.each { |k, v|
                      EmptyField(v, "[#{k}]", event)
                  }
              '
          }
      split {
        field => "question"
      }
      mutate {
        remove_field => [ "[question]" ]
      }
    }
22

The first thing that the EmptyField function tests is if object, so if ! object inside that branch cannot possibly test true.

input { generator { count => 1 lines => [ '{ "question39": { "dec": null, "question": "Sup Brah?", "bool": null, "date": null, "text": null, "int": null } }' ] codec => json } }
filter {
    ruby {
        code => '
            def EmptyField(object, name, event)
                if object.kind_of?(Hash) and object != {}
                    object.each { |k, v| EmptyField(v, "#{name}[#{k}]", event) }
                elsif object.kind_of?(Array) and object != []
                    object.each_index { |i|
                        EmptyField(object[i], "#{name}[#{i}]", event)
                    }
                else
                    if ! object
                        event.remove(name)
                    end
                end
            end

            event.to_hash.each { |k, v|
                EmptyField(v, "[#{k}]", event)
            }
        '
    }
}
output { stdout { codec => rubydebug { metadata => false } } }

results in

"question39" => {
    "question" => "Sup Brah?"
}
1 Like
23

Awesome thank you. This is where my inability to understand Ruby really shines, lol. Just one of the many things to add to the list of needing to give time to learn.

24

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.