Lowercase all fields and sub fields

Elastic Stack Logstash
2

Took a while to figure out this puzzle, but if you create a file containing this script in /home/user/toLower.rb

def register(params)
    if params['ignore']
        @ignore = params['ignore']
    else
        @ignore = [ "path", "@timestamp", "@metadata", "host", "@version" ]
    end

end

def processArray(a)
    newArray = []
    a.each { |x|
        newArray << processObject(x)
    }
    newArray
end

def processHash(h)
    newHash = {}
    h.each { |k, v|
        newHash[k.downcase] = processObject(v)
    }
    newHash
end

def processObject(v)
    if v.kind_of?(Array)
        processArray(v)
    elsif v.kind_of?(Hash)
        processHash(v)
    elsif v.kind_of?(String)
        v.downcase
    else
        v
    end
end

def filter(event)
    event.to_hash.each { |k, v|
        unless @ignore.include?(k)
            event.remove(k)
            event.set(k.downcase, processObject(v))
        end
    }
    [event]
end

you can call it using

 ruby { path => "/home/user/toLower.rb" }

It will handle arrays of hashes and hashes of array, data types of boolean, float, integer, string...

There may be other fields that should be ignored by default. Perhaps the stuff added by beats.

Note that it relies on the downcase function of the String class, which in 2.3.3 does not handle accented characters.

5 Likes