Moving from logstash-1.5.4 to 6.x | how do i convert filter?

[Pierre_es (Pierre Englund)]

We have been running elk on Windows for years and its come to the point where we cant upgrade. The filters dont work at all.
instead of using beats i tried to mount the shares again but the performance is terrible.
If i go without filters in beats i get ~6k /s indexing. with filtering it went down to 5-15 / s.

Made a new linux machine to have everything in as its easier to maintain.
Converted the filter to work on linux if i mount shares and use that for input but when using beats it stops working.

grok-patterns
GREEDYXML (.<?xml.)
GREEDYASMXML (.<asm.)

filter:

filter {
if ("reqlog" in [tags]) {
grok {
patterns_dir => ["/etc/logstash/grok-patterns"]
match => [ "message", "%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{TIMESTAMP_ISO8601:timestampLocal}%{SPACE}%{WORD:thread}%{SPACE}%{WORD:nodeId}%{SPACE}%{WORD:type}\s%{GREEDYDATA:module}\s%{WORD:service}\s%{INT:nicelevel}\s%{INT:total_ms}\s%{INT:execution_ms}\s%{INT:transaction_ms}\s%{INT:statement_ms}\s%{INT:commit_ms}\s%{INT:istakeover}\s%{INT:status}" ,
"message", "%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{TIMESTAMP_ISO8601:timestampLocal}%{SPACE}%{WORD:thread}%{SPACE}Conn:%{WORD:nodeId}%{SPACE}%{WORD:type}\s%{GREEDYDATA:module}\s%{WORD:service}\s%{INT:nicelevel}\s%{INT:total_ms}\s%{INT:execution_ms}\s%{INT:transaction_ms}\s%{INT:statement_ms}\s%{INT:commit_ms}\s%{INT:istakeover}\s%{INT:status}",
"message", "%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{TIMESTAMP_ISO8601:timestampLocal}%{SPACE}%{WORD:thread}%{SPACE}%{WORD:nodeId}%{SPACE}%{WORD:type}%{SPACE}%{WORD:takeovernode}%{SPACE}%{WORD:module}.%{WORD:service}\s%{INT:nicelevel}\s%{INT:total_ms}\s%{INT:execution_ms}\s%{INT:transaction_ms}\s%{INT:statement_ms}\s%{INT:commit_ms}\s%{INT:istakeover}\s%{INT:status}"]
}
date {
match => [ "timestamp", "ISO8601" ]
}
mutate {
convert => { "nicelevel" => "integer" }
convert => { "total_ms" => "integer" }
convert => { "execution_ms" => "integer" }
convert => { "transaction_ms" => "integer" }
convert => { "statement_ms" => "integer" }
convert => { "commit_ms" => "integer" }
convert => { "istakeover" => "integer" }
convert => { "status" => "integer" }
}
}
else {
multiline {
pattern => "^#"
negate => true
what => "previous"
}
if [message] =~ /.+/ {
grok {
patterns_dir => ["/etc/logstash/grok-patterns"]
match => [ "message", "(?m)%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{WORD:thread}%{SPACE}%{WORD:nodeid}%{SPACE}%{LOGLEVEL:level}\s%{DATA:class}\s%{GREEDYDATA:messagetext}\s%{GREEDYXML:messagexml}" ,
"message", "(?m)%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{WORD:thread}%{SPACE}Conn:%{WORD:nodeid}%{SPACE}%{LOGLEVEL:level}\s%{DATA:class}\s%{GREEDYDATA:messagetext}\s%{GREEDYXML:messagexml}" ,
"message", "(?m)%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{WORD:thread}%{SPACE}%{WORD:nodeid}%{SPACE}%{LOGLEVEL:level}\s%{DATA:class}\s%{GREEDYDATA:messagetext}",
"message", "(?m)%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{WORD:thread}%{SPACE}Conn:%{WORD:nodeid}%{SPACE}%{LOGLEVEL:level}\s%{DATA:class}\s%{GREEDYDATA:messagetext}",
"message", "(?m)%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{WORD:thread}%{SPACE}Conn:%{WORD:nodeid}%{SPACE}IP:%{WORD:ip}\s%{LOGLEVEL:level}\s%{DATA:class}\s%{GREEDYDATA:messagetext}"
]
}

[Pierre_es (Pierre Englund)]

                           date {
                                    match => [ "timestamp", "ISO8601" ]
                            }

                            if ("false" in "true") and ([messagexml]) and ("_grokparsefailure" not in [tags]) {
                                    mutate {
                                            gsub => ['messagexml', "\n", ""]
                                            gsub => ['messagexml', "\r", ""]
                                            gsub => ['messagexml', "<!\[CDATA\[", ""]
                                            gsub => ['messagexml', "\]\]>", ""]
                                    }

                                    xml {
                                            source => "messagexml"
                                            target => "messagexml"
                                    }

                                    mutate {
                                            replace => [ "messagexml", "%{[messagexml]}" ]
                                    }
                                    if ("_xmlparsefailure" not in [tags]) {
                                            ruby {
                                                    init => "

                                                    def to_number(val)
                                                    begin
                                                            i = Integer(val)
                                                    rescue
                                                            return val
                                                    end
                                                    return i
                                                    end

                                                    def parse_json obj, pname=nil, event

                                                    obj = JSON.parse(obj) unless obj.is_a? Hash
                                                    obj = obj.to_hash unless obj.is_a? Hash

                                                    obj.each {|k,v|
                                                            p = pname.nil?? k : [pname,k].join('.')
                                                            if v.is_a? Array
                                                            v.each_with_index {|oo,ii|

                                                                    parse_json_array(oo,ii,p,event)
                                                            }
                                                            elsif v.is_a? Hash
                                                            parse_json(v,p,event)
                                                            else
                                                            p = pname.nil?? k : [pname,k].join('.')
                                                            event[p] = to_number(v)
                                                            end
                                                    }

                                                    end
                                                    def parse_json_array obj, i,pname, event
                                                    begin
                                                    obj = JSON.parse(obj) unless obj.is_a? Hash
                                                    pname_ = pname
                                                    if obj.is_a? Hash
                                                            obj.each {|k,v|

                                                            p=[pname_,i,k].join('.')
                                                            if v.is_a? Array
                                                                    v.each_with_index {|oo,ii|
                                                                    parse_json_array(oo,ii,p,event)
                                                                    }
                                                            elsif v.is_a? Hash
                                                                    parse_json(v,p, event)
                                                            else
                                                                    event[p] = to_number(v)
                                                                                                            end
                                                            }
                                                    else
                                                            n = [pname_, i].join('.')
                                                            event[n] = 'hej'
                                                    end
                                                    rescue
                                                            event['tags'] << '_jsonarrayparsefailure' << pname_ << i << obj
                                                    end
                                                    end
                                            "
                                            code => "parse_json(event['messagexml'].to_s,nil,event) if event['messagexml'].to_s.include? ':'"
                                            }

                                    }
                            }
                    }
            }
    metrics {
            meter => [ "message" ]
            add_tag => "metric"
    }

}