Urldecode useragent cloudfront

Elastic Stack Logstash
1

I' m having problems with the urldecode filter and can't get it to work on any field so far and I can't figure out what I am doing wrong.

I have a grok filter that is parsing the agent field just fine, but when adding urldecode { field => "agent" } the data sent to ES is still url encoded.

input {
    s3 {
            bucket => "mybucket"
            delete => false
            interval => 60 # seconds
            prefix => "cf-logs/"
            region => "myregion"
            type => "cloudfront"
            codec => "plain"
    }
}
filter {
    if [type] == "cloudfront" {
            if ( ("#Version: 1.0" in [message]) or ("#Fields: date" in [message])) {
                    drop {}
            }
            grok {
                    tag_on_failure => "message-err"
                    match => { "message" => "%{DATE_EU:date}\t%{TIME:time}\t%{DATA:x_edge_location}\t(?:%{NUMBER:bytes}|-)\t%{IPORHOST:clientip}\t%{WORD:verb}\t%{HOSTNAME:cs_host}\t%{NOTSPACE:request}\t%{NUMBER:response}\t(?:%{NOTSPACE:referrer}|-)\t(?:%{GREEDYDATA:agent}|-)\t(?:%{GREEDYDATA:cs_uri_stem}|-)\t%{GREEDYDATA:cookies}\t%{WORD:x_edge_result_type}\t%{NOTSPACE:x_edge_request_id}\t%{HOSTNAME:x_host_header}\t%{URIPROTO:protocol}\t%{INT:bytes}\t%{NUMBER:responsetime:float}\t(?:%{NOTSPACE:x_forwarded_for}|)\t%{GREEDYDATA:ssl_protocol}\t%{GREEDYDATA:ssl_cipher}\t%{GREEDYDATA:x_edge_response_result_type}\t%{GREEDYDATA:httpversion}" }
            }
            urldecode { field => "agent" }
    }
}
output {
    elasticsearch {
            hosts => ["ES:80"]
            }
}

Still the data send to ES is url endocded e.g:
Mozilla/5.0%20(Windows%20NT%206.1;%20Win64;%20x64)%20AppleWebKit/537.36%20(KHTML,%20like%20Gecko)%20Chrome/62.0.3202.94%20Safari/537.36

2

Just needed to run it twice on agent field. Missed it was %2520 and it could not be handled in one run of urldecode filter

3

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.