I am having a constant issue while sending Windows DNS logs from filebeat to logstash.
This is the config I use during debug:
input {
beats {
port => XXXX
}
}
filter {
if [type] =="Win_DNS" {
grok {
patterns_dir => "/etc/logstash/patterns/dns"
match => { "message" => "%{WINDNS}" }
break_on_match => false
}
mutate {
add_field => { "log_timestamp" => "%{log_date} %{log_time}" }
lowercase => "dns_query_name"
}
if [dns_client_address] != "xx.yyy.zz.aaa" {
mutate {
add_field => { "dest_url" => "%{dns_query_name}" }
}
}
else {
drop { }
}
if [dest_url] == /^10\((?:2|3)\)/ or "dns_query_name" in [dest_url] {
drop { }
}
mutate {
gsub => [ "dest_url","\([1-9][0-9]?\)", "." ]
gsub => [ "dest_url","\(0\)", " " ]
strip => "dest_url"
}
if [dest_url] =~ /10\.(?:[0-9]{1,2}|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.(?:[0-9]{1,2}|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.(?:[1-9][0-9]?|1[0-9]{2}|2[0-4][0-9]|25[0-4])\-(?:[0-9]{1,2}|1[0-9]{2}|2[0-4][0-9]|25[0-5])/ {
drop { }
}
mutate {
add_field => { "dest_ip" => "%{dest_url}" }
}
dns {
resolve => [ "dest_ip" ]
action => "replace"
nameserver => "10.110.24.4"
}
if [dest_ip] !~ /[A-Za-z-_\/\\]/ {
geoip {
source => "dest_ip"
}
} else {
grok {
remove_field => [ "dest_ip" ]
}
}
date {
match => [ "log_timestamp", "yyyyMMdd HH:mm:ss" ]
}
}
}
The config works fine for a while, but the error that follows is what happens once it receives a certain line. I have not been able to track down the line that describes/causes the issue.
{:timestamp=>"2016-04-27T10:55:53.670000-0400", :message=>"Exception in pipelineworker, the pipeline stopped processing new events, please check your filter configuration and restart Logstash.", "exception"=>#<Encoding::CompatibilityError: incompatible character encodings: UTF-8 and ASCII-8BIT>, "backtrace"=>["org/jruby/RubyString.java:3936:in
[]='", "/opt/logstash/vendor/jruby/lib/ruby/1.9/resolv.rb:792:insender'", "/opt/logstash/vendor/jruby/lib/ruby/1.9/resolv.rb:508:ineach_resource'", "/opt/logstash/vendor/jruby/lib/ruby/1.9/resolv.rb:1035:inresolv'", "org/jruby/RubyArray.java:1613:ineach'", "/opt/logstash/vendor/jruby/lib/ruby/1.9/resolv.rb:1034:inresolv'", "org/jruby/RubyArray.java:1613:ineach'", "/opt/logstash/vendor/jruby/lib/ruby/1.9/resolv.rb:1033:inresolv'", "org/jruby/RubyArray.java:1613:ineach'", "/opt/logstash/vendor/jruby/lib/ruby/1.9/resolv.rb:1031:inresolv'", "/opt/logstash/vendor/jruby/lib/ruby/1.9/resolv.rb:503:ineach_resource'", "/opt/logstash/vendor/jruby/lib/ruby/1.9/resolv.rb:502:ineach_resource'", "/opt/logstash/vendor/jruby/lib/ruby/1.9/resolv.rb:396:ineach_address'", "/opt/logstash/vendor/jruby/lib/ruby/1.9/resolv.rb:120:ineach_address'", "org/jruby/RubyArray.java:1613:ineach'", "/opt/logstash/vendor/jruby/lib/ruby/1.9/resolv.rb:119:ineach_address'", "/opt/logstash/vendor/jruby/lib/ruby/1.9/resolv.rb:97:ingetaddress'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-filter-dns-2.1.3/lib/logstash/filters/dns.rb:249:ingetaddress'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-filter-dns-2.1.3/lib/logstash/filters/dns.rb:238:inretriable_getaddress'", "org/jruby/RubyProc.java:281:incall'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-filter-dns-2.1.3/lib/logstash/filters/dns.rb:216:inretriable_request'", "org/jruby/ext/timeout/Timeout.java:115:intimeout'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-filter-dns-2.1.3/lib/logstash/filters/dns.rb:215:inretriable_request'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-filter-dns-2.1.3/lib/logstash/filters/dns.rb:237:inretriable_getaddress'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-filter-dns-2.1.3/lib/logstash/filters/dns.rb:120:inresolve'", "org/jruby/RubyArray.java:1613:ineach'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-filter-dns-
Any help would be great.
Thank you.
/opt/logstash/bin/logstash --version
logstash 2.3.1
