Working on migrating Windows DNS logs to data stream and seeing a lot of errors.
[2022-05-26T13:43:37,457][INFO ][logstash.outputs.elasticsearch][filebeat-dns-dhcp][7fe0474f97756847637aa742a5ac38f62e0e1460d0c70da893dabe9614baac8f] Retrying failed action {:status=>500, :action=>["create", {:_id=>nil, :_index=>"logs-dns-default", :routing=>nil}, {"dns.subdomainname"=>"my", "input"=>{"type"=>"filestream"}, "dns.subdomain"=>"my.salesforce.com", "Flags"=>{"AuthoritativeAnswer"=>"False", "RecursionAvailable"=>"True", "TruncatedResponse"=>"False", "RecursionDesired"=>"True"}, "host"=>{"name"=>"KIKMRKSEC01"}, "packetIdentifier"=>"0000006C3EA11110", "responseCode"=>"NOERROR", "protocol"=>"UDP", "agent"=>{"ephemeral_id"=>"04c7ff0b-2de2-4c87-85fa-197a7f3e1d7f", "name"=>"KIKMRKSEC01", "version"=>"8.1.3", "type"=>"filebeat", "id"=>"8121e146-08af-4ca8-9cb4-1bcfefe696ec"}, "remoteIP"=>"172.20.6.4", "context"=>"PACKET", "ecs"=>{"version"=>"8.0.0"}, "dns.TLD"=>"com", "direction"=>"Snd", "flagsHex"=>"8081", "log"=>{"file"=>{"path"=>"D:\\logs\\dnslog\\dns.log"}, "offset"=>18816}, "dns.questionName"=>"westlakechemical.my.salesforce.com", "tags"=>["beats_input_codec_plain_applied"], "event"=>{}, "xid"=>"0559", "dns.questionType"=>"A", "dns.domainname"=>"salesforce", "dns.domain"=>"salesforce.com", "queryResponse"=>"Response", "flags"=>"DR ", "opcode"=>"Standard Query", "@version"=>"1", "@timestamp"=>2022-05-26T14:57:09Z, "treadId"=>"1A00", "data_stream"=>{"type"=>"logs", "dataset"=>"dns", "namespace"=>"default"}}], :error=>{"type"=>"null_pointer_exception", "reason"=>"Cannot invoke \"String.length()\" because \"str\" is null"}}
Current config is
filter{
if "dnslog" in [log][file][path] {
grok {
patterns_dir => ["/etc/logstash/patterns"]
match => { "message" => "(?<timestamp>%{DATE_US} %{TIME} (?:AM|PM))\s+%{DATA:treadId}\s+%{WORD:context}\s+%{DATA:packetIdentifier}\s+%{WORD:protocol}\s+%{WORD:direction}\s+%{IP:remoteIP}\s+%{BASE16NUM:xid}\s+(%{WORD:queryResponse})?\s+%{DATA:opcode}\s+\[%{BASE16NUM:flagsHex}\s+%{GREEDYDATA:flags}\s+%{WORD:responseCode}\]\s+(%{WORD:dns.questionType}\s+)?(%{GREEDYDATA:dns.questionName})?"}
}
### parse timestamp
date { match => [ "[timestamp]", "M/dd/YYYY hh:mm:ss a" ] }
### parse query/response type
if [queryResponse] == "R" {
mutate {update => { "queryResponse" => "Response" }}
}
else {
mutate {add_field => {"queryResponse" => "Query"}}
}
### parse opcode
if [opcode] == "Q" {
mutate {update => { "opcode" => "Standard Query" }}
} else if [opcode] == "N" {
mutate {update => { "opcode" => "Notify" }}
} else if [opcode] == "U" {
mutate {update => { "opcode" => "Update" }}
} else if [opcode] == "?" {
mutate {update => { "opcode" => "Unknown" }}
}
### parse flags
if "A" in [flags] {
mutate {add_field => {"[Flags][AuthoritativeAnswer]" => True}}
} else {
mutate {add_field => {"[Flags][AuthoritativeAnswer]" => False}}
}
if "T" in [flags] {
mutate {add_field => {"[Flags][TruncatedResponse]" => True}}
} else {
mutate {add_field => {"[Flags][TruncatedResponse]" => False}}
}
if "D" in [flags] {
mutate {add_field => {"[Flags][RecursionDesired]" => True}}
} else {
mutate {add_field => {"[Flags][RecursionDesired]" => False}}
}
if "R" in [flags] {
mutate {add_field => {"[Flags][RecursionAvailable]" => True}}
} else {
mutate {add_field => {"[Flags][RecursionAvailable]" => False}}
}
### parse dns question name
mutate {
gsub => [ #(4)maps(17)google(5)com(0)
"dns.questionName", "^\s*", "", #remove leading space
"dns.questionName", "\s*$", "", #remove tailing space
"dns.questionName", "^\(\d+\)", "", #remove leading (xx)
"dns.questionName", "\(\d\)$", "", #remove tailing (xx)
"dns.questionName", "\(\d+\)", "." #replace (xx) with .
]
lowercase => ["dns.questionName"]
}
if [dns.questionName] {
if "." in [dns.questionName] {
grok {
match => { "dns.questionName" => "(((.*)\.)?(?<dns.subdomainname>.*)\.)?(?<dns.domainname>.*)\.(?<dns.TLD>\w+$)"}
}
mutate {
add_field => {"[dns.domain]" => "%{dns.domainname}.%{dns.TLD}"}
}
if [dns.subdomainname] {
mutate {
add_field => {"[dns.subdomain]" => "%{dns.subdomainname}.%{dns.domainname}.%{dns.TLD}"}
}
}
} else {
mutate { add_field => {"[dns][TLD]" => "%{dns.questionName}"} }
}
}
}
}
Please advise.