Hi,
I have created the logstash CIDR filter that adds the field name "network_name".
cidr {
add_tag => [ "home_network" ]
add_field => { "network_name" => "home_network" }
address => [ "%{destination_ip}", "%{source_ip}" ]
network => [ "192.168.1.0/24" ]
id => "cidr3"
}
I have also modified the logstash template to add this field to the mappings
"network_name":{
"type":"text",
"fields":{
"keyword":{
"type":"keyword"
}
}
},
Logstash successfully maps the field, but it's not "cached"
I've tried the "Refresh Field" but with no luck
I've even created a new index looking at only log file, with a fresh index and that does not identify "network_name" as a cached field, meaning I can't use it in filters.
I've had a chat on the Logstash forums and the feedback is that Logstash config is ok
Does this suggest that I've missed a field/config/other which would allow Elastic?
I'm using Security Onion and they confirmed "Fields are statically mapped with our templates to prevent field explosion".
For reference:
The complete Elastic results including "network_name" for a record:
{
"took" : 30,
"timed_out" : false,
"_shards" : {
"total" : 120,
"successful" : 120,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : 1,
"max_score" : 2.0,
"hits" : [
{
"_index" : "logstash-bro-2020.06.01",
"_type" : "doc",
"_id" : "3aTPcXIBGLNnyZGQ0wPh",
"_score" : 2.0,
"_source" : {
"answers" : [
"87.194.89.8",
"87.194.8.8",
"87.194.9.8"
],
"destination_ips" : "8.8.4.4",
"source_ip" : "192.168.1.3",
"protocol" : "udp",
"destination_ip" : "8.8.4.4",
"event_type" : "bro_dns",
"parent_domain_length" : 11,
"syslog-facility" : "user",
"host" : "gateway",
"query_class" : 1,
"aa" : false,
"transaction_id" : 5364,
"syslog-priority" : "notice",
"query" : "epdg.epc.mnc010.mcc234.pub.3gppnetwork.org",
"network_name" : "home_network",
"rcode" : 0,
"query_type" : 1,
"creation_date" : "2002-05-15T13:35:38.000Z",
"ips" : [
"192.168.1.3",
"8.8.4.4"
],
"subdomain_frequency_score" : 2.16,
"syslog-host" : "seconion-NU691",
"ra" : true,
"tags" : [
"syslogng",
"bro",
"dns",
"external_destination",
"internal_source",
"home_network"
],
"ttls" : [
95.0,
95.0,
95.0
],
"rd" : true,
"port" : 55732,
"frequency_scores" : [
"3.9616",
"3.8966",
"2.16"
],
"subdomain" : "epdg.epc.mnc010.mcc234.pub",
"parent_domain_frequency_score" : 3.8966,
"syslog-tags" : ".source.s_bro_dns",
"syslog-host_from" : "seconion-nu691",
"parent_domain" : "3gppnetwork",
"syslog-sourceip" : "127.0.0.1",
"query_class_name" : "C_INTERNET",
"highest_registered_domain" : "3gppnetwork.org",
"destination_port" : 53,
"rejected" : false,
"top_level_domain" : "org",
"source_ips" : "192.168.1.3",
"uid" : "CPd9873rVfIU2iH4Bf",
"highest_registered_domain_frequency_score" : 3.9616,
"destination_geo" : {
"ip" : "8.8.4.4",
"latitude" : 37.751,
"country_name" : "United States",
"country_code2" : "US",
"continent_code" : "NA",
"country_code3" : "US",
"location" : {
"lon" : -97.822,
"lat" : 37.751
},
"longitude" : -97.822
},
"source_port" : 61982,
"syslog-file_name" : "/nsm/bro/logs/current/dns.log",
"@version" : "1",
"timestamp" : "2020-06-01T21:35:44.996Z",
"logstash_time" : 0.022396087646484375,
"message" : """{"ts":"2020-06-01T21:35:43.991106Z","uid":"CPd9873rVfIU2iH4Bf","id.orig_h":"192.168.1.3","id.orig_p":61982,"id.resp_h":"8.8.4.4","id.resp_p":53,"proto":"udp","trans_id":5364,"rtt":0.008337974548339844,"query":"epdg.epc.mnc010.mcc234.pub.3gppnetwork.org","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["87.194.89.8","87.194.8.8","87.194.9.8"],"TTLs":[95.0,95.0,95.0],"rejected":false}""",
"tld" : {
"subdomain" : "epdg.epc.mnc010.mcc234.pub.3gppnetwork.org"
},
"subdomain_length" : 26,
"rcode_name" : "NOERROR",
"tc" : "false",
"site" : "0",
"query_length" : 42,
"@timestamp" : "2020-06-01T21:35:43.991Z",
"rtt" : 0.008337974548339844,
"query_type_name" : "A",
"z" : 0
}
}
]
}
}
And looking at the mappings for the index:
GET /*logstash*/_mapping
...
"network_name" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword"
}
}
},
Many thanks
Andy