Refresh Field List not updating

Elastic Stack Elasticsearch
1

Hi,

I have created the logstash CIDR filter that adds the field name "network_name".

      cidr {
        add_tag => [ "home_network" ]
        add_field => { "network_name" => "home_network" }
        address => [ "%{destination_ip}", "%{source_ip}" ]
        network => [ "192.168.1.0/24" ]
        id => "cidr3"
      }

I have also modified the logstash template to add this field to the mappings

        "network_name":{
          "type":"text",
          "fields":{
            "keyword":{
              "type":"keyword"
            }
          }
        },

Logstash successfully maps the field, but it's not "cached"
image

I've tried the "Refresh Field" but with no luck
image

I've even created a new index looking at only log file, with a fresh index and that does not identify "network_name" as a cached field, meaning I can't use it in filters.

I've had a chat on the Logstash forums and the feedback is that Logstash config is ok

Does this suggest that I've missed a field/config/other which would allow Elastic?
I'm using Security Onion and they confirmed "Fields are statically mapped with our templates to prevent field explosion".

For reference:

The complete Elastic results including "network_name" for a record:

{
  "took" : 30,
  "timed_out" : false,
  "_shards" : {
    "total" : 120,
    "successful" : 120,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : 1,
    "max_score" : 2.0,
    "hits" : [
      {
        "_index" : "logstash-bro-2020.06.01",
        "_type" : "doc",
        "_id" : "3aTPcXIBGLNnyZGQ0wPh",
        "_score" : 2.0,
        "_source" : {
          "answers" : [
            "87.194.89.8",
            "87.194.8.8",
            "87.194.9.8"
          ],
          "destination_ips" : "8.8.4.4",
          "source_ip" : "192.168.1.3",
          "protocol" : "udp",
          "destination_ip" : "8.8.4.4",
          "event_type" : "bro_dns",
          "parent_domain_length" : 11,
          "syslog-facility" : "user",
          "host" : "gateway",
          "query_class" : 1,
          "aa" : false,
          "transaction_id" : 5364,
          "syslog-priority" : "notice",
          "query" : "epdg.epc.mnc010.mcc234.pub.3gppnetwork.org",
          "network_name" : "home_network",
          "rcode" : 0,
          "query_type" : 1,
          "creation_date" : "2002-05-15T13:35:38.000Z",
          "ips" : [
            "192.168.1.3",
            "8.8.4.4"
          ],
          "subdomain_frequency_score" : 2.16,
          "syslog-host" : "seconion-NU691",
          "ra" : true,
          "tags" : [
            "syslogng",
            "bro",
            "dns",
            "external_destination",
            "internal_source",
            "home_network"
          ],
          "ttls" : [
            95.0,
            95.0,
            95.0
          ],
          "rd" : true,
          "port" : 55732,
          "frequency_scores" : [
            "3.9616",
            "3.8966",
            "2.16"
          ],
          "subdomain" : "epdg.epc.mnc010.mcc234.pub",
          "parent_domain_frequency_score" : 3.8966,
          "syslog-tags" : ".source.s_bro_dns",
          "syslog-host_from" : "seconion-nu691",
          "parent_domain" : "3gppnetwork",
          "syslog-sourceip" : "127.0.0.1",
          "query_class_name" : "C_INTERNET",
          "highest_registered_domain" : "3gppnetwork.org",
          "destination_port" : 53,
          "rejected" : false,
          "top_level_domain" : "org",
          "source_ips" : "192.168.1.3",
          "uid" : "CPd9873rVfIU2iH4Bf",
          "highest_registered_domain_frequency_score" : 3.9616,
          "destination_geo" : {
            "ip" : "8.8.4.4",
            "latitude" : 37.751,
            "country_name" : "United States",
            "country_code2" : "US",
            "continent_code" : "NA",
            "country_code3" : "US",
            "location" : {
              "lon" : -97.822,
              "lat" : 37.751
            },
            "longitude" : -97.822
          },
          "source_port" : 61982,
          "syslog-file_name" : "/nsm/bro/logs/current/dns.log",
          "@version" : "1",
          "timestamp" : "2020-06-01T21:35:44.996Z",
          "logstash_time" : 0.022396087646484375,
          "message" : """{"ts":"2020-06-01T21:35:43.991106Z","uid":"CPd9873rVfIU2iH4Bf","id.orig_h":"192.168.1.3","id.orig_p":61982,"id.resp_h":"8.8.4.4","id.resp_p":53,"proto":"udp","trans_id":5364,"rtt":0.008337974548339844,"query":"epdg.epc.mnc010.mcc234.pub.3gppnetwork.org","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["87.194.89.8","87.194.8.8","87.194.9.8"],"TTLs":[95.0,95.0,95.0],"rejected":false}""",
          "tld" : {
            "subdomain" : "epdg.epc.mnc010.mcc234.pub.3gppnetwork.org"
          },
          "subdomain_length" : 26,
          "rcode_name" : "NOERROR",
          "tc" : "false",
          "site" : "0",
          "query_length" : 42,
          "@timestamp" : "2020-06-01T21:35:43.991Z",
          "rtt" : 0.008337974548339844,
          "query_type_name" : "A",
          "z" : 0
        }
      }
    ]
  }
}

And looking at the mappings for the index:

GET /*logstash*/_mapping
...
          "network_name" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "type" : "keyword"
              }
            }
          },

Many thanks
Andy

2

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.