Trouble with template and field mapping


(Dave) #1

Hello Guy and Gals. I am rather new and think I am missing something simplistic. I am running BRO IDS, grabbing the connection log file with logstash and sending it to my ES cluster to index => "logstash-bro-conn-%{+YYYY.MM.dd}"
. I tried to created a template called logstash-bro-conn to specify the field types that are coming from the bro log to set the field mapping. I am trying to have this template apply to the new daily index that is created "logstash-bro-conn-YYYYMMDD"

I created the template with the following

_PUT template/logstash-bro-conn
{
"order": 1,
"template" : "logstash-bro-conn-*",
"settings": {
"index": {
"mappings": {
"ts": { "type": "date" },
"uid": { "type": "string" },
"id_resp_h": { "type": "ip" },
"id_resp_p": { "type": "int" },
"id_orig_h": { "type": "ip" },
"id_orig_p": { "type": "int" },
"proto": { "type": "string" },
"duration": { "type": "long" },
"local_orig": { "type": "bool" },
"conn_state": { "type": "string" },
"history": { "type": "string" },
"local_resp": { "type": "bool" },
"missed_bytes": { "type": "byte" },
"orig_bytes": { "type": "byte" },
"orig_cc": { "type": "string" },
"orig_ip_bytes": { "type": "byte" },
"orig_pkts": { "type": "long" },
"resp_bytes": { "type": "byte" },
"resp_cc": { "type": "string" },
"resp_ip_bytes": { "type": "byte" },
"resp_pkts": { "type": "long" },
"sensorname": { "type": "string" },
"service": { "type": "string" },
"tunnel_parents": { "type": "string" }
},
"refresh_interval": "30s",
"number_of_shards": "2",
"number_of_replicas": "2"
},


(Dave) #2

When I look at the index settings all of the field types seem to be set

_GET logstash-bro-conn-2016.03.16/settings

{
"logstash-bro-conn-2016.03.16": {
"settings": {
"index": {
"mappings": {
"resp_pkts": {
"type": "long"
},
"resp_cc": {
"type": "string"
},
"orig_cc": {
"type": "string"
},
"sensorname": {
"type": "string"
},
"id_orig_p": {
"type": "int"
},
"duration": {
"type": "long"
},
"local_resp": {
"type": "bool"
},
"default": {
"dynamic_templates": [
{
"message_field": {
"match": "message",
"match_mapping_type": "string",
"mapping": {
"index": "analyzed",
"omit_norms": "true",
"fielddata": {
"format": "disabled"
},
"type": "string"
}
}
},
{

              }
            }
          }
        ],
        "_all": {
          "omit_norms": "true",
          "enabled": "true"
        },
        "properties": {
          "@version": {
            "type": "string",
            "index": "not_analyzed",
            "doc_values": "true"
          },
          "@timestamp": {
            "type": "date",
            "doc_values": "true"
          },
          "geoip": {
            "type": "object",
            "dynamic": "true",
            "properties": {
              "location": {
                "type": "geo_point",
                "doc_values": "true"
              },
              "ip": {
                "type": "ip",
                "doc_values": "true"
              },
              "latitude": {
                "type": "float",
                "doc_values": "true"
              },
              "longitude": {
                "type": "float",
                "doc_values": "true"
              }
            }
          }
        }
      },
      "uid": {
        "type": "string"
      },
      "conn_state": {
        "type": "string"
      },
      "id_orig_h": {
        "type": "ip"
      },
      "id_resp_h": {
        "type": "ip"
      },
      "id_resp_p": {
        "type": "int"
      },
      "resp_ip_bytes": {
        "type": "byte"
      },
      "orig_bytes": {
        "type": "byte"
      },
      "local_orig": {
        "type": "bool"
      },
      "orig_ip_bytes": {
        "type": "byte"
      },
      "orig_pkts": {
        "type": "long"
      },
      "history": {
        "type": "string"
      },
      "missed_bytes": {
        "type": "byte"
      },
      "tunnel_parents": {
        "type": "string"
      },
      "resp_bytes": {
        "type": "byte"
      },
      "service": {
        "type": "string"
      },
      "proto": {
        "type": "string"
      },
      "ts": {
        "type": "date"
      }
    },
    "refresh_interval": "30s",
    "number_of_shards": "2",
    "creation_date": "1458141964034",
    "number_of_replicas": "2",
    "uuid": "K8tWMImQSSeEDjp8-8_d5w",
    "version": {
      "created": "2020099"
    }
  }
}

}


(Dave) #3

When I look at the mappings for the index none of the mapping that were set in the template are correct

_GET logstash-bro-conn-2016.03.16/mapping

{
"logstash-bro-conn-2016.03.16": {
"mappings": {
"default": {
"_all": {
"enabled": true,
"omit_norms": true
},

      "id_resp_h": {
        **"type": "string"**,
        "norms": {
          "enabled": false

      "id_resp_p": {
        **"type": "string"**,
        "norms": {
          "enabled": false
       
        }
      },
      "local_orig": {
        **"type": "string",**
        "norms": {
          "enabled": false
        },

/////////

Thanks in advance!

David


(system) #4