Hi,
I recently moved an existing configuration of Logstash indexing to a setup with Logstash using a data stream with component templates. As recommended the component template was broken down into two templates, one for index settings and one for mappings. I imported my mappings from the previous set-up, but I'm now suffering field duplication with field and field.keyword.
As the data represents logs i believe i just want the keyword.
Is there a reason for the difference in behaviour between a standard logstash template and datastream component template?
thanks
My template looks as follows;
"mappings" : {
"_data_stream_timestamp" : {
"enabled" : true
},
"dynamic_templates" : [ ],
"properties" : {
"@timestamp" : {
"type" : "date"
},
"@version" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"client" : {
"properties" : {
"address" : {
"type" : "ip"
},
"ip" : {
"type" : "ip"
}
}
},
"custom" : {
"properties" : {
"timestamp" : {
"properties" : {
"day" : {
"type" : "short"
},
"hour" : {
"type" : "short"
},
"minute" : {
"type" : "short"
},
"month" : {
"type" : "short"
},
"timezone" : {
"type" : "keyword"
},
"week_number" : {
"type" : "short"
},
"weekday" : {
"type" : "short"
}
}
}
}
},
"data_stream" : {
"properties" : {
"dataset" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"namespace" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"type" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"destination" : {
"properties" : {
"address" : {
"type" : "ip"
},
"as" : {
"properties" : {
"as_org" : {
"type" : "keyword"
},
"asn" : {
"type" : "long"
},
"ip" : {
"type" : "ip"
}
}
},
"geo" : {
"properties" : {
"city_name" : {
"type" : "keyword"
},
"continent_code" : {
"type" : "keyword"
},
"country_code2" : {
"type" : "keyword"
},
"country_code3" : {
"type" : "keyword"
},
"country_name" : {
"type" : "keyword"
},
"dma_code" : {
"type" : "long"
},
"ip" : {
"type" : "ip"
},
"latitude" : {
"type" : "float"
},
"location" : {
"type" : "geo_point"
},
"longitude" : {
"type" : "float"
},
"postal_code" : {
"type" : "keyword"
},
"region_code" : {
"type" : "keyword"
},
"region_name" : {
"type" : "keyword"
},
"timezone" : {
"type" : "keyword"
}
}
},
"ip" : {
"type" : "ip"
}
}
},
"ecs" : {
"properties" : {
"version" : {
"type" : "keyword"
}
}
},
"event" : {
"properties" : {
"action" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"category" : {
"type" : "keyword"
},
"kind" : {
"type" : "keyword"
},
"module" : {
"type" : "keyword"
},
"original" : {
"type" : "keyword"
},
"severity" : {
"type" : "short"
},
"type" : {
"type" : "keyword"
}
}
},
"observer" : {
"properties" : {
"product" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"type" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"vendor" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"organization" : {
"properties" : {
"id" : {
"type" : "keyword"
},
"name" : {
"type" : "keyword"
}
}
},
"request" : {
"properties" : {
"categories" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"referer" : {
"type" : "keyword"
},
"request_size" : {
"type" : "long"
},
"url" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"response" : {
"properties" : {
"blocked_categories" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"bytes" : {
"type" : "long"
},
"content" : {
"properties" : {
"amp_disposition" : {
"type" : "keyword"
},
"amp_malware_name" : {
"type" : "keyword"
},
"amp_score" : {
"type" : "short"
},
"av_detections" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
},
"index_options" : "docs",
"norms" : false
},
"puas" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"sha" : {
"type" : "keyword"
}
}
},
"content_type" : {
"type" : "keyword"
},
"response_body_size" : {
"type" : "long"
},
"status_code" : {
"type" : "short"
},
"verdict" : {
"type" : "keyword"
}
}
},
"server" : {
"properties" : {
"address" : {
"type" : "ip"
},
"ip" : {
"type" : "ip"
}
}
},
"source" : {
"properties" : {
"address" : {
"type" : "ip"
},
"as" : {
"properties" : {
"as_org" : {
"type" : "keyword"
},
"asn" : {
"type" : "long"
},
"ip" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"ip" : {
"type" : "ip"
},
"nat" : {
"properties" : {
"geo" : {
"properties" : {
"city_name" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"continent_code" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"country_code2" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"country_code3" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"country_name" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"dma_code" : {
"type" : "long"
},
"ip" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"latitude" : {
"type" : "float"
},
"location" : {
"type" : "geo_point"
},
"longitude" : {
"type" : "float"
},
"postal_code" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"region_code" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"region_name" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"timezone" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"ip" : {
"type" : "ip"
}
}
},
"user" : {
"properties" : {
"identities" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"identity_type" : {
"type" : "keyword"
}
}
}
}
},
"tags" : {
"type" : "keyword"
},
"type" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"url" : {
"properties" : {
"full" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"original" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"user_agent" : {
"properties" : {
"build" : {
"type" : "keyword"
},
"device" : {
"type" : "keyword"
},
"major" : {
"type" : "keyword"
},
"minor" : {
"type" : "keyword"
},
"name" : {
"type" : "keyword"
},
"original" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"os" : {
"type" : "keyword"
},
"os_full" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"os_major" : {
"type" : "keyword"
},
"os_minor" : {
"type" : "keyword"
},
"os_name" : {
"type" : "keyword"
},
"os_patch" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"os_version" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"patch" : {
"type" : "keyword"
},
"version" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
}
}
}