# Duplicate field.keyword and field in data stream within Observability

**URL:** <https://discuss.elastic.co/t/duplicate-field-keyword-and-field-in-data-stream-within-observability/311383>\
**Category:** Elasticsearch\
**Created:** [August 4, 2022, 8:27am UTC](https://discuss.elastic.co/t/duplicate-field-keyword-and-field-in-data-stream-within-observability/311383 "2022-08-04T08:27:01Z")\
**Posts on this page:** 3\
**Page:** 1

<div class="post-metadata">

**Author:** ![Mark\_Rodman](https://avatars.discourse-cdn.com/v4/letter/m/57b2e6/32.png) [@Mark\_Rodman](https://discuss.elastic.co/u/Mark_Rodman)\
**Post date:** [August 4, 2022, 8:27am UTC](https://discuss.elastic.co/t/duplicate-field-keyword-and-field-in-data-stream-within-observability/311383/1 "2022-08-04T08:27:01Z")

</div>

Hi,  
I recently moved an existing configuration of Logstash indexing to a setup with Logstash using a data stream with component templates. As recommended the component template was broken down into two templates, one for index settings and one for mappings. I imported my mappings from the previous set-up, but I'm now suffering field duplication with field and field.keyword.  
As the data represents logs i believe i just want the keyword.

Is there a reason for the difference in behaviour between a standard logstash template and datastream component template?

thanks

My template looks as follows;

```auto
   "mappings" : {
      "_data_stream_timestamp" : {
        "enabled" : true
      },
      "dynamic_templates" : [],
      "properties" : {
        "@timestamp" : {
          "type" : "date"
        },
        "@version" : {
          "type" : "text",
          "fields" : {
            "keyword" : {
              "type" : "keyword",
              "ignore_above" : 256
            }
          }
        },
        "client" : {
          "properties" : {
            "address" : {
              "type" : "ip"
            },
            "ip" : {
              "type" : "ip"
            }
          }
        },
        "custom" : {
          "properties" : {
            "timestamp" : {
              "properties" : {
                "day" : {
                  "type" : "short"
                },
                "hour" : {
                  "type" : "short"
                },
                "minute" : {
                  "type" : "short"
                },
                "month" : {
                  "type" : "short"
                },
                "timezone" : {
                  "type" : "keyword"
                },
                "week_number" : {
                  "type" : "short"
                },
                "weekday" : {
                  "type" : "short"
                }
              }
            }
          }
        },
        "data_stream" : {
          "properties" : {
            "dataset" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            },
            "namespace" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            },
            "type" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            }
          }
        },
        "destination" : {
          "properties" : {
            "address" : {
              "type" : "ip"
            },
            "as" : {
              "properties" : {
                "as_org" : {
                  "type" : "keyword"
                },
                "asn" : {
                  "type" : "long"
                },
                "ip" : {
                  "type" : "ip"
                }
              }
            },
            "geo" : {
              "properties" : {
                "city_name" : {
                  "type" : "keyword"
                },
                "continent_code" : {
                  "type" : "keyword"
                },
                "country_code2" : {
                  "type" : "keyword"
                },
                "country_code3" : {
                  "type" : "keyword"
                },
                "country_name" : {
                  "type" : "keyword"
                },
                "dma_code" : {
                  "type" : "long"
                },
                "ip" : {
                  "type" : "ip"
                },
                "latitude" : {
                  "type" : "float"
                },
                "location" : {
                  "type" : "geo_point"
                },
                "longitude" : {
                  "type" : "float"
                },
                "postal_code" : {
                  "type" : "keyword"
                },
                "region_code" : {
                  "type" : "keyword"
                },
                "region_name" : {
                  "type" : "keyword"
                },
                "timezone" : {
                  "type" : "keyword"
                }
              }
            },
            "ip" : {
              "type" : "ip"
            }
          }
        },
        "ecs" : {
          "properties" : {
            "version" : {
              "type" : "keyword"
            }
          }
        },
        "event" : {
          "properties" : {
            "action" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            },
            "category" : {
              "type" : "keyword"
            },
            "kind" : {
              "type" : "keyword"
            },
            "module" : {
              "type" : "keyword"
            },
            "original" : {
              "type" : "keyword"
            },
            "severity" : {
              "type" : "short"
            },
            "type" : {
              "type" : "keyword"
            }
          }
        },
        "observer" : {
          "properties" : {
            "product" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            },
            "type" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            },
            "vendor" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            }
          }
        },
        "organization" : {
          "properties" : {
            "id" : {
              "type" : "keyword"
            },
            "name" : {
              "type" : "keyword"
            }
          }
        },
        "request" : {
          "properties" : {
            "categories" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            },
            "referer" : {
              "type" : "keyword"
            },
            "request_size" : {
              "type" : "long"
            },
            "url" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            }
          }
        },
        "response" : {
          "properties" : {
            "blocked_categories" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            },
            "bytes" : {
              "type" : "long"
            },
            "content" : {
              "properties" : {
                "amp_disposition" : {
                  "type" : "keyword"
                },
                "amp_malware_name" : {
                  "type" : "keyword"
                },
                "amp_score" : {
                  "type" : "short"
                },
                "av_detections" : {
                  "type" : "text",
                  "fields" : {
                    "keyword" : {
                      "type" : "keyword",
                      "ignore_above" : 256
                    }
                  },
                  "index_options" : "docs",
                  "norms" : false
                },
                "puas" : {
                  "type" : "text",
                  "fields" : {
                    "keyword" : {
                      "type" : "keyword",
                      "ignore_above" : 256
                    }
                  }
                },
                "sha" : {
                  "type" : "keyword"
                }
              }
            },
            "content_type" : {
              "type" : "keyword"
            },
            "response_body_size" : {
              "type" : "long"
            },
            "status_code" : {
              "type" : "short"
            },
            "verdict" : {
              "type" : "keyword"
            }
          }
        },
        "server" : {
          "properties" : {
            "address" : {
              "type" : "ip"
            },
            "ip" : {
              "type" : "ip"
            }
          }
        },
        "source" : {
          "properties" : {
            "address" : {
              "type" : "ip"
            },
            "as" : {
              "properties" : {
                "as_org" : {
                  "type" : "keyword"
                },
                "asn" : {
                  "type" : "long"
                },
                "ip" : {
                  "type" : "text",
                  "fields" : {
                    "keyword" : {
                      "type" : "keyword",
                      "ignore_above" : 256
                    }
                  }
                }
              }
            },
            "ip" : {
              "type" : "ip"
            },
            "nat" : {
              "properties" : {
                "geo" : {
                  "properties" : {
                    "city_name" : {
                      "type" : "text",
                      "fields" : {
                        "keyword" : {
                          "type" : "keyword",
                          "ignore_above" : 256
                        }
                      }
                    },
                    "continent_code" : {
                      "type" : "text",
                      "fields" : {
                        "keyword" : {
                          "type" : "keyword",
                          "ignore_above" : 256
                        }
                      }
                    },
                    "country_code2" : {
                      "type" : "text",
                      "fields" : {
                        "keyword" : {
                          "type" : "keyword",
                          "ignore_above" : 256
                        }
                      }
                    },
                    "country_code3" : {
                      "type" : "text",
                      "fields" : {
                        "keyword" : {
                          "type" : "keyword",
                          "ignore_above" : 256
                        }
                      }
                    },
                    "country_name" : {
                      "type" : "text",
                      "fields" : {
                        "keyword" : {
                          "type" : "keyword",
                          "ignore_above" : 256
                        }
                      }
                    },
                    "dma_code" : {
                      "type" : "long"
                    },
                    "ip" : {
                      "type" : "text",
                      "fields" : {
                        "keyword" : {
                          "type" : "keyword",
                          "ignore_above" : 256
                        }
                      }
                    },
                    "latitude" : {
                      "type" : "float"
                    },
                    "location" : {
                      "type" : "geo_point"
                    },
                    "longitude" : {
                      "type" : "float"
                    },
                    "postal_code" : {
                      "type" : "text",
                      "fields" : {
                        "keyword" : {
                          "type" : "keyword",
                          "ignore_above" : 256
                        }
                      }
                    },
                    "region_code" : {
                      "type" : "text",
                      "fields" : {
                        "keyword" : {
                          "type" : "keyword",
                          "ignore_above" : 256
                        }
                      }
                    },
                    "region_name" : {
                      "type" : "text",
                      "fields" : {
                        "keyword" : {
                          "type" : "keyword",
                          "ignore_above" : 256
                        }
                      }
                    },
                    "timezone" : {
                      "type" : "text",
                      "fields" : {
                        "keyword" : {
                          "type" : "keyword",
                          "ignore_above" : 256
                        }
                      }
                    }
                  }
                },
                "ip" : {
                  "type" : "ip"
                }
              }
            },
            "user" : {
              "properties" : {
                "identities" : {
                  "type" : "text",
                  "fields" : {
                    "keyword" : {
                      "type" : "keyword",
                      "ignore_above" : 256
                    }
                  }
                },
                "identity_type" : {
                  "type" : "keyword"
                }
              }
            }
          }
        },
        "tags" : {
          "type" : "keyword"
        },
        "type" : {
          "type" : "text",
          "fields" : {
            "keyword" : {
              "type" : "keyword",
              "ignore_above" : 256
            }
          }
        },
        "url" : {
          "properties" : {
            "full" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            },
            "original" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            }
          }
        },
        "user_agent" : {
          "properties" : {
            "build" : {
              "type" : "keyword"
            },
            "device" : {
              "type" : "keyword"
            },
            "major" : {
              "type" : "keyword"
            },
            "minor" : {
              "type" : "keyword"
            },
            "name" : {
              "type" : "keyword"
            },
            "original" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            },
            "os" : {
              "type" : "keyword"
            },
            "os_full" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            },
            "os_major" : {
              "type" : "keyword"
            },
            "os_minor" : {
              "type" : "keyword"
            },
            "os_name" : {
              "type" : "keyword"
            },
            "os_patch" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            },
            "os_version" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            },
            "patch" : {
              "type" : "keyword"
            },
            "version" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            }
          }
        }
      }
    }

```

---

<div class="post-metadata">

**Author:** ![Mark\_Rodman](https://avatars.discourse-cdn.com/v4/letter/m/57b2e6/32.png) [@Mark\_Rodman](https://discuss.elastic.co/u/Mark_Rodman)\
**Post date:** [August 4, 2022, 12:44pm UTC](https://discuss.elastic.co/t/duplicate-field-keyword-and-field-in-data-stream-within-observability/311383/2 "2022-08-04T12:44:26Z")

</div>

found the issue. was of course the template.

---

<div class="post-metadata">

**Author:** ![system](https://us1.discourse-cdn.com/elastic/original/3X/1/a/1ac57faf039f6b580b3f104ef42a2a89e41014de.png) [@system](https://discuss.elastic.co/u/system)\
**Post date:** [September 1, 2022, 12:45pm UTC](https://discuss.elastic.co/t/duplicate-field-keyword-and-field-in-data-stream-within-observability/311383/3 "2022-09-01T12:45:02Z")

</div>

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.