Duplicate field.keyword and field in data stream within Observability

Hi,
I recently moved an existing configuration of Logstash indexing to a setup with Logstash using a data stream with component templates. As recommended the component template was broken down into two templates, one for index settings and one for mappings. I imported my mappings from the previous set-up, but I'm now suffering field duplication with field and field.keyword.
As the data represents logs i believe i just want the keyword.

Is there a reason for the difference in behaviour between a standard logstash template and datastream component template?

thanks

My template looks as follows;

   "mappings" : {
      "_data_stream_timestamp" : {
        "enabled" : true
      },
      "dynamic_templates" : [ ],
      "properties" : {
        "@timestamp" : {
          "type" : "date"
        },
        "@version" : {
          "type" : "text",
          "fields" : {
            "keyword" : {
              "type" : "keyword",
              "ignore_above" : 256
            }
          }
        },
        "client" : {
          "properties" : {
            "address" : {
              "type" : "ip"
            },
            "ip" : {
              "type" : "ip"
            }
          }
        },
        "custom" : {
          "properties" : {
            "timestamp" : {
              "properties" : {
                "day" : {
                  "type" : "short"
                },
                "hour" : {
                  "type" : "short"
                },
                "minute" : {
                  "type" : "short"
                },
                "month" : {
                  "type" : "short"
                },
                "timezone" : {
                  "type" : "keyword"
                },
                "week_number" : {
                  "type" : "short"
                },
                "weekday" : {
                  "type" : "short"
                }
              }
            }
          }
        },
        "data_stream" : {
          "properties" : {
            "dataset" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            },
            "namespace" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            },
            "type" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            }
          }
        },
        "destination" : {
          "properties" : {
            "address" : {
              "type" : "ip"
            },
            "as" : {
              "properties" : {
                "as_org" : {
                  "type" : "keyword"
                },
                "asn" : {
                  "type" : "long"
                },
                "ip" : {
                  "type" : "ip"
                }
              }
            },
            "geo" : {
              "properties" : {
                "city_name" : {
                  "type" : "keyword"
                },
                "continent_code" : {
                  "type" : "keyword"
                },
                "country_code2" : {
                  "type" : "keyword"
                },
                "country_code3" : {
                  "type" : "keyword"
                },
                "country_name" : {
                  "type" : "keyword"
                },
                "dma_code" : {
                  "type" : "long"
                },
                "ip" : {
                  "type" : "ip"
                },
                "latitude" : {
                  "type" : "float"
                },
                "location" : {
                  "type" : "geo_point"
                },
                "longitude" : {
                  "type" : "float"
                },
                "postal_code" : {
                  "type" : "keyword"
                },
                "region_code" : {
                  "type" : "keyword"
                },
                "region_name" : {
                  "type" : "keyword"
                },
                "timezone" : {
                  "type" : "keyword"
                }
              }
            },
            "ip" : {
              "type" : "ip"
            }
          }
        },
        "ecs" : {
          "properties" : {
            "version" : {
              "type" : "keyword"
            }
          }
        },
        "event" : {
          "properties" : {
            "action" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            },
            "category" : {
              "type" : "keyword"
            },
            "kind" : {
              "type" : "keyword"
            },
            "module" : {
              "type" : "keyword"
            },
            "original" : {
              "type" : "keyword"
            },
            "severity" : {
              "type" : "short"
            },
            "type" : {
              "type" : "keyword"
            }
          }
        },
        "observer" : {
          "properties" : {
            "product" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            },
            "type" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            },
            "vendor" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            }
          }
        },
        "organization" : {
          "properties" : {
            "id" : {
              "type" : "keyword"
            },
            "name" : {
              "type" : "keyword"
            }
          }
        },
        "request" : {
          "properties" : {
            "categories" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            },
            "referer" : {
              "type" : "keyword"
            },
            "request_size" : {
              "type" : "long"
            },
            "url" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            }
          }
        },
        "response" : {
          "properties" : {
            "blocked_categories" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            },
            "bytes" : {
              "type" : "long"
            },
            "content" : {
              "properties" : {
                "amp_disposition" : {
                  "type" : "keyword"
                },
                "amp_malware_name" : {
                  "type" : "keyword"
                },
                "amp_score" : {
                  "type" : "short"
                },
                "av_detections" : {
                  "type" : "text",
                  "fields" : {
                    "keyword" : {
                      "type" : "keyword",
                      "ignore_above" : 256
                    }
                  },
                  "index_options" : "docs",
                  "norms" : false
                },
                "puas" : {
                  "type" : "text",
                  "fields" : {
                    "keyword" : {
                      "type" : "keyword",
                      "ignore_above" : 256
                    }
                  }
                },
                "sha" : {
                  "type" : "keyword"
                }
              }
            },
            "content_type" : {
              "type" : "keyword"
            },
            "response_body_size" : {
              "type" : "long"
            },
            "status_code" : {
              "type" : "short"
            },
            "verdict" : {
              "type" : "keyword"
            }
          }
        },
        "server" : {
          "properties" : {
            "address" : {
              "type" : "ip"
            },
            "ip" : {
              "type" : "ip"
            }
          }
        },
        "source" : {
          "properties" : {
            "address" : {
              "type" : "ip"
            },
            "as" : {
              "properties" : {
                "as_org" : {
                  "type" : "keyword"
                },
                "asn" : {
                  "type" : "long"
                },
                "ip" : {
                  "type" : "text",
                  "fields" : {
                    "keyword" : {
                      "type" : "keyword",
                      "ignore_above" : 256
                    }
                  }
                }
              }
            },
            "ip" : {
              "type" : "ip"
            },
            "nat" : {
              "properties" : {
                "geo" : {
                  "properties" : {
                    "city_name" : {
                      "type" : "text",
                      "fields" : {
                        "keyword" : {
                          "type" : "keyword",
                          "ignore_above" : 256
                        }
                      }
                    },
                    "continent_code" : {
                      "type" : "text",
                      "fields" : {
                        "keyword" : {
                          "type" : "keyword",
                          "ignore_above" : 256
                        }
                      }
                    },
                    "country_code2" : {
                      "type" : "text",
                      "fields" : {
                        "keyword" : {
                          "type" : "keyword",
                          "ignore_above" : 256
                        }
                      }
                    },
                    "country_code3" : {
                      "type" : "text",
                      "fields" : {
                        "keyword" : {
                          "type" : "keyword",
                          "ignore_above" : 256
                        }
                      }
                    },
                    "country_name" : {
                      "type" : "text",
                      "fields" : {
                        "keyword" : {
                          "type" : "keyword",
                          "ignore_above" : 256
                        }
                      }
                    },
                    "dma_code" : {
                      "type" : "long"
                    },
                    "ip" : {
                      "type" : "text",
                      "fields" : {
                        "keyword" : {
                          "type" : "keyword",
                          "ignore_above" : 256
                        }
                      }
                    },
                    "latitude" : {
                      "type" : "float"
                    },
                    "location" : {
                      "type" : "geo_point"
                    },
                    "longitude" : {
                      "type" : "float"
                    },
                    "postal_code" : {
                      "type" : "text",
                      "fields" : {
                        "keyword" : {
                          "type" : "keyword",
                          "ignore_above" : 256
                        }
                      }
                    },
                    "region_code" : {
                      "type" : "text",
                      "fields" : {
                        "keyword" : {
                          "type" : "keyword",
                          "ignore_above" : 256
                        }
                      }
                    },
                    "region_name" : {
                      "type" : "text",
                      "fields" : {
                        "keyword" : {
                          "type" : "keyword",
                          "ignore_above" : 256
                        }
                      }
                    },
                    "timezone" : {
                      "type" : "text",
                      "fields" : {
                        "keyword" : {
                          "type" : "keyword",
                          "ignore_above" : 256
                        }
                      }
                    }
                  }
                },
                "ip" : {
                  "type" : "ip"
                }
              }
            },
            "user" : {
              "properties" : {
                "identities" : {
                  "type" : "text",
                  "fields" : {
                    "keyword" : {
                      "type" : "keyword",
                      "ignore_above" : 256
                    }
                  }
                },
                "identity_type" : {
                  "type" : "keyword"
                }
              }
            }
          }
        },
        "tags" : {
          "type" : "keyword"
        },
        "type" : {
          "type" : "text",
          "fields" : {
            "keyword" : {
              "type" : "keyword",
              "ignore_above" : 256
            }
          }
        },
        "url" : {
          "properties" : {
            "full" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            },
            "original" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            }
          }
        },
        "user_agent" : {
          "properties" : {
            "build" : {
              "type" : "keyword"
            },
            "device" : {
              "type" : "keyword"
            },
            "major" : {
              "type" : "keyword"
            },
            "minor" : {
              "type" : "keyword"
            },
            "name" : {
              "type" : "keyword"
            },
            "original" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            },
            "os" : {
              "type" : "keyword"
            },
            "os_full" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            },
            "os_major" : {
              "type" : "keyword"
            },
            "os_minor" : {
              "type" : "keyword"
            },
            "os_name" : {
              "type" : "keyword"
            },
            "os_patch" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            },
            "os_version" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            },
            "patch" : {
              "type" : "keyword"
            },
            "version" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            }
          }
        }
      }
    }

found the issue. was of course the template.

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.