Duplication data in the event

Elastic Stack Elasticsearch
1

Hi guys
I have some log from filebeat--logstash--> elastic
I see logs in kibana and it is strange
in JSON for example:

{
  "_index": "test-fbt-2021.11",
  "_type": "_doc",
  "_id": "dvMV_3wBOQF85F2Xl-Y4",
  "_score": 1,
  "_source": {
    "endOfBatch": false,
    "agent": {
      "ephemeral_id": "44437e45-9644-4524-b63a-2d708bc20cee",
      "version": "7.13.2",
      "hostname": "c-t-sandbox-k8s-wrk",
      "name": "c-t-sandbox-k8s-wrk",
      "type": "filebeat",
      "id": "68056566-714e-4566-9108-5713735a413f"
    },
    "ecs": {
      "version": "1.8.0"
    },
    
  },
  "fields": {
    "agent.version.keyword": [
      "7.13.2"
    ],
    "endOfBatch": [
      false
    ],
    "agent.hostname.keyword": [
      "c-t-sandbox-k8s-wrk"
    ],
    "ecs.version.keyword": [
      "1.8.0"
    ],
    "agent.name": [
      "c-t-sandbox-k8s-wrk"
    ],
    "agent.id.keyword": [
      "68056566-714e-4566-9108-5713735a413f"
    ],
    "agent.hostname": [
      "c-t-sandbox-k8s-wrk"
    ],
    "agent.id": [
      "68056566-714e-4566-9108-5713735a413f"
    ],
    "ecs.version": [
      "1.8.0"
    ],
    "agent.version": [
      "7.13.2"
    ],
    "agent.type": [
      "filebeat"
    ],
    "agent.type.keyword": [
      "filebeat"
    ],
    "agent.ephemeral_id.keyword": [
      "44437e45-9644-4524-b63a-2d708bc20cee"
    ],
    "agent.name.keyword": [
      "c-t-sandbox-k8s-wrk"
    ],
   "agent.ephemeral_id": [
      "44437e45-9644-4524-b63a-2d708bc20cee"
    ],
  }
}

data duplicate, index size bigger
I check filebeat and logstash it's OK

Please help. Can I drop this duplicate field "fields"?

2

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.