Collecting logs from filebeats at k8s to logstash, can't filter fields

hi there!! sorry i'm very new to this technology. ELK got installed at k8s cluster, working fine. from beats to logstash i get these things (see code below). which i'm happy about. but i need to get rid of several fields. not successful so far. tried filter {kv {exclude_keys}}, filter {json {remove_fields}} etc... either my syntax is wrong or just this approach is not correct. could you please turn me into right direction? a snippet would be very helpful and appreciated. thank you!

{
  "@timestamp": [
    "2024-08-16T05:51:20.818Z"
  ],
  "@version": [
    "1"
  ],
  "@version.keyword": [
    "1"
  ],
  "agent.ephemeral_id": [
    "8ed493c2-6f7f-42bd-82ee-19222ef8d23b"
  ],
  "agent.ephemeral_id.keyword": [
    "8ed493c2-6f7f-42bd-82ee-19222ef8d23b"
  ],
  "agent.id": [
    "75f5e133-9544-442d-9571-c433f182afb6"
  ],
  "agent.id.keyword": [
    "75f5e133-9544-442d-9571-c433f182afb6"
  ],
  "agent.name": [
    "BLC-master-01"
  ],
  "agent.name.keyword": [
    "BLC-master-01"
  ],
  "agent.type": [
    "filebeat"
  ],
  "agent.type.keyword": [
    "filebeat"
  ],
  "agent.version": [
    "8.14.3"
  ],
  "agent.version.keyword": [
    "8.14.3"
  ],
  "container.id": [
    "1a585e1ebde78cb33c5a3bf34f1d21ce0874a72d11b699711c10304b421cc311"
  ],
  "container.id.keyword": [
    "1a585e1ebde78cb33c5a3bf34f1d21ce0874a72d11b699711c10304b421cc311"
  ],
  "container.image.name": [
    "registry.k8s.io/coredns/coredns:v1.11.1"
  ],
  "container.image.name.keyword": [
    "registry.k8s.io/coredns/coredns:v1.11.1"
  ],
  "container.runtime": [
    "cri-o"
  ],
  "container.runtime.keyword": [
    "cri-o"
  ],
  "ecs.version": [
    "8.0.0"
  ],
  "ecs.version.keyword": [
    "8.0.0"
  ],
  "event.original": [
    "[INFO] 10.244.97.167:39456 - 54038 \"A IN telegram-alertmanager. udp 50 false 1232\" NXDOMAIN qr,aa,rd,ra 114 0.000273943s"
  ],
  "event.original.keyword": [
    "[INFO] 10.244.97.167:39456 - 54038 \"A IN telegram-alertmanager. udp 50 false 1232\" NXDOMAIN qr,aa,rd,ra 114 0.000273943s"
  ],
  "host.name": [
    "BLC-master-01"
  ],
  "host.name.keyword": [
    "BLC-master-01"
  ],
  "input.type": [
    "container"
  ],
  "input.type.keyword": [
    "container"
  ],
  "kubernetes.container.name": [
    "coredns"
  ],
  "kubernetes.container.name.keyword": [
    "coredns"
  ],
  "kubernetes.labels.k8s-app": [
    "kube-dns"
  ],
  "kubernetes.labels.k8s-app.keyword": [
    "kube-dns"
  ],
  "kubernetes.labels.pod-template-hash": [
    "76f75df574"
  ],
  "kubernetes.labels.pod-template-hash.keyword": [
    "76f75df574"
  ],
  "kubernetes.namespace": [
    "kube-system"
  ],
  "kubernetes.namespace_labels.kubernetes_io/metadata_name": [
    "kube-system"
  ],
  "kubernetes.namespace_labels.kubernetes_io/metadata_name.keyword": [
    "kube-system"
  ],
  "kubernetes.namespace_uid": [
    "5f3cf5b1-b6a3-42e7-8e78-5c4385738b54"
  ],
  "kubernetes.namespace_uid.keyword": [
    "5f3cf5b1-b6a3-42e7-8e78-5c4385738b54"
  ],
  "kubernetes.namespace.keyword": [
    "kube-system"
  ],
  "kubernetes.node.hostname": [
    "blc-master-01"
  ],
  "kubernetes.node.hostname.keyword": [
    "blc-master-01"
  ],
  "kubernetes.node.labels.beta_kubernetes_io/arch": [
    "amd64"
  ],
  "kubernetes.node.labels.beta_kubernetes_io/arch.keyword": [
    "amd64"
  ],
  "kubernetes.node.labels.beta_kubernetes_io/os": [
    "linux"
  ],
  "kubernetes.node.labels.beta_kubernetes_io/os.keyword": [
    "linux"
  ],
  "kubernetes.node.labels.kubernetes_io/arch": [
    "amd64"
  ],
  "kubernetes.node.labels.kubernetes_io/arch.keyword": [
    "amd64"
  ],
  "kubernetes.node.labels.kubernetes_io/hostname": [
    "blc-master-01"
  ],
  "kubernetes.node.labels.kubernetes_io/hostname.keyword": [
    "blc-master-01"
  ],
  "kubernetes.node.labels.kubernetes_io/os": [
    "linux"
  ],
  "kubernetes.node.labels.kubernetes_io/os.keyword": [
    "linux"
  ],
  "kubernetes.node.labels.node_kubernetes_io/exclude-from-external-load-balancers": [
    ""
  ],
  "kubernetes.node.labels.node_kubernetes_io/exclude-from-external-load-balancers.keyword": [
    ""
  ],
  "kubernetes.node.labels.node-role_kubernetes_io/control-plane": [
    ""
  ],
  "kubernetes.node.labels.node-role_kubernetes_io/control-plane.keyword": [
    ""
  ],
  "kubernetes.node.name": [
    "blc-master-01"
  ],
  "kubernetes.node.name.keyword": [
    "blc-master-01"
  ],
  "kubernetes.node.uid": [
    "64006ab0-3f85-4a72-a801-129eb65cb0f0"
  ],
  "kubernetes.node.uid.keyword": [
    "64006ab0-3f85-4a72-a801-129eb65cb0f0"
  ],
  "kubernetes.pod.ip": [
    "10.244.194.134"
  ],
  "kubernetes.pod.ip.keyword": [
    "10.244.194.134"
  ],
  "kubernetes.pod.name": [
    "coredns-76f75df574-q825g"
  ],
  "kubernetes.pod.name.keyword": [
    "coredns-76f75df574-q825g"
  ],
  "kubernetes.pod.uid": [
    "7323e973-1d67-4908-b99a-9fe7381756d9"
  ],
  "kubernetes.pod.uid.keyword": [
    "7323e973-1d67-4908-b99a-9fe7381756d9"
  ],
  "kubernetes.replicaset.name": [
    "coredns-76f75df574"
  ],
  "kubernetes.replicaset.name.keyword": [
    "coredns-76f75df574"
  ],
  "log.file.path": [
    "/var/log/containers/coredns-76f75df574-q825g_kube-system_coredns-1a585e1ebde78cb33c5a3bf34f1d21ce0874a72d11b699711c10304b421cc311.log"
  ],
  "log.file.path.keyword": [
    "/var/log/containers/coredns-76f75df574-q825g_kube-system_coredns-1a585e1ebde78cb33c5a3bf34f1d21ce0874a72d11b699711c10304b421cc311.log"
  ],
  "log.offset": [
    10364251
  ],
  "message": [
    "[INFO] 10.244.97.167:39456 - 54038 \"A IN telegram-alertmanager. udp 50 false 1232\" NXDOMAIN qr,aa,rd,ra 114 0.000273943s"
  ],
  "message.keyword": [
    "[INFO] 10.244.97.167:39456 - 54038 \"A IN telegram-alertmanager. udp 50 false 1232\" NXDOMAIN qr,aa,rd,ra 114 0.000273943s"
  ],
  "stream": [
    "stdout"
  ],
  "stream.keyword": [
    "stdout"
  ],
  "tags": [
    "beats_input_codec_plain_applied"
  ],
  "tags.keyword": [
    "beats_input_codec_plain_applied"
  ],
  "_id": "Y7i9WZEB_ARFySvclBpR",
  "_index": "filebeat-8.14.3",
  "_score": null
}

Which ones?

"agent.ephemeral_id": [
    "8ed493c2-6f7f-42bd-82ee-19222ef8d23b"
  ],
  "agent.ephemeral_id.keyword": [
    "8ed493c2-6f7f-42bd-82ee-19222ef8d23b"
  ],
  "agent.id": [
    "75f5e133-9544-442d-9571-c433f182afb6"
  ],
  "agent.id.keyword": [
    "75f5e133-9544-442d-9571-c433f182afb6"
  ],

for example

You appear to be showing us a document fetched from elasticsearch, so the first thing to understand is that the .keyword fields are added by elasticsearch and do not exist in logstash. Your index mapping (possibly a default one you did not configure) is creating them. You can read more about .keyword fields here.

agent.emphemeral_id and agent.id are fields within the agent object. In elasticsearch or kibana you would refer to them as agent.id, but in logstash you have to refer to them as [agent][emphemeral_id] and [agent][id]. The logstash syntax allows unambiguous references to fields with names that contain dots, so it can distinguish [a.b][c] and [a][b][c], which both have the same name in elasticsearch.

If you do not send [agent][id] to elasticsearch then [agent][id][keyword] will not be created.

You could remove these with

mutate { remove_fields => [ "[agent][emphemeral_id]", "[agent][id]" ] }
1 Like

didn't change so far.. fields are still there.

  logstash.conf: |
    input {
      beats {
        port => 5044
      }
    }
    mutate { remove_fields => [ "[agent][ephemeral_id]", "[agent][id]" ] }
    output {
      elasticsearch {
        index => "%{[@metadata][beat]}-%{[@metadata][version]}" 
        hosts => [ "${ES_HOSTS}" ]
        user => "${ES_USER}"
        password => "${ES_PASSWORD}"
        cacert => '/etc/logstash/certificates/ca.crt'
      }
    }

message:

{
  "_index": "filebeat-8.14.3",
  "_id": "YikLXJEBHbspKTFxy4FU",
  "_version": 1,
  "_score": null,
  "fields": {
    "agent.version.keyword": [
      "8.14.3"
    ],
    "kubernetes.node.uid": [
      "64006ab0-3f85-4a72-a801-129eb65cb0f0"
    ],
    "kubernetes.namespace_uid.keyword": [
      "5f3cf5b1-b6a3-42e7-8e78-5c4385738b54"
    ],
    "host.name.keyword": [
      "BLC-master-01"
    ],
    "kubernetes.namespace_uid": [
      "5f3cf5b1-b6a3-42e7-8e78-5c4385738b54"
    ],
    "kubernetes.node.labels.kubernetes_io/os": [
      "linux"
    ],
    "container.id": [
      "1a585e1ebde78cb33c5a3bf34f1d21ce0874a72d11b699711c10304b421cc311"
    ],
    "kubernetes.node.labels.kubernetes_io/os.keyword": [
      "linux"
    ],
    "ecs.version.keyword": [
      "8.0.0"
    ],
    "container.image.name": [
      "registry.k8s.io/coredns/coredns:v1.11.1"
    ],
    "kubernetes.container.name.keyword": [
      "coredns"
    ],
    "kubernetes.namespace": [
      "kube-system"
    ],
    "kubernetes.node.labels.beta_kubernetes_io/os": [
      "linux"
    ],
    "kubernetes.pod.name.keyword": [
      "coredns-76f75df574-q825g"
    ],
    "agent.name": [
      "BLC-master-01"
    ],
    "host.name": [
      "BLC-master-01"
    ],
    "event.original": [
      "[INFO] 10.244.97.167:50871 - 6108 \"A IN telegram-alertmanager. udp 50 false 1232\" NXDOMAIN qr,aa,rd,ra 114 0.000079522s"
    ],
    "kubernetes.node.labels.kubernetes_io/hostname.keyword": [
      "blc-master-01"
    ],
    "agent.id.keyword": [
      "75f5e133-9544-442d-9571-c433f182afb6"
    ],
    "kubernetes.node.labels.node-role_kubernetes_io/control-plane.keyword": [
      ""
    ],
    "@version.keyword": [
      "1"
    ],
    "input.type": [
      "container"
    ],
    "kubernetes.node.uid.keyword": [
      "64006ab0-3f85-4a72-a801-129eb65cb0f0"
    ],
    "log.offset": [
      73929
    ],
    "tags": [
      "beats_input_codec_plain_applied"
    ],
    "container.runtime": [
      "cri-o"
    ],
    "agent.id": [
      "75f5e133-9544-442d-9571-c433f182afb6"
    ],
    "ecs.version": [
      "8.0.0"
    ],
    "message.keyword": [
      "[INFO] 10.244.97.167:50871 - 6108 \"A IN telegram-alertmanager. udp 50 false 1232\" NXDOMAIN qr,aa,rd,ra 114 0.000079522s"
    ],
    "kubernetes.node.labels.node-role_kubernetes_io/control-plane": [
      ""
    ],
    "agent.version": [
      "8.14.3"
    ],
    "kubernetes.namespace.keyword": [
      "kube-system"
    ],
    "kubernetes.node.name": [
      "blc-master-01"
    ],
    "input.type.keyword": [
      "container"
    ],
    "stream.keyword": [
      "stdout"
    ],
    "kubernetes.node.hostname": [
      "blc-master-01"
    ],
    "tags.keyword": [
      "beats_input_codec_plain_applied"
    ],
    "kubernetes.node.name.keyword": [
      "blc-master-01"
    ],
    "kubernetes.pod.uid": [
      "7323e973-1d67-4908-b99a-9fe7381756d9"
    ],
    "kubernetes.node.hostname.keyword": [
      "blc-master-01"
    ],
    "agent.type": [
      "filebeat"
    ],
    "stream": [
      "stdout"
    ],
    "kubernetes.node.labels.kubernetes_io/arch.keyword": [
      "amd64"
    ],
    "@version": [
      "1"
    ],
    "kubernetes.pod.name": [
      "coredns-76f75df574-q825g"
    ],
    "container.image.name.keyword": [
      "registry.k8s.io/coredns/coredns:v1.11.1"
    ],
    "log.file.path.keyword": [
      "/var/log/containers/coredns-76f75df574-q825g_kube-system_coredns-1a585e1ebde78cb33c5a3bf34f1d21ce0874a72d11b699711c10304b421cc311.log"
    ],
    "agent.type.keyword": [
      "filebeat"
    ],
    "kubernetes.pod.ip": [
      "10.244.194.134"
    ],
    "agent.ephemeral_id.keyword": [
      "1854056d-b9e7-41ca-9f92-bf4bb37898c0"
    ],
    "kubernetes.container.name": [
      "coredns"
    ],
    "agent.name.keyword": [
      "BLC-master-01"
    ],
    "kubernetes.node.labels.beta_kubernetes_io/arch.keyword": [
      "amd64"
    ],
    "kubernetes.replicaset.name": [
      "coredns-76f75df574"
    ],
    "kubernetes.namespace_labels.kubernetes_io/metadata_name": [
      "kube-system"
    ],
    "message": [
      "[INFO] 10.244.97.167:50871 - 6108 \"A IN telegram-alertmanager. udp 50 false 1232\" NXDOMAIN qr,aa,rd,ra 114 0.000079522s"
    ],
    "kubernetes.node.labels.kubernetes_io/hostname": [
      "blc-master-01"
    ],
    "kubernetes.node.labels.beta_kubernetes_io/arch": [
      "amd64"
    ],
    "@timestamp": [
      "2024-08-16T16:35:58.897Z"
    ],
    "kubernetes.pod.uid.keyword": [
      "7323e973-1d67-4908-b99a-9fe7381756d9"
    ],
    "kubernetes.replicaset.name.keyword": [
      "coredns-76f75df574"
    ],
    "container.runtime.keyword": [
      "cri-o"
    ],
    "kubernetes.namespace_labels.kubernetes_io/metadata_name.keyword": [
      "kube-system"
    ],
    "event.original.keyword": [
      "[INFO] 10.244.97.167:50871 - 6108 \"A IN telegram-alertmanager. udp 50 false 1232\" NXDOMAIN qr,aa,rd,ra 114 0.000079522s"
    ],
    "kubernetes.node.labels.node_kubernetes_io/exclude-from-external-load-balancers.keyword": [
      ""
    ],
    "log.file.path": [
      "/var/log/containers/coredns-76f75df574-q825g_kube-system_coredns-1a585e1ebde78cb33c5a3bf34f1d21ce0874a72d11b699711c10304b421cc311.log"
    ],
    "agent.ephemeral_id": [
      "1854056d-b9e7-41ca-9f92-bf4bb37898c0"
    ],
    "kubernetes.node.labels.kubernetes_io/arch": [
      "amd64"
    ],
    "container.id.keyword": [
      "1a585e1ebde78cb33c5a3bf34f1d21ce0874a72d11b699711c10304b421cc311"
    ],
    "kubernetes.node.labels.beta_kubernetes_io/os.keyword": [
      "linux"
    ],
    "kubernetes.node.labels.node_kubernetes_io/exclude-from-external-load-balancers": [
      ""
    ],
    "kubernetes.pod.ip.keyword": [
      "10.244.194.134"
    ]
  },
  "sort": [
    "2024-08-16T16:35:58.897Z",
    20057782
  ]
}

didn't change. fields are still there.
config:

  logstash.conf: |
    input {
      beats {
        port => 5044
      }
    }
    mutate { remove_fields => [ "[agent][ephemeral_id]", "[agent][id]" ] }
    output {
      elasticsearch {
        index => "%{[@metadata][beat]}-%{[@metadata][version]}" 
        hosts => [ "${ES_HOSTS}" ]
        user => "${ES_USER}"
        password => "${ES_PASSWORD}"
        cacert => '/etc/logstash/certificates/ca.crt'
      }
    }

message:

{
  "@timestamp": [
    "2024-08-16T16:35:58.897Z"
  ],
  "@version": [
    "1"
  ],
  "@version.keyword": [
    "1"
  ],
  "agent.ephemeral_id": [
    "1854056d-b9e7-41ca-9f92-bf4bb37898c0"
  ],
  "agent.ephemeral_id.keyword": [
    "1854056d-b9e7-41ca-9f92-bf4bb37898c0"
  ],
  "agent.id": [
    "75f5e133-9544-442d-9571-c433f182afb6"
  ],
  "agent.id.keyword": [
    "75f5e133-9544-442d-9571-c433f182afb6"
  ],
  "agent.name": [
    "BLC-master-01"
  ],
  "agent.name.keyword": [
    "BLC-master-01"
  ],
  "agent.type": [
    "filebeat"
  ],
  "agent.type.keyword": [
    "filebeat"
  ],
  "agent.version": [
    "8.14.3"
  ],
  "agent.version.keyword": [
    "8.14.3"
  ],
  "container.id": [
    "1a585e1ebde78cb33c5a3bf34f1d21ce0874a72d11b699711c10304b421cc311"
  ],
  "container.id.keyword": [
    "1a585e1ebde78cb33c5a3bf34f1d21ce0874a72d11b699711c10304b421cc311"
  ],
  "container.image.name": [
    "registry.k8s.io/coredns/coredns:v1.11.1"
  ],
  "container.image.name.keyword": [
    "registry.k8s.io/coredns/coredns:v1.11.1"
  ],
  "container.runtime": [
    "cri-o"
  ],
  "container.runtime.keyword": [
    "cri-o"
  ],
  "ecs.version": [
    "8.0.0"
  ],
  "ecs.version.keyword": [
    "8.0.0"
  ],
  "event.original": [
    "[INFO] 10.244.97.167:50871 - 6108 \"A IN telegram-alertmanager. udp 50 false 1232\" NXDOMAIN qr,aa,rd,ra 114 0.000079522s"
  ],
  "event.original.keyword": [
    "[INFO] 10.244.97.167:50871 - 6108 \"A IN telegram-alertmanager. udp 50 false 1232\" NXDOMAIN qr,aa,rd,ra 114 0.000079522s"
  ],
  "host.name": [
    "BLC-master-01"
  ],
  "host.name.keyword": [
    "BLC-master-01"
  ],
  "input.type": [
    "container"
  ],
  "input.type.keyword": [
    "container"
  ],
  "kubernetes.container.name": [
    "coredns"
  ],
  "kubernetes.container.name.keyword": [
    "coredns"
  ],
  "kubernetes.namespace": [
    "kube-system"
  ],
  "kubernetes.namespace_labels.kubernetes_io/metadata_name": [
    "kube-system"
  ],
  "kubernetes.namespace_labels.kubernetes_io/metadata_name.keyword": [
    "kube-system"
  ],
  "kubernetes.namespace_uid": [
    "5f3cf5b1-b6a3-42e7-8e78-5c4385738b54"
  ],
  "kubernetes.namespace_uid.keyword": [
    "5f3cf5b1-b6a3-42e7-8e78-5c4385738b54"
  ],
  "kubernetes.namespace.keyword": [
    "kube-system"
  ],
  "kubernetes.node.hostname": [
    "blc-master-01"
  ],
  "kubernetes.node.hostname.keyword": [
    "blc-master-01"
  ],
  "kubernetes.node.labels.beta_kubernetes_io/arch": [
    "amd64"
  ],
  "kubernetes.node.labels.beta_kubernetes_io/arch.keyword": [
    "amd64"
  ],
  "kubernetes.node.labels.beta_kubernetes_io/os": [
    "linux"
  ],
  "kubernetes.node.labels.beta_kubernetes_io/os.keyword": [
    "linux"
  ],
  "kubernetes.node.labels.kubernetes_io/arch": [
    "amd64"
  ],
  "kubernetes.node.labels.kubernetes_io/arch.keyword": [
    "amd64"
  ],
  "kubernetes.node.labels.kubernetes_io/hostname": [
    "blc-master-01"
  ],
  "kubernetes.node.labels.kubernetes_io/hostname.keyword": [
    "blc-master-01"
  ],
  "kubernetes.node.labels.kubernetes_io/os": [
    "linux"
  ],
  "kubernetes.node.labels.kubernetes_io/os.keyword": [
    "linux"
  ],
  "kubernetes.node.labels.node_kubernetes_io/exclude-from-external-load-balancers": [
    ""
  ],
  "kubernetes.node.labels.node_kubernetes_io/exclude-from-external-load-balancers.keyword": [
    ""
  ],
  "kubernetes.node.labels.node-role_kubernetes_io/control-plane": [
    ""
  ],
  "kubernetes.node.labels.node-role_kubernetes_io/control-plane.keyword": [
    ""
  ],
  "kubernetes.node.name": [
    "blc-master-01"
  ],
  "kubernetes.node.name.keyword": [
    "blc-master-01"
  ],
  "kubernetes.node.uid": [
    "64006ab0-3f85-4a72-a801-129eb65cb0f0"
  ],
  "kubernetes.node.uid.keyword": [
    "64006ab0-3f85-4a72-a801-129eb65cb0f0"
  ],
  "kubernetes.pod.ip": [
    "10.244.194.134"
  ],
  "kubernetes.pod.ip.keyword": [
    "10.244.194.134"
  ],
  "kubernetes.pod.name": [
    "coredns-76f75df574-q825g"
  ],
  "kubernetes.pod.name.keyword": [
    "coredns-76f75df574-q825g"
  ],
  "kubernetes.pod.uid": [
    "7323e973-1d67-4908-b99a-9fe7381756d9"
  ],
  "kubernetes.pod.uid.keyword": [
    "7323e973-1d67-4908-b99a-9fe7381756d9"
  ],
  "kubernetes.replicaset.name": [
    "coredns-76f75df574"
  ],
  "kubernetes.replicaset.name.keyword": [
    "coredns-76f75df574"
  ],
  "log.file.path": [
    "/var/log/containers/coredns-76f75df574-q825g_kube-system_coredns-1a585e1ebde78cb33c5a3bf34f1d21ce0874a72d11b699711c10304b421cc311.log"
  ],
  "log.file.path.keyword": [
    "/var/log/containers/coredns-76f75df574-q825g_kube-system_coredns-1a585e1ebde78cb33c5a3bf34f1d21ce0874a72d11b699711c10304b421cc311.log"
  ],
  "log.offset": [
    73929
  ],
  "message": [
    "[INFO] 10.244.97.167:50871 - 6108 \"A IN telegram-alertmanager. udp 50 false 1232\" NXDOMAIN qr,aa,rd,ra 114 0.000079522s"
  ],
  "message.keyword": [
    "[INFO] 10.244.97.167:50871 - 6108 \"A IN telegram-alertmanager. udp 50 false 1232\" NXDOMAIN qr,aa,rd,ra 114 0.000079522s"
  ],
  "stream": [
    "stdout"
  ],
  "stream.keyword": [
    "stdout"
  ],
  "tags": [
    "beats_input_codec_plain_applied"
  ],
  "tags.keyword": [
    "beats_input_codec_plain_applied"
  ],
  "_id": "YikLXJEBHbspKTFxy4FU",
  "_index": "filebeat-8.14.3",
  "_score": null
}