hi there!! sorry i'm very new to this technology. ELK got installed at k8s cluster, working fine. from beats to logstash i get these things (see code below). which i'm happy about. but i need to get rid of several fields. not successful so far. tried filter {kv {exclude_keys}}, filter {json {remove_fields}} etc... either my syntax is wrong or just this approach is not correct. could you please turn me into right direction? a snippet would be very helpful and appreciated. thank you!
{
"@timestamp": [
"2024-08-16T05:51:20.818Z"
],
"@version": [
"1"
],
"@version.keyword": [
"1"
],
"agent.ephemeral_id": [
"8ed493c2-6f7f-42bd-82ee-19222ef8d23b"
],
"agent.ephemeral_id.keyword": [
"8ed493c2-6f7f-42bd-82ee-19222ef8d23b"
],
"agent.id": [
"75f5e133-9544-442d-9571-c433f182afb6"
],
"agent.id.keyword": [
"75f5e133-9544-442d-9571-c433f182afb6"
],
"agent.name": [
"BLC-master-01"
],
"agent.name.keyword": [
"BLC-master-01"
],
"agent.type": [
"filebeat"
],
"agent.type.keyword": [
"filebeat"
],
"agent.version": [
"8.14.3"
],
"agent.version.keyword": [
"8.14.3"
],
"container.id": [
"1a585e1ebde78cb33c5a3bf34f1d21ce0874a72d11b699711c10304b421cc311"
],
"container.id.keyword": [
"1a585e1ebde78cb33c5a3bf34f1d21ce0874a72d11b699711c10304b421cc311"
],
"container.image.name": [
"registry.k8s.io/coredns/coredns:v1.11.1"
],
"container.image.name.keyword": [
"registry.k8s.io/coredns/coredns:v1.11.1"
],
"container.runtime": [
"cri-o"
],
"container.runtime.keyword": [
"cri-o"
],
"ecs.version": [
"8.0.0"
],
"ecs.version.keyword": [
"8.0.0"
],
"event.original": [
"[INFO] 10.244.97.167:39456 - 54038 \"A IN telegram-alertmanager. udp 50 false 1232\" NXDOMAIN qr,aa,rd,ra 114 0.000273943s"
],
"event.original.keyword": [
"[INFO] 10.244.97.167:39456 - 54038 \"A IN telegram-alertmanager. udp 50 false 1232\" NXDOMAIN qr,aa,rd,ra 114 0.000273943s"
],
"host.name": [
"BLC-master-01"
],
"host.name.keyword": [
"BLC-master-01"
],
"input.type": [
"container"
],
"input.type.keyword": [
"container"
],
"kubernetes.container.name": [
"coredns"
],
"kubernetes.container.name.keyword": [
"coredns"
],
"kubernetes.labels.k8s-app": [
"kube-dns"
],
"kubernetes.labels.k8s-app.keyword": [
"kube-dns"
],
"kubernetes.labels.pod-template-hash": [
"76f75df574"
],
"kubernetes.labels.pod-template-hash.keyword": [
"76f75df574"
],
"kubernetes.namespace": [
"kube-system"
],
"kubernetes.namespace_labels.kubernetes_io/metadata_name": [
"kube-system"
],
"kubernetes.namespace_labels.kubernetes_io/metadata_name.keyword": [
"kube-system"
],
"kubernetes.namespace_uid": [
"5f3cf5b1-b6a3-42e7-8e78-5c4385738b54"
],
"kubernetes.namespace_uid.keyword": [
"5f3cf5b1-b6a3-42e7-8e78-5c4385738b54"
],
"kubernetes.namespace.keyword": [
"kube-system"
],
"kubernetes.node.hostname": [
"blc-master-01"
],
"kubernetes.node.hostname.keyword": [
"blc-master-01"
],
"kubernetes.node.labels.beta_kubernetes_io/arch": [
"amd64"
],
"kubernetes.node.labels.beta_kubernetes_io/arch.keyword": [
"amd64"
],
"kubernetes.node.labels.beta_kubernetes_io/os": [
"linux"
],
"kubernetes.node.labels.beta_kubernetes_io/os.keyword": [
"linux"
],
"kubernetes.node.labels.kubernetes_io/arch": [
"amd64"
],
"kubernetes.node.labels.kubernetes_io/arch.keyword": [
"amd64"
],
"kubernetes.node.labels.kubernetes_io/hostname": [
"blc-master-01"
],
"kubernetes.node.labels.kubernetes_io/hostname.keyword": [
"blc-master-01"
],
"kubernetes.node.labels.kubernetes_io/os": [
"linux"
],
"kubernetes.node.labels.kubernetes_io/os.keyword": [
"linux"
],
"kubernetes.node.labels.node_kubernetes_io/exclude-from-external-load-balancers": [
""
],
"kubernetes.node.labels.node_kubernetes_io/exclude-from-external-load-balancers.keyword": [
""
],
"kubernetes.node.labels.node-role_kubernetes_io/control-plane": [
""
],
"kubernetes.node.labels.node-role_kubernetes_io/control-plane.keyword": [
""
],
"kubernetes.node.name": [
"blc-master-01"
],
"kubernetes.node.name.keyword": [
"blc-master-01"
],
"kubernetes.node.uid": [
"64006ab0-3f85-4a72-a801-129eb65cb0f0"
],
"kubernetes.node.uid.keyword": [
"64006ab0-3f85-4a72-a801-129eb65cb0f0"
],
"kubernetes.pod.ip": [
"10.244.194.134"
],
"kubernetes.pod.ip.keyword": [
"10.244.194.134"
],
"kubernetes.pod.name": [
"coredns-76f75df574-q825g"
],
"kubernetes.pod.name.keyword": [
"coredns-76f75df574-q825g"
],
"kubernetes.pod.uid": [
"7323e973-1d67-4908-b99a-9fe7381756d9"
],
"kubernetes.pod.uid.keyword": [
"7323e973-1d67-4908-b99a-9fe7381756d9"
],
"kubernetes.replicaset.name": [
"coredns-76f75df574"
],
"kubernetes.replicaset.name.keyword": [
"coredns-76f75df574"
],
"log.file.path": [
"/var/log/containers/coredns-76f75df574-q825g_kube-system_coredns-1a585e1ebde78cb33c5a3bf34f1d21ce0874a72d11b699711c10304b421cc311.log"
],
"log.file.path.keyword": [
"/var/log/containers/coredns-76f75df574-q825g_kube-system_coredns-1a585e1ebde78cb33c5a3bf34f1d21ce0874a72d11b699711c10304b421cc311.log"
],
"log.offset": [
10364251
],
"message": [
"[INFO] 10.244.97.167:39456 - 54038 \"A IN telegram-alertmanager. udp 50 false 1232\" NXDOMAIN qr,aa,rd,ra 114 0.000273943s"
],
"message.keyword": [
"[INFO] 10.244.97.167:39456 - 54038 \"A IN telegram-alertmanager. udp 50 false 1232\" NXDOMAIN qr,aa,rd,ra 114 0.000273943s"
],
"stream": [
"stdout"
],
"stream.keyword": [
"stdout"
],
"tags": [
"beats_input_codec_plain_applied"
],
"tags.keyword": [
"beats_input_codec_plain_applied"
],
"_id": "Y7i9WZEB_ARFySvclBpR",
"_index": "filebeat-8.14.3",
"_score": null
}