Please help on this and stuck in this from last 5 days and not able to create index for each pod in Elasticsearch using filebeat.
Elastic search version: 7.10.1 AWS
Filebeat version: 7.12.1
apiVersion: v1
kind: ConfigMap
metadata:
name: filebeat-config
namespace: filebeat
labels:
k8s-app: filebeat
data:
filebeat.yml: |-
filebeat.inputs:
- type: container
paths:
- /var/lib/kubelet/pods/*_communication-service_*
tags: ["communication-service"]
processors:
- add_kubernetes_metadata:
host: ${NODE_NAME}
matchers:
- logs_path:
logs_path: "/var/lib/kubelet/pods/"
resource_type: "pod"
default_indexers.enabled: false
default_matchers.enabled: false
indexers:
- pod_name:
- type: container
paths:
- /var/lib/kubelet/pods/*_bgv-dl-verification_*
tags: ["bgv-dl-verification"]
processors:
- add_kubernetes_metadata:
host: ${NODE_NAME}
matchers:
- logs_path:
logs_path: "/var/lib/kubelet/pods/"
resource_type: "pod"
default_indexers.enabled: false
default_matchers.enabled: false
indexers:
- pod_name:
output.elasticsearch:
hosts: ['${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:443}']
username: ${ELASTICSEARCH_USERNAME}
password: ${ELASTICSEARCH_PASSWORD}
protocol: "https"
indices:
- index: "logs-%{[kubernetes.pod.name]}-%{+yyyy.MM.dd}"
when.contains:
tags: "communication-service"
- index: "logs-%{[kubernetes.pod.name]}-%{+yyyy.MM.dd}"
when.contains:
tags: "bgv-dl-verification"
- index: "logs-%{[kubernetes.pod.name]}-%{+yyyy.MM.dd}"
when.contains:
tags: "reports-generator"
- index: "logs-%{[kubernetes.pod.name]}-%{+yyyy.MM.dd}"
when.contains:
nd the file beat logs says index is created but dont see any index with name of the pod
DEBUG [kubernetes] add_kubernetes_metadata/kubernetes.go:206 Adding kubernetes pod: logging/fluentd-8cvhz {"libbeat.processor": "add_kubernetes_metadata"}
2024-07-05T13:45:46.712Z DEBUG [kubernetes] add_kubernetes_metadata/kubernetes.go:296 Created index logging/fluentd-8cvhz for pod logging/fluentd-8cvhz {"libbeat.processor": "add_kubernetes_metadata"}
2024-07-05T13:45:46.712Z DEBUG [kubernetes] add_kubernetes_metadata/kubernetes.go:206 Adding kubernetes pod: kube-system/ebs-csi-node-fkj9g {"libbeat.processor": "add_kubernetes_metadata"}
2024-07-05T13:45:46.712Z DEBUG [kubernetes] add_kubernetes_metadata/kubernetes.go:296 Created index kube-system/ebs-csi-node-fkj9g for pod kube-system/ebs-csi-node-fkj9g {"libbeat.processor": "add_kubernetes_metadata"}
2024-07-05T13:45:46.712Z DEBUG [kubernetes] add_kubernetes_metadata/kubernetes.go:206 Adding kubernetes pod: dev/insurance-service-dev-deployment-649c79c86d-nbrvw {"libbeat.processor": "add_kubernetes_metadata"}
2024-07-05T13:45:46.718Z DEBUG [kubernetes] add_kubernetes_metadata/kubernetes.go:296 Created index dev/
)