Filebeat autodiscovery not working on AKS cluster
failing with below error as soon i include auto discovery
<
ax_events":4096}}},"registrar":{"states":{"cleanup":0,"current":0,"update":51},"writes":{"fail":0,"success":0,"total":0}},"system":{"cpu":{"cores":16},"load":{"1":0.81,"15":0.56,"5":0.53,"norm":{"1":0.0506,"15":0.035,"5":0.0331}}}},"ecs.version":"1.6.0"}}
{"log.level":"info","@timestamp":"2024-02-23T06:13:07.650Z","log.logger":"monitoring","log.origin":{"file.name":"log/log.go","file.line":196},"message":"Uptime: 1m43.898682829s","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2024-02-23T06:13:07.650Z","log.logger":"monitoring","log.origin":{"file.name":"log/log.go","file.line":163},"message":"Stopping metrics logging.","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2024-02-23T06:13:07.650Z","log.origin":{"file.name":"instance/beat.go","file.line":531},"message":"filebeat stopped.","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"error","@timestamp":"2024-02-23T06:13:07.650Z","log.origin":{"file.name":"instance/beat.go","file.line":1307},"message":"Exiting: error in autodiscover provider settings: error setting up kubernetes autodiscover provider: missing field accessing 'filebeat.autodiscover.providers.0.node' (source:'/etc/beat.yml')","service.name":"filebeat","ecs.version":"1.6.0"}
Exiting: error in autodiscover provider settings: error setting up kubernetes autodiscover provider: missing field accessing 'filebeat.autodiscover.providers.0.node' (source:'/etc/beat.yml')
{"log.level":"info","@timestamp":"2024-02-23T06:13:07.650Z","log.logger":"input","log.origin":{"file.name":"log/input.go","file.line":798},"message":"input outlet closed","service.name":"filebeat","input_id":"36e836f3-f5e8-4838-8fe0-85d1add712c4","ecs.version":"1.6.0"}
/>
Deployment
apiVersion: beat.k8s.elastic.co/v1beta1
kind: Beat
metadata:
name: elasticsearch
spec:
type: filebeat
version: 8.11.4
elasticsearchRef:
name: elasticsearch
kibanaRef:
name: elasticsearch
config:
filebeat.inputs:
- type: container
exclude_lines: ['^DBG', 'GIT']
paths:
- /var/log/containers/*.log
- /var/log/pods/*.log
- /var/lib/docker/containers/*/*.log
processors:
- decode_json_fields:
fields: ["message"]
target: ""
overwrite_keys: true
max_depth: 1
add_error_key: true
filebeat.autodiscover:
providers:
- type: kubernetes
node: ${NODE_NAME}
templates:
- condition:
contains:
kubernetes.container.name: "no-json-logging"
config:
- type: container
paths:
- "/var/log/containers/*-${data.kubernetes.container.id}.log"
- condition:
contains:
kubernetes.container.name: "json-logging"
config:
- type: container
paths:
- "/var/log/containers/*-${data.kubernetes.container.id}.log"
json.keys_under_root: true
json.add_error_key: true
json.message_key: message
daemonSet:
podTemplate:
spec:
dnsPolicy: ClusterFirstWithHostNet
hostNetwork: true
securityContext:
runAsUser: 0
containers:
- name: filebeat
volumeMounts:
- name: varlogcontainers
mountPath: /var/log/containers
- name: varlogpods
mountPath: /var/log/pods
- name: varlibdockercontainers
mountPath: /var/lib/docker/containers
volumes:
- name: varlogcontainers
hostPath:
path: /var/log/containers
- name: varlogpods
hostPath:
path: /var/log/pods
- name: varlibdockercontainers
hostPath:
path: /var/lib/docker/containers