When no log was written within 5 minutes,filebeat cat not erich with kubenetes metadata.
After I had change logger.level to debug,the debug file like below:
kubernetes: Querying for pod failed with error: pods \"worker3.ocp1.example.com\" not found
Couldn't collect info from any of the files in /etc/machine-id /var/lib/dbus/machine-id
Strange: worker3.ocp1.example.com is node name not pod name。
The full log:
{"log.level":"debug","@timestamp":"2023-10-19T06:20:18.308Z","log.logger":"input.filestream","log.origin":{"file.name":"filestream/filestream.go","file.line":131},"message":"End of file reached: /var/lib/kubelet/pods/64ea8a31-9347-49d4-92ea-a5e54ccac25c/volumes/kubernetes.io~empty-dir/log/4.log; Backoff now.","service.name":"filebeat","id":"myid","source_file":"filestream::myid::native::576717991-64516","path":"/var/lib/kubelet/pods/64ea8a31-9347-49d4-92ea-a5e54ccac25c/volumes/kubernetes.io~empty-dir/log/4.log","state-id":"native::576717991-64516","ecs.version":"1.6.0"}
{"log.level":"debug","@timestamp":"2023-10-19T06:20:18.310Z","log.logger":"kubernetes","log.origin":{"file.name":"kubernetes/util.go","file.line":132},"message":"kubernetes: Querying for pod failed with error: pods \"worker3.ocp1.example.com\" not found","service.name":"filebeat","libbeat.processor":"add_kubernetes_metadata","ecs.version":"1.6.0"}
{"log.level":"debug","@timestamp":"2023-10-19T06:20:18.310Z","log.logger":"kubernetes","log.origin":{"file.name":"kubernetes/util.go","file.line":141},"message":"kubernetes: Couldn't collect info from any of the files in /etc/machine-id /var/lib/dbus/machine-id","service.name":"filebeat","libbeat.processor":"add_kubernetes_metadata","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2023-10-19T06:20:18.310Z","log.logger":"kubernetes","log.origin":{"file.name":"kubernetes/util.go","file.line":146},"message":"kubernetes: Node worker3.ocp5-t.sz.bocsys.cn discovered by NODE_NAME environment variable","service.name":"filebeat","libbeat.processor":"add_kubernetes_metadata","ecs.version":"1.6.0"}
{"log.level":"debug","@timestamp":"2023-10-19T06:20:18.310Z","log.logger":"kubernetes","log.origin":{"file.name":"add_kubernetes_metadata/kubernetes.go","file.line":181},"message":"Initializing a new Kubernetes watcher using host: worker3.ocp5-t.sz.bocsys.cn","service.name":"filebeat","libbeat.processor":"add_kubernetes_metadata","ecs.version":"1.6.0"}
{"log.level":"debug","@timestamp":"2023-10-19T06:20:18.418Z","log.logger":"kubernetes","log.origin":{"file.name":"kubernetes/watcher.go","file.line":189},"message":"cache sync done","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"debug","@timestamp":"2023-10-19T06:20:18.519Z","log.logger":"kubernetes","log.origin":{"file.name":"kubernetes/watcher.go","file.line":189},"message":"cache sync done","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"debug","@timestamp":"2023-10-19T06:20:18.619Z","log.logger":"kubernetes","log.origin":{"file.name":"kubernetes/watcher.go","file.line":189},"message":"cache sync done","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"debug","@timestamp":"2023-10-19T06:20:18.619Z","log.logger":"kubernetes","log.origin":{"file.name":"add_kubernetes_metadata/kubernetes.go","file.line":222},"message":"Adding kubernetes pod: openshift-network-diagnostics/network-check-target-bc6tc","service.name":"filebeat","libbeat.processor":"add_kubernetes_metadata","ecs.version":"1.6.0"}
{"log.level":"debug","@timestamp":"2023-10-19T06:20:18.619Z","log.logger":"kubernetes","log.origin":{"file.name":"add_kubernetes_metadata/kubernetes.go","file.line":319},"message":"Created index 1275efe8-e287-4193-bfe2-91c6bc219dd2 for pod openshift-network-diagnostics/network-check-target-bc6tc","service.name":"filebeat","libbeat.processor":"add_kubernetes_metadata","ecs.version":"1.6.0"}
The filebeat config:
apiVersion: v1
kind: ConfigMap
metadata:
name: filebeat-config
namespace: kube-system
labels:
k8s-app: filebeat
data:
filebeat.yml: |-
- type: filestream
id: myid
enabled: true
paths:
- /var/lib/kubelet/pods/*/volumes/kubernetes.io~empty-dir/log/*.log
processors:
- add_kubernetes_metadata:
default_indexers.enabled: false
default_matchers.enabled: false
host: ${NODE_NAME}
sync_period: 1m
indexers:
- pod_uid:
matchers:
- logs_path:
logs_path: /var/lib/kubelet/pods/
resource_type: pod
add_resource_metadata:
namespace:
include_annotations: ["type"]
logging.level: debug
processors:
- add_host_metadata:
output.console:
pretty: true