I have some problems with Filebeats 8.12.0.
in the Kubernetes cluster.
My filebeat.yml:
filebeat.inputs:
- type: filestream
paths:
- /var/log/pods/**/*.log
parsers:
- container: ~
prospector:
scanner:
fingerprint.enabled: true
symlinks: true
file_identity.fingerprint: ~
processors:
- add_kubernetes_metadata:
host: ${NODE_NAME}
matchers:
- logs_path:
logs_path: "/var/log/pods/**/*.log"
# To enable hints based autodiscover, remove `filebeat.inputs` configuration and uncomment this:
# filebeat.autodiscover:
# providers:
# - type: kubernetes
# node: ${NODE_NAME}
# hints.enabled: true
# hints.default_config:
# type: filestream
# id: kubernetes-container-logs-${data.kubernetes.pod.name}-${data.kubernetes.container.id}
# paths:
# - /var/log/containers/*-${data.kubernetes.container.id}.log
# parsers:
# - container: ~
# prospector:
# scanner:
# fingerprint.enabled: true
# symlinks: true
# file_identity.fingerprint: ~
processors:
- add_cloud_metadata:
- add_host_metadata:
cloud.id: ${ELASTIC_CLOUD_ID}
cloud.auth: ${ELASTIC_CLOUD_AUTH}
output.elasticsearch:
hosts: ['${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}']
username: ${ELASTICSEARCH_USERNAME}
password: ${ELASTICSEARCH_PASSWORD}
index: "cnv-k8s-filebeat-%{+yyyy.MM.dd}"
setup.template:
name: "cnv-k8s-filebeat"
pattern: "cnv-k8s-filebeat-*"
enabled: false
# Debug logging config.
logging.level: debug
Logs are not sent to Elasticsearch. You can see error message below.
{"log.level":"error","@timestamp":"2024-01-31T14:48:37.170Z","log.logger":"index-management.ilm","log.origin":{"function":"github.com/elastic/beats/v7/libbeat/idxmgmt/lifecycle.(*stdManager).EnsurePolicy","file.name":"lifecycle/standard_manager.go","file.line":117},"message":"lifecycle policy filebeat creation failed: error creating policy from config: error submitting policy: error creating lifecycle policy: got 400 from elasticsearch: {\"error\":{\"root_cause\":[{\"type\":\"x_content_parse_exception\",\"reason\":\"[1:52] [rollover] unknown field [max_primary_shard_size]\"}],\"type\":\"x_content_parse_exception\",\"reason\":\"[1:77] [put_lifecycle_request] failed to parse field [policy]\",\"caused_by\":{\"type\":\"x_content_parse_exception\",\"reason\":\"[1:77] [lifecycle_policy] failed to parse field [phases]\",\"caused_by\":{\"type\":\"x_content_parse_exception\",\"reason\":\"[1:77] [phases] failed to parse field [hot]\",\"caused_by\":{\"type\":\"x_content_parse_exception\",\"reason\":\"[1:77] [phase] failed to parse field [actions]\",\"caused_by\":{\"type\":\"x_content_parse_exception\",\"reason\":\"[1:77] [actions] failed to parse field [rollover]\",\"caused_by\":{\"type\":\"x_content_parse_exception\",\"reason\":\"[1:52] [rollover] unknown field [max_primary_shard_size]\"}}}}}},\"status\":400}","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"debug","@timestamp":"2024-01-31T14:48:40.170Z","log.logger":"esclientleg","log.origin":{"function":"github.com/elastic/elastic-agent-libs/transport.(*loggingConn).Read","file.name":"transport/logging.go","file.line":50},"message":"Error reading from connection: read tcp 172.16.124.251:56046->172.16.104.49:9200: use of closed network connection","service.name":"filebeat","network":"tcp","address":"172.16.104.49:9200","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2024-01-31T14:48:41.389Z","log.logger":"monitoring","log.origin":{"function":"github.com/elastic/beats/v7/libbeat/monitoring/report/log.(*reporter).logSnapshot","file.name":"log/log.go","file.line":187},"message":"Non-zero metrics in the last 30s","service.name":"filebeat","monitoring":{"metrics":{"beat":{"cgroup":{"cpu":{"cfs":{"period":{"us":100000}},"id":"/"},"cpuacct":{"id":"/","total":{"ns":1474961633}},"memory":{"id":"/","mem":{"limit":{"bytes":209715200},"usage":{"bytes":126844928}}}},"cpu":{"system":{"ticks":190,"time":{"ms":190}},"total":{"ticks":1450,"time":{"ms":1450},"value":1450},"user":{"ticks":1260,"time":{"ms":1260}}},"handles":{"limit":{"hard":1048576,"soft":1048576},"open":46},"info":{"ephemeral_id":"4b69da0d-fea9-4229-945d-b1bbdc2f900c","name":"filebeat","uptime":{"ms":30082},"version":"8.12.0"},"memstats":{"gc_next":147767280,"memory_alloc":93108904,"memory_sys":136364312,"memory_total":261403504,"rss":161968128},"runtime":{"goroutines":1250}},"filebeat":{"events":{"active":3223,"added":3224,"done":1},"harvester":{"closed":1,"open_files":36,"running":36,"started":37}},"libbeat":{"config":{"module":{"running":0}},"output":{"events":{"active":0},"read":{"bytes":9162,"errors":3},"type":"elasticsearch","write":{"bytes":9476}},"pipeline":{"clients":36,"events":{"active":3223,"failed":1,"published":3200,"retry":6400,"total":3224},"queue":{"max_events":3200}}},"registrar":{"states":{"current":0}},"system":{"cpu":{"cores":4},"load":{"1":0.12,"15":0.22,"5":0.19,"norm":{"1":0.03,"15":0.055,"5":0.0475}}}},"ecs.version":"1.6.0"}}
{"log.level":"debug","@timestamp":"2024-01-31T14:48:41.535Z","log.logger":"file_watcher","log.origin":{"function":"github.com/elastic/beats/v7/filebeat/input/filestream.(*fileWatcher).watch","file.name":"filestream/fswatch.go","file.line":120},"message":"Start next scan","service.name":"filebeat","ecs.version":"1.6.0"}