Hi,
We are trying to move from filebeats version 6.8 to 7.10 however, the way to collect docker logs seems to have changed and following the new autodiscover method docker logs are not appearing in elasticsearch but normal non docker logs are. Our logstash version 7.16 processor adds a few things then passes to ES.
Is there an issue with our filebeat.yml file? Not sure on the best way to debug this is as im not seeing much in any log files on the boxes. filebeat test config returns Config OK
OLD CONFIG
filebeat.inputs:
- type: docker
containers.ids : '*'
containers.stream: all
fields:
pipeline_marker: "application"
service: "Monitoring"
server_name: "AWSServerName"
processors:
- add_docker_metadata: ~
json.keys_under_root: true
json.add_error_key: true
json.message_key: message
NEW CONFIG
filebeat.autodiscover:
providers:
- type: docker
templates:
config:
- type: container
paths:
- /var/lib/docker/containers/${data.docker.container.id}/*.log
fields:
pipeline_marker: "application"
service: "Monitoring"
server_name: "AWSServerName"
processors:
- add_docker_metadata: ~
json.keys_under_root: true
json.add_error_key: true
json.message_key: message
PROCESSOR
input {
pipeline {
address => "processor-simple"
}
}
filter {
mutate {
add_field => { "[innerindex]" => "service-%{[fields][service]}-%{+YYYY.MM.dd}" }
add_field => { "[logstashProcessor]" => "processor-simple" }
# Teams message
add_field => {"[@metadata][teams_webhook_url]" => ""}
add_field => {"[@metadata][teams_message]" => '{}'
}
}
output {
if [notifications][raiseNotification] {
pipeline { send_to => "dispatcher-teams" }
}
pipeline { send_to => "dispatcher-elasticsearch" }
}