Hi Badre
Thanks for your reply. Here is most of my filebeat config file.
( Feel free to suggest other improvements if you see some. I am still new to filebeat )
apiVersion: v1
kind: ConfigMap
metadata:
name: filebeat-config
namespace: default
labels:
k8s-app: xxx
app: xxx
data:
filebeat.yml: |-
processors:
- add_cloud_metadata:
- add_host_metadata:
- add_docker_metadata:
labels.dedot: true
annotations.dedot: true
- drop_fields:
#fields: ["agent.ephemeral_id", "agent.hostname", "agent.id", "agent.type", "agent.version", "host.name", "ecs.version", "input.type"]
fields:
- "agent.ephemeral_id"
- "agent.hostname"
- "agent.id"
- "agent.type"
- "agent.version"
- "host.name"
- "ecs.version"
- "input.type"
ignore_missing: true
filebeat.autodiscover:
providers:
- type: docker
# this input section is for module eg: mongo/nginx specific
templates:
- condition:
contains:
docker.container.name: echo
config:
- module: nginx
enabled: true
access:
input:
type: container
stream: "stdout"
containers.ids:
- ${data.docker.container.id}
error:
input:
type: container
stream: "stdout"
containers.ids:
- ${data.docker.container.id}
- condition:
contains:
docker.container.name: darwin
config:
- module: mongodb
enabled: true
log:
input:
type: docker
containers.ids:
- ${data.docker.container.id}
filebeat.inputs:
- type: log
# this input section is for system log files
enabled: true
paths:
- /var/log/*.log
- /var/log/messages
- type: container
# this input section is for general application docker/containers logs
enabled: true
stream: all
paths:
- /var/lib/docker/containers/*/*.log
include_lines: ['"pod":']
json.keys_under_root: true
json.ignore_decoding_error: true
json.add_error_key: true
json.overwrite_keys: true
json.message_key: log
# Avoid parsing exceptions due to name conflicts
processors:
- rename:
fields:
- from: "service"
to: "service-name"
- from: "error"
to: "service-error"
- from: "url"
to: "service-url"
ignore_missing: true
fail_on_error: false