I have problem with my filebeat configuration, it doesnt filter logs to separate index in elasticsearch.
My configuration looks like:
logging:
level: error
metrics:
enabled: false
filebeat.modules:
- module: mongodb
- module: kafka
- module: traefik
filebeat.autodiscover:
providers:
- type: docker
labels.dedot: true
templates:
- condition:
equals:
docker.container.labels.jsonLog: 'true'
config:
- type: container
format: docker
paths:
- "/var/lib/docker/containers/${data.docker.container.id}/*.log"
exclude_lines: [".*eureka endpoints.*", "DEBUG"]
processors:
- decode_json_fields:
when.equals:
docker.container.labels.jsonLog: 'true'
fields: ["message"]
target: "logstash"
overwrite_keys: true
- add_docker_metadata: ~
- script:
lang: javascript
id: shorten_message
source: >
function process(event) {
var message = event.Get("logstash.message");
if (message != null && message.length > 900000) {
event.Put("logstash.message", message.substring(0, 900000));
}
}
- drop_fields:
fields: ["message"]
- condition:
equals:
docker.container.labels.proxy: 'true'
config:
- type: container
format: docker
paths:
- "/var/lib/docker/containers/${data.docker.container.id}/*.log"
processors:
- decode_json_fields:
when.equals:
docker.container.labels.proxy: 'true'
fields: ["message"]
target: "logstash"
overwrite_keys: true
- add_docker_metadata: ~
- drop_fields:
fields: ["message"]
output.elasticsearch:
hosts: ["elasticsearch-master:9200", "elasticsearch-slave:9200"]
pipelines:
- pipeline: "logstash-json-timestamp"
when.equals:
docker.container.labels.jsonLog: 'true'
indices:
- index: "filebeat-%{[beat.version]}-%{[docker.container.labels.com.docker.swarm.service.name]}-%{+yyyy.MM.dd}"
when:
or:
- equals:
docker.container.labels.jsonLog: 'true'
- equals:
container.labels.jsonLog: 'true'
- contains:
docker.container.labels.jsonLog: 'tru'
- contains:
container.labels.jsonLog: 'tru'
- has_fields: ['docker.container.labels.jsonLog']
- index: "traefik-%{[beat.version]}-%{[docker.container.labels.com.docker.swarm.service.name]}-%{+yyyy.MM.dd}"
when:
or:
- equals:
docker.container.labels.proxy: 'true'
- equals:
container.labels.proxy: 'true'
- equals:
docker.container.labels.proxy: 'tru'
- equals:
container.labels.proxy: 'tru'
- has_fields: ['docker.container.labels.proxy']