Hi,
I'm struggling with filebeat and logstash because, i want to handle different logs ( logs and json file) as inputs and i want to have a dedicated index for all logs (first_index and another one for json datas ( on second_index). but as a result, first_index is ok but , on my second_index i have all logs datas where i just expected amneo-connector.ndjson datas.
My filebeat config is
filebeat.inputs:
- type: log
paths:
- /var/log/apache2/access-toto-magento.log
fields:
file_name: access-toto-magento.log
log_type: apache
environment: in01
- type: log
paths:
- /var/log/apache2/error-toto-magento.log
fields:
file_name: error-toto-magento.log
log_type: apache
environment: in01
- type: log
paths:
- /var/www/toto-magento/current/var/log/system.log
fields:
file_name: system.log
log_type: magento
environment: in01
- type: log
paths:
- /var/www/toto-magento/current/var/log/amneo-connector.ndjson
fields:
file_name: amneo-connector.ndjson
log_type: json
environment: in01
json.keys_under_root: true
json.add_error_key: true
processors:
- decode_json_fields:
fields: ["message"]
#----------------------------- Logstash output --------------------------------
output.logstash:
# Boolean flag to enable or disable the output module.
enabled: true
# The Logstash hosts
hosts: ["amneo-corenonprod-logstash.amneo-corenonprod.int:5044"]
logging.level: warning
logging.to_files: true
# Send all logging output to syslog. The default is false.
logging.to_syslog: false
logging.files:
path: /var/log/filebeat
name: filebeat.log
keepfiles: 7
permissions: 0644
and my logstash config is
input {
beats {
port => "5044"
ssl => false
}
}
filter {
if "access-toto-magento.log" in [fields][file_name] {
grok {
patterns_dir => "/etc/logstash/patterns"
match => { "message" => "%{COMBINEDAPACHELOG}" }
}
geoip {
source => "clientip"
}
mutate {
remove_field => ["beat"]
}
mutate {
add_tag => ["logs"]
}
}
if "error-toto-magento.log" in [fields][file_name] {
grok {
patterns_dir => "/etc/logstash/patterns"
match => { "message" => "%{APACHE_ERROR_LOG}" }
}
mutate {
remove_field => ["beat"]
}
mutate {
add_tag => ["logs"]
}
}
if "system.log" in [fields][file_name] {
grok {
patterns_dir => "/etc/logstash/patterns"
match => { "message" => "%{MAGENTO_LOGS}" }
}
mutate {
remove_field => ["beat"]
}
mutate {
add_tag => ["logs"]
}
}
if "tesb.log" in [fields][file_name] {
grok {
patterns_dir => "/etc/logstash/patterns"
match => { "message" => "%{TALEND_TESB_LOGS}" }
}
mutate {
remove_field => ["beat"]
}
mutate {
add_tag => ["logs"]
}
}
if "toto.log" in [fields][file_name] {
grok {
patterns_dir => "/etc/logstash/patterns"
match => { "message" => "%{TALEND_TOTO_LOGS}" }
}
mutate {
remove_field => ["beat"]
}
mutate {
add_tag => ["logs"]
}
}
if "amneo-connector.ndjson" in [fields][file_name] {
grok {
match => { "message" => "%{GREEDYDATA:json_data}" }
}
mutate {
add_tag => ["ndjson"]
}
}
}
output {
if "logs" in [tags] {
amazon_es {
hosts => "search-amneo-corenonprod-1-hr2touhg2mcgdis4f6y4jvktli.eu-west-1.es.amazonaws.com"
region => "eu-west-1"
index => "first_index-%{+YYYY.MM.dd}"
#max_bulk_bytes => 9999999
}
}
if "ndjson" in [tags] {
amazon_es {
hosts => "search-amneo-corenonprod-1-hr2touhg2mcgdis4f6y4jvktli.eu-west-1.es.amazonaws.com"
region => "eu-west-1"
index => "second_index-%{+YYYY.MM.dd}"
#max_bulk_bytes => 9999999
}
}
}
can you help understand why my logs datas are appearing in second_index ??