Hi Team,
We are frequently facing logs lagging in Kibana due to other application team while doing performance testing and also they are sending a bulk data at a time.
In that scenario, that would be taking too much time to reach the normal flow of logs. We would like to know if we can control the log flow at Logstash?
This is the Logs Flow - FileBeat/MetricBeat -> AWS Kafka -> Logstash -> ES -> Kibana to visualize the logs in kibana.
Please find the below Logstash config,
input {
kafka {
bootstrap_servers => "b-2.kafka-ej-development.gv3pju.c6.kafka.eu-west-1.amazonaws.com:9094,b-3.kafka-ej-development.gv3pju.c6.kafka.eu-west-1.amazonaws.com:9094,b-1.kafka-ej-development.gv3pju.c6.kafka.eu-west-1.amazonaws.com:9094"
topics => ["easyjet_lc_dev_app_topic","easyjet_lc_dev_linuxlog_topic","easyjet_lc_dev_monitoring_topic","easyjet_lc_dev_linux_topic
"]
consumer_threads => 3
group_id => "fb-lc-grp"
security_protocol => "SSL"
ssl_truststore_location => "/usr/share/softwares/cert/kafka.client.truststore.jks"
ssl_truststore_password => "changeit"
codec => "json"
}
}
filter {
# json {
# source => "message"
# }
#mutate {
# remove_field => [ "message" ]
# }
fingerprint {
source => [ "fingerprint", "log" ]
target => "[@metadata][fingerprint]"
method => "SHA1"
concatenate_sources => true
}
}
output {
if [logtype] == "linuxlog" {
elasticsearch {
index => "ej-lc16-%{[logtype]}-%{+YYYY.MM.dd}"
hosts => ["##"]
api_key => ["##"]
ssl => true
ilm_rollover_alias => "ej-lc16-linuxlog"
ilm_pattern => "000001"
ilm_policy => "DeleteIndexAfterOneDay"
document_id => "%{[@metadata][fingerprint]}"
}
}
if [logtype] == "applog" {
elasticsearch {
index => "ej-lc-%{[logtype]}-%{+YYYY.MM.dd}"
hosts => ["##"]
api_key => ["##"]
ssl => true
ilm_rollover_alias => "ej-lc-applog"
ilm_pattern => "000001"
ilm_policy => "DeleteIndexAfterOneDay"
document_id => "%{[@metadata][fingerprint]}"
}
}
if [logtype] == "linuxlog" {
elasticsearch {
index => "ej-lc-%{[logtype]}-%{+YYYY.MM.dd}"
hosts => ["##"]
api_key => ["##"]
ssl => true
ilm_rollover_alias => "ej-lc-linuxlog"
ilm_pattern => "000001"
ilm_policy => "DeleteIndexAfterOneDay"
document_id => "%{[@metadata][fingerprint]}"
}
}
}
Regards
Anushya