Hello guys,
I need a help in specific case. My environment have a DLL application who create messages and topics in Kafka, than logstash get messages of the all topics. But problem is new topic only collect when a next is create. I tested propertities auto_offset_reset but not work. Someone have idea how solve this problem?
[logstash.conf]
input{
kafka{
topics_pattern => ".*"
bootstrap_servers => "SERVER_IP_01:PORT"
sasl_jaas_config => "org.apache.kafka.common.security.plain.PlainLoginModule required username='${KK_USR}' password='${KK_PWD}';"
security_protocol => SASL_PLAINTEXT
sasl_mechanism => PLAIN
codec => "json"
}
kafka{
topics_pattern => ".*"
bootstrap_servers => "SERVER_IP_02:PORT"
sasl_jaas_config => "org.apache.kafka.common.security.plain.PlainLoginModule required username='${KK_USR}' password='${KK_PWD}';"
security_protocol => SASL_PLAINTEXT
sasl_mechanism => PLAIN
decorate_events => true
codec => "json"
auto_offset_reset => "earliest"
}
}
output {
elasticsearch{
hosts => ["ServerElasticsearch-01:9200", "ServerElasticsearch-02:9200", "ServerElasticsearch-03:9200"]
password => "${ES_PWD}"
#cacert => "/etc/logstash/logstash.pem"
index => "%{+YYYY.MM.dd}"
#ssl => true
#ssl_certificate_verification => true
user => "${ES_USR}"
}
stdout {
codec => rubydebug {metadata => true}
}
}
```