Problem with output elasticsearch data duplicate on index

[eldiablo62]

Hi,
I have a problem, i have configured 2 datasources and i want to index the data different index but my data are not on the good index.

my /etc/logstash/conf.d/config_mysql_glpi_tickets.conf

input {
jdbc {
    jdbc_driver_library => "/usr/share/logstash/vendor/jdbc-mysql/mysql-connector-java-5.1.44-bin.jar"
    jdbc_driver_class => "com.mysql.jdbc.Driver"
    jdbc_connection_string => "jdbc:mysql://ip:3306/glpi"
    jdbc_user => "user"
    jdbc_password => "pass"
    schedule => "*/5 * * * *"
    jdbc_page_size => 50000
    jdbc_paging_enabled => true
    statement => "SELECT * FROM view_tickets"
    use_column_value => true
    tracking_column => "id"
    tracking_column_type => "numeric"
}

}
output {
elasticsearch {
hosts => ["http://localhost:9200"]
index => "glpi_tickets"
document_type => "tickets"
document_id => "%{id}"
}
}

And my /etc/logstash/conf.d/config_mysql_glpi_tasks.conf

input {
jdbc {
    jdbc_driver_library => "/usr/share/logstash/vendor/jdbc-mysql/mysql-connector-java-5.1.44-bin.jar"
    jdbc_driver_class => "com.mysql.jdbc.Driver"
    jdbc_connection_string => "jdbc:mysql://ip:3306/glpi"
    jdbc_user => "user"
    jdbc_password => "pass"
    schedule => "*/5 * * * *"
    jdbc_page_size => 50000
    jdbc_paging_enabled => true
    statement => "SELECT * FROM view_tickets"
    use_column_value => true
    tracking_column => "id"
    tracking_column_type => "numeric"
}

}
output {
elasticsearch {
hosts => ["http://localhost:9200"]
index => "glpi_tickets"
document_type => "tickets"
document_id => "%{id}"
}
}

On my glpi_tickets index i have tasks document. I don't understand my mistake

[Christian_Dahlqvist]

When you put multiple config files in a directory for Logstash to use, they will all be concatenated. This means that if you do not add a tag/field with each input and use conditionals abided on this tag/field, all data will go to all output plugins.

[eldiablo62]

hum, now i understand why But for solved can i put a value like this data_type => tickets in my input block and do on if( data_type == 'tickets') {} on my output block ?

[Christian_Dahlqvist]

You can use the add_field parameter in the input to create a data_type field and then your conditional in the output section.

[eldiablo62]

Thank for your help
I share my config files for other people in same case

input {
jdbc {
jdbc_driver_library => "/usr/share/logstash/vendor/jdbc-mysql/mysql-connector-java-5.1.44-bin.jar"
jdbc_driver_class => "com.mysql.jdbc.Driver"
jdbc_connection_string => "jdbc:mysql://ip:3306/glpi"
jdbc_user => "user"
jdbc_password => "password"
schedule => "* * * * *"
jdbc_page_size => 50000
jdbc_paging_enabled => true
statement => "SELECT * FROM view_tickets"
use_column_value => true
tracking_column => "id"
tracking_column_type => "numeric"
add_field => { "data_source" => "glpi_tickets" }

}
}
output {
if [data_source] == "glpi_tickets" {
elasticsearch {
hosts => ["http://localhost:9200"]
index => "glpi_tickets"
document_type => "tickets"
user => "elastic"
password => "changeme"
document_id => "%{id}"
}
}
}