In my use case, I have around 20 csv to upload, so I prepare 20 logstash config files and add an unique tag for each of them to filter and output conditionally. However, when I run logstash in the directory, only one of the config is being inserted in elastic, obviously another one is being skipped in the "if" statement. I would be appropriated if somebody could point out what did I do wrong here.
Below is the single config file I prepared to simulate the concated config file.
input {
file {
tags => ["test1"]
id => "test1"
path => "D:/ELK/operation/input/a.csv"
start_position => "beginning"
sincedb_path => "D:/ELK/.sincedb"
mode => "read"
}
file {
tags => ["test2"]
id => "test2"
path => "D:/ELK/operation/input/b.csv"
start_position => "beginning"
sincedb_path => "D:/ELK/.sincedb"
mode => "read"
}
}
filter {
if "test1" in [tags] {
csv {
separator => ","
skip_header => "true"
columns => ["a","b","c","d"]
}
}
if "test2" in [tags] {
csv {
separator => ","
skip_header => "true"
columns => ["a","b","c"]
}
}
}
output {
if "test1" in [tags] {
elasticsearch {
"hosts" => "https://localhost:443"
index => "idx_test1"
action => "create"
doc_as_upsert => true
cacert => "D:/ELK/ca_logstash.cer"
ssl_certificate_verification => false
}
stdout { codec => rubydebug { metadata => true } }
}
if "test2" in [tags] {
elasticsearch {
"hosts" => "https://localhost:443"
index => "idx_test2"
action => "create"
doc_as_upsert => true
cacert => "D:/ELK/ca_logstash.cer"
ssl_certificate_verification => false
}
stdout { codec => rubydebug { metadata => true } }
}
}