These 2 configs :
input {
file {
path => "/srv/edm/ftp/not_reversed/*.csv"
start_position => "beginning"
ignore_older => 17280000
sincedb_path => "/dev/null"
codec => plain {
charset => "ANSI_X3.4-1968"
}
}
}
filter {
csv {
separator => ";"
columns => ["sender", "receiver", "flow_type", "start_traitement", "end_traitement", "size_start_f", "size_end_f", "format_start", "format_end", "platform_start", "platform_end", "transport_start", "transport_end", "prod"]
skip_header => true
}
ruby {
code => 'event.set("date_start_formated", event.get("start_traitement").ljust(21, "0"))'
}
ruby {
code => 'event.set("date_end_formated", event.get("end_traitement").ljust(21, "0"))'
}
ruby {
code => 'event.set("date_start_formated_cut", ((event.get("date_start_formated"))[0,21]))'
}
ruby {
code => 'event.set("date_end_formated_cut", ((event.get("date_end_formated"))[0,21]))'
}
ruby {
code => 'event.set("date_start", (event.get("date_start_formated_cut").gsub(",",".")))'
}
ruby {
code => 'event.set("date_end", (event.get("date_end_formated_cut").gsub(",",".")))'
}
ruby {
code => 'event.set("size_start", event.get("size_start_f").to_i)'
}
ruby {
code => 'event.set("size_end", event.get("size_end_f").to_i)'
}
date {
match => ["date_start", "dd/MM/yy HH:mm:ss.SSS"]
target => "start_traitement_true"
}
date {
match => ["date_end", "dd/MM/yy HH:mm:ss.SSS"]
target => "end_traitement_true"
}
ruby {
code => 'event.set("date_epoch_end", event.get("end_traitement_true").to_i)'
}
ruby {
code => 'event.set("date_epoch_start", event.get("start_traitement_true").to_i)'
}
ruby {
code => 'event.set("time_between", ((event.get("date_epoch_end"))-(event.get("date_epoch_start"))).abs)'
}
ruby {
code => 'event.set("fixedProd", (event.get("prod")).tr("\r", ""))'
}
ruby {
code => 'event.set("cat", "BCP")'
}
mutate {
convert => {
"sender" => "string"
"receiver" => "string"
"flow_type" => "string"
"size_start" => "integer"
"size_end" => "integer"
"format_start" => "string"
"format_end" => "string"
"platform_start" => "string"
"platform_end" => "string"
"date_epoch_end" => "float"
"date_epoch_start" => "float"
"transport_start" => "string"
"transport_end" => "string"
"prod" => "string"
}
}
prune{
blacklist_names => ["message","date_end_formated_cut","date_start_formated_cut","date_start_formated","date_end_formated","date_start","date_end"]
}
}
output {
stdout{}
elasticsearch {
hosts => "http://XXXXXXX:9200"
index => "index_bcp"
user => "XXXXX"
password => "XXXXXXX"
ssl_certificate_verification => false
}
}
AND
input {
file {
path => "/srv/edm/ftp/bcp/*.csv"
start_position => "beginning"
ignore_older => 17280000
sincedb_path => "/dev/null"
codec => plain {
charset => "ANSI_X3.4-1968"
}
}
}
filter {
csv {
separator => ";"
columns => ["sender", "receiver", "flow_type", "end_traitement", "start_traitement", "size_end_f", "size_start_f", "format_end", "format_start", "platform_end", "platform_start", "transport_end", "transport_start", "prod"]
skip_header => true
}
ruby {
code => 'event.set("date_start_formated", event.get("start_traitement").ljust(21, "0"))'
}
ruby {
code => 'event.set("date_end_formated", event.get("end_traitement").ljust(21, "0"))'
}
ruby {
code => 'event.set("date_start_formated_cut", ((event.get("date_start_formated"))[0,21]))'
}
ruby {
code => 'event.set("date_end_formated_cut", ((event.get("date_end_formated"))[0,21]))'
}
ruby {
code => 'event.set("date_start", (event.get("date_start_formated_cut").gsub(",",".")))'
}
ruby {
code => 'event.set("date_end", (event.get("date_end_formated_cut").gsub(",",".")))'
}
ruby {
code => 'event.set("size_start", event.get("size_start_f").to_i)'
}
ruby {
code => 'event.set("size_end", event.get("size_end_f").to_i)'
}
date {
match => ["date_start", "dd/MM/yy HH:mm:ss.SSS"]
target => "start_traitement_true"
}
date {
match => ["date_end", "dd/MM/yy HH:mm:ss.SSS"]
target => "end_traitement_true"
}
ruby {
code => 'event.set("date_epoch_end", event.get("end_traitement_true").to_i)'
}
ruby {
code => 'event.set("date_epoch_start", event.get("start_traitement_true").to_i)'
}
ruby {
code => 'event.set("time_between", ((event.get("date_epoch_end"))-(event.get("date_epoch_start"))).abs)'
}
ruby {
code => 'event.set("fixedProd", (event.get("prod")).tr("\r", ""))'
}
ruby {
code => 'event.set("cat", "BCP")'
}
mutate {
convert => {
"sender" => "string"
"receiver" => "string"
"flow_type" => "string"
"size_start" => "integer"
"size_end" => "integer"
"format_start" => "string"
"format_end" => "string"
"platform_start" => "string"
"platform_end" => "string"
"date_epoch_end" => "float"
"date_epoch_start" => "float"
"transport_start" => "string"
"transport_end" => "string"
"prod" => "string"
}
}
prune{
blacklist_names => ["message","date_end_formated_cut","date_start_formated_cut","date_start_formated","date_end_formated","date_start","date_end"]
}
}
output {
stdout{}
elasticsearch {
hosts => "http://XXXXXX:9200"
index => "index_bcp"
user => "XXXXXXX"
password => "XXXXXXX"
ssl_certificate_verification => false
}
}
Both hosts, user and password are the same.