i found the solution ( add the ruby filter to see if it's empty field it will remove it else if it's full it will keep it )
input{
file{
path => "/home/user/data/test-536-Copie.csv"
start_position => "beginning"
sincedb_path => "/dev/null"
}
}
filter{
csv{
separator => ","
columns => ["Service Fee Period Warning Played Flag",........,"Community Id 3"]
}
ruby {
code => "
hash = event.to_hash
hash.each do |k,v|
if v == nil
event.remove(k)
end
end
"
}
mutate {
convert => {
"Supervision Period Warning Played Flag" => "integer"
..........
"Community Id 3" => "integer"
}
}
mutate {
add_field => {"D"=> ["2020-01-01"]}
}
date {
match => [ "Supervision Period Expiry Date", "YYYY-MM-dd" ]
target => "Supervision Period Expiry Date"
}
}
output{
elasticsearch{
hosts => "localhost"
index => "sub122"
document_type => "subscriber"
}
stdout{codec => rubydebug }
}