my logstash configuration working.But the putput is not placeholders with empty values.
sample input :
0216042000018087881|ELECTRONIC WAYS|2017-12-16|8100.00||4812|Telecom||Primary
Actual output :
0216042000018087881,ELECTRONIC WAYS,2017-12-16,8100.00,%{description},4812,Telecom,Primary
Expected output :
0216042000018087881,ELECTRONIC WAYS,2017-12-16,8100.00,,4812,Telecom,Primary
conf file :
input {
file {
path => "/Volumes/External/RnD/logstash/input-datas/cc/txn.csv"
type => "transactions"
start_position => "beginning"
sincedb_path => "/Volumes/External/RnD/logstash/logstash-conf/sincedb.txt"
}
filter {
if [type] == "transactions" {
grok {
match => { "message" => "%{WORD:cust_id}\|(?<merchant_name>[^\|]*)\|(?<txn_date>\d{4}-\d{2}-\d{2})\|(?<amount>[^\|]*)\|(?<description>[^\|]*)\|(?<mcc_code>[^\|]*)\|(?<merchant_category>[^\|]*)\|(?<card_flag>[^\|]*)" }
}
}
}
output {
if [type] == "transactions" {
if "_grokparsefailure" in [tags] {
file {
path => "/Volumes/External/RnD/logstash/output-datas/cc/failed.transactions.txt"
codec => line { format => "%{message}"}
}
} else {
webhdfs {
host => "127.0.0.1" # (required)
port => 50070 # (optional, default: 50070)
path => "/Users/pkd/HadoopFolder/transactions/success.csv"
user => "pkd" # (required)
flush_size => 50000
idle_flush_time => 2
# retry_interval => 3
retry_interval => 3
#codec => line { format => "%{cust_id},%{merchant_name},%{txn_date},%{amount},%{description},%{mcc_code},%{merchant_category},%{card_flag}"}
}
}
}
}
}