Hi
My sample data is like follow
TransactionsPK,InsertTime,RetrivalRefNo,MsgType,TimeLocal,DateLocal,DateCaptured,TerminalID,CardAcqID,SystemTraceNo,Amount,TransType,PrimaryAccNo,PrimaryAccNoEnc,IssCode,ProcessingCode,PosCondition,Reciept,AcqCode,SettlementType
D41E16F4-BAEB-E411-80C6-0050569F2C47,2015-04-26 06:51:33.763,320000050291,0210,06:51:32,2015-04-26,2015-04-26,C0000255,M00000000000093,000159,1000,1,505416******2848,AaocywiSZBzCuhkEE4sfeA==,505416,000000,14,NULL,581672132,0001
7DEC913D-BCEB-E411-80C6-0050569F2C47,2015-04-26 07:00:46.003,320000050292,0210,07:00:45,2015-04-26,2015-04-26,C0000388,M00000000000187,000057,25000,1,603799******9794,vpYphJ4axQW9conblV39nQ==,603799,000000,14,NULL,581672132,0001
and my filter is :
input {
file {
path => '/root/datasample/transaction.csv'
start_position => beginning
}
}
filter
{
csv{
columns =>[
'transactionspk',
'inserttime',
'retrivalrefno',
'msgtype',
'timeLocal',
'dateLocal',
'datecaptured',
'terminalid',
'cardacqid',
'systemtraceNo',
'amount',
'transtype',
'primaryaccNo',
'primaryaccNoEnc',
'isscode',
'processingcode',
'poscondition',
'reciept',
'acqcode',
'settlementtype'
]
separator => ','
remove_field => ['message']
}
date {
match => ['inserttime', 'yyyy-MM-dd HH:mm:ss.SSS']
match => ['timeLocal', 'HH:mm:ss']
match => ['dateLocal', 'yyyy-MM-dd']
match => ['datecaptured', 'yyyy-MM-dd']
}
mutate {
convert =>{
"transactionspk" => "string"
"retrivalrefno" => "float"
"msgtype" => "float"
"terminalid" => "string"
"cardacqid" => "string"
"systemtraceNo" =>"float"
"amount" => "float"
"transtype" => "float"
"primaryaccNo" => "string"
"primaryaccNoEnc" => "string"
"isscode" => "float"
"processingcode" => "float"
"poscondition" => "float"
"reciept" => "string"
"acqcode" => "float"
"settlementtype" => "float" }
}
}
output {
elasticsearch {
hosts => 'localhost:9200'
action => 'index'
}
}