Hi,
Logstash giving parser error whenever CSV file gets updated with new records. However logstash works fine when you delete the old csv file and replace with new updated CSV file, but in that scenerio logstash all records from the beginning. Please help.
#Exception in logstash
[2020-11-09T13:22:54,359][WARN ][logstash.filters.csv ][main][c4f81f54b5e067137ee87e67fd5f5dfde69428e1fed1f393fcd23c6f6c2faf76] Error parsing csv {:field=>"message", :source=>""," LS"\r", :exception=>#<CSV::MalformedCSVError: Unclosed quoted field on line 1.>}
{
"path" => "Q:/DataFileLogstash/XDViews/TotatOrderOfDay.csv",
"@version" => "1",
"@timestamp" => 2020-11-09T18:22:54.168Z,
"host" => "PC",
"tags" => [
[0] "_csvparsefailure"
]
}
Below is my CSV file;
input {
file {
path => "Q:/DataFileLogstash/XDViews/TotatOrderOfDay.csv"
start_position => "beginning"
sincedb_path => "Q:/DataFileLogstash/logstash.db"
#stat_interval => 1
#discover_interval => 1
#sincedb_path => "NULL"
}
}
filter {
csv {
separator => ","
skip_header => "true"
columns => [ "Order Key","Cust #","Doc Type","Doc #","Order #","Order Type","Order Complete?","Order Date","Total Order Value","Total Dollar Value","TotalOrderReceivedToday","Inv #","Inv Date","Expected Ship Date","Shipment Date","TotalShipped","ShippedYesterday","OrderYesterday","Salesperson","Salesperson name","Loc" ]
#autodetect_column_names => true
}
date {
match => ["Order Date", "dd-MMM-yy"]
target => "Order Date"
}
date {
match => ["Inv Date", "dd-MMM-yy"]
target => "Inv Date"
}
date {
match => ["Expected Ship Date", "dd-MMM-yy"]
target => "Expected Ship Date"
}
date {
match => ["Shipment Date", "dd-MMM-yy"]
target => "Shipment Date"
}
mutate {
convert => {
"Cust #" => "integer"
"Doc #" => "integer"
"Order #" => "integer"
"Total Order Value" => "float"
"Total Dollar Value" => "float"
"TotalOrderReceivedToday" => "integer"
"TotalShipped" => "integer"
"OrderYesterday" => "integer"
"ShippedYesterday" => "integer"
"Salesperson" => "string"
"Salesperson name" => "string"
}
remove_field => ["message"]
}
}
output {
elasticsearch {
hosts => "http://localhost:9200"
document_id => "%{Doc #}"
index => "sales"
}
stdout {}
}
CSV file records.
"Order Key","Cust #","Doc Type","Doc #","Order #","Order Type","Order Complete?","Order Date","Total Order Value","Total Dollar Value","TotalOrderReceivedToday","Inv #","Inv Date","Expected Ship Date","Shipment Date","TotalShipped","ShippedYesterday","OrderYesterday","Salesperson","Salesperson name","Loc"
"C 2214",322324,"C"," 2314"," 172105","O","True",18-Sep-07,0,86.29,0," 207999",18-Sep-07,,18-Sep-07,0,0,0," SPZ","name 11"," LS"
"C 2227",222773,"C"," 2327"," 172386","O","True",15-Oct-07,0,51.3,0," 209211",15-Oct-07,,15-Oct-07,0,0,0," JP","name 2"," LS"
"C 2247",929339,"C"," 2347"," 172637","O","True",12-Dec-07,0,43.26,0," 213733",12-Dec-07,,12-Dec-07,0,0,0," PP","** Salesperson not on file "," LS"
"C 2248"," 280","C"," 2348"," 279677","O","True",13-Dec-07,0,51.12,0," 213754",13-Dec-07,,13-Dec-07,0,0,0," SPZ","name 3"," GS"
"C 2258"," 3","C"," 2358"," 2297","O","True",14-Jan-08,0,101.7,0," 214996",14-Jan-08,,14-Jan-08,0,0,0," MS"," Salesperson not on file **"," LS"
"C 2268"," 82331","C"," 2368"," 182491","O","True",29-Feb-08,0,5775,0," 216673",29-Feb-08,,29-Feb-08,0,0,0," MM","name 4"," GS"
"C 2269"," 82331","C"," 2369"," 182959","O","True",29-Feb-08,0,8225.23,0," 218095",29-Feb-08,,29-Feb-08,0,0,0," HOUSE","HOUSE ACCOUNT"," LS"
"C 2271",999299,"C"," 2371"," 184623","O","True",13-Mar-08,0,44.06,0," 218761",13-Mar-08,,13-Mar-08,0,0,0," MP","name 1"," LS"
"C 2285"," 3","C"," 2385"," 182581","O","True",29-Apr-08,0,867.83,0," 221598",29-Apr-08,,29-Apr-08,0,0,0," HOUSE","name 5"," LS"