Hi
Maybe You have idea why in this case autodetect_column_names doesn't work as independent.
input:
"CLLI","SWREL","RPTDATE","RPTIME","TZ","RPTTYPE","RPTPD","IVALDATE","IVALSTART","IVALEND","NUMENTIDS"
"wifi06","EAGLE 47.0.0.0.0-79.13.0","2023-02-24","13:24:23","CET ","AVAILABILITY MEASUREMENTS ON LINK","LAST","2023-02-24","13:00:00","13:15:00",285
"STATUS","LSN","LOC","LINK","LNKTYPE","NEARMGIH","FARMGINH","NMDCLFLR","DRDCLFLR","SURCVERR","DRLKINHB","NDCFLABN","NDCLFSYNC","NDCFLXDA","NDCFLXER","NDCFLXDC","NDCLFALP","NDCLFINTR","NMFEPRO","NMLCLPRO","DRFEPRO","DRLCLPRO","SUSRECVD","SUSTRAN","PCRN1N2EXC"
"K","eip61sccip","1207","A ","IPVL",0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
"K","eip05sccip","1207","B ","IPVL",0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
"K","wifi41ip","1208","B ","IPVL",0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
"K","wifi41ip","1211","A ","IPVL",0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
input {
file {
mode => read
path => "/opt/data/input/new_test1.csv"
sincedb_path => "/dev/null"
start_position => beginning
file_completed_action => "log"
file_completed_log_path => "/opt/data/logstash_files/fin_eir.log"
}
}
filter{
ruby { code => 'event.set("[@metadata][fields]", 1 + event.get("message").count(","))' }
if [@metadata][fields] == 11 {
csv {
separator => ","
columns => ["CLLI","SWREL","RPTDATE","RPTIME","TZ","RPTTYPE","RPTPD","IVALDATE","IVALSTART","IVALEND","NUMENTIDS"]
}
# ruby { code => "event.set('event_timestamp', event.get('RPTDATE') + ' ' + event.get('RPTIME'))"}
# date {
# match => ["event_timestamp", "dd-MM-yyyy H:m:s.SSS", "yyyy-MM-dd H:m:s.SSS", "ISO8601"]
# timezone => "Europe/Warsaw"
# target => "@timestamp"}
} else {
csv {
separator => ","
autodetect_column_names => true
skip_empty_columns => true
skip_header => true
}
}
mutate {
remove_field => ["message"]
}
}
output {
stdout { codec => rubydebug }
}
output:
[INFO ] 2023-02-24 13:30:02.708 [Agent thread] agent - Pipelines running {:count=>1, :running_pipelines=>[:stp_test], :non_running_pipelines=>[]}
[WARN ] 2023-02-24 13:30:03.152 [[stp_test]>worker0] csv - Error parsing csv {:field=>"message", :source=>"\"STATUS\",\"LSN\",\"LOC\",\"LINK\",\"LNKTYPE\",\"NEARMGIH\",\"FARMGINH\",\"NMDCLFLR\",\"DRDCLFLR\",\"SURCVERR\",\"DRLKINHB\",\"NDCFLABN\",\"NDCLFSYNC\",\"NDCFLXDA\",\"NDCFLXER\",\"NDCFLXDC\",\"NDCLFALP\",\"NDCLFINTR\",\"NMFEPRO\",\"NMLCLPRO\",\"DRFEPRO\",\"DRLCLPRO\",\"SUSRECVD\",\"SUSTRAN\",\"PCRN1N2EXC\"", :exception=>#<NoMethodError: undefined method `empty?' for nil:NilClass>}
[WARN ] 2023-02-24 13:30:03.154 [[stp_test]>worker0] csv - Error parsing csv {:field=>"message", :source=>"", :exception=>#<NoMethodError: undefined method `empty?' for nil:NilClass>}
[WARN ] 2023-02-24 13:30:03.156 [[stp_test]>worker0] csv - Error parsing csv {:field=>"message", :source=>"\"K\",\"eip61sccip\",\"1207\",\"A \",\"IPVL\",0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0", :exception=>#<NoMethodError: undefined method `empty?' for nil:NilClass>}
[WARN ] 2023-02-24 13:30:03.158 [[stp_test]>worker0] csv - Error parsing csv {:field=>"message", :source=>"\"K\",\"eip05sccip\",\"1207\",\"B \",\"IPVL\",0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0", :exception=>#<NoMethodError: undefined method `empty?' for nil:NilClass>}
[WARN ] 2023-02-24 13:30:03.160 [[stp_test]>worker0] csv - Error parsing csv {:field=>"message", :source=>"\"K\",\"wifi41ip\",\"1208\",\"B \",\"IPVL\",0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0", :exception=>#<NoMethodError: undefined method `empty?' for nil:NilClass>}
[WARN ] 2023-02-24 13:30:03.163 [[stp_test]>worker0] csv - Error parsing csv {:field=>"message", :source=>"\"K\",\"wifi41ip\",\"1211\",\"A \",\"IPVL\",0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0", :exception=>#<NoMethodError: undefined method `empty?' for nil:NilClass>}
[WARN ] 2023-02-24 13:30:03.164 [[stp_test]>worker0] csv - Error parsing csv {:field=>"message", :source=>"", :exception=>#<NoMethodError: undefined method `empty?' for nil:NilClass>}
{
"IVALEND" => "13:15:00",
"@version" => "1",
"CLLI" => "stp06",
"RPTTYPE" => "AVAILABILITY MEASUREMENTS ON LINK",
"RPTPD" => "LAST",
"SWREL" => "EAGLE 47.0.0.0.0-79.13.0",
"@timestamp" => 2023-02-24T13:01:04.567304Z,
"RPTDATE" => "2023-02-24",
"RPTIME" => "13:24:23",
"IVALSTART" => "13:00:00",
"NUMENTIDS" => "285",
"host" => "0.0.0.0",
"TZ" => "CET ",
"IVALDATE" => "2023-02-24",
"path" => "/opt/data/input/new_test1.csv"
}
{
"@version" => "1",
"tags" => [
[0] "_csvparsefailure"
],
"host" => "0.0.0.0",
"@timestamp" => 2023-02-24T13:01:04.568660Z,
"path" => "/opt/data/input/new_test1.csv"
}
{
"@version" => "1",
"tags" => [
[0] "_csvparsefailure"
],
"host" => "0.0.0.0",
"@timestamp" => 2023-02-24T13:01:04.569210Z,
"path" => "/opt/data/input/new_test1.csv"
}
{
"@version" => "1",
"tags" => [
[0] "_csvparsefailure"
],
"host" => "0.0.0.0",
"@timestamp" => 2023-02-24T13:01:04.567866Z,
"path" => "/opt/data/input/new_test1.csv"
}
{
"@version" => "1",
"tags" => [
[0] "_csvparsefailure"
],
"host" => "0.0.0.0",
"@timestamp" => 2023-02-24T13:01:04.568395Z,
"path" => "/opt/data/input/new_test1.csv"
}
{
"@version" => "1",
"tags" => [
[0] "_csvparsefailure"
],
"host" => "0.0.0.0",
"@timestamp" => 2023-02-24T13:01:04.568938Z,
"path" => "/opt/data/input/new_test1.csv"