The debug helps somewhat to at least see that the filters are loaded and that the config file entries have been loaded in each filter.
The stdout output. If i remove a field via one the filter section via remove_filed , should it not be removed from stdout?
Why is the filed still shown in the output?
example input: 1'st line is the header followed by 4 data lines.
"Obs","Portfolio","Appln Name","PORT Ord","APPL Ord","TRAN Ord","Trans Class","Req Type","Class Desc","Obs Date","Obs Hour","Obs Int","Total #of Reqs","Reqs / Sec","GPP Load (MIPS)","zIIP Load (MIPS)","GPP Pathlen (Mils Instr)","zIIP Pathlen (Mils Instr)","Avg Resp (Sec)","Avg Input Wait","Avg Queue Wait","Avg Dispatched Time","Avg Finish Wait","Avg Data Received (bytes)","Avg Data Sent (bytes)","Avg Network Traffic (KBs/sec)","Avg Concurrent Requests","Total CPU Seconds Used","Total CPU Sec on zIIP","Total Bytes Received","Total Bytes Sent","Finished TS - Received TS","QueueAdd TS - Received TS","Dispatched TS - QueueAdd TS","Completed TS - Dispatched TS","Finish TS - Completed TS","Total CPU Load (MIPS)","Total Pathlen (Mils Instr)","Normalized to UPS"
"1","01=Commercial","01=eManifest",1,1,1,"CB_EZ04S","HTTP","CDEM Synch","03OCT17",0,0,9633,10.703,27.610,104.10,2.58,9.73,0.04,0.00014,.000053557,0.0367,0.00,962.43,9203.52,106.26,0.3975,197.566390,156.151939,9271044,88657520,357.7128,1.364966,0.515914,353.8543,1.977569,131.71,12.31,600
"2","01=Commercial","01=eManifest",1,1,1,"CB_EZ04S","HTTP","CDEM Synch","03OCT17",0,1,36357,40.397,37.811,390.48,0.94,9.67,0.04,0.00016,.000063536,0.0407,0.00,964.11,8686.69,380.72,1.6633,642.440499,585.723468,35052056,315821992,1496.934,5.774812,2.309995,1480.557,8.292170,428.29,10.60,600
"3","01=Commercial","01=eManifest",1,1,1,"CB_EZ04S","HTTP","CDEM Synch","03OCT17",0,2,23083,25.648,34.436,250.10,1.34,9.75,0.04,0.00014,.000052828,0.0402,0.00,970.14,9045.69,250.86,1.0421,426.807992,375.154415,22393667,208801760,937.8940,3.258630,1.219431,928.8089,4.606977,284.54,11.09,600
"4","01=Commercial","01=eManifest",1,1,1,"CB_EZ04S","HTTP","CDEM Synch","03OCT17",0,3,23374,25.971,31.207,236.30,1.20,9.10,0.04,0.00014,.000054100,0.0348,0.00,962.92,9025.80,253.34,0.9131,401.253328,354.442849,22507232,210969041,821.8078,3.204272,1.264542,812.8629,4.476091,267.50,10.30,600
The config that i tried to use:
input {
file {
path => "/home/pxg110/Elastic/conf/test.csv"
start_position => "beginning"
sincedb_path => "/dev/null"
}
}
filter {
csv {
separator => ","
columns => [
"Obs" ,
"Portfolio" ,
"Appln Name",
"PORT Ord",
"APPL Ord",
"TRAN Ord",
"Trans Class",
"Req Type",
"Class Desc",
"Obs Date",
"Obs Hour",
"Obs Int",
"Total #of Reqs",
"Reqs / Sec",
"GPP Load (MIPS)",
"zIIP Load (MIPS)",
"GPP Pathlen (Mils Instr)",
"zIIP Pathlen (Mils Instr)",
"Avg Resp (Sec)",
"Avg Input Wait",
"Avg Queue Wait",
"Avg Dispatched Time",
"Avg Finish Wait",
"Avg Data Received (bytes)",
"Avg Data Sent (bytes)",
"Avg Network Traffic (KBs/sec)",
"Avg Concurrent Requests",
"Total CPU Seconds Used",
"Total CPU Sec on zIIP",
"Total Bytes Received",
"Total Bytes Sent",
"Finished TS - Received TS",
"QueueAdd TS - Received TS",
"Dispatched TS - QueueAdd TS",
"Completed TS - Dispatched TS",
"Finish TS - Completed TS",
"Total CPU Load (MIPS)",
"Total Pathlen (Mils Instr)",
"Normalized to UPS"
]
convert => {
"Obs" => "integer"
"PORT Ord" => "integer"
"APPL Ord" => "integer"
"TRAN Ord" => "integer"
"Obs Date" => "date"
"Obs Hour" => "integer"
"Obs Int" => "integer"
"Total #of Reqs" => "integer"
"Reqs / Sec" => "float"
"GPP Load (MIPS)" => "float"
"zIIP Load (MIPS)" => "float"
"GPP Pathlen (Mils Instr)" => "float"
"zIIP Pathlen (Mils Instr)" => "float"
"Avg Resp (Sec)" => "float"
"Avg Input Wait" => "float"
"Avg Queue Wait" => "float"
"Avg Dispatched Time" => "float"
"Avg Finish Wait" => "float"
"Avg Data Received (bytes)" => "float"
"Avg Data Sent (bytes)" => "float"
"Avg Network Traffic (KBs/sec)" => "float"
"Avg Concurrent Requests" => "float"
"Total CPU Seconds Used" => "float"
"Total CPU Sec on zIIP" => "float"
"Total Bytes Received" => "float"
"Total Bytes Sent" => "float"
"Finished TS - Received TS" => "float"
"QueueAdd TS - Received TS" => "float"
"Dispatched TS - QueueAdd TS" => "float"
"Completed TS - Dispatched TS" => "float"
"Finish TS - Completed TS" => "float"
"Total CPU Load (MIPS)" => "float"
"Total Pathlen (Mils Instr)" => "float"
"Normalized to UPS" => "integer"
}
}
if ["Obs Int"] == 0 {
mutate {
add_field => { "timestamp" => "%{Obs Date} %{Obs Hour} 0" }
remove_field => [ "Obs Int" ]
}
} else if ["Obs Int"] == 1 {
mutate {
add_field => { "timestamp" => "%{Obs Date} %{Obs Hour} 15" }
remove_field => [ "Obs Int" ]
}
} else if ["Obs Int"] == 2 {
mutate {
add_field => { "timestamp" => "%{Obs Date} %{Obs Hour} 30" }
remove_field => [ "Obs Int" ]
}
} else if ["Obs Int"] == 3 {
mutate {
add_field => { "timestamp" => "%{Obs Date} %{Obs Hour} 45" }
remove_field => [ "Obs Int" ]
}
} else {
mutate { add_field => { "timestamp" => "%{Obs Date} %{Obs Hour} 0" } }
}
date {
match => [ "timestamp", "dd MMM yy HH" ]
}
kv {
remove_char_key => "[0-9]\="
}
mutate {
remove_field => ["Obs", "PORT Ord", "APPL Ord", "TRAN Ord", "Normalized to UPS", "Obs Date", "Obs Hour"]
}
}
output {
elasticsearch {
action => "index"
hosts => "localhost:9200"
index => "test"
document_type => "test_record"
}
stdout {}
}