I am using a fresh install of the ELK stack 5.4.0
Elasticsearch, Logstash, kibana with X-pack
Now when i fire a configuration given below
input
{
file{
path => "C:\somepath\path\file.csv"
start_position => beginning
# sincedb_path => "NUL"
sincedb_path => "F:\somepath\testsince"
}
}
filter {
csv {
columns => ["IPID",
"EVENT_ID",
"AREA_ID",
"DATE_AND_TIME","EVENT_TEXT","DATE_AND_TIME_UTC","RTU_TIME","RTU_TIME_UTC",
"COLOR",
"CRITICALITY",
"TYPE",
"VALUE",
"COMMENT_NBR",
"INDEX_NBR",
"RTU",
"IPTYPE",
"DPADR",
"NBR",
"Area_code",
"RTU_ID",
"PIPESIZE",
"B4",
"B5",
"PIS",
"DIMENSION",
"STATE_ZERO",
"STATE_ONE",
"ARCHIVE_CLASS",
"GENTIME",
"UPPER_IPID",
"PART",
"PLANT",
"IT",
"TC_ADR_ASDU",
"TC_ADR_IO",
"PROFILE_TYPE",
"TEXT",
"CLIENT",
"AREA",
"AREA_CODE",
"PVNAME",
"Connection_type",
"PRJTXT2T","PRJTXT3T","PRJTXT4T","PRJTXT5T","PRJTXT6T","PRJTXT7T","PRJTXT8T","PRJTXT9T",
"LATITUDE","LONGITUDE",
"DEC_PLACES",
"ALARM_LIMIT_LOW","ALARM_LIMIT_HIGH","WARNING_LIMIT_LOW","WARNING_LIMIT_HIGH","MIN_VALUE","MAX_VALUE"]
separator => ","
remove_field => ["PRJTXT2T","PRJTXT3T","PRJTXT4T","PRJTXT5T","PRJTXT6T","PRJTXT7T","PRJTXT8T","PRJTXT9T"]
convert => {"DATE_AND_TIME" => "date_time"}
convert => {"DATE_AND_TIME_UTC" => "date_time"}
convert => {"RTU_TIME" => "date_time"}
convert => {"RTU_TIME_UTC" => "date_time"}
convert => {"ALARM_LIMIT_LOW" => "float"}
convert => {"ALARM_LIMIT_HIGH" => "float"}
convert => {"WARNING_LIMIT_LOW" => "float"}
convert => {"WARNING_LIMIT_HIGH" => "float"}
convert => {"MIN_VALUE" => "float"}
convert => {"MAX_VALUE" => "float"}
convert => {"LATITUDE" => "float"}
convert => {"LONGITUDE" => "float"}
add_field => ["location","%{LATITUDE},%{LONGITUDE}"]
convert => {"location" => "float"}
}
}
output{
elasticsearch {
hosts => ["127.0.0.1:9200"]
action => "index"
index => "scada2"
}
stdout {}
}
I could see the events been processed by logstash on the command line,
however there are no documents found in the ES
May i know what is wrong with the configuration, may be X-pack? since this was working fine without it when using 5.3.0