Hi everyone
I am a new user to ELK stack. I tried to setup elastic stack on my local machine but logstash doesn't work for me. I tried to debug, it showed an error "Exception in pipelineworker". Please help !
My logstash configuration are following:
input {
file{
path => "/Users/aztahir/Downloads/data/cars.csv"
start_position => "beginning"
sincedb_path => "/dev/null"
}
}
filter {
csv {
separator => ","
columns => [ "maker", "model", "mileage", "manufacture_year", "engine_displacement",
"engine_power", "body_type", "color_slug", "stk_year", "transmission", "door_count",
"seat_count", "fuel_type", "date_created", "date_last_seen", "price_eur" ]
}
mutate {convert => ["mileage", "integer"] }
mutate {convert => ["price_eur", "float"] }
mutate {convert => ["engine_power", "integer"] }
mutate {convert => ["door_count", "integer"] }
mutate {convert => ["seat_count", "integer"] }
}
output{
elasticsearch {
hosts => "localhost"
index => "cars"
document_type => "sold_cars"
}
stdout {}
}
[2017-10-25T09:47:03,770][DEBUG][logstash.outputs.stdout ] config LogStash::Outputs::Stdout/@workers = 1
[2017-10-25T09:47:03,770][DEBUG][logstash.agent ] starting agent
[2017-10-25T09:47:03,770][DEBUG][logstash.agent ] starting pipeline {:id=>"main"}
[2017-10-25T09:47:03,786][DEBUG][logstash.outputs.elasticsearch] Normalizing http path {:path=>nil, :normalized=>nil}
[2017-10-25T09:47:04,004][INFO ][logstash.outputs.elasticsearch] Elasticsearch pool URLs updated {:changes=>{:removed=>[], :added=>[http://localhost:9200/]}}
[2017-10-25T09:47:04,004][INFO ][logstash.outputs.elasticsearch] Running health check to see if an Elasticsearch connection is working {:healthcheck_url=>http://localhost:9200/, :path=>"/"}
[2017-10-25T09:47:04,087][WARN ][logstash.outputs.elasticsearch] Restored connection to ES instance {:url=>"http://localhost:9200/"}
[2017-10-25T09:47:04,165][INFO ][logstash.outputs.elasticsearch] Using mapping template from {:path=>nil}
[2017-10-25T09:47:04,165][INFO ][logstash.outputs.elasticsearch] Attempting to install template {:manage_template=>{"template"=>"logstash-", "version"=>50001, "settings"=>{"index.refresh_interval"=>"5s"}, "mappings"=>{"default"=>{"_all"=>{"enabled"=>true, "norms"=>false}, "dynamic_templates"=>[{"message_field"=>{"path_match"=>"message", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false}}}, {"string_fields"=>{"match"=>"", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false, "fields"=>{"keyword"=>{"type"=>"keyword", "ignore_above"=>256}}}}}], "properties"=>{"@timestamp"=>{"type"=>"date", "include_in_all"=>false}, "@version"=>{"type"=>"keyword", "include_in_all"=>false}, "geoip"=>{"dynamic"=>true, "properties"=>{"ip"=>{"type"=>"ip"}, "location"=>{"type"=>"geo_point"}, "latitude"=>{"type"=>"half_float"}, "longitude"=>{"type"=>"half_float"}}}}}}}}
[2017-10-25T09:47:04,165][DEBUG][logstash.outputs.elasticsearch] Found existing Elasticsearch template. Skipping template management {:name=>"logstash"}
[2017-10-25T09:47:04,165][INFO ][logstash.outputs.elasticsearch] New Elasticsearch output {:class=>"LogStash::Outputs::ElasticSearch", :hosts=>["//localhost"]}
[2017-10-25T09:47:04,168][DEBUG][logstash.filters.csv ] CSV parsing options {:col_sep=>",", :quote_char=>"""}
[2017-10-25T09:47:04,168][INFO ][logstash.pipeline ] Starting pipeline {"id"=>"main", "pipeline.workers"=>4, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>5, "pipeline.max_inflight"=>500}
[2017-10-25T09:47:04,170][ERROR][logstash.pipeline ] Exception in pipelineworker, the pipeline stopped processing new events, please check your filter configuration and restart Logstash. {"exception"=>"undefined method to_hash' for []:Array", "backtrace"=>["(eval):190:infilter_func'", "C:/Users/aztahir/Downloads/ELK-Stack/Logstash-5.6.3/logstash-core/lib/logstash/pipeline.rb:398:in filter_batch'", "C:/Users/aztahir/Downloads/ELK-Stack/Logstash-5.6.3/logstash-core/lib/logstash/pipeline.rb:379:inworker_loop'", "C:/Users/aztahir/Downloads/ELK-Stack/Logstash-5.6.3/logstash-core/lib/logstash/pipeline.rb:342:in start_workers'"]} [2017-10-25T09:47:04,204][ERROR][logstash.pipeline ] Exception in pipelineworker, the pipeline stopped processing new events, please check your filter configuration and restart Logstash. {"exception"=>"undefined methodto_hash' for []:Array", "backtrace"=>["(eval):190:in filter_func'", "C:/Users/aztahir/Downloads/ELK-Stack/Logstash-5.6.3/logstash-core/lib/logstash/pipeline.rb:398:infilter_batch'", "C:/Users/aztahir/Downloads/ELK-Stack/Logstash-5.6.3/logstash-core/lib/logstash/pipeline.rb:379:in worker_loop'", "C:/Users/aztahir/Downloads/ELK-Stack/Logstash-5.6.3/logstash-core/lib/logstash/pipeline.rb:342:instart_workers'"]}
[FATAL] 2017-10-25 09:47:04.204 [LogStash::Runner] runner - An unexpected error occurred! {:error=>#<NoMethodError: undefined method to_hash' for []:Array>, :backtrace=>["(eval):190:infilter_func'", "C:/Users/aztahir/Downloads/ELK-Stack/Logstash-5.6.3/logstash-core/lib/logstash/pipeline.rb:398:in filter_batch'", "C:/Users/aztahir/Downloads/ELK-Stack/Logstash-5.6.3/logstash-core/lib/logstash/pipeline.rb:379:inworker_loop'", "C:/Users/aztahir/Downloads/ELK-Stack/Logstash-5.6.3/logstash-core/lib/logstash/pipeline.rb:342:in start_workers'"]} The signal INT is in use by the JVM and will not work correctly on this platform [2017-10-25T09:47:04,220][ERROR][logstash.pipeline ] Exception in pipelineworker, the pipeline stopped processing new events, please check your filter configuration and restart Logstash. {"exception"=>"undefined methodto_hash' for []:Array", "backtrace"=>["(eval):190:in filter_func'", "C:/Users/aztahir/Downloads/ELK-Stack/Logstash-5.6.3/logstash-core/lib/logstash/pipeline.rb:398:infilter_batch'", "C:/Users/aztahir/Downloads/ELK-Stack/Logstash-5.6.3/logstash-core/lib/logstash/pipeline.rb:379:in worker_loop'", "C:/Users/aztahir/Downloads/ELK-Stack/Logstash-5.6.3/logstash-core/lib/logstash/pipeline.rb:342:instart_workers'"]}