I have 1000 records but i am just getting 997 records.
csv-input.conf
input
{
file {
path => "/usr/share/logstash/user_details.csv"
start_position => "beginning"
#sincedb_path => "/usr/share/logstash/dbteste"
}
}
filter {
csv {
columns => ["user_id","firstname","lastname","email","profession"]
separator => ","
convert => {
"user_id" => "integer"
}
}
}
csv-output.conf
output {
elasticsearch {
hosts => ["http://localhost:9200"]
manage_template => false
index => "log-csv-%{+YYYY.MM.dd}"
}
}
[2021-12-16T12:51:23,081][INFO ][logstash.agent ] Pipelines running {:count=>1, :running_pipelines=>[:csv], :non_running_pipelines=>[]}
[2021-12-16T12:51:23,238][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>9600}
[2021-12-16T12:51:23,860][WARN ][logstash.outputs.elasticsearch][csv][d3518d752450cf7068967aabf6be12bdafe058b833b41be9efceee00d0e4f8e3] Could not index event to Elasticsearch. {:status=>400, :action=>["index", {:_id=>nil, :_index=>"log-csv-2021.12.16", :routing=>nil, :_type=>"_doc"}, #<LogStash::Event:0x4e52fc59>], :response=>{"index"=>{"_index"=>"log-csv-2021.12.16", "_type"=>"_doc", "_id"=>"eSIdwn0BDjowTWK3-Os-", "status"=>400, "error"=>{"type"=>"illegal_argument_exception", "reason"=>"mapper [user_id] cannot be changed from type [text] to [long]"}}}}
[2021-12-16T12:51:23,872][WARN ][logstash.outputs.elasticsearch][csv][d3518d752450cf7068967aabf6be12bdafe058b833b41be9efceee00d0e4f8e3] Could not index event to Elasticsearch. {:status=>400, :action=>["index", {:_id=>nil, :_index=>"log-csv-2021.12.16", :routing=>nil, :_type=>"_doc"}, #<LogStash::Event:0x58a27cd7>], :response=>{"index"=>{"_index"=>"log-csv-2021.12.16", "_type"=>"_doc", "_id"=>"_CIdwn0BDjowTWK3-OoO", "status"=>400, "error"=>{"type"=>"illegal_argument_exception", "reason"=>"mapper [user_id] cannot be changed from type [text] to [long]"}}}}
[2021-12-16T12:51:23,872][WARN ][logstash.outputs.elasticsearch][csv][d3518d752450cf7068967aabf6be12bdafe058b833b41be9efceee00d0e4f8e3] Could not index event to Elasticsearch. {:status=>400, :action=>["index", {:_id=>nil, :_index=>"log-csv-2021.12.16", :routing=>nil, :_type=>"_doc"}, #<LogStash::Event:0x3d65df2a>], :response=>{"index"=>{"_index"=>"log-csv-2021.12.16", "_type"=>"_doc", "_id"=>"AiIdwn0BDjowTWK3-OoN", "status"=>400, "error"=>{"type"=>"illegal_argument_exception", "reason"=>"mapper [user_id] cannot be changed from type [text] to [long]"}}}}
I don't why i am getting this warning and with this issue i am getting less doc_counts
template for this index pattern
{
"csv" : {
"order" : 0,
"index_patterns" : [
"log-csv-*"
],
"settings" : {
"index" : {
"number_of_replicas" : "0"
}
},
"mappings" : { },
"aliases" : { }
}
}