Hello there
I've recently started using the ELK stack but ran into a problem when using templates to map the types. But when doing that I found the following errors in the logstash logs:
[2018-05-03T10:53:45,201][WARN ][logstash.outputs.elasticsearch] Could not index event to Elasticsearch. {:status=>400, :action=>["index", {:_id=>nil, :_index=>"acc_app_haproxy-2018.05.03", :_type=>"doc", :_routing=>nil}, #LogStash::Event:0x7e7d48ea], :response=>{"index"=>{"_index"=>"acc_app_haproxy-2018.05.03", "_type"=>"doc", "_id"=>"NXw1JWMB_gYFHDTjEu5J", "status"=>400, "error"=>{"type"=>"illegal_argument_exception", "reason"=>"Field [_id] is defined twice in [doc]"}}}}
[2018-05-03T10:53:45,201][WARN ][logstash.outputs.elasticsearch] Could not index event to Elasticsearch. {:status=>400, :action=>["index", {:_id=>nil, :_index=>"acc_app_haproxy-2018.05.03", :_type=>"doc", :_routing=>nil}, #LogStash::Event:0x2137a00f], :response=>{"index"=>{"_index"=>"acc_app_haproxy-2018.05.03", "_type"=>"doc", "_id"=>"MXw1JWMB_gYFHDTjEu4o", "status"=>400, "error"=>{"type"=>"illegal_argument_exception", "reason"=>"Field [_id] is defined twice in [doc]"}}}}
[2018-05-03T10:53:45,201][WARN ][logstash.outputs.elasticsearch] Could not index event to Elasticsearch. {:status=>400, :action=>["index", {:_id=>nil, :_index=>"acc_app_haproxy-2018.05.03", :_type=>"doc", :_routing=>nil}, #LogStash::Event:0x46ead929], :response=>{"index"=>{"_index"=>"acc_app_haproxy-2018.05.03", "_type"=>"doc", "_id"=>"NHw1JWMB_gYFHDTjEu46", "status"=>400, "error"=>{"type"=>"illegal_argument_exception", "reason"=>"Field [_id] is defined twice in [doc]"}}}}
[2018-05-03T10:53:45,202][WARN ][logstash.outputs.elasticsearch] Could not index event to Elasticsearch. {:status=>400, :action=>["index", {:_id=>nil, :_index=>"acc_app_haproxy-2018.05.03", :_type=>"doc", :_routing=>nil}, #LogStash::Event:0x12884733], :response=>{"index"=>{"_index"=>"acc_app_haproxy-2018.05.03", "_type"=>"doc", "_id"=>"M3w1JWMB_gYFHDTjEu4w", "status"=>400, "error"=>{"type"=>"illegal_argument_exception", "reason"=>"Field [_id] is defined twice in [doc]"}}}}
[2018-05-03T10:53:45,202][WARN ][logstash.outputs.elasticsearch] Could not index event to Elasticsearch. {:status=>400, :action=>["index", {:_id=>nil, :_index=>"acc_app_haproxy-2018.05.03", :_type=>"doc", :_routing=>nil}, #LogStash::Event:0x261e4ac1], :response=>{"index"=>{"_index"=>"acc_app_haproxy-2018.05.03", "_type"=>"doc", "_id"=>"Mnw1JWMB_gYFHDTjEu4q", "status"=>400, "error"=>{"type"=>"illegal_argument_exception", "reason"=>"Field [_id] is defined twice in [doc]"}}}}
Template:
{
"template": "_app_haproxy",
"settings": {
"index.refresh_interval": "5s"
},
"mappings": {
"default": {
"dynamic": "false",
"properties": {
"@timestamp": {
"type": "date"
},
"backend": {
"type": "keyword"
},
"bytes": {
"type": "integer"
},
"concurrentConnectionsBackend": {
"type": "integer"
},
"concurrentConnectionsFrontend": {
"type": "integer"
},
"concurrentConnectionsProcess": {
"type": "integer"
},
"concurrentConnectionsServer": {
"type": "integer"
},
"connectionsBackend": {
"type": "integer"
},
"connectionsServer": {
"type": "integer"
},
"file": {
"type": "text"
},
"frontend": {
"type": "keyword"
},
"host": {
"type": "keyword"
},
"message": {
"type": "text"
},
"port": {
"type": "integer"
},
"request": {
"type": "text"
},
"retries": {
"type": "integer"
},
"server": {
"type": "keyword"
},
"source": {
"type": "ip"
},
"statusCode": {
"type": "integer"
},
"timeClientRequest": {
"type": "integer"
},
"timeQueue": {
"type": "integer"
},
"timeServer": {
"type": "integer"
},
"timeTCP": {
"type": "integer"
},
"timeTotal": {
"type": "integer"
},
"userAgent": {
"type": "text"
},
"verb": {
"type": "keyword"
}
}
}
}
}
And the logstash configuration file (output part is as follows:
} else if [@metadata][app] == "haproxy" {
elasticsearch {
hosts => "localhost:9200"
index => "%{[@metadata][env]}app%{[@metadata][app]}-%{+YYYY.MM.dd}"
template => "/etc/logstash/pipeline/templates/haproxy-logs.json"
template_name => "haproxy_logs"
template_overwrite => true
}
}
Before using the templates I didn't have this problem but then I couldn't map the sourceIP to an ip type.
What am I doing wrong?
Thank you