Hello,
This issue is already opened in git https://github.com/logstash-plugins/logstash-output-elasticsearch/issues/129
I'm facing the issue while indexing child docs. I tried the code as mentioned by Vordoom in above link, but didn't make it to work. Would appreciate if someone from the community could help in this regard.
Here's the mapping and code change suggested in the issue
Elasticsearch Version: 1.50
Logstash Version: 1.5.0
Added below line of code inside elasticsearch.rb at line 426
parent = @parent ? event.sprintf(@parent) : nil
buffer_receive([event.sprintf(@action), { :_id => document_id, :_index => index, :_type => type, :_routing => routing, :_parent => parent }, event])
Installed the plugin after gem build inside standalone logstash
./bin/plugin install /home/vagrant/logstash-output-elasticsearch/logstash-output-elasticsearch-0.2.7-java.gem
Logstash configuration looks like below:
input {
elasticsearch {
hosts => ["localhost:9200"]
index => "shop"
size => 500
scroll => "5m"
docinfo => true
}
}
filter {
mutate {
remove_field => [ "@version" ]
remove_field => ["@timestamp"]
}
}
output {
elasticsearch {
index => "shop_logstash"
host => "localhost"
protocol => "http"
index_type => "%{[@metadata][_type]}"
document_id => "%{[@metadata][_id]}"
workers => 1
manage_template => false
}
stdout { codec => rubydebug }
}
Index Structure : Shop is parent and variation is child type
PUT /shop/cloth/_mapping -d
{
"cloth" : {
"properties" : {
"name" : {"type" : "string"}
}
}
}
PUT /shop/variation/_mapping
{
"variation" : {
"_parent" : { "type" : "cloth" },
"properties" : {
"size" : {"type" : "string", "index" : "not_analyzed"},
"color" : {"type" : "string", "index" : "not_analyzed"}
}
}
}
Logstash debug output
Logstash startup completed
output received {:event=>{"name"=>"Test shirt"}, :level=>:debug, :file=>"(eval)", :line=>"50", :method=>"output_func"}
{
"name" => "Test shirt"
}
output received {:event=>{"color"=>"red", "size"=>"XXL"}, :level=>:debug, :file=>"(eval)", :line=>"50", :method=>"output_func"}
{
"color" => "red",
"size" => "XXL"
}
output received {:event=>{"color"=>"black", "size"=>"XL"}, :level=>:debug, :file=>"(eval)", :line=>"50", :method=>"output_func"}
{
"color" => "black",
"size" => "XL"
}
Flushing output {:outgoing_count=>3, :time_since_last_flush=>0.049, :outgoing_events=>{nil=>[["index", {:_id=>"1", :_index=>"shop_logstash", :_type=>"cloth", :_routing=>nil, :parent=>nil}, #"shop", "_type"=>"cloth", "_id"=>"1", "retry_count"=>0}, @lut={"[_type]"=>[{"_index"=>"shop", "_type"=>"cloth", "_id"=>"1", "retry_count"=>0}, "_type"], "[_id]"=>[{"_index"=>"shop", "_type"=>"cloth", "_id"=>"1", "retry_count"=>0}, "_id"]}>, @cancelled=false, @data={"name"=>"Test shirt"}, @metadata={"_index"=>"shop", "_type"=>"cloth", "_id"=>"1", "retry_count"=>0}, @accessors=#"Test shirt"}, @lut={"@version"=>[{"name"=>"Test shirt"}, "@version"], "@timestamp"=>[{"name"=>"Test shirt"}, "@timestamp"]}>>], ["index", {:_id=>"1000", :_index=>"shop_logstash", :_type=>"variation", :_routing=>nil, :parent=>nil}, #"shop", "_type"=>"variation", "_id"=>"1000", "retry_count"=>0}, @lut={"[_type]"=>[{"_index"=>"shop", "_type"=>"variation", "_id"=>"1000", "retry_count"=>0}, "_type"], "[_id]"=>[{"_index"=>"shop", "_type"=>"variation", "_id"=>"1000", "retry_count"=>0}, "_id"]}>, @cancelled=false, @data={"color"=>"red", "size"=>"XXL"}, @metadata={"_index"=>"shop", "_type"=>"variation", "_id"=>"1000", "retry_count"=>0}, @accessors=#"red", "size"=>"XXL"}, @lut={"@version"=>[{"color"=>"red", "size"=>"XXL"}, "@version"], "@timestamp"=>[{"color"=>"red", "size"=>"XXL"}, "@timestamp"]}>>], ["index", {:_id=>"1001", :_index=>"shop_logstash", :_type=>"variation", :_routing=>nil, :parent=>nil}, #"shop", "_type"=>"variation", "_id"=>"1001", "retry_count"=>0}, @lut={"[_type]"=>[{"_index"=>"shop", "_type"=>"variation", "_id"=>"1001", "retry_count"=>0}, "_type"], "[_id]"=>[{"_index"=>"shop", "_type"=>"variation", "_id"=>"1001", "retry_count"=>0}, "_id"]}>, @cancelled=false, @data={"color"=>"black", "size"=>"XL"}, @metadata={"_index"=>"shop", "_type"=>"variation", "_id"=>"1001", "retry_count"=>0}, @accessors=#"black", "size"=>"XL"}, @lut={"@version"=>[{"color"=>"black", "size"=>"XL"}, "@version"], "@timestamp"=>[{"color"=>"black", "size"=>"XL"}, "@timestamp"]}>>]]}, :batch_timeout=>1, :force=>true, :final=>true, :level=>:debug, :file=>"stud/buffer.rb", :line=>"207", :method=>"buffer_flush"}
failed action with response of 400, dropping action: ["index", {:_id=>"1000", :_index=>"shop_logstash", :_type=>"variation", :_routing=>nil, :parent=>nil}, #"shop", "_type"=>"variation", "_id"=>"1000", "retry_count"=>0}, @lut={"[_type]"=>[{"_index"=>"shop", "_type"=>"variation", "_id"=>"1000", "retry_count"=>0}, "_type"], "[_id]"=>[{"_index"=>"shop", "_type"=>"variation", "_id"=>"1000", "retry_count"=>0}, "_id"]}>, @cancelled=false, @data={"color"=>"red", "size"=>"XXL"}, @metadata={"_index"=>"shop", "_type"=>"variation", "_id"=>"1000", "retry_count"=>0}, @accessors=#"red", "size"=>"XXL"}, @lut={"@version"=>[{"color"=>"red", "size"=>"XXL"}, "@version"], "@timestamp"=>[{"color"=>"red", "size"=>"XXL"}, "@timestamp"]}>>] {:level=>:warn, :file=>"logstash/outputs/elasticsearch.rb", :line=>"451", :method=>"submit"}
failed action with response of 400, dropping action: ["index", {:_id=>"1001", :_index=>"shop_logstash", :_type=>"variation", :_routing=>nil, :parent=>nil}, #"shop", "_type"=>"variation", "_id"=>"1001", "retry_count"=>0}, @lut={"[_type]"=>[{"_index"=>"shop", "_type"=>"variation", "_id"=>"1001", "retry_count"=>0}, "_type"], "[_id]"=>[{"_index"=>"shop", "_type"=>"variation", "_id"=>"1001", "retry_count"=>0}, "_id"]}>, @cancelled=false, @data={"color"=>"black", "size"=>"XL"}, @metadata={"_index"=>"shop", "_type"=>"variation", "_id"=>"1001", "retry_count"=>0}, @accessors=#"black", "size"=>"XL"}, @lut={"@version"=>[{"color"=>"black", "size"=>"XL"}, "@version"], "@timestamp"=>[{"color"=>"black", "size"=>"XL"}, "@timestamp"]}>>] {:level=>:warn, :file=>"logstash/outputs/elasticsearch.rb", :line=>"451", :method=>"submit"}
Shifting current elasticsearch client {:level=>:debug, :file=>"logstash/outputs/elasticsearch.rb", :line=>"469", :method=>"flush"}
Switched current elasticsearch client to #0 at localhost {:level=>:debug, :file=>"logstash/outputs/elasticsearch.rb", :line=>"519", :method=>"shift_client"}
Plugin is finished {:plugin=>, :level=>:info, :file=>"logstash/plugin.rb", :line=>"61", :method=>"finished"}
Pipeline shutdown complete. {:level=>:info, :file=>"logstash/pipeline.rb", :line=>"100", :method=>"run"}
Logstash shutdown completed
Just wondering if this issue is planned to be fixed in upcoming release? Thanks
