Elasticsearch output does not support parent-child indexing


(Sumit Arora) #1

Hello,

This issue is already opened in git https://github.com/logstash-plugins/logstash-output-elasticsearch/issues/129

I'm facing the issue while indexing child docs. I tried the code as mentioned by Vordoom in above link, but didn't make it to work. Would appreciate if someone from the community could help in this regard.

Here's the mapping and code change suggested in the issue

Elasticsearch Version: 1.50
Logstash Version: 1.5.0

Added below line of code inside elasticsearch.rb at line 426

parent = @parent ? event.sprintf(@parent) : nil
buffer_receive([event.sprintf(@action), { :_id => document_id, :_index => index, :_type => type, :_routing => routing, :_parent => parent }, event])

Installed the plugin after gem build inside standalone logstash

./bin/plugin install /home/vagrant/logstash-output-elasticsearch/logstash-output-elasticsearch-0.2.7-java.gem

Logstash configuration looks like below:

input {
elasticsearch {
hosts => ["localhost:9200"]
index => "shop"
size => 500
scroll => "5m"
docinfo => true
}
}

filter {
mutate {
remove_field => [ "@version" ]
remove_field => ["@timestamp"]
}
}

output {
elasticsearch {
index => "shop_logstash"
host => "localhost"
protocol => "http"
index_type => "%{[@metadata][_type]}"
document_id => "%{[@metadata][_id]}"
workers => 1
manage_template => false
}
stdout { codec => rubydebug }
}

Index Structure : Shop is parent and variation is child type

PUT /shop/cloth/_mapping -d
{
"cloth" : {
"properties" : {
"name" : {"type" : "string"}
}
}
}

PUT /shop/variation/_mapping
{
"variation" : {
"_parent" : { "type" : "cloth" },
"properties" : {
"size" : {"type" : "string", "index" : "not_analyzed"},
"color" : {"type" : "string", "index" : "not_analyzed"}
}
}
}

Logstash debug output

Logstash startup completed
output received {:event=>{"name"=>"Test shirt"}, :level=>:debug, :file=>"(eval)", :line=>"50", :method=>"output_func"}
{
"name" => "Test shirt"
}
output received {:event=>{"color"=>"red", "size"=>"XXL"}, :level=>:debug, :file=>"(eval)", :line=>"50", :method=>"output_func"}
{
"color" => "red",
"size" => "XXL"
}
output received {:event=>{"color"=>"black", "size"=>"XL"}, :level=>:debug, :file=>"(eval)", :line=>"50", :method=>"output_func"}
{
"color" => "black",
"size" => "XL"
}

Flushing output {:outgoing_count=>3, :time_since_last_flush=>0.049, :outgoing_events=>{nil=>[["index", {:_id=>"1", :_index=>"shop_logstash", :_type=>"cloth", :_routing=>nil, :parent=>nil}, #"shop", "_type"=>"cloth", "_id"=>"1", "retry_count"=>0}, @lut={"[_type]"=>[{"_index"=>"shop", "_type"=>"cloth", "_id"=>"1", "retry_count"=>0}, "_type"], "[_id]"=>[{"_index"=>"shop", "_type"=>"cloth", "_id"=>"1", "retry_count"=>0}, "_id"]}>, @cancelled=false, @data={"name"=>"Test shirt"}, @metadata={"_index"=>"shop", "_type"=>"cloth", "_id"=>"1", "retry_count"=>0}, @accessors=#"Test shirt"}, @lut={"@version"=>[{"name"=>"Test shirt"}, "@version"], "@timestamp"=>[{"name"=>"Test shirt"}, "@timestamp"]}>>], ["index", {:_id=>"1000", :_index=>"shop_logstash", :_type=>"variation", :_routing=>nil, :parent=>nil}, #"shop", "_type"=>"variation", "_id"=>"1000", "retry_count"=>0}, @lut={"[_type]"=>[{"_index"=>"shop", "_type"=>"variation", "_id"=>"1000", "retry_count"=>0}, "_type"], "[_id]"=>[{"_index"=>"shop", "_type"=>"variation", "_id"=>"1000", "retry_count"=>0}, "_id"]}>, @cancelled=false, @data={"color"=>"red", "size"=>"XXL"}, @metadata={"_index"=>"shop", "_type"=>"variation", "_id"=>"1000", "retry_count"=>0}, @accessors=#"red", "size"=>"XXL"}, @lut={"@version"=>[{"color"=>"red", "size"=>"XXL"}, "@version"], "@timestamp"=>[{"color"=>"red", "size"=>"XXL"}, "@timestamp"]}>>], ["index", {:_id=>"1001", :_index=>"shop_logstash", :_type=>"variation", :_routing=>nil, :parent=>nil}, #"shop", "_type"=>"variation", "_id"=>"1001", "retry_count"=>0}, @lut={"[_type]"=>[{"_index"=>"shop", "_type"=>"variation", "_id"=>"1001", "retry_count"=>0}, "_type"], "[_id]"=>[{"_index"=>"shop", "_type"=>"variation", "_id"=>"1001", "retry_count"=>0}, "_id"]}>, @cancelled=false, @data={"color"=>"black", "size"=>"XL"}, @metadata={"_index"=>"shop", "_type"=>"variation", "_id"=>"1001", "retry_count"=>0}, @accessors=#"black", "size"=>"XL"}, @lut={"@version"=>[{"color"=>"black", "size"=>"XL"}, "@version"], "@timestamp"=>[{"color"=>"black", "size"=>"XL"}, "@timestamp"]}>>]]}, :batch_timeout=>1, :force=>true, :final=>true, :level=>:debug, :file=>"stud/buffer.rb", :line=>"207", :method=>"buffer_flush"}
failed action with response of 400, dropping action: ["index", {:_id=>"1000", :_index=>"shop_logstash", :_type=>"variation", :_routing=>nil, :parent=>nil}, #"shop", "_type"=>"variation", "_id"=>"1000", "retry_count"=>0}, @lut={"[_type]"=>[{"_index"=>"shop", "_type"=>"variation", "_id"=>"1000", "retry_count"=>0}, "_type"], "[_id]"=>[{"_index"=>"shop", "_type"=>"variation", "_id"=>"1000", "retry_count"=>0}, "_id"]}>, @cancelled=false, @data={"color"=>"red", "size"=>"XXL"}, @metadata={"_index"=>"shop", "_type"=>"variation", "_id"=>"1000", "retry_count"=>0}, @accessors=#"red", "size"=>"XXL"}, @lut={"@version"=>[{"color"=>"red", "size"=>"XXL"}, "@version"], "@timestamp"=>[{"color"=>"red", "size"=>"XXL"}, "@timestamp"]}>>] {:level=>:warn, :file=>"logstash/outputs/elasticsearch.rb", :line=>"451", :method=>"submit"}

failed action with response of 400, dropping action: ["index", {:_id=>"1001", :_index=>"shop_logstash", :_type=>"variation", :_routing=>nil, :parent=>nil}, #"shop", "_type"=>"variation", "_id"=>"1001", "retry_count"=>0}, @lut={"[_type]"=>[{"_index"=>"shop", "_type"=>"variation", "_id"=>"1001", "retry_count"=>0}, "_type"], "[_id]"=>[{"_index"=>"shop", "_type"=>"variation", "_id"=>"1001", "retry_count"=>0}, "_id"]}>, @cancelled=false, @data={"color"=>"black", "size"=>"XL"}, @metadata={"_index"=>"shop", "_type"=>"variation", "_id"=>"1001", "retry_count"=>0}, @accessors=#"black", "size"=>"XL"}, @lut={"@version"=>[{"color"=>"black", "size"=>"XL"}, "@version"], "@timestamp"=>[{"color"=>"black", "size"=>"XL"}, "@timestamp"]}>>] {:level=>:warn, :file=>"logstash/outputs/elasticsearch.rb", :line=>"451", :method=>"submit"}
Shifting current elasticsearch client {:level=>:debug, :file=>"logstash/outputs/elasticsearch.rb", :line=>"469", :method=>"flush"}
Switched current elasticsearch client to #0 at localhost {:level=>:debug, :file=>"logstash/outputs/elasticsearch.rb", :line=>"519", :method=>"shift_client"}
Plugin is finished {:plugin=>, :level=>:info, :file=>"logstash/plugin.rb", :line=>"61", :method=>"finished"}
Pipeline shutdown complete. {:level=>:info, :file=>"logstash/pipeline.rb", :line=>"100", :method=>"run"}
Logstash shutdown completed

Just wondering if this issue is planned to be fixed in upcoming release? Thanks


(Mark Walkom) #2

You'll likely get a better response on the GH issue :slight_smile:


(Sumit Arora) #3

The issue is resolved and could be tracked here


(system) #4