I have been trying to resolve this issue which is preventing me from upgrading from 1.7.1 to 2.x due to this error:
The _uid, _id, _type, _source, _all, _parent, _field_names, _routing, _index, _size, _timestamp, and _ttl field names are reserved and can no longer be used in the document _source., in type: logs.
At the ElasticOn NY couple of days ago, I worked with Chris Earle and he suggested to make these changes in logstash and re-indexed but those indices when ran through the migration tool still show the same errors.
Please help
filter {
ruby {
init => "
def remove_dots hash
new = Hash.new
hash.each { |k,v|
if v.is_a? Hash
v = remove_dots(v)
end
new[ k.gsub('.','_') ] = v
if v.is_a? Array
v.each { |elem|
if elem.is_a? Hash
elem = remove_dots(elem)
end
key = k.gsub('.','_')
if key.starts_with?('_')
key = key[1..-1]
end
new[ key ] = elem
} unless v.nil?
end
} unless hash.nil?
return new
end
"
code => "
event.instance_variable_set(:@data,remove_dots(event.to_hash))
"
}
mutate {
rename => { "_uid" => "uid" }
rename => { "_action" => "action" }
rename => { "_caps" => "caps" }
rename => { "_cldee" => "cldee" }
rename => { "_ga" => "ga" }
rename => { "_mbox" => "mbox" }
rename => { "_partner" => "partner" }
rename => { "_rand" => "rand" }
rename => { "_rdr" => "rdr" }
rename => { "_sm_au_" => "sm_au_" }
rename => { "_suid" => "suid" }
rename => { "_task" => "task" }
}
}
output {
elasticsearch {
hosts => [ "xxxxxxxxxx.com" ]
index => "%{[@metadata][_index]}"
document_type => "%{[@metadata][_type]}"
document_id => "%{[@metadata][_id]}"
}