Updating the datastream from logstash

Elastic Stack Elasticsearch
1

Hi Team,

we are trying to lookup the data in one datastream with anothe data in another datastream based on a key using logstash elasticseach input plugin and elasticsearch filter as shown in the sample config below.

My both indexes - index1, index2 are datastreams. Could you please help me how can i achieve the requirement to update the index1 datastream.

input {
  elasticsearch {
    cloud_id => "xxxxxxxxxxxxxxxxxxxx"
    index =>  "index1"
    query => '{"query": {"match_phrase": {"shop": "12345"}},"_source": ["Price", "shop", "source"]}'
    ssl => true
    user => "xxxxxxxxxx"
    password => "xxxxxxxxxxxxx"
  }
}
 
 
filter {
    mutate {
      add_field => {
        "common_key" => "%{shop}"
        "Secondary" => "%{source}"
        "price" => "%{price}"
        "amount" => "%{amount}"
      }
    }
 
 elasticsearch {
        cloud_id => "xxxxxxxxxx"
    user => "xxxxxxxxxx"
    password => "xxxxxxxxxxxxx"
        index =>  "index2"
        query => "sourcecode:%{[source]}"
        ssl => true
        fields => {
                "[shop]" => "[shop]"
                "[price]" => "[price]"
                "[amount]" => "[amount]"
                }
    }
}
 
output {
   stdout {
        codec => rubydebug
   }
    elasticsearch {
        cloud_id => "xxxxxxxxxxxx"
        proxy => "xxxxxxxxxxxx"
        index => "index1"
        ssl => true
        user => "xxxxxxxxxxxx"
        password => "xxxxxxxxxxx"
        action => "update"
    }
}

i am getting below error while trying
"reason"=>"only write ops with an op_type of create are allowed in data streams"

2

Data streams are designed for immutable data so if you need to update your data you may be better off switching to normal indices. If you need to perform updates you may need to use update-by-query which can not be triggered from Logstash.

1 Like