I am trying to perform delta delete for old data based on date field range, I tried few options but not working.
Conf file:
input {
s3 {
bucket => "#####"
region => "#####"
prefix => "#####"
}
}
filter {
csv {
separator => ","
columns => [ "time_id","timezone_id","city","community","sub_community","customer_id","agent_id","user_id","feed_id","customer_name","company_name","customer_master_name","device_source","b2c_c2c","featured","promoted","verified","vertical_id","vertical_name","category_l1_name","category_l2_name","category_l3_name","bedrooms","data_source","nnl","dal"
]
quote_char => "`"
}
mutate {
convert => { "timezone_id" => "integer" }
convert => { "customer_id" => "integer" }
convert => { "agent_id" => "integer" }
convert => { "feed_id" => "integer" }
convert => { "featured" => "integer" }
convert => { "promoted" => "integer" }
convert => { "verified" => "integer" }
convert => { "vertical_id" => "integer" }
convert => { "bedrooms" => "integer" }
convert => { "nnl" => "integer" }
convert => { "dal" => "integer" }
}
}
output {
amazon_es {
hosts => ["search-meabi-es-pub-ixvztcxb4kahwtuphho4odpfni.eu-west-1.es.amazonaws.com"]
region => "eu-west-1"
document_id => "ps_20150502"
index => "v_ft_d_popularity_data_supply"
}
}
output {
amazon_es {
hosts => ["####"]
region => "eu-west-1"
document_id => "ps_20150502"
index => "v_ft_d_popularity_data_supply"
}
}
tried the below delete command but no luck
curl -XDELETE https://search-meabi-es-pub-ixvztcxb4kahwtuphho4odpfni.eu-west-1.es.amazonaws.com/v_ft_d_popularity_data_supply/logs/_query -d '{"query": {"range":{ "time_id": {"from":"2019-05-02","to":"2019-05-05" }}}}' -H 'Content-Type: application/json'
Result:
{"_index":"v_ft_d_popularity_data_supply","_type":"logs","_id":"_query","_version":1,"result":"not_found","_shards":{"total":2,"successful":1,"failed":0},"_seq_no":4356