Hello,
I have a logstash configuration with multiple http_poller input plugins.
say
input {
http_poller {
id => "s1-input"
urls => {
sector_api => {
method => "POST"
url => "url1"
headers => {
"Content-Type" => "application/json"
}
body=> '{"SecretKey":"k1"}'
}
}
request_timeout => 120
schedule => {cron => "30 6 * * * UTC"}
codec => "json"
tags => ["se-api-s1"]
}
http_poller {
id => "s2-input"
urls => {
mf_scheme_api => {
method => "POST"
url => "url2"
headers => {
"Content-Type" => "application/json"
}
body=> '{"SecretKey":"k2"}'
}
}
request_timeout => 120
schedule => {cron => "30 6 * * * UTC"}
codec => "json"
tags => ["se-api-s2"]
}
http_poller {
id => "s3-input"
urls => {
industry_api => {
method => "POST"
url => "url3"
headers => {
"Content-Type" => "application/json"
}
body=> '{"SecretKey":"k3"}'
}
}
request_timeout => 120
schedule => {cron => "32 6 * * * UTC"}
codec => "json"
tags => ["se-api-s3"]
}
}
filter {
mutate {
remove_field => [ "[message]" ]
remove_field => [ "[@version]" ]
remove_field => [ "[event]" ]
}
if [tags][0] =~ /failure$/ or [tags][0] =~ /exception$/ or [@metadata][input][http_poller][response][status_code] != 200
{
mutate {
add_field => { "Status Message" => "%{[@metadata][input][http_poller][response][status_message]}" }
}
}
if ([@metadata][input][http_poller][response][status_code] == 200 and ![field1])
{
mutate {
add_field => { "field1" => 1 }
}
mutate {
convert => ["field1","integer"]
}
}
}
output {
if "se-api-s1" in [tags] and [@metadata][input][http_poller][response][status_code] == 200
{
elasticsearch {
id => "s1-output"
hosts => ["https://localhost:9200"]
user => "usr"
password => "pass"
ssl => false
ssl_certificate_verification => false
index => "s1"
document_id => "%{id}"
doc_as_upsert => true
action => "update"
}
}
if "se-api-s2" in [tags] and [@metadata][input][http_poller][response][status_code] == 200
{
elasticsearch {
id => "s2-output"
hosts => ["https://localhost:9200"]
user => "usr"
password => "pass"
ssl => false
ssl_certificate_verification => false
index => "s2"
document_id => "%{id}"
doc_as_upsert => true
action => "update"
}
}
if "se-api-s3" in [tags] and [@metadata][input][http_poller][response][status_code] == 200
{
elasticsearch {
id => "s3-output"
hosts => ["https://localhost:9200"]
user => "usr"
password => "pass"
ssl => false
ssl_certificate_verification => false
index => "s3"
document_id => "%{id}"
doc_as_upsert => true
action => "update"
}
}
if [tags][0] =~ /failure$/ or [tags][0] =~ /exception$/ or [@metadata][input][http_poller][response][status_code] != 200
{
file {
path => "error-log\dir1\%{+YYYYMMdd}-se-logstash-error-log.csv"
}
}
stdout { codec => rubydebug }
}
here upsert is true so all the document for indices s1 s2 s3 should be updated and @timestamp field should be updated to the latest date.
but only 2 indices are updated but not the third one. here I have take only 3 poller configurations. but in actual i have 20 poller config and from them 12 are running properly as per schedule but not the rest.
I have scheduled then at at interval of 2 minutes.
suppose s1 - schedule => {cron => "30 6 * * * UTC"}
s2 - schedule => {cron => "30 6 * * * UTC"}
s3 - schedule => {cron => "32 6 * * * UTC"}
s4 - schedule => {cron => "32 6 * * * UTC"}
s5 - schedule => {cron => "34 6 * * * UTC"}
s6 - schedule => {cron => "36 6 * * * UTC"}
'
'
'
like that....
I am not getting the resaon why it is happening.....Please help.