Logstash configuration with multiple http_poller did'nt ran for some indices

Hello,
I have a logstash configuration with multiple http_poller input plugins.
say

input {
    http_poller {
	id => "s1-input"
    urls => {
	sector_api => {
        method => "POST"
        url => "url1"
		headers => {
                    "Content-Type" => "application/json"
                }
        body=> '{"SecretKey":"k1"}'
      }
	}	
	request_timeout => 120
	schedule => {cron => "30 6 * * * UTC"}
	codec => "json"
	tags => ["se-api-s1"]
  }
  http_poller {
	id => "s2-input"
    urls => {
	mf_scheme_api => {
        method => "POST"
        url => "url2"
		headers => {
                    "Content-Type" => "application/json"
                }
        body=> '{"SecretKey":"k2"}'
      }
	}	
	request_timeout => 120
	schedule => {cron => "30 6 * * * UTC"}
	codec => "json"
	tags => ["se-api-s2"]
  }
  http_poller {
	id => "s3-input"
    urls => {
	industry_api => {
        method => "POST"
        url => "url3"
		headers => {
                    "Content-Type" => "application/json"
                }
        body=> '{"SecretKey":"k3"}'
      }
	}	
	request_timeout => 120
	schedule => {cron => "32 6 * * * UTC"}
	codec => "json"
	tags => ["se-api-s3"]
  }
}
filter { 
	mutate {
		remove_field => [ "[message]" ]
		remove_field => [ "[@version]" ]
		remove_field => [ "[event]" ]
	}
	if [tags][0] =~ /failure$/ or [tags][0] =~ /exception$/ or [@metadata][input][http_poller][response][status_code] != 200
	{
		mutate {
		add_field => { "Status Message" => "%{[@metadata][input][http_poller][response][status_message]}" }
		}
	}
	if ([@metadata][input][http_poller][response][status_code] == 200 and ![field1])
	{
		mutate {
			add_field => { "field1" => 1 }
		}
		mutate {
			convert => ["field1","integer"]
		}
	}
}
output {
	if "se-api-s1" in [tags] and [@metadata][input][http_poller][response][status_code] == 200
	{
		elasticsearch {
			id => "s1-output"
			hosts => ["https://localhost:9200"]
			user => "usr"
			password => "pass"
			ssl => false
			ssl_certificate_verification => false
			index => "s1"
			document_id => "%{id}"
			doc_as_upsert => true
			action => "update"
		 }
	}
  	if "se-api-s2" in [tags] and [@metadata][input][http_poller][response][status_code] == 200
	{
		elasticsearch {
			id => "s2-output"
			hosts => ["https://localhost:9200"]
			user => "usr"
			password => "pass"
			ssl => false
			ssl_certificate_verification => false
			index => "s2"
			document_id => "%{id}"
			doc_as_upsert => true
			action => "update"
		}
	}
  	if "se-api-s3" in [tags] and [@metadata][input][http_poller][response][status_code] == 200
	{
		elasticsearch {
			id => "s3-output"
			hosts => ["https://localhost:9200"]
			user => "usr"
			password => "pass"
			ssl => false
			ssl_certificate_verification => false
			index => "s3"
			document_id => "%{id}"
			doc_as_upsert => true
			action => "update"
		}
	}
	
	if [tags][0] =~ /failure$/ or [tags][0] =~ /exception$/ or [@metadata][input][http_poller][response][status_code] != 200
	{
		file {
			path => "error-log\dir1\%{+YYYYMMdd}-se-logstash-error-log.csv"
		}
	}
	stdout { codec => rubydebug }
}

here upsert is true so all the document for indices s1 s2 s3 should be updated and @timestamp field should be updated to the latest date.

but only 2 indices are updated but not the third one. here I have take only 3 poller configurations. but in actual i have 20 poller config and from them 12 are running properly as per schedule but not the rest.

I have scheduled then at at interval of 2 minutes.
suppose s1 - schedule => {cron => "30 6 * * * UTC"}
s2 - schedule => {cron => "30 6 * * * UTC"}
s3 - schedule => {cron => "32 6 * * * UTC"}
s4 - schedule => {cron => "32 6 * * * UTC"}
s5 - schedule => {cron => "34 6 * * * UTC"}
s6 - schedule => {cron => "36 6 * * * UTC"}
'
'
'
like that....

I am not getting the resaon why it is happening.....Please help.

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.