Logstash input plugin http_poller only loading 300 records in a run

Elastic Stack Logstash
1

I am trying to use the http_poller input plugin to load data from sap CDC into Elasticsearch.
But Logstash input plugin http_poller is only loading 300 records in a run.

  1. How can I load bulk data using the http_poller plugin?
  2. If the bulk load is impossible, how can I dynamically pass an incremental date value for "created" in the below query for every run in the schedule?
input {
http_poller {
  urls => {
    test_url =>{
      url => "${cdcurl}"
      method => post
      user => "${devuser}"
      password => "${password}"
      headers => {
                "Accept" => "application/json"
              }
      params => {
          query => "select * from accounts where created >= '2022-09-17T15:00:00.010Z'"
        }
      }
  }
  codec => "json"
  request_timeout => 60
  schedule => { "every" => "20s" }
}
}

filter {
 json {
    source => "message"
  }
  split {
    field => "[results]"
  }
  prune {
        whitelist_names => ["^results$"]
      }
}


output {
    elasticsearch {
        cloud_id => '${ELASTICSEARCH_CLOUD_ID}'
        cloud_auth => '${ELASTICSEARCH_CLOUD_AUTH}'
        index => "sapcdc"
        action => "index"
        document_id => "%{[results][UID]}"
    }
    stdout { codec => rubydebug }
}```
2

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.