Hi Team,
I am trying to load Servicenow tables in Elasticsearch using Logstash HTTP Poller Plugin.I am able to load a table with 42 records into Elasticsearch index without any issue.But when I tried to load a table with 8000+ and 10000+ records I am getting below error.I am attaching the error and config file for your reference.Please help.
Config File:
input {
http_poller {
urls => {
url => "https://dev63285.service-now.com/incident_metric_list.do?JSONv2&display_value=True&sysparm_exclude_reference_link=True&sysparm_fields=number%2Cstate%2Copened_at%2Cclosed_at%2Cpriority%2Cassigned_to%2Cassignment_group%2Cactive%2Cimpact%2Curgency%2Cseverity&sysparm_limit=1sysparm_view=json_view"
}
connect_timeout => 1000000
request_timeout => 900000
socket_timeout => 180
user => "admin"
password => "Snow2020*"
schedule => { cron => "* * * * *"}
codec => "json"
metadata_target => "http_poller_metadata"
}
}
filter
{
split {
field => "records"
}
}
output {
elasticsearch {
hosts => ["localhost:9200"]
index => "servicenowinc"
document_id => "%{[records][inc_number]}"
}
stdout {
codec => rubydebug
}
}
Error in Logstash:
Sending Logstash logs to C:/Elastic_Stack/logstash-7.5.1/logs which is now configured via log4j2.properties
[2020-02-05T17:21:48,560][WARN ][logstash.config.source.multilocal] Ignoring the 'pipelines.yml' file because modules or command line options are specified
[2020-02-05T17:21:48,724][INFO ][logstash.runner ] Starting Logstash {"logstash.version"=>"7.5.1"}
[2020-02-05T17:21:54,137][INFO ][org.reflections.Reflections] Reflections took 155 ms to scan 1 urls, producing 20 keys and 40 values
[2020-02-05T17:21:58,405][INFO ][logstash.outputs.elasticsearch][main] Elasticsearch pool URLs updated {:changes=>{:removed=>, :added=>[http://localhost:9200/]}}
[2020-02-05T17:21:59,020][WARN ][logstash.outputs.elasticsearch][main] Restored connection to ES instance {:url=>"http://localhost:9200/"}
[2020-02-05T17:21:59,141][INFO ][logstash.outputs.elasticsearch][main] ES Output version determined {:es_version=>7}
[2020-02-05T17:21:59,151][WARN ][logstash.outputs.elasticsearch][main] Detected a 6.x and above cluster: the type event field won't be used to determine the document _type {:es_version=>7}
[2020-02-05T17:21:59,252][INFO ][logstash.outputs.elasticsearch][main] New Elasticsearch output {:class=>"LogStash::Outputs::ElasticSearch", :hosts=>["//localhost:9200"]}
[2020-02-05T17:21:59,353][INFO ][logstash.outputs.elasticsearch][main] Using default mapping template
[2020-02-05T17:21:59,430][WARN ][org.logstash.instrument.metrics.gauge.LazyDelegatingGauge][main] A gauge metric of an unknown type (org.jruby.specialized.RubyArrayOneObject) has been create for key: cluster_uuids. This may result in invalid serialization. It is recommended to log an issue to the responsible developer/development team.
[2020-02-05T17:21:59,444][INFO ][logstash.javapipeline ][main] Starting pipeline {:pipeline_id=>"main", "pipeline.workers"=>4, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>50, "pipeline.max_inflight"=>500, "pipeline.sources"=>["C:/Elastic_Stack/logstash-7.5.1/bin/logstash_incident_test2.conf"], :thread=>"#<Thread:0x247dcc35 run>"}
[2020-02-05T17:21:59,563][INFO ][logstash.outputs.elasticsearch][main] Attempting to install template {:manage_template=>{"index_patterns"=>"logstash-", "version"=>60001, "settings"=>{"index.refresh_interval"=>"5s", "number_of_shards"=>1}, "mappings"=>{"dynamic_templates"=>[{"message_field"=>{"path_match"=>"message", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false}}}, {"string_fields"=>{"match"=>"", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false, "fields"=>{"keyword"=>{"type"=>"keyword", "ignore_above"=>256}}}}}], "properties"=>{"@timestamp"=>{"type"=>"date"}, "@version"=>{"type"=>"keyword"}, "geoip"=>{"dynamic"=>true, "properties"=>{"ip"=>{"type"=>"ip"}, "location"=>{"type"=>"geo_point"}, "latitude"=>{"type"=>"half_float"}, "longitude"=>{"type"=>"half_float"}}}}}}}
[2020-02-05T17:21:59,640][INFO ][logstash.inputs.http_poller][main] Registering http_poller Input {:type=>nil, :schedule=>{"cron"=>"* * * * *"}, :timeout=>nil}
[2020-02-05T17:21:59,686][INFO ][logstash.javapipeline ][main] Pipeline started {"pipeline.id"=>"main"}
[2020-02-05T17:22:00,075][INFO ][logstash.agent ] Pipelines running {:count=>1, :running_pipelines=>[:main], :non_running_pipelines=>}
C:/Elastic_Stack/logstash-7.5.1/vendor/bundle/jruby/2.5.0/gems/rufus-scheduler-3.0.9/lib/rufus/scheduler/cronline.rb:77: warning: constant ::Fixnum is deprecated
[2020-02-05T17:22:02,407][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>9600}
[2020-02-05T17:28:25,304][WARN ][logstash.filters.split ][main] Only String and Array types are splittable. field:records is of type = NilClass
C:/Elastic_Stack/logstash-7.5.1/vendor/bundle/jruby/2.5.0/gems/awesome_print-1.7.0/lib/awesome_print/formatters/base_formatter.rb:31: warning: constant ::Fixnum is deprecated
{
"@version" => "1",
"@timestamp" => 2020-02-05T11:58:25.101Z,
"http_poller_metadata" => {
"runtime_seconds" => nil,
"request" => {
"method" => "get",
"url" => "https://dev63285.service-now.com/incident_metric_list.do?JSONv2&display_value=True&sysparm_exclude_reference_link=True&sysparm_fields=number%2Cstate%2Copened_at%2Cclosed_at%2Cpriority%2Cassigned_to%2Cassignment_group%2Cactive%2Cimpact%2Curgency%2Cseverity&sysparm_limit=1sysparm_view=json_view"
},
"host" => "DESKTOP-JRAAHA2",
"name" => "url"
},
"tags" => [
[0] "_http_request_failure",
[1] "_split_type_failure"
],
"http_request_failure" => {
"backtrace" => [
[0] "C:/Elastic_Stack/logstash-7.5.1/vendor/bundle/jruby/2.5.0/gems/manticore-0.6.4-java/lib/manticore/response.rb:122:in body'", [1] "C:/Elastic_Stack/logstash-7.5.1/vendor/bundle/jruby/2.5.0/gems/logstash-input-http_poller-5.0.1/lib/logstash/inputs/http_poller.rb:162:in handle_success'",
[2] "C:/Elastic_Stack/logstash-7.5.1/vendor/bundle/jruby/2.5.0/gems/logstash-input-http_poller-5.0.1/lib/logstash/inputs/http_poller.rb:154:in block in request_async'", [3] "C:/Elastic_Stack/logstash-7.5.1/vendor/bundle/jruby/2.5.0/gems/manticore-0.6.4-java/lib/manticore/response.rb:268:in handleResponse'",
[4] "C:/Elastic_Stack/logstash-7.5.1/vendor/bundle/jruby/2.5.0/gems/manticore-0.6.4-java/lib/manticore/response.rb:50:in `call'"
],
"runtime_seconds" => 181.838,
"request" => {
"method" => "get",
"url" => "https://dev63285.service-now.com/incident_metric_list.do?JSONv2&display_value=True&sysparm_exclude_reference_link=True&sysparm_fields=number%2Cstate%2Copened_at%2Cclosed_at%2Cpriority%2Cassigned_to%2Cassignment_group%2Cactive%2Cimpact%2Curgency%2Cseverity&sysparm_limit=1sysparm_view=json_view"
},
"error" => "Could not read from stream: Connection reset",
"name" => "url"
}
}
[2020-02-05T17:32:36,209][WARN ][logstash.filters.split ][main] Only String and Array types are splittable. field:records is of type = NilClass
Many Thanks,
Ramya