_source field gets: "http_request_failure"


(Rym Guerbi Michaut) #1

Hi there,
I want to fetch JSON formatted logs from a URL.
I'm using HTTP_POLLER. Sending it to Elasticsearch output under the name: "json_10meilleurs_logs"

Here's my config file:

input {
 http_poller {
    urls => {
        test => {
        method => get
        user => "me"
        password => "mypassword"
        url => "myurl"
        headers => {
        Accept => "application/json"
        }
     }
    }
    request_timeout => 120
    schedule => { cron => "*/03 * * * * UTC"}
    codec => "json"
    metadata_target => "http_poller_metadata"
  }
}

output {
  	elasticsearch {
	hosts => ["localhost:9200"] 
	index => "json_10meilleurs_logs"
	user => "elastic"
	password => "changeme"
	}
}

but the answer I get is an error mentioning "http_request_failure":

{
    "took": 0,
    "timed_out": false,
    "_shards": {
        "total": 5,
        "successful": 5,
        "skipped": 0,
        "failed": 0
    },
    "hits": {
        "total": 1,
        "max_score": 1,
        "hits": [
            {
                "_index": "json_10meilleurs_logs",
                "_type": "logs",
                "_id": "AWBKBGYk-gt4iIR0On40",
                "_score": 1,
                "_source": {
                    "http_request_failure": {
                        "name": "test",
                        "request": {
                            "headers": {
                                "Accept": "application/json"
                            },
                            "method": "get",
                            "auth": {
                                "eager": true,
                                "user": "me",
                                "pass": "mypassword"
                            },
                            "url": "myurl"
                        },
                        "error": "Read timed out",
                        "backtrace": null,
                        "runtime_seconds": 10.289
                    },
                    "@timestamp": "2017-12-12T09:18:10.416Z",
                    "@version": "1",
                    "http_poller_metadata": {
                        "name": "test",
                        "host": "kda-CTBR3G2",
                        "request": {
                            "headers": {
                                "Accept": "application/json"
                            },
                            "method": "get",
                            "auth": {
                                "eager": true,
                                "user": "me",
                                "pass": "mypassword"
                            },
                            "url": "myurl"
                        },
                        "runtime_seconds": null
                    },
                    "tags": [
                        "_http_request_failure"
                    ]
                }
            }
        ]
    }
}

Could anyone help please? I'm stuck on this for a while :frowning:


(Magnus Bäck) #2

Well, it looks like there's a ten-second timeout somewhere that the request is overstepping. Is the request supposed to take that long? What if you try the same thing with e.g. curl?


(Rym Guerbi Michaut) #3

@magnusbaeck, when you say "curl", are you talking about the method?
If so, it doesn't work, I get this error on Logstash command terminal:

{ 211110 rufus-scheduler intercepted an error:
  211110   job:
  211110     Rufus::Scheduler::CronJob "*/03 * * * * UTC" {}
  211110   error:
  211110     211110
  211110     NoMethodError
  211110     undefined method `curl' for #<Manticore::Client::AsyncProxy:0xa67a3a5>
  211110       /home/mmichaut/logstash-5.6.3/vendor/bundle/jruby/1.9/gems/logstash-input-http_poller-3.3.4/lib/logstash/inputs/http_poller.rb:180:in `request_async'
  211110       /home/mmichaut/logstash-5.6.3/vendor/bundle/jruby/1.9/gems/logstash-input-http_poller-3.3.4/lib/logstash/inputs/http_poller.rb:168:in `run_once'
  211110       org/jruby/RubyHash.java:1342:in `each'
  211110       /home/mmichaut/logstash-5.6.3/vendor/bundle/jruby/1.9/gems/logstash-input-http_poller-3.3.4/lib/logstash/inputs/http_poller.rb:167:in `run_once'
  211110       /home/mmichaut/logstash-5.6.3/vendor/bundle/jruby/1.9/gems/logstash-input-http_poller-3.3.4/lib/logstash/inputs/http_poller.rb:162:in `setup_schedule'
  211110       org/jruby/RubyProc.java:281:in `call'
  211110       /home/mmichaut/logstash-5.6.3/vendor/bundle/jruby/1.9/gems/rufus-scheduler-3.0.9/lib/rufus/scheduler/jobs.rb:234:in `do_call'
  211110       /home/mmichaut/logstash-5.6.3/vendor/bundle/jruby/1.9/gems/rufus-scheduler-3.0.9/lib/rufus/scheduler/jobs.rb:258:in `do_trigger'
  211110       /home/mmichaut/logstash-5.6.3/vendor/bundle/jruby/1.9/gems/rufus-scheduler-3.0.9/lib/rufus/scheduler/jobs.rb:300:in `start_work_thread'
  211110       org/jruby/RubyProc.java:281:in `call'
  211110       /home/mmichaut/logstash-5.6.3/vendor/bundle/jruby/1.9/gems/rufus-scheduler-3.0.9/lib/rufus/scheduler/jobs.rb:299:in `start_work_thread'
  211110       org/jruby/RubyKernel.java:1479:in `loop'
  211110       /home/mmichaut/logstash-5.6.3/vendor/bundle/jruby/1.9/gems/rufus-scheduler-3.0.9/lib/rufus/scheduler/jobs.rb:289:in `start_work_thread'
  211110   tz:
  211110     ENV['TZ']: 
  211110     Time.now: 2017-12-12 11:06:00 +0100
  211110   scheduler:
  211110     object_id: 209606
  211110     opts:
  211110       {:max_work_threads=>1}
  211110       frequency: 0.3
  211110       scheduler_lock: #<Rufus::Scheduler::NullLock:0x2e4b26e4>
  211110       trigger_lock: #<Rufus::Scheduler::NullLock:0x53a6ef4b>
  211110     uptime: 74.425 (1m14s424)
  211110     down?: false
  211110     threads: 2
  211110       thread: #<Thread:0x10c559e9>
  211110       thread_key: rufus_scheduler_209606
  211110       work_threads: 1
  211110         active: 1
  211110         vacant: 0
  211110         max_work_threads: 1
  211110       mutexes: {}
  211110     jobs: 1
  211110       at_jobs: 0
  211110       in_jobs: 0
  211110       every_jobs: 0
  211110       interval_jobs: 0
  211110       cron_jobs: 1
  211110     running_jobs: 1
  211110     work_queue: 0

(Magnus Bäck) #4

No, I'm referring to the curl command (https://en.wikipedia.org/wiki/CURL).


(Rym Guerbi Michaut) #5

Hi,
When I do a curl in the Linux terminal, I get :
[1] 6406

I'm new, so, I guess this means that the URL is correctly fetched, no?
This means that the problem isn't in the URL itself but in the Logstash config file, no?


(Magnus Bäck) #6

I'm new, so, I guess this means that the URL is correctly fetched, no?

No, it looks like you backgrounded the curl process because the URL contains &.


(system) #7

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.