How to update a document in elasticsearch with logstash http output plugin

Elastic Stack Logstash
1

Hi all....
Here i am trying to update a document in elasticsearch with the help of output logstash http plugin
here is my logstash conf

input { 
  elasticsearch { 
    hosts => "192.168.1.59:9233" 
    index => "titanic_kfk_shrink" 
    query => '{ "query":{ "match" : {            "PassengerId" : "200"        } }, "sort": [ "_doc" ] }' 
    size => 5000 
    scroll => "5m"
    docinfo => true 
   docinfo_target => "@metadata" 
} 
 }

output {
stdout { codec => rubydebug { metadata => true } }
http {
       format => "json"
       http_method => "post"
       headers => ["X-My-Header","Content-Type: application/json"]
#       headers => [ "X-My-Header", "Content-Type: application/json"]
       url => "http://192.168.1.59:9233/%{[@metadata][_index]}/%{[@metadata][_type]}/%{[@metadata][_id]}/_update -d' { "doc" : { "Name" : "Jagan B" } }'"
#mapping => {{ "doc" : { "Name" : "Jagan B" } }}
#mapping => { \"doc\" : { \"Name\" : \"Jagan B\" } }

    }
}

but it throws an error like this

[2017-11-16T23:34:53,681][ERROR][logstash.agent           ] Cannot create pipeline {:reason=>"Expected one of #, {, } at line 26, column 121 (byte 1220) after output {\nstdout { codec => rubydebug { metadata => true } }\nhttp {\n       format => \"json\"\n       http_method => \"post\"\n       headers => [\"X-My-Header\",\"Content-Type: application/json\"]\n#       headers => [ \"X-My-Header\", \"Content-Type: application/json\"]\n       url => \"http://192.168.1.59:9233/%{[@metadata][_index]}/%{[@metadata][_type]}/%{[@metadata][_id]}/_update -d' { \"", :backtrace=>["/home/sgpl/logstash-5.4.0/logstash-core/lib/logstash/pipeline.rb:50:in `initialize'", "/home/sgpl/logstash-5.4.0/logstash-core/lib/logstash/pipeline.rb:145:in `initialize'", "/home/sgpl/logstash-5.4.0/logstash-core/lib/logstash/agent.rb:286:in `create_pipeline'", "/home/sgpl/logstash-5.4.0/logstash-core/lib/logstash/agent.rb:95:in `register_pipeline'", "/home/sgpl/logstash-5.4.0/logstash-core/lib/logstash/runner.rb:274:in `execute'", "/home/sgpl/logstash-5.4.0/vendor/bundle/jruby/1.9/gems/clamp-0.6.5/lib/clamp/command.rb:67:in `run'", "/home/sgpl/logstash-5.4.0/logstash-core/lib/logstash/runner.rb:185:in `run'", "/home/sgpl/logstash-5.4.0/vendor/bundle/jruby/1.9/gems/clamp-0.6.5/lib/clamp/command.rb:132:in `run'", "/home/sgpl/logstash-5.4.0/lib/bootstrap/environment.rb:71:in `(root)'"]}

after that i have escaped double quotes with \ in url tag
then i have got the following error

[2017-11-16T23:37:58,897][ERROR][logstash.outputs.http    ] Error in http output loop {:class=>"Java::JavaNet::URISyntaxException", :message=>"Illegal character in path at index 59: http://192.168.1.59:9233/titanic_kfk_shrink/doc/200/_update -d' { \\\"doc\\\" : { \\\"Name\\\" : \\\"Jagan B\\\" } }'", :backtrace=>["java.net.URI$Parser.fail(java/net/URI.java:2848)", "java.net.URI$Parser.checkChars(java/net/URI.java:3021)", "java.net.URI$Parser.parseHierarchical(java/net/URI.java:3105)", "java.net.URI$Parser.parse(java/net/URI.java:3053)", "java.net.URI.<init>(java/net/URI.java:588)", "org.apache.http.client.utils.URIBuilder.<init>(org/apache/http/client/utils/URIBuilder.java:82)", "java.lang.reflect.Constructor.newInstance(java/lang/reflect/Constructor.java:422)", "RUBY.uri_from_url_and_options(/home/sgpl/logstash-5.4.0/vendor/bundle/jruby/1.9/gems/manticore-0.6.1-java/lib/manticore/client.rb:450)", "RUBY.request_from_options(/home/sgpl/logstash-5.4.0/vendor/bundle/jruby/1.9/gems/manticore-0.6.1-java/lib/manticore/client.rb:457)", "RUBY.request(/home/sgpl/logstash-5.4.0/vendor/bundle/jruby/1.9/gems/manticore-0.6.1-java/lib/manticore/client.rb:418)", "RUBY.post(/home/sgpl/logstash-5.4.0/vendor/bundle/jruby/1.9/gems/manticore-0.6.1-java/lib/manticore/client.rb:265)", "RUBY.post(/home/sgpl/logstash-5.4.0/vendor/bundle/jruby/1.9/gems/manticore-0.6.1-java/lib/manticore/client/proxies.rb:53)", "RUBY.send_event(/home/sgpl/logstash-5.4.0/vendor/bundle/jruby/1.9/gems/logstash-output-http-5.1.1/lib/logstash/outputs/http.rb:210)", "RUBY.send_events(/home/sgpl/logstash-5.4.0/vendor/bundle/jruby/1.9/gems/logstash-output-http-5.1.1/lib/logstash/outputs/http.rb:145)", "RUBY.multi_receive(/home/sgpl/logstash-5.4.0/vendor/bundle/jruby/1.9/gems/logstash-output-http-5.1.1/lib/logstash/outputs/http.rb:116)", "RUBY.multi_receive(/home/sgpl/logstash-5.4.0/logstash-core/lib/logstash/output_delegator_strategies/shared.rb:13)", "RUBY.multi_receive(/home/sgpl/logstash-5.4.0/logstash-core/lib/logstash/output_delegator.rb:47)", "RUBY.output_batch(/home/sgpl/logstash-5.4.0/logstash-core/lib/logstash/pipeline.rb:407)", "org.jruby.RubyHash.each(org/jruby/RubyHash.java:1342)", "RUBY.output_batch(/home/sgpl/logstash-5.4.0/logstash-core/lib/logstash/pipeline.rb:406)", "RUBY.worker_loop(/home/sgpl/logstash-5.4.0/logstash-core/lib/logstash/pipeline.rb:352)", "RUBY.start_workers(/home/sgpl/logstash-5.4.0/logstash-core/lib/logstash/pipeline.rb:317)", "java.lang.Thread.run(java/lang/Thread.java:745)"]}
Exception in thread "[main]>worker0" java.net.URISyntaxException: Illegal character in path at index 59: http://192.168.1.59:9233/titanic_kfk_shrink/doc/200/_update -d' { \"doc\" : { \"Name\" : \"Jagan B\" } }'
	at java.net.URI$Parser.fail(java/net/URI.java:2848)
	at java.net.URI$Parser.checkChars(java/net/URI.java:3021)
	at java.net.URI$Parser.parseHierarchical(java/net/URI.java:3105)
	at java.net.URI$Parser.parse(java/net/URI.java:3053)
	at java.net.URI.<init>(java/net/URI.java:588)
	at org.apache.http.client.utils.URIBuilder.<init>(org/apache/http/client/utils/URIBuilder.java:82)
	at java.lang.reflect.Constructor.newInstance(java/lang/reflect/Constructor.java:422)
	at RUBY.uri_from_url_and_options(/home/sgpl/logstash-5.4.0/vendor/bundle/jruby/1.9/gems/manticore-0.6.1-java/lib/manticore/client.rb:450)

Please suggest me how can i update a field in elasticsearch with LS output http plugin.

Thank You

2

Uh, the -d <JSON stuff> syntax isn't valid in an HTTP request, it's part of a curl command. Perhaps you can use the message option of the http output to set the post body? Or maybe use an exec output and invoke curl?

3

Hi..
Now i have changed my logstash conf like this

input { 
  elasticsearch { 
    hosts => "192.168.1.59:9233" 
    index => "titanic_kfk_shrink" 
    query => '{ "query":{ "match" : {            "PassengerId" : "200"        } }, "sort": [ "_doc" ] }' 
    size => 5000 
    scroll => "5m"
    docinfo => true 
   docinfo_target => "@metadata" 
} 
 }

output {
stdout { codec => rubydebug { metadata => true } }
http {
       format => "json"
       http_method => "post"
       headers => ["X-My-Header","Content-Type: application/json"]
#       headers => [ "X-My-Header", "Content-Type: application/json"]
       url => "http://192.168.1.59:9233/%{[@metadata][_index]}/%{[@metadata][_type]}/%{[@metadata][_id]}/_update -d' { "doc" : { "Name" : "Jagan B" } }'"
#mapping => {{ "doc" : { "Name" : "Jagan B" } }}
#mapping => { \"doc\" : { \"Name\" : \"Jagan B\" } }
message => "{ \"doc\" : { \"Name\" : \"Jagan B\" } }"


    }
}

where i am getting following error

[2017-11-17T00:50:36,023][DEBUG][logstash.pipeline        ] Shutdown waiting for worker thread #<Thread:0x2483d800>
[2017-11-17T00:50:36,166][ERROR][logstash.outputs.http    ] [HTTP Output Failure] Encountered non-2xx HTTP code 400 {:response_code=>400, :url=>"http://192.168.1.59:9233/titanic_kfk_shrink/doc/200/_update", :event=>2017-10-21T05:37:13.114Z elasticsearch_masternode 2, :will_retry=>false}Henriette (""Mrs Harbeck"")",female,24,0,0,248747,13,,S
4

I don't think the escaped double quotes in the message option turn out as expected. Try this instead:

message => '{"doc" : { "Name" : "Jagan B" } }'

If that also fails, use Wireshark or a similar tool to inspect the network traffic.

5

Hi i got same result again like this

[2017-11-17T01:25:10,090][ERROR][logstash.outputs.http    ] [HTTP Output Failure] Encountered non-2xx HTTP code 400 {:response_code=>400, :url=>"http://192.168.1.59:9233/titanic_kfk_shrink/doc/200/_update", :event=>2017-10-21T05:37:13.114Z elasticsearch_masternode 2, :will_retry=>false}Henriette (""Mrs Harbeck"")",female,24,0,0,248747,13,,S
[2017-11-17T01:25:10,098][DEBUG][logstash.filters.mutate  ] closing {:plugin=>"LogStash::Filters::Mutate"}
[2017-11-17T01:25:10,098][DEBUG][logstash.outputs.stdout  ] closing {:plugin=>"LogStash::Outputs::Stdout"}

Right now i have no idea on Wireshark
So now i'll look into Wireshark and then i'll update you

Thank You

6

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.