Send data from Logstash to hadoop

Elastic Stack Logstash
1

hi can help me how can i send data to hadoop or hive using logstash this is my config file

input{
stdin{}

}

filter{
grok {
# patterns_dir => ["/etc/logstash/conf.d/patterns"]
match => { "message" => "%{TIMESTAMP_ISO8601:xxxxxxx},%{IP:client_xxx.xxx.xxx.xx},%{WORD:xxxxx}&&%{NUMBER:xxxxxxx:int},%{DATESTAMP:xxxxxxx},%{NUMBER:xxxxxxx:float},%{NUMBER:xxxxx:float},%{NUMBER:xxxx:int},%{NUMBER:xxxxxx:int}(:%{NUMBER:xxxxx:int})?,%{WORD:xxxx},%{NUMBER:xxxxxx:float},%{NUMBER:xxxxx:float},%{NUMBER:xxxxxx:float},%{NUMBER:xxxxxx:float},%{NUMBER:xxxx:int},%{NUMBER:xxxxxx:int},%{NUMBER:xxxxxxxx:int},%{NUMBER:xxxxxxx:float},%{NUMBER:xxxxxx:int},%{NUMBER:xxxxxx:int},%{NUMBER:xxxxx:int},%{NUMBER:xxxxxx:int}" }
}
}
output{
webhdfs{
host => "10.200.234.88"
port => 50070
user => "hduser"
path => "/tmp/logstash.log"
}
}

2

im getting this error in log file of logstash

[2018-05-31T15:32:43,149][INFO ][logstash.modules.scaffold] Initializing module {:module_name=>"fb_apache", :directory=>"/usr/share/logstash/modules/fb_apache/configuration"}
[2018-05-31T15:32:43,156][INFO ][logstash.modules.scaffold] Initializing module {:module_name=>"netflow", :directory=>"/usr/share/logstash/modules/netflow/configuration"}
[2018-05-31T15:32:43,891][ERROR][logstash.outputs.webhdfs ] Webhdfs check request failed. (namenode: 10.200.234.88:50070, Exception: Failed to connect to host 10.200.234.88:50070, Connection refused - Connection refused)
[2018-05-31T15:32:43,900][ERROR][logstash.pipeline ] Error registering plugin {:plugin=>"#<LogStash::OutputDelegator:0x3489f9a8 @namespaced_metric=#<LogStash::Instrument::NamespacedMetric:0x5ff48819 @metric=#<LogStash::Instrument::Metric:0x3c8a383e @collector=#<LogStash::Instrument::Collector:0xe5a4479 @agent=nil, @metric_store=#<LogStash::Instrument::MetricStore:0x75e5a8a3 @store=#<Concurrent::map:0x0000000006566c entries=2 default_proc=nil>, @structured_lookup_mutex=#Mutex:0xe2a5246, @fast_lookup=#<Concurrent::map:0x00000000065670 entries=57 default_proc=nil>>>>, @namespace_name=[:stats, :pipelines, :main, :plugins, :outputs, :"bf64a8297df974f5b72780197009d24db4775b71-3"]>, @metric=#<LogStash::Instrument::NamespacedMetric:0x3fa939f1 @metric=#<LogStash::Instrument::Metric:0x3c8a383e @collector=#<LogStash::Instrument::Collector:0xe5a4479 @agent=nil, @metric_store=#<LogStash::Instrument::MetricStore:0x75e5a8a3 @store=#<Concurrent::map:0x0000000006566c entries=2 default_proc=nil>, @structured_lookup_mutex=#Mutex:0xe2a5246, @fast_lookup=#<Concurrent::map:0x00000000065670 entries=57 default_proc=nil>>>>, @namespace_name=[:stats, :pipelines, :main, :plugins, :outputs]>, @logger=#<LogStash::Logging::Logger:0x3bb5f8d9 @logger=#Java::OrgApacheLoggingLog4jCore::Logger:0x251b1058>, @out_counter=LogStash::Instrument::MetricType::Counter - namespaces: [:stats, :pipelines, :main, :plugins, :outputs, :"bf64a8297df974f5b72780197009d24db4775b71-3", :events] key: out value: 0, @in_counter=LogStash::Instrument::MetricType::Counter - namespaces: [:stats, :pipelines, :main, :plugins, :outputs, :"bf64a8297df974f5b72780197009d24db4775b71-3", :events] key: in value: 0, @strategy=#<LogStash::OutputDelegatorStrategies::Legacy:0x266076a1 @worker_count=1, @workers=[<LogStash::Outputs::WebHdfs host=>"10.200.234.88", user=>"hduser", path=>"/tmp/logstash.log", id=>"bf64a8297df974f5b72780197009d24db4775b71-3", enable_metric=>true, codec=><LogStash::Codecs::Line id=>"line_7416d2ae-6cb5-487f-b886-6a6fefce4a75", enable_metric=>true, charset=>"UTF-8", delimiter=>"\n">, workers=>1, port=>50070, standby_host=>false, standby_port=>50070, idle_flush_time=>1, flush_size=>500, open_timeout=>30, read_timeout=>30, use_httpfs=>false, single_file_per_thread=>false, retry_known_errors=>true, retry_interval=>0.5, retry_times=>5, compression=>"none", snappy_bufsize=>32768, snappy_format=>"stream", use_kerberos_auth=>false, use_ssl_auth=>false>], @worker_queue=#SizedQueue:0x703f45ee>, @id="bf64a8297df974f5b72780197009d24db4775b71-3", @time_metric=LogStash::Instrument::MetricType::Counter - namespaces: [:stats, :pipelines, :main, :plugins, :outputs, :"bf64a8297df974f5b72780197009d24db4775b71-3", :events] key: duration_in_millis value: 0, @metric_events=#<LogStash::Instrument::NamespacedMetric:0x4263c5ac @metric=#<LogStash::Instrument::Metric:0x3c8a383e @collector=#<LogStash::Instrument::Collector:0xe5a4479 @agent=nil, @metric_store=#<LogStash::Instrument::MetricStore:0x75e5a8a3 @store=#<Concurrent::map:0x0000000006566c entries=2 default_proc=nil>, @structured_lookup_mutex=#Mutex:0xe2a5246, @fast_lookup=#<Concurrent::map:0x00000000065670 entries=57 default_proc=nil>>>>, @namespace_name=[:stats, :pipelines, :main, :plugins, :outputs, :"bf64a8297df974f5b72780197009d24db4775b71-3", :events]>, @output_class=LogStash::Outputs::WebHdfs>", :error=>"Failed to connect to host 10.200.234.88:50070, Connection refused - Connection refused"}
[2018-05-31T15:32:43,904][ERROR][logstash.agent ] Pipeline aborted due to error {:exception=>#<WebHDFS::ServerError: Failed to connect to host 10.200.234.88:50070, Connection refused - Connection refused>, :backtrace=>["/usr/share/logstash/vendor/bundle/jruby/1.9/gems/webhdfs-0.8.0/lib/webhdfs/client_v1.rb:351:in request'", "/usr/share/logstash/vendor/bundle/jruby/1.9/gems/webhdfs-0.8.0/lib/webhdfs/client_v1.rb:275:inoperate_requests'", "/usr/share/logstash/vendor/bundle/jruby/1.9/gems/webhdfs-0.8.0/lib/webhdfs/client_v1.rb:138:in list'", "/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-output-webhdfs-3.0.6/lib/logstash/outputs/webhdfs_helper.rb:49:intest_client'", "/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-output-webhdfs-3.0.6/lib/logstash/outputs/webhdfs.rb:155:in register'", "org/jruby/RubyArray.java:1613:ineach'", "/usr/share/logstash/logstash-core/lib/logstash/output_delegator_strategies/legacy.rb:17:in register'", "/usr/share/logstash/logstash-core/lib/logstash/output_delegator.rb:43:inregister'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:290:in register_plugin'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:301:inregister_plugins'", "org/jruby/RubyArray.java:1613:in each'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:301:inregister_plugins'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:310:in start_workers'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:235:inrun'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:408:in `start_pipeline'"]}

3

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.