Hello,
Can some one help with the below error
[2020-09-14T19:28:25,643][WARN ][logstash.config.source.multilocal] Ignoring the 'pipelines.yml' file because modules or command line options are specified
[2020-09-14T19:28:25,760][INFO ][logstash.runner ] Starting Logstash {"logstash.version"=>"7.7.0"}
[2020-09-14T19:28:27,858][INFO ][org.reflections.Reflections] Reflections took 44 ms to scan 1 urls, producing 21 keys and 41 values
[2020-09-14T19:28:29,917][ERROR][logstash.plugins.registry] Problems loading a plugin with {:type=>"input", :name=>"azureblob", :path=>"logstash/inputs/azureblob", :error_message=>"no implicit conversion of nil into String", :error_class=>TypeError, :error_backtrace=>["org/jruby/RubyKernel.java:976:in require'", "C:/logstash-7.7.0/vendor/bundle/jruby/2.5.0/gems/polyglot-0.3.5/lib/polyglot.rb:65:in require'", "C:/logstash-7.7.0/vendor/bundle/jruby/2.5.0/gems/logstash-input-azureblob-0.9.13-java/lib/com/microsoft/json-parser.rb:3:in <main>'", "org/jruby/RubyKernel.java:974:in require'", "C:/logstash-7.7.0/vendor/bundle/jruby/2.5.0/gems/polyglot-0.3.5/lib/polyglot.rb:65:in require'", "C:/logstash-7.7.0/vendor/bundle/jruby/2.5.0/gems/logstash-input-azureblob-0.9.13-java/lib/logstash/inputs/azureblob.rb:10:in '", "org/jruby/RubyKernel.java:974:in require'", "C:/logstash-7.7.0/vendor/bundle/jruby/2.5.0/gems/polyglot-0.3.5/lib/polyglot.rb:65:in require'", "C:/logstash-7.7.0/logstash-core/lib/logstash/plugins/registry.rb:207:in legacy_lookup'", "C:/logstash-7.7.0/logstash-core/lib/logstash/plugins/registry.rb:182:in block in lookup'", "org/jruby/ext/thread/Mutex.java:164:in synchronize'", "C:/logstash-7.7.0/logstash-core/lib/logstash/plugins/registry.rb:178:in lookup'", "C:/logstash-7.7.0/logstash-core/lib/logstash/plugins/registry.rb:232:in lookup_pipeline_plugin'", "C:/logstash-7.7.0/logstash-core/lib/logstash/plugin.rb:160:in lookup'", "org/logstash/plugins/PluginFactoryExt.java:224:in plugin'", "org/logstash/execution/JavaBasePipelineExt.java:80:in initialize'", "C:/logstash-7.7.0/logstash-core/lib/logstash/java_pipeline.rb:43:in initialize'", "C:/logstash-7.7.0/logstash-core/lib/logstash/pipeline_action/create.rb:52:in execute'", "C:/logstash-7.7.0/logstash-core/lib/logstash/agent.rb:342:in block in converge_state'"]} [2020-09-14T19:28:29,932][ERROR][logstash.agent ] Failed to execute action {:action=>LogStash::PipelineAction::Create/pipeline_id:main, :exception=>"Java::JavaLang::IllegalStateException", :message=>"Unable to configure plugins: (PluginLoadingError) Couldn't find any input plugin named 'azureblob'. Are you sure this is correct? Trying to load the azureblob input plugin resulted in this error: Problems loading the requested plugin named azureblob of type input. Error: TypeError no implicit conversion of nil into String", :backtrace=>["org.logstash.config.ir.CompiledPipeline.<init>(CompiledPipeline.java:126)", "org.logstash.execution.JavaBasePipelineExt.initialize(JavaBasePipelineExt.java:80)", "org.logstash.execution.JavaBasePipelineExt$INVOKER$i$1$0$initialize.call(JavaBasePipelineExt$INVOKER$i$1$0$initialize.gen)", "org.jruby.internal.runtime.methods.JavaMethod$JavaMethodN.call(JavaMethod.java:837)", "org.jruby.ir.runtime.IRRuntimeHelpers.instanceSuper(IRRuntimeHelpers.java:1169)", "org.jruby.ir.runtime.IRRuntimeHelpers.instanceSuperSplatArgs(IRRuntimeHelpers.java:1156)", "org.jruby.ir.targets.InstanceSuperInvokeSite.invoke(InstanceSuperInvokeSite.java:39)", "C_3a_.logstash_minus_7_dot_7_dot_0.logstash_minus_core.lib.logstash.java_pipeline.RUBY$method$initialize$0(C:/logstash-7.7.0/logstash-core/lib/logstash/java_pipeline.rb:43)", "org.jruby.internal.runtime.methods.CompiledIRMethod.call(CompiledIRMethod.java:82)", "org.jruby.internal.runtime.methods.MixedModeIRMethod.call(MixedModeIRMethod.java:70)", "org.jruby.runtime.callsite.CachingCallSite.cacheAndCall(CachingCallSite.java:332)", "org.jruby.runtime.callsite.CachingCallSite.call(CachingCallSite.java:86)", "org.jruby.RubyClass.newInstance(RubyClass.java:939)", "org.jruby.RubyClass$INVOKER$i$newInstance.call(RubyClass$INVOKER$i$newInstance.gen)", "org.jruby.ir.targets.InvokeSite.invoke(InvokeSite.java:207)", "C_3a_.logstash_minus_7_dot_7_dot_0.logstash_minus_core.lib.logstash.pipeline_action.create.RUBY$method$execute$0(C:/logstash-7.7.0/logstash-core/lib/logstash/pipeline_action/create.rb:52)", "C_3a_.logstash_minus_7_dot_7_dot_0.logstash_minus_core.lib.logstash.pipeline_action.create.RUBY$method$execute$0$__VARARGS__(C:/logstash-7.7.0/logstash-core/lib/logstash/pipeline_action/create.rb)", "org.jruby.internal.runtime.methods.CompiledIRMethod.call(CompiledIRMethod.java:82)", "org.jruby.internal.runtime.methods.MixedModeIRMethod.call(MixedModeIRMethod.java:70)", "org.jruby.ir.targets.InvokeSite.invoke(InvokeSite.java:207)", "C_3a_.logstash_minus_7_dot_7_dot_0.logstash_minus_core.lib.logstash.agent.RUBY$block$converge_state$2(C:/logstash-7.7.0/logstash-core/lib/logstash/agent.rb:342)", "org.jruby.runtime.CompiledIRBlockBody.callDirect(CompiledIRBlockBody.java:138)", "org.jruby.runtime.IRBlockBody.call(IRBlockBody.java:58)", "org.jruby.runtime.IRBlockBody.call(IRBlockBody.java:52)", "org.jruby.runtime.Block.call(Block.java:139)", "org.jruby.RubyProc.call(RubyProc.java:318)", "org.jruby.internal.runtime.RubyRunnable.run(RubyRunnable.java:105)", "java.base/java.lang.Thread.run(Thread.java:832)"]} warning: thread "Converge PipelineAction::Create<main>" terminated with exception (report_on_exception is true): LogStash::Error: Don't know how to handle Java::JavaLang::IllegalStateExceptionforPipelineAction::Create create at org/logstash/execution/ConvergeResultExt.java:129 add at org/logstash/execution/ConvergeResultExt.java:57 converge_state at C:/logstash-7.7.0/logstash-core/lib/logstash/agent.rb:355 [2020-09-14T19:28:29,943][ERROR][logstash.agent ] An exception happened when converging configuration {:exception=>LogStash::Error, :message=>"Don't know how to handleJava::JavaLang::IllegalStateExceptionforPipelineAction::Create", :backtrace=>["org/logstash/execution/ConvergeResultExt.java:129:in create'", "org/logstash/execution/ConvergeResultExt.java:57:in add'", "C:/logstash-7.7.0/logstash-core/lib/logstash/agent.rb:355:in block in converge_state'"]}
[2020-09-14T19:28:30,009][FATAL][logstash.runner ] An unexpected error occurred! {:error=>#<LogStash::Error: Don't know how to handle Java::JavaLang::IllegalStateException for PipelineAction::Create<main>>, :backtrace=>["org/logstash/execution/ConvergeResultExt.java:129:in create'", "org/logstash/execution/ConvergeResultExt.java:57:in add'", "C:/logstash-7.7.0/logstash-core/lib/logstash/agent.rb:355:in `block in converge_state'"]}
[2020-09-14T19:28:30,030][ERROR][org.logstash.Logstash ] java.lang.IllegalStateException: Logstash stopped processing because of an error: (SystemExit) exit
I'm using the below conf file. FYI, I use windows machine,
input {
azureblob
{
storage_account_name => "mystorageaccount"
storage_access_key => "VGhpcyBpcyBhIGZha2Uga2V5Lg=="
container => "insights-logs-networksecuritygroupflowevent"
codec => "json"
# Refer https://docs.microsoft.com/azure/network-watcher/network-watcher-read-nsg-flow-logs
# Typical numbers could be 21/9 or 12/2 depends on the nsg log file types
file_head_bytes => 12
file_tail_bytes => 2
# Enable / tweak these settings when event is too big for codec to handle.
# break_json_down_policy => "with_head_tail"
# break_json_batch_count => 2
}
}
filter {
split { field => "[records]" }
split { field => "[records][properties][flows]"}
split { field => "[records][properties][flows][flows]"}
split { field => "[records][properties][flows][flows][flowTuples]"}
mutate{
split => { "[records][resourceId]" => "/"}
add_field => {"Subscription" => "%{[records][resourceId][2]}"
"ResourceGroup" => "%{[records][resourceId][4]}"
"NetworkSecurityGroup" => "%{[records][resourceId][8]}"}
convert => {"Subscription" => "string"}
convert => {"ResourceGroup" => "string"}
convert => {"NetworkSecurityGroup" => "string"}
split => { "[records][properties][flows][flows][flowTuples]" => ","}
add_field => {
"unixtimestamp" => "%{[records][properties][flows][flows][flowTuples][0]}"
"srcIp" => "%{[records][properties][flows][flows][flowTuples][1]}"
"destIp" => "%{[records][properties][flows][flows][flowTuples][2]}"
"srcPort" => "%{[records][properties][flows][flows][flowTuples][3]}"
"destPort" => "%{[records][properties][flows][flows][flowTuples][4]}"
"protocol" => "%{[records][properties][flows][flows][flowTuples][5]}"
"trafficflow" => "%{[records][properties][flows][flows][flowTuples][6]}"
"traffic" => "%{[records][properties][flows][flows][flowTuples][7]}"
"flowstate" => "%{[records][properties][flows][flows][flowTuples][8]}"
"packetsSourceToDest" => "%{[records][properties][flows][flows][flowTuples][9]}"
"bytesSentSourceToDest" => "%{[records][properties][flows][flows][flowTuples][10]}"
"packetsDestToSource" => "%{[records][properties][flows][flows][flowTuples][11]}"
"bytesSentDestToSource" => "%{[records][properties][flows][flows][flowTuples][12]}"
}
convert => {"unixtimestamp" => "integer"}
convert => {"srcPort" => "integer"}
convert => {"destPort" => "integer"}
}
date{
match => ["unixtimestamp" , "UNIX"]
}
}
output {
stdout { codec => rubydebug }
elasticsearch {
hosts => "localhost"
index => "nsg-flow-logs"
}
}