Hey guys, all right?
I need help with an issue I've been experiencing when collecting data using jdbc_streaming, running conf locally through docker the data collection by logstash is performed successfully, but when applying conf in the cloud service the following error is returned:
Blockquote [2021-11-05T14:47:47,945][ERROR][logstash.javapipeline ][any_index] Pipeline worker error, the pipeline will be stopped {:pipeline_id=>"any_index", :error=>"Missing Converter handling for full class name=org.jruby.ext.date.RubyDateTime, simple name=RubyDateTime", :exception=>Java::OrgLogstash::MissingConverterException, :backtrace=>["org.logstash.Valuefier.fallbackConvert(Valuefier.java:118)", "org.logstash.Valuefier.convert(Valuefier.java:96)", "org.logstash.ConvertedMap$1.visit(ConvertedMap.java:55)", "org.logstash.ConvertedMap$1.visit(ConvertedMap.java:49)", "org.jruby.RubyHash.visitLimited(RubyHash.java:698)", "org.jruby.RubyHash.visitAll(RubyHash.java:683)", "org.logstash.ConvertedMap.newFromRubyHash(ConvertedMap.java:89)", "org.logstash.ConvertedMap.newFromRubyHash(ConvertedMap.java:84)", "org.logstash.Valuefier.lambda$initConverters$12(Valuefier.java:171)", "org.logstash.Valuefier.convert(Valuefier.java:94)", "org.logstash.ConvertedList.newFromRubyArray(ConvertedList.java:64)", "org.logstash.Valuefier.lambda$initConverters$16(Valuefier.java:183)", "org.logstash.Valuefier.convert(Valuefier.java:94)", "org.logstash.ext.JrubyEventExtLibrary$RubyEvent.safeValueifierConvert(JrubyEventExtLibrary.java:355)", "org.logstash.ext.JrubyEventExtLibrary$RubyEvent.ruby_set_field(JrubyEventExtLibrary.java:121)", "org.logstash.ext.JrubyEventExtLibrary$RubyEvent$INVOKER$i$2$0$ruby_set_field.call(JrubyEventExtLibrary$RubyEvent$INVOKER$i$2$0$ruby_set_field.gen)", "org.jruby.internal.runtime.methods.JavaMethod$JavaMethodN.call(JavaMethod.java:835)", "org.jruby.ir.targets.InvokeSite.invoke(InvokeSite.java:207)", "usr.share.logstash.vendor.bundle.jruby.$2_dot_5_dot_0.gems.logstash_minus_integration_minus_jdbc_minus_5_dot_0_dot_6.lib.logstash.filters.jdbc_streaming.RUBY$method$process_event$0(/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-integration-jdbc-5.0.6/lib/logstash/filters/jdbc_streaming.rb:163)", "usr.share.logstash.vendor.bundle.jruby.$2_dot_5_dot_0.gems.logstash_minus_integration_minus_jdbc_minus_5_dot_0_dot_6.lib.logstash.filters.jdbc_streaming.RUBY$method$process_event$0$VARARGS(/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-integration-jdbc-5.0.6/lib/logstash/filters/jdbc_streaming.rb)", "org.jruby.internal.runtime.methods.CompiledIRMethod.call(CompiledIRMethod.java:80)", "org.jruby.internal.runtime.methods.MixedModeIRMethod.call(MixedModeIRMethod.java:70)", "org.jruby.ir.targets.InvokeSite.invoke(InvokeSite.java:207)", "usr.share.logstash.vendor.bundle.jruby.$2_dot_5_dot_0.gems.logstash_minus_integration_minus_jdbc_minus_5_dot_0_dot_6.lib.logstash.filters.jdbc_streaming.RUBY$method$filter$0(/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-integration-jdbc-5.0.6/lib/logstash/filters/jdbc_streaming.rb:136)", "usr.share.logstash.vendor.bundle.jruby.$2_dot_5_dot_0.gems.logstash_minus_integration_minus_jdbc_minus_5_dot_0_dot_6.lib.logstash.filters.jdbc_streaming.RUBY$method$filter$0$VARARGS(/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-integration-jdbc-5.0.6/lib/logstash/filters/jdbc_streaming.rb)", "org.jruby.internal.runtime.methods.CompiledIRMethod.call(CompiledIRMethod.java:80)", "org.jruby.internal.runtime.methods.MixedModeIRMethod.call(MixedModeIRMethod.java:70)", "org.jruby.ir.targets.InvokeSite.invoke(InvokeSite.java:207)", "usr.share.logstash.logstash_minus_core.lib.logstash.filters.base.RUBY$method$do_filter$0(/usr/share/logstash/logstash-core/lib/logstash/filters/base.rb:159)", "usr.share.logstash.logstash_minus_core.lib.logstash.filters.base.RUBY$method$do_filter$0$VARARGS(/usr/share/logstash/logstash-core/lib/logstash/filters/base.rb)", "org.jruby.internal.runtime.methods.CompiledIRMethod.call(CompiledIRMethod.java:80)", "org.jruby.internal.runtime.methods.MixedModeIRMethod.call(MixedModeIRMethod.java:70)", "org.jruby.ir.targets.InvokeSite.invoke(InvokeSite.java:197)", "usr.share.logstash.logstash_minus_core.lib.logstash.filters.base.RUBY$block$multi_filter$1(/usr/share/logstash/logstash-core/lib/logstash/filters/base.rb:178)", "org.jruby.runtime.CompiledIRBlockBody.yieldDirect(CompiledIRBlockBody.java:148)", "org.jruby.runtime.BlockBody.yield(BlockBody.java:106)", "org.jruby.runtime.Block.yield(Block.java:184)", "org.jruby.RubyArray.each(RubyArray.java:1809)", "usr.share.logstash.logstash_minus_core.lib.logstash.filters.base.RUBY$method$multi_filter$0(/usr/share/logstash/logstash-core/lib/logstash/filters/base.rb:175)", "org.jruby.internal.runtime.methods.CompiledIRMethod.call(CompiledIRMethod.java:106)", "org.jruby.internal.runtime.methods.MixedModeIRMethod.call(MixedModeIRMethod.java:140)", "org.jruby.internal.runtime.methods.DynamicMethod.call(DynamicMethod.java:200)", "org.logstash.config.ir.compiler.FilterDelegatorExt.doMultiFilter(FilterDelegatorExt.java:127)", "org.logstash.config.ir.compiler.AbstractFilterDelegatorExt.multiFilter(AbstractFilterDelegatorExt.java:134)", "org.logstash.generated.CompiledDataset3.compute(Unknown Source)", "org.logstash.generated.CompiledDataset3.compute(Unknown Source)", "org.logstash.generated.CompiledDataset3.compute(Unknown Source)", "org.logstash.generated.CompiledDataset7.compute(Unknown Source)", "org.logstash.generated.CompiledDataset3.compute(Unknown Source)", "org.logstash.generated.CompiledDataset5.compute(Unknown Source)", "org.logstash.generated.CompiledDataset3.compute(Unknown Source)", "org.logstash.config.ir.CompiledPipeline$CompiledUnorderedExecution.compute(CompiledPipeline.java:329)", "org.logstash.config.ir.CompiledPipeline$CompiledUnorderedExecution.compute(CompiledPipeline.java:323)", "org.logstash.execution.WorkerLoop.run(WorkerLoop.java:83)", "java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)", "java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)", "java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)", "java.base/java.lang.reflect.Method.invoke(Method.java:566)", "org.jruby.javasupport.JavaMethod.invokeDirectWithExceptionHandling(JavaMethod.java:441)", "org.jruby.javasupport.JavaMethod.invokeDirect(JavaMethod.java:305)", "org.jruby.java.invokers.InstanceMethodInvoker.call(InstanceMethodInvoker.java:32)", "usr.share.logstash.logstash_minus_core.lib.logstash.java_pipeline.RUBY$block$start_workers$5(/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:295)", "org.jruby.runtime.CompiledIRBlockBody.callDirect(CompiledIRBlockBody.java:138)", "org.jruby.runtime.IRBlockBody.call(IRBlockBody.java:58)", "org.jruby.runtime.IRBlockBody.call(IRBlockBody.java:52)", "org.jruby.runtime.Block.call(Block.java:139)", "org.jruby.RubyProc.call(RubyProc.java:318)", "org.jruby.internal.runtime.RubyRunnable.run(RubyRunnable.java:105)", "java.base/java.lang.Thread.run(Thread.java:834)"], :thread=>"#<Thread:0x11dec9f4 sleep>"}
[2021-11-05T14:47:48,022][WARN ][logstash.javapipeline ][any_index] Waiting for input plugin to close {:pipeline_id=>"any_index", :thread=>"#<Thread:0x11dec9f4 run>"}
conf:
input {
jdbc {
jdbc_driver_library => "${LOGSTASH_CONFIG_PATH}/jconn4.jar"
jdbc_driver_class => "com.sybase.jdbc4.jdbc.SybDriver"
jdbc_connection_string => "${DB_CONNECTION_BCO_STRING}"
jdbc_user => "${DB_USER}"
jdbc_password => "${DB_PASS}"
schedule => "0 3 * * 1-5"
statement => "select * from dbx.dbo.Tb1"
}
}
filter {
mutate {
add_field => {"param" => "%{param1}"}
add_field => {"anyParam" => "%{param2}"}
add_field => {"dt_apuracao" => "%{dtantesvirada}"}
}
jdbc_streaming {
jdbc_driver_library => "${LOGSTASH_CONFIG_PATH}/jconn4.jar"
jdbc_driver_class => "com.sybase.jdbc4.jdbc.SybDriver"
jdbc_connection_string => "${DB_CONNECTION_SPB_STRING}"
jdbc_user => "${DB_USER}"
jdbc_password => "${DB_PASS}"
parameters => {"ini_range" => "param" "fim_range" => "anyParam" "dt_apuracao" => "dt_apuracao"}
statement => "
SET NOCOUNT ON
Declare @IniRange numeric, @FimRange numeric, @Datah varchar(100)
BEGIN
SET @IniRange = CONVERT(numeric, :ini_range)
SET @FimRange = CONVERT(numeric, :fim_range)
select * from dbx.dbo.Tb2 ths WHERE ths.NumParam BETWEEN @IniRange and @FimRange
and ths.AnyDate = :dt_apuracao
END
"
use_cache => false
periodic_flush => true
target => "results"
}
split { field => "results" }
ruby {
code =>'
event.get("results").each {|k,v|
if v.respond_to?(:strftime)
event.set(k, LogStash::Timestamp.new(v))
else
event.set(k,v)
end
}
'
}
}
output {
stdout { codec => rubydebug { metadata => true }}
elasticsearch {
index => "any_index"
hosts => ["${ELASTIC_HOST}"]
user => ["${ELASTIC_USER}"]
password => ["${ELASTIC_PASS}"]
document_id => "%{[@metadata][_id]}"
ssl_certificate_verification => true
cacert => "${LOGSTASH_CONFIG_PATH}/ca.crt"
}
}
I need a lot of help, I've been in this activity for days
Blockquote note 1: The ruby code has been added to handle this error, but conf continues to fail.
note 2: The version of the stack I use to run locally is the same as the one running in the cloud