Missing Converter handling for full class name=org.jruby.ext.date.RubyDateTime

Hey guys, all right?

I need help with an issue I've been experiencing when collecting data using jdbc_streaming, running conf locally through docker the data collection by logstash is performed successfully, but when applying conf in the cloud service the following error is returned:

Blockquote [2021-11-05T14:47:47,945][ERROR][logstash.javapipeline ][any_index] Pipeline worker error, the pipeline will be stopped {:pipeline_id=>"any_index", :error=>"Missing Converter handling for full class name=org.jruby.ext.date.RubyDateTime, simple name=RubyDateTime", :exception=>Java::OrgLogstash::MissingConverterException, :backtrace=>["org.logstash.Valuefier.fallbackConvert(Valuefier.java:118)", "org.logstash.Valuefier.convert(Valuefier.java:96)", "org.logstash.ConvertedMap$1.visit(ConvertedMap.java:55)", "org.logstash.ConvertedMap$1.visit(ConvertedMap.java:49)", "org.jruby.RubyHash.visitLimited(RubyHash.java:698)", "org.jruby.RubyHash.visitAll(RubyHash.java:683)", "org.logstash.ConvertedMap.newFromRubyHash(ConvertedMap.java:89)", "org.logstash.ConvertedMap.newFromRubyHash(ConvertedMap.java:84)", "org.logstash.Valuefier.lambda$initConverters$12(Valuefier.java:171)", "org.logstash.Valuefier.convert(Valuefier.java:94)", "org.logstash.ConvertedList.newFromRubyArray(ConvertedList.java:64)", "org.logstash.Valuefier.lambda$initConverters$16(Valuefier.java:183)", "org.logstash.Valuefier.convert(Valuefier.java:94)", "org.logstash.ext.JrubyEventExtLibrary$RubyEvent.safeValueifierConvert(JrubyEventExtLibrary.java:355)", "org.logstash.ext.JrubyEventExtLibrary$RubyEvent.ruby_set_field(JrubyEventExtLibrary.java:121)", "org.logstash.ext.JrubyEventExtLibrary$RubyEvent$INVOKER$i$2$0$ruby_set_field.call(JrubyEventExtLibrary$RubyEvent$INVOKER$i$2$0$ruby_set_field.gen)", "org.jruby.internal.runtime.methods.JavaMethod$JavaMethodN.call(JavaMethod.java:835)", "org.jruby.ir.targets.InvokeSite.invoke(InvokeSite.java:207)", "usr.share.logstash.vendor.bundle.jruby.$2_dot_5_dot_0.gems.logstash_minus_integration_minus_jdbc_minus_5_dot_0_dot_6.lib.logstash.filters.jdbc_streaming.RUBY$method$process_event$0(/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-integration-jdbc-5.0.6/lib/logstash/filters/jdbc_streaming.rb:163)", "usr.share.logstash.vendor.bundle.jruby.$2_dot_5_dot_0.gems.logstash_minus_integration_minus_jdbc_minus_5_dot_0_dot_6.lib.logstash.filters.jdbc_streaming.RUBY$method$process_event$0$VARARGS(/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-integration-jdbc-5.0.6/lib/logstash/filters/jdbc_streaming.rb)", "org.jruby.internal.runtime.methods.CompiledIRMethod.call(CompiledIRMethod.java:80)", "org.jruby.internal.runtime.methods.MixedModeIRMethod.call(MixedModeIRMethod.java:70)", "org.jruby.ir.targets.InvokeSite.invoke(InvokeSite.java:207)", "usr.share.logstash.vendor.bundle.jruby.$2_dot_5_dot_0.gems.logstash_minus_integration_minus_jdbc_minus_5_dot_0_dot_6.lib.logstash.filters.jdbc_streaming.RUBY$method$filter$0(/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-integration-jdbc-5.0.6/lib/logstash/filters/jdbc_streaming.rb:136)", "usr.share.logstash.vendor.bundle.jruby.$2_dot_5_dot_0.gems.logstash_minus_integration_minus_jdbc_minus_5_dot_0_dot_6.lib.logstash.filters.jdbc_streaming.RUBY$method$filter$0$VARARGS(/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-integration-jdbc-5.0.6/lib/logstash/filters/jdbc_streaming.rb)", "org.jruby.internal.runtime.methods.CompiledIRMethod.call(CompiledIRMethod.java:80)", "org.jruby.internal.runtime.methods.MixedModeIRMethod.call(MixedModeIRMethod.java:70)", "org.jruby.ir.targets.InvokeSite.invoke(InvokeSite.java:207)", "usr.share.logstash.logstash_minus_core.lib.logstash.filters.base.RUBY$method$do_filter$0(/usr/share/logstash/logstash-core/lib/logstash/filters/base.rb:159)", "usr.share.logstash.logstash_minus_core.lib.logstash.filters.base.RUBY$method$do_filter$0$VARARGS(/usr/share/logstash/logstash-core/lib/logstash/filters/base.rb)", "org.jruby.internal.runtime.methods.CompiledIRMethod.call(CompiledIRMethod.java:80)", "org.jruby.internal.runtime.methods.MixedModeIRMethod.call(MixedModeIRMethod.java:70)", "org.jruby.ir.targets.InvokeSite.invoke(InvokeSite.java:197)", "usr.share.logstash.logstash_minus_core.lib.logstash.filters.base.RUBY$block$multi_filter$1(/usr/share/logstash/logstash-core/lib/logstash/filters/base.rb:178)", "org.jruby.runtime.CompiledIRBlockBody.yieldDirect(CompiledIRBlockBody.java:148)", "org.jruby.runtime.BlockBody.yield(BlockBody.java:106)", "org.jruby.runtime.Block.yield(Block.java:184)", "org.jruby.RubyArray.each(RubyArray.java:1809)", "usr.share.logstash.logstash_minus_core.lib.logstash.filters.base.RUBY$method$multi_filter$0(/usr/share/logstash/logstash-core/lib/logstash/filters/base.rb:175)", "org.jruby.internal.runtime.methods.CompiledIRMethod.call(CompiledIRMethod.java:106)", "org.jruby.internal.runtime.methods.MixedModeIRMethod.call(MixedModeIRMethod.java:140)", "org.jruby.internal.runtime.methods.DynamicMethod.call(DynamicMethod.java:200)", "org.logstash.config.ir.compiler.FilterDelegatorExt.doMultiFilter(FilterDelegatorExt.java:127)", "org.logstash.config.ir.compiler.AbstractFilterDelegatorExt.multiFilter(AbstractFilterDelegatorExt.java:134)", "org.logstash.generated.CompiledDataset3.compute(Unknown Source)", "org.logstash.generated.CompiledDataset3.compute(Unknown Source)", "org.logstash.generated.CompiledDataset3.compute(Unknown Source)", "org.logstash.generated.CompiledDataset7.compute(Unknown Source)", "org.logstash.generated.CompiledDataset3.compute(Unknown Source)", "org.logstash.generated.CompiledDataset5.compute(Unknown Source)", "org.logstash.generated.CompiledDataset3.compute(Unknown Source)", "org.logstash.config.ir.CompiledPipeline$CompiledUnorderedExecution.compute(CompiledPipeline.java:329)", "org.logstash.config.ir.CompiledPipeline$CompiledUnorderedExecution.compute(CompiledPipeline.java:323)", "org.logstash.execution.WorkerLoop.run(WorkerLoop.java:83)", "java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)", "java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)", "java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)", "java.base/java.lang.reflect.Method.invoke(Method.java:566)", "org.jruby.javasupport.JavaMethod.invokeDirectWithExceptionHandling(JavaMethod.java:441)", "org.jruby.javasupport.JavaMethod.invokeDirect(JavaMethod.java:305)", "org.jruby.java.invokers.InstanceMethodInvoker.call(InstanceMethodInvoker.java:32)", "usr.share.logstash.logstash_minus_core.lib.logstash.java_pipeline.RUBY$block$start_workers$5(/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:295)", "org.jruby.runtime.CompiledIRBlockBody.callDirect(CompiledIRBlockBody.java:138)", "org.jruby.runtime.IRBlockBody.call(IRBlockBody.java:58)", "org.jruby.runtime.IRBlockBody.call(IRBlockBody.java:52)", "org.jruby.runtime.Block.call(Block.java:139)", "org.jruby.RubyProc.call(RubyProc.java:318)", "org.jruby.internal.runtime.RubyRunnable.run(RubyRunnable.java:105)", "java.base/java.lang.Thread.run(Thread.java:834)"], :thread=>"#<Thread:0x11dec9f4 sleep>"}
[2021-11-05T14:47:48,022][WARN ][logstash.javapipeline ][any_index] Waiting for input plugin to close {:pipeline_id=>"any_index", :thread=>"#<Thread:0x11dec9f4 run>"}

conf:


input {

  jdbc {

    jdbc_driver_library => "${LOGSTASH_CONFIG_PATH}/jconn4.jar"

    jdbc_driver_class => "com.sybase.jdbc4.jdbc.SybDriver"

    jdbc_connection_string => "${DB_CONNECTION_BCO_STRING}"

    jdbc_user => "${DB_USER}"

    jdbc_password => "${DB_PASS}"

    schedule => "0 3 * * 1-5"

    statement => "select * from dbx.dbo.Tb1"

  }

}

filter {

  mutate {

    add_field => {"param" => "%{param1}"}

    add_field => {"anyParam" => "%{param2}"}

    add_field => {"dt_apuracao" => "%{dtantesvirada}"}

  }

  jdbc_streaming {

    jdbc_driver_library => "${LOGSTASH_CONFIG_PATH}/jconn4.jar"

    jdbc_driver_class => "com.sybase.jdbc4.jdbc.SybDriver"

    jdbc_connection_string => "${DB_CONNECTION_SPB_STRING}"

    jdbc_user => "${DB_USER}"

    jdbc_password => "${DB_PASS}"

    parameters => {"ini_range" => "param" "fim_range" => "anyParam" "dt_apuracao" => "dt_apuracao"}

    statement => "

                  SET NOCOUNT ON

                  Declare @IniRange numeric, @FimRange numeric, @Datah varchar(100)

                  BEGIN

                   

                    SET @IniRange = CONVERT(numeric, :ini_range)

                    SET @FimRange = CONVERT(numeric, :fim_range)

                    select * from dbx.dbo.Tb2 ths WHERE ths.NumParam BETWEEN  @IniRange and  @FimRange

                    and ths.AnyDate = :dt_apuracao

                  END  

                "

    use_cache => false

    periodic_flush => true

    target => "results"

  }

  split { field => "results" }

  ruby {

    code =>'

      event.get("results").each {|k,v|

        if  v.respond_to?(:strftime)

          event.set(k, LogStash::Timestamp.new(v))

        else

          event.set(k,v)

        end

      }

    '

  }

}

output {

  stdout { codec => rubydebug { metadata => true }}

  elasticsearch {

    index => "any_index"

    hosts => ["${ELASTIC_HOST}"]

    user => ["${ELASTIC_USER}"]

    password => ["${ELASTIC_PASS}"]

    document_id => "%{[@metadata][_id]}"

    ssl_certificate_verification => true

    cacert => "${LOGSTASH_CONFIG_PATH}/ca.crt"

  }

}

I need a lot of help, I've been in this activity for days

Blockquote note 1: The ruby code has been added to handle this error, but conf continues to fail.
note 2: The version of the stack I use to run locally is the same as the one running in the cloud

Do not allow Logstash to take care of fields serialization
Use convert as maximum possible in your SQL statement to convert most of fields into VARCHAR, then using logstash filters you can convert as you want to date, float ....

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.