Hi Team,
ELK : 7.3.0 on Kubernetes
MSSQL JDBC Driver : mssql-jdbc-7.4.1.jre11.jar
OpenJDK Version: openjdk 11.0.4 2019-07-16 LTS
I'm trying to deploy Logstash 7.3.0 pod into my on-premise Kubernetes cluster with a pipeline configuration to load data from input MSSQL server to output RabbitMQ. But ended up following error.
[2019-08-22T11:59:19,446][INFO ][logstash.runner ] Starting Logstash {"logstash.version"=>"7.3.0"}
[2019-08-22T11:59:20,862][INFO ][org.reflections.Reflections] Reflections took 30 ms to scan 1 urls, producing 19 keys and 39 values
[2019-08-22T11:59:21,657][INFO ][logstash.outputs.rabbitmq] Connected to RabbitMQ at
[2019-08-22T11:59:21,711][INFO ][logstash.outputs.rabbitmq] Connected to RabbitMQ at
[2019-08-22T11:59:21,751][WARN ][org.logstash.instrument.metrics.gauge.LazyDelegatingGauge] A gauge metric of an unknown type (org.jruby.RubyArray) has been create for key: cluster_uuids. This may result in invalid serialization. It is recommended to log an issue to the responsible developer/development team.
[2019-08-22T11:59:21,753][INFO ][logstash.javapipeline ] Starting pipeline {:pipeline_id=>"main", "pipeline.workers"=>1, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>50, "pipeline.max_inflight"=>125, :thread=>"#<Thread:0x10184c75 run>"}
[2019-08-22T11:59:21,845][INFO ][logstash.javapipeline ] Pipeline started {"pipeline.id"=>"main"}
[2019-08-22T11:59:21,964][INFO ][logstash.agent ] Pipelines running {:count=>1, :running_pipelines=>[:main], :non_running_pipelines=>[]}
[2019-08-22T11:59:22,153][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>9600}
[2019-08-22T11:59:22,357][ERROR][logstash.inputs.jdbc ] Failed to load /usr/share/logstash/tools/sqljdbc7/mssql-jdbc-7.4.1.jre11.jar {:exception=>#<TypeError: failed to coerce jdk.internal.loader.ClassLoaders$AppClassLoader to java.net.URLClassLoader>}
[2019-08-22T11:59:22,375][ERROR][logstash.inputs.jdbc ] Failed to load /usr/share/logstash/tools/sqljdbc7/mssql-jdbc-7.4.1.jre11.jar {:exception=>#<TypeError: failed to coerce jdk.internal.loader.ClassLoaders$AppClassLoader to java.net.URLClassLoader>}
[2019-08-22T11:59:22,396][ERROR][logstash.javapipeline ] A plugin had an unrecoverable error. Will restart this plugin.
Pipeline_id:main
Plugin: <LogStash::Inputs::Jdbc jdbc_user=>"mssqluser", last_run_metadata_path=>"/usr/share/logstash/tools/jdbc-last-run/Users/userroles/.logstash_jdbc_last_run", tracking_column_type=>"timestamp", record_last_run=>true, tags=>["UserRoles"], use_column_value=>true, tracking_column=>"updated", jdbc_validate_connection=>true, jdbc_password=><password>, statement=>" SELECT * FROM [Users].[dbo].[UserRoles] WHERE Updated > :sql_last_value ", clean_run=>false, jdbc_driver_library=>"/usr/share/logstash/tools/sqljdbc7/mssql-jdbc-7.4.1.jre11.jar", id=>"userroles", jdbc_connection_string=>"jdbc:sqlserver://mssqlserver;database=Users;user=mssqluser;password=mssqlpassword", jdbc_driver_class=>"com.microsoft.sqlserver.jdbc.SQLServerDriver", lowercase_column_names=>true, enable_metric=>true, codec=><LogStash::Codecs::Plain id=>"plain_4c846847-8dc9-4dde-8663-dc352d383bf9", enable_metric=>true, charset=>"UTF-8">, jdbc_paging_enabled=>false, jdbc_page_size=>100000, jdbc_validation_timeout=>3600, jdbc_pool_timeout=>5, sql_log_level=>"info", connection_retry_attempts=>1, connection_retry_attempts_wait_time=>0.5, parameters=>{"sql_last_value"=>1970-01-01 00:00:00 UTC}>
Error: com.microsoft.sqlserver.jdbc.SQLServerDriver not loaded. Are you sure you've included the correct jdbc driver in :jdbc_driver_library?
Exception: LogStash::ConfigurationError
Stack: /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-jdbc-4.3.13/lib/logstash/plugin_mixins/jdbc/jdbc.rb:163:in `open_jdbc_connection'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-jdbc-4.3.13/lib/logstash/plugin_mixins/jdbc/jdbc.rb:221:in `execute_statement'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-jdbc-4.3.13/lib/logstash/inputs/jdbc.rb:277:in `execute_query'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-jdbc-4.3.13/lib/logstash/inputs/jdbc.rb:263:in `run'
/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:309:in `inputworker'
/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:302:in `block in start_input'
[2019-08-22T11:59:22,397][ERROR][logstash.javapipeline ] A plugin had an unrecoverable error. Will restart this plugin.
Below is my pipeline conf file
input {
jdbc {
id => "users"
tags => ["Users"]
jdbc_connection_string => "jdbc:sqlserver://mssqlserver;database=Users;user=mssqluser;password=mssqlpassword"
jdbc_driver_library => "/usr/share/logstash/tools/sqljdbc7/mssql-jdbc-7.4.1.jre11.jar"
jdbc_driver_class => "com.microsoft.sqlserver.jdbc.SQLServerDriver"
jdbc_user => "mssqluser"
jdbc_password => "mssqlpassword"
jdbc_validate_connection => true
jdbc_page_size => "50000"
use_column_value => true
tracking_column => "updated"
tracking_column_type => "timestamp"
clean_run => false
record_last_run => true
lowercase_column_names => true
last_run_metadata_path => "/usr/share/logstash/tools/jdbc-last-run/Users/users/.logstash_jdbc_last_run"
statement => " select cu.id, cu.userid, cu.username, cu.displayname, cu.emailid, cu.roleid, cu.created, cu.updated, ur.rolename, cu.status, cu.updatedby
from [users].[dbo].[users] cu
join [users].[dbo].[userroles] ur
on cu.roleid=ur.roleid
where cu.updated > :sql_last_value order by cu.updated asc "
}
}
output {
if "Users" in [tags] {
rabbitmq {
id => "ls-users"
host => "rabbitmq"
user => "mquser"
password => "passw0rd"
port => 5672
exchange => "ls-users-exchange"
exchange_type => "topic"
key => "lsusersroutingkey"
durable => true
codec => "json"
}
}
}
Could you please assist me with this ?
Thanks!