I am facing below issue while streaming message from logstash to kiabana:
[2021-06-09T10:15:01,047][INFO ][logstash.inputs.jdbc][main] (0.067667s) SELECT * FROM GV$ACTIVE_SESSION_HISTORY WHERE SAMPLE_TIME > TIMESTAMP '2021-06-09 00:00:00.000000 +00:00'
[2021-06-09T10:15:02,029][ERROR][logstash.outputs.elasticsearch][main] An unknown error occurred sending a bulk request to Elasticsearch (will retry indefinitely) {:message=>""\x85" from ASCII-8BIT to UTF-8", :exception=>LogStash::Json::GeneratorError, :backtrace=>["path/logstash-core/lib/logstash/json.rb:43:in jruby_dump'", "path/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-11.0.2-java/lib/logstash/outputs/elasticsearch/http_client.rb:122:in block in bulk'", "org/jruby/RubyArray.java:2577:in map'", "path/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-11.0.2-java/lib/logstash/outputs/elasticsearch/http_client.rb:122:in block in bulk'", "org/jruby/RubyArray.java:1809:in each'", "org/jruby/RubyEnumerable.java:1258:in each_with_index'", "path/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-11.0.2-java/lib/logstash/outputs/elasticsearch/http_client.rb:120:in bulk'", "path/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-11.0.2-java/lib/logstash/plugin_mixins/elasticsearch/common.rb:287:in safe_bulk'", "path/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-11.0.2-java/lib/logstash/plugin_mixins/elasticsearch/common.rb:219:in submit'", "path/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-11.0.2-java/lib/logstash/plugin_mixins/elasticsearch/common.rb:168:in retrying_submit'", "path/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-11.0.2-java/lib/logstash/outputs/elasticsearch.rb:333:in multi_receive'", "org/logstash/config/ir/compiler/OutputStrategyExt.java:143:in multi_receive'", "org/logstash/config/ir/compiler/AbstractOutputDelegatorExt.java:121:in multi_receive'", "path/logstash-core/lib/logstash/java_pipeline.rb:295:in block in start_workers'"]}
logstash config file:
Sample Logstash configuration for creating a simple
Beats -> Logstash -> Elasticsearch pipeline.
input {
jdbc {
jdbc_validate_connection => true
jdbc_connection_string => "jdbc:oracle:thin:@IP:PORT/SID"
jdbc_user => "username"
jdbc_password => "password"
jdbc_driver_library => "path to /ojdbc7.jar"
jdbc_driver_class => "Java::oracle.jdbc.driver.OracleDriver"
statement => "SELECT * FROM GV$ACTIVE_SESSION_HISTORY WHERE SAMPLE_TIME > TIMESTAMP '2021-06-09 11:00:00.000000 +00:00'"
last_run_metadata_path => "path/logstash-oradb.lastrun"
record_last_run => true
schedule => "*/5 * * * *"
}
}
filter {
# Set the timestamp to that of the ASH sample, not current time.
mutate { convert => [ "sample_time" , "string" ]}
date { match => ["sample_time", "ISO8601"]}
# Drop "force_matching_signature" because its value sometimes blows the limits of the long data type
mutate { remove_field => [ "force_matching_signature" ] }
}
output {
elasticsearch {
hosts => ["https://ip:port"]
user => "username"
password => "password"
cacert => "path to cer"
ssl => true
ssl_certificate_verification => false
data_stream => "true"
}
stdout { codec => rubydebug }
}