I've been trying to use s3 input plugin in Logstash to pull data from the Tencent COS (cloud object storage). They're claiming they're using the AWS SDK. It works fine, I'm able to pull the COS data just fine, but it times out after a while. Not sure why.
input {
s3 {
"type" => "tencent-cos"
"access_key_id" => "XXXXXXXXXX"
"secret_access_key" => "XXXXXXXXXX"
"bucket" => "XXXXXXXXXX"
"endpoint" => "https://cos.ap-shanghai.myqcloud.com"
"region" => "ap-shanghai"
"interval" => 600
"prefix" => "XXXXXXXXXX"
"watch_for_new_files" => true
"include_object_properties" => true
"temporary_directory" => "/var/log/logstash"
"additional_settings" => {
"force_path_style" => true
"follow_redirects" => false
}
}
}
filter {
json {
source => "message"
}
mutate {
add_field => { "cos.filename" => "%{[@metadata][s3][key]}" }
add_field => { "cos.lastmodify" => "%{[@metadata][s3][last_modified]}" }
}
}
These are the errors I'm getting. It will need some time to time out and crash (I'd say maybe 4-8h).
[2020-08-21T09:56:00,664][ERROR][logstash.javapipeline ][main][97a05e0dbeadd6457fb7a501a18fd6a6df01cc2fc9fdc8faf2c338ecc8221a29] A plugin had an unrecoverable error. Will restart this plugin.
Pipeline_id:main
Plugin: <REDACTED>
Error: execution expired
Exception: Seahorse::Client::NetworkingError
Stack: org/jruby/ext/socket/RubyTCPSocket.java:144:in `initialize'
org/jruby/RubyIO.java:1156:in `open'
uri:classloader:/META-INF/jruby.home/lib/ruby/stdlib/net/http.rb:941:in `block in connect'
org/jruby/ext/timeout/Timeout.java:114:in `timeout'
org/jruby/ext/timeout/Timeout.java:90:in `timeout'
uri:classloader:/META-INF/jruby.home/lib/ruby/stdlib/net/http.rb:939:in `connect'
uri:classloader:/META-INF/jruby.home/lib/ruby/stdlib/net/http.rb:924:in `do_start'
uri:classloader:/META-INF/jruby.home/lib/ruby/stdlib/net/http.rb:919:in `start'
uri:classloader:/META-INF/jruby.home/lib/ruby/stdlib/delegate.rb:83:in `method_missing'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-core-2.11.540/lib/seahorse/client/net_http/connection_pool.rb:285:in `start_session'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-core-2.11.540/lib/seahorse/client/net_http/connection_pool.rb:92:in `session_for'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-core-2.11.540/lib/seahorse/client/net_http/handler.rb:119:in `session'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-core-2.11.540/lib/seahorse/client/net_http/handler.rb:71:in `transmit'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-core-2.11.540/lib/seahorse/client/net_http/handler.rb:45:in `call'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-core-2.11.540/lib/seahorse/client/plugins/content_length.rb:12:in `call'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-core-2.11.540/lib/aws-sdk-core/plugins/s3_host_id.rb:14:in `call'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-core-2.11.540/lib/aws-sdk-core/xml/error_handler.rb:8:in `call'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-core-2.11.540/lib/aws-sdk-core/plugins/helpful_socket_errors.rb:10:in `call'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-core-2.11.540/lib/aws-sdk-core/plugins/s3_request_signer.rb:65:in `call'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-core-2.11.540/lib/aws-sdk-core/plugins/retry_errors.rb:139:in `retry_request'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-core-2.11.540/lib/aws-sdk-core/plugins/retry_errors.rb:122:in `retry_if_possible'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-core-2.11.540/lib/aws-sdk-core/plugins/retry_errors.rb:110:in `call'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-core-2.11.540/lib/aws-sdk-core/plugins/s3_dualstack.rb:32:in `call'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-core-2.11.540/lib/aws-sdk-core/plugins/s3_accelerate.rb:49:in `call'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-core-2.11.540/lib/aws-sdk-core/plugins/s3_md5s.rb:31:in `call'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-core-2.11.540/lib/aws-sdk-core/plugins/s3_expect_100_continue.rb:21:in `call'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-core-2.11.540/lib/aws-sdk-core/plugins/s3_bucket_name_restrictions.rb:12:in `call'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-core-2.11.540/lib/aws-sdk-core/rest/handler.rb:7:in `call'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-core-2.11.540/lib/aws-sdk-core/plugins/user_agent.rb:12:in `call'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-core-2.11.540/lib/seahorse/client/plugins/endpoint.rb:41:in `call'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-core-2.11.540/lib/aws-sdk-core/plugins/param_validator.rb:21:in `call'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-core-2.11.540/lib/seahorse/client/plugins/raise_response_errors.rb:14:in `call'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-core-2.11.540/lib/aws-sdk-core/plugins/s3_sse_cpk.rb:19:in `call'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-core-2.11.540/lib/aws-sdk-core/plugins/s3_dualstack.rb:24:in `call'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-core-2.11.540/lib/aws-sdk-core/plugins/s3_accelerate.rb:34:in `call'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-core-2.11.540/lib/aws-sdk-core/plugins/jsonvalue_converter.rb:20:in `call'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-core-2.11.540/lib/aws-sdk-core/plugins/idempotency_token.rb:18:in `call'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-core-2.11.540/lib/aws-sdk-core/plugins/param_converter.rb:20:in `call'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-core-2.11.540/lib/aws-sdk-core/plugins/response_paging.rb:26:in `call'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-core-2.11.540/lib/seahorse/client/plugins/response_target.rb:21:in `call'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-core-2.11.540/lib/seahorse/client/request.rb:70:in `send_request'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-core-2.11.540/lib/seahorse/client/base.rb:207:in `block in define_operation_methods'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-resources-2.11.540/lib/aws-sdk-resources/request.rb:24:in `call'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-resources-2.11.540/lib/aws-sdk-resources/operations.rb:139:in `all_batches'
org/jruby/RubyEnumerator.java:396:in `each'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-resources-2.11.540/lib/aws-sdk-resources/collection.rb:18:in `each'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-s3-3.5.0/lib/logstash/inputs/s3.rb:132:in `list_new_files'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-s3-3.5.0/lib/logstash/inputs/s3.rb:172:in `process_files'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-s3-3.5.0/lib/logstash/inputs/s3.rb:123:in `block in run'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/stud-0.0.23/lib/stud/interval.rb:20:in `interval'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-s3-3.5.0/lib/logstash/inputs/s3.rb:122:in `run'
/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:374:in `inputworker'
/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:365:in `block in start_input'
[2020-08-21T09:57:23,252][ERROR][logstash.javapipeline ][main] . [97a05e0dbeadd6457fb7a501a18fd6a6df01cc2fc9fdc8faf2c338ecc8221a29] A plugin had an unrecoverable error. Will restart this plugin.
Pipeline_id:main
Plugin: <REDACTED>
Error: Net::OpenTimeout
Exception: Seahorse::Client::NetworkingError
Stack: uri:classloader:/META-INF/jruby.home/lib/ruby/stdlib/net/protocol.rb:41:in `ssl_socket_connect'
uri:classloader:/META-INF/jruby.home/lib/ruby/stdlib/net/http.rb:985:in `connect'
uri:classloader:/META-INF/jruby.home/lib/ruby/stdlib/net/http.rb:924:in `do_start'
uri:classloader:/META-INF/jruby.home/lib/ruby/stdlib/net/http.rb:919:in `start'
uri:classloader:/META-INF/jruby.home/lib/ruby/stdlib/delegate.rb:83:in `method_missing'
uri:classloader:/META-INF/jruby.home/lib/ruby/stdlib/net/http.rb:985:in `connect'
uri:classloader:/META-INF/jruby.home/lib/ruby/stdlib/net/http.rb:924:in `do_start'
uri:classloader:/META-INF/jruby.home/lib/ruby/stdlib/net/http.rb:919:in `start'
uri:classloader:/META-INF/jruby.home/lib/ruby/stdlib/delegate.rb:83:in `method_missing'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-core-2.11.540/lib/seahorse/client/net_http/connection_pool.rb:285:in `start_session'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-core-2.11.540/lib/seahorse/client/net_http/connection_pool.rb:92:in `session_for'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-core-
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-core-2.11.540/lib/aws-sdk-core/plugins/helpful_socket_errors.rb:10:in `call'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-core-2.11.540/lib/aws-sdk-core/plugins/s3_request_signer.rb:65:in `call'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-core-2.11.540/lib/aws-sdk-
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-core-2.11.540/lib/aws-sdk-core/plugins/retry_errors.rb:122:in `retry_if_possible'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-core-2.11.540/lib/aws-sdk-core/plugins/retry_errors.rb:110:in `call'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-core-2.11.540/lib/aws-sdk-core/plugins/retry_errors.rb:139:in `retry_request'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-core-2.11.540/lib/aws-sdk-core/plugins/retry_errors.rb:122:in `retry_if_possible'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-core-2.11.540/lib/aws-sdk-core/plugins/retry_errors.rb:110:in `call'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-core-2.11.540/lib/aws-sdk-core/plugins/s3_dualstack.rb:32:in `call'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-core-2.11.540/lib/aws-sdk-core/plugins/s3_accelerate.rb:49:in `call'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-core-2.11.540/lib/aws-sdk-
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-core-2.11.540/lib/seahorse/client/plugins/response_target.rb:21:in `call'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-core-2.11.540/lib/seahorse/client/request.rb:70:in `send_request'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-core-2.11.540/lib/seahorse/client/base.rb:207:in `block in define_operation_methods'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-resources-2.11.540/lib/aws-sdk-resources/request.rb:24:in `call'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-resources-2.11.540/lib/aws-sdk-resources/operations.rb:139:in `all_batches'
org/jruby/RubyEnumerator.java:396:in `each'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/aws-sdk-resources-2.11.540/lib/aws-sdk-resources/collection.rb:18:in `each'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-s3-3.5.0/lib/logstash/inputs/s3.rb:132:in `list_new_files'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-s3-3.5.0/lib/logstash/inputs/s3.rb:172:in `process_files'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-s3-3.5.0/lib/logstash/inputs/s3.rb:123:in `block in run'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/stud-0.0.23/lib/stud/interval.rb:20:in `interval'
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-s3-3.5.0/lib/logstash/inputs/s3.rb:122:in `run'
/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:374:in `inputworker'
/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:365:in `block in start_input'