I have an logstash conf file like
input {
s3 {
access_key_id => "redacted"
secret_access_key => "redacted"
bucket => "aws-cloudtrail-logs-905418385260-a48e6842"
delete => false
interval => 30 # seconds
prefix => "AWSLogs/905418385260/"
type => "cloudtrail"
codec => "cloudtrail"
# credentials => "/etc/logstash/s3_credentials.ini"
# sincedb_path => "/opt/logstash_cloudtrail/sincedb"
}
}
output {
stdout{
# codec => "rubydebug"
}
s3 {
access_key_id => "redacted"
secret_access_key => "redacted"
region => "us-east-1"
bucket => "logstash-test-random"
prefix => "logs/%{+YYYY.MM.dd}"
retry_count => 1
additional_settings => {
"force_path_style" => true
"follow_redirects" => false
}
time_file => 4
codec => "json"
rotation_strategy => "time"
temporary_directory => "C:\elk\logstash-8.13.1\config"
}
}```
whenever i run this conf i get the following warning
[2024-06-04T11:45:59,223][WARN ][logstash.outputs.s3 ][main] Uploading failed, retrying (#674 of Infinity) {:exception=>ArgumentError, :message=>":key must not be blank", :path=>"C:\elk\logstash-8.13.1\config/jvm.options", :backtrace=>["C:/elk/logstash-8.13.1/vendor/bundle/jruby/3.1.0/gems/aws-sdk-core-3.191.4/lib/aws-sdk-core/rest/request/endpoint.rb:42:in param_value_for_placeholder'", "C:/elk/logstash-8.13.1/vendor/bundle/jruby/3.1.0/gems/aws-sdk-core-3.191.4/lib/aws-sdk-core/rest/request/endpoint.rb:35:in block in apply_path_params'", "org/jruby/RubyString.java:3186:in gsub'", "C:/elk/logstash-8.13.1/vendor/bundle/jruby/3.1.0/gems/aws-sdk-core-3.191.4/lib/aws-sdk-core/rest/request/endpoint.rb:34:in apply_path_params'", "C:/elk/logstash-8.13.1/vendor/bundle/jruby/3.1.0/gems/aws-sdk-core-3.191.4/lib/aws-sdk-core/rest/request/endpoint.rb:25:in uri'", "C:/elk/logstash-8.13.1/vendor/bundle/jruby/3.1.0/gems/aws-sdk-core-3.191.4/lib/aws-sdk-core/rest/request/builder.rb:25:in populate_endpoint'", "C:/elk/logstash-8.13.1/vendor/bundle/jruby/3.1.0/gems/aws-sdk-core-3.191.4/lib/aws-sdk-core/rest/request/builder.rb:10:in apply'", "C:/elk/logstash-8.13.1/vendor/bundle/jruby/3.1.0/gems/aws-sdk-core-3.191.4/lib/aws-sdk-core/rest/handler.rb:9:in call'", "C:/elk/logstash-8.13.1/vendor/bundle/jruby/3.1.0/gems/aws-sdk-core-3.191.4/lib/aws-sdk-core/plugins/recursion_detection.rb:18:in call'", "C:/elk/logstash-8.13.1/vendor/bundle/jruby/3.1.0/gems/aws-sdk-s3-1.146.0/lib/aws-sdk-s3/plugins/endpoints.rb:49:in call'", "C:/elk/logstash-8.13.1/vendor/bundle/jruby/3.1.0/gems/aws-sdk-core-3.191.4/lib/aws-sdk-core/plugins/endpoint_discovery.rb:84:in call'", "C:/elk/logstash-8.13.1/vendor/bundle/jruby/3.1.0/gems/aws-sdk-core-3.191.4/lib/seahorse/client/plugins/endpoint.rb:47:in call'", "C:/elk/logstash-8.13.1/vendor/bundle/jruby/3.1.0/gems/aws-sdk-core-3.191.4/lib/aws-sdk-core/plugins/param_validator.rb:26:in call'", "C:/elk/logstash-8.13.1/vendor/bundle/jruby/3.1.0/gems/aws-sdk-core-3.191.4/lib/seahorse/client/plugins/raise_response_errors.rb:16:in call'", "C:/elk/logstash-8.13.1/vendor/bundle/jruby/3.1.0/gems/aws-sdk-s3-1.146.0/lib/aws-sdk-s3/plugins/sse_cpk.rb:24:in call'", "C:/elk/logstash-8.13.1/vendor/bundle/jruby/3.1.0/gems/aws-sdk-s3-1.146.0/lib/aws-sdk-s3/plugins/dualstack.rb:21:in call'", "C:/elk/logstash-8.13.1/vendor/bundle/jruby/3.1.0/gems/aws-sdk-s3-1.146.0/lib/aws-sdk-s3/plugins/accelerate.rb:43:in call'", "C:/elk/logstash-8.13.1/vendor/bundle/jruby/3.1.0/gems/aws-sdk-core-3.191.4/lib/aws-sdk-core/plugins/checksum_algorithm.rb:111:in call'", "C:/elk/logstash-8.13.1/vendor/bundle/jruby/3.1.0/gems/aws-sdk-core-3.191.4/lib/aws-sdk-core/plugins/jsonvalue_converter.rb:16:in call'", "C:/elk/logstash-8.13.1/vendor/bundle/jruby/3.1.0/gems/aws-sdk-core-3.191.4/lib/aws-sdk-core/plugins/idempotency_token.rb:19:in call'", "C:/elk/logstash-8.13.1/vendor/bundle/jruby/3.1.0/gems/aws-sdk-core-3.191.4/lib/aws-sdk-core/plugins/param_converter.rb:26:in call'", "C:/elk/logstash-8.13.1/vendor/bundle/jruby/3.1.0/gems/aws-sdk-core-3.191.4/lib/seahorse/client/plugins/request_callback.rb:89:in call'", "C:/elk/logstash-8.13.1/vendor/bundle/jruby/3.1.0/gems/aws-sdk-core-3.191.4/lib/aws-sdk-core/plugins/response_paging.rb:12:in call'", "C:/elk/logstash-8.13.1/vendor/bundle/jruby/3.1.0/gems/aws-sdk-core-3.191.4/lib/seahorse/client/plugins/response_target.rb:24:in call'", "C:/elk/logstash-8.13.1/vendor/bundle/jruby/3.1.0/gems/aws-sdk-core-3.191.4/lib/seahorse/client/request.rb:72:in send_request'", "C:/elk/logstash-8.13.1/vendor/bundle/jruby/3.1.0/gems/aws-sdk-s3-1.146.0/lib/aws-sdk-s3/client.rb:15579:in put_object'", "C:/elk/logstash-8.13.1/vendor/bundle/jruby/3.1.0/gems/aws-sdk-s3-1.146.0/lib/aws-sdk-s3/file_uploader.rb:66:in block in put_object'", "C:/elk/logstash-8.13.1/vendor/bundle/jruby/3.1.0/gems/aws-sdk-s3-1.146.0/lib/aws-sdk-s3/file_uploader.rb:55:in block in open_file'", "org/jruby/RubyIO.java:1179:in open'", "C:/elk/logstash-8.13.1/vendor/bundle/jruby/3.1.0/gems/aws-sdk-s3-1.146.0/lib/aws-sdk-s3/file_uploader.rb:55:in open_file'", "C:/elk/logstash-8.13.1/vendor/bundle/jruby/3.1.0/gems/aws-sdk-s3-1.146.0/lib/aws-sdk-s3/file_uploader.rb:65:in put_object'", "C:/elk/logstash-8.13.1/vendor/bundle/jruby/3.1.0/gems/aws-sdk-s3-1.146.0/lib/aws-sdk-s3/file_uploader.rb:46:in block in upload'", "C:/elk/logstash-8.13.1/vendor/bundle/jruby/3.1.0/gems/aws-sdk-core-3.191.4/lib/aws-sdk-core/plugins/user_agent.rb:28:in feature'", "C:/elk/logstash-8.13.1/vendor/bundle/jruby/3.1.0/gems/aws-sdk-s3-1.146.0/lib/aws-sdk-s3/file_uploader.rb:40:in upload'", "C:/elk/logstash-8.13.1/vendor/bundle/jruby/3.1.0/gems/aws-sdk-s3-1.146.0/lib/aws-sdk-s3/customizations/object.rb:477:in block in upload_file'", "C:/elk/logstash-8.13.1/vendor/bundle/jruby/3.1.0/gems/aws-sdk-core-3.191.4/lib/aws-sdk-core/plugins/user_agent.rb:28:in feature'", "C:/elk/logstash-8.13.1/vendor/bundle/jruby/3.1.0/gems/aws-sdk-s3-1.146.0/lib/aws-sdk-s3/customizations/object.rb:476:in upload_file'", "C:/elk/logstash-8.13.1/vendor/bundle/jruby/3.1.0/gems/logstash-integration-aws-7.1.6-java/lib/logstash/outputs/s3/uploader.rb:41:in upload'", "C:/elk/logstash-8.13.1/vendor/bundle/jruby/3.1.0/gems/logstash-integration-aws-7.1.6-java/lib/logstash/outputs/s3/uploader.rb:30:in block in upload_async'", "C:/elk/logstash-8.13.1/vendor/bundle/jruby/3.1.0/gems/concurrent-ruby-1.1.9/lib/concurrent-ruby/concurrent/executor/java_executor_service.rb:79:in run'"]} ```
Things to note
- using codec json , the files are getting uploaded to s3 even though the warning still persists
- i have tried to do retry_count = 0 it still showed the warning
- logs are getting generated (verified by stdout) and temp file is also getting made
Another issue
using codec cloudtrail in s3 output, i get the same warning about 90 something times and then the program catches some uncaught error and stops( idk why if somebody could tell why would be great)