Hi all,
I am trying to get familiar with S3 plugin in Logstash in two steps :
1 - Pushing logs to S3 as output
2 - Getting logs from S3 as input
1 - The Logstash conf file looks like this :
output {
s3{
access_key_id => "***"
secret_access_key => "******"
endpoint_region => "**-***-*"
bucket => "bucket-name"
size_file => 2048
codec => "json"
canned_acl => "private"
}
}
The logs I am sending look like this :
{"_id":"55fee3c726045db681acd139","index":0,"guid":"199836ff-bbfd-4a2f-9e66-452e72342e24","isActive":false,"balance":"$3,565.23","picture":"http://placehold.it/32x32","age":38,"eyeColor":"green","name":{"first":"Brandie","last":"Blackwell"}}
...
Which seems to look pretty fine as I have logs stored in S3 in different files with 2K size :
{"_id":"55fee3c726045db681acd139","index":0,"guid":"199836ff-bbfd-4a2f-9e66-452e72342e24","isActive":false,"balance":"$3,565.23","picture":"http://placehold.it/32x32","age":38,"eyeColor":"green","name":{"first":"Brandie","last":"Blackwell"},"@version":"1","@timestamp":"2015-09-29T12:20:12.679Z","host":"*.*.*.*"}
...
2 - However when getting logs as input an try to send them to Elasticsearch issues came across :
The config file :
input {
s3{
access_key_id => "****"
secret_access_key => "*******"
region => "**-***-*"
bucket => "bucket-name"
codec => "json"
}
}
output {
elasticsearch {
host => "*.*.*.*"
protocol => "http"
}
}
Losgtash is not able to retrieve logs from S3 and send them correctly to Elasticsearch :
failed action with response of 400, dropping action: ["index", {:_id=>nil, :_index=>"logstash-2015.09.29", :_type=>"logs", :_routing=>nil}, #<LogStash::Event:0x324fbb79 @metadata_accessors=#<LogStash::Util::Accessors:0x6f0fb4c6 @store={"retry_count"=>0}, @lut={}>, @cancelled=false, @data={"_id"=>"55fee3c7643281f83e914515", "index"=>70, "guid"=>"891dfae1-c48d-483a-a065-f61f37ed41a3", "isActive"=>true, "balance"=>"$3,263.89", "picture"=>"http://placehold.it/32x32", "age"=>26, "eyeColor"=>"green", "name"=>{"first"=>"Parrish", "last"=>"Baxter"}, "@version"=>"1", "@timestamp"=>"2015-09-29T12:59:31.543Z", "host"=>"..."}, @metadata={"retry_count"=>0}, @accessors=#<LogStash::Util::Accessors:0x4c9b374a @store={"_id"=>"55fee3c7643281f83e914515", "index"=>70, "guid"=>"891dfae1-c48d-483a-a065-f61f37ed41a3", "isActive"=>true, "balance"=>"$3,263.89", "picture"=>"http://placehold.it/32x32", "age"=>26, "eyeColor"=>"green", "name"=>{"first"=>"Parrish", "last"=>"Baxter"}, "@version"=>"1", "@timestamp"=>"2015-09-29T12:59:31.543Z", "host"=>"..."}, @lut={"message"=>[{"_id"=>"55fee3c7643281f83e914515", "index"=>70, "guid"=>"891dfae1-c48d-483a-a065-f61f37ed41a3", "isActive"=>true, "balance"=>"$3,263.89", "picture"=>"http://placehold.it/32x32", "age"=>26, "eyeColor"=>"green", "name"=>{"first"=>"Parrish", "last"=>"Baxter"}, "@version"=>"1", "@timestamp"=>"2015-09-29T12:59:31.543Z", "host"=>"..."}, "message"], "type"=>[{"_id"=>"55fee3c7643281f83e914515", "index"=>70, "guid"=>"891dfae1-c48d-483a-a065-f61f37ed41a3", "isActive"=>true, "balance"=>"$3,263.89", "picture"=>"http://placehold.it/32x32", "age"=>26, "eyeColor"=>"green", "name"=>{"first"=>"Parrish", "last"=>"Baxter"}, "@version"=>"1", "@timestamp"=>"2015-09-29T12:59:31.543Z", "host"=>"..."}, "type"]}>>] {:level=>:warn}
Could you please help me find out what I am missing here?