Hi ,
This is my logstash conf
input {
s3 {
access_key_id => "xxxx"
secret_access_key => "xxx"
#bucket => "mybucketname/2020/01/16"
bucket => "sftlcloudtrail"
sincedb_path => "/tmp/last-s3-file-s3-access-logs"
additional_settings => {
force_path_style => true
follow_redirects => false
}
}
}
output {
elasticsearch {
hosts => ["http://192.168.1.72:9200","http://192.168.1.62:9200"]
index => "s3flowlogs-16012020"
}
stdout { codec => rubydebug }
}
The logstash is starting fine and the logs from the bucket are not pulled to the elasticsearch. I can't figure is there any error in the conf or what ?
Sending Logstash logs to /opt/logstash-7.4.0/logs which is now configured via log4j2.properties
[2020-01-17T19:07:41,613][WARN ][logstash.config.source.multilocal] Ignoring the 'pipelines.yml' file because modules or command line options are specified
[2020-01-17T19:07:41,632][INFO ][logstash.runner ] Starting Logstash {"logstash.version"=>"7.4.0"}
[2020-01-17T19:07:51,180][INFO ][org.reflections.Reflections] Reflections took 278 ms to scan 1 urls, producing 20 keys and 40 values
[2020-01-17T19:08:52,140][INFO ][logstash.outputs.elasticsearch][main] Elasticsearch pool URLs updated {:changes=>{:removed=>, :added=>[http://192.168.1.72:9200/, http://192.168.1.62:9200/]}}
[2020-01-17T19:08:52,811][WARN ][logstash.outputs.elasticsearch][main] Restored connection to ES instance {:url=>"http://192.168.1.72:9200/"}
[2020-01-17T19:08:52,899][INFO ][logstash.outputs.elasticsearch][main] ES Output version determined {:es_version=>7}
[2020-01-17T19:08:52,906][WARN ][logstash.outputs.elasticsearch][main] Detected a 6.x and above cluster: thetypeevent field won't be used to determine the document _type {:es_version=>7}
[2020-01-17T19:08:52,916][WARN ][logstash.outputs.elasticsearch][main] Restored connection to ES instance {:url=>"http://192.168.1.62:9200/"}
[2020-01-17T19:08:52,971][INFO ][logstash.outputs.elasticsearch][main] New Elasticsearch output {:class=>"LogStash::Outputs::Elasticsearch", :hosts=>["http://192.168.1.72:9200", "http://192.168.1.62:9200"]}
[2020-01-17T19:08:53,330][INFO ][logstash.outputs.elasticsearch][main] Using default mapping template
[2020-01-17T19:08:53,629][INFO ][logstash.outputs.elasticsearch][main] Attempting to install template {:manage_template=>{"index_patterns"=>"logstash-", "version"=>60001, "settings"=>{"index.refresh_interval"=>"5s", "number_of_shards"=>1}, "mappings"=>{"dynamic_templates"=>[{"message_field"=>{"path_match"=>"message", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false}}}, {"string_fields"=>{"match"=>"", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false, "fields"=>{"keyword"=>{"type"=>"keyword", "ignore_above"=>256}}}}}], "properties"=>{"@timestamp"=>{"type"=>"date"}, "@version"=>{"type"=>"keyword"}, "geoip"=>{"dynamic"=>true, "properties"=>{"ip"=>{"type"=>"ip"}, "location"=>{"type"=>"geo_point"}, "latitude"=>{"type"=>"half_float"}, "longitude"=>{"type"=>"half_float"}}}}}}}
[2020-01-17T19:08:53,723][WARN ][org.logstash.instrument.metrics.gauge.LazyDelegatingGauge][main] A gauge metric of an unknown type (org.jruby.specialized.RubyArrayOneObject) has been create for key: cluster_uuids. This may result in invalid serialization. It is recommended to log an issue to the responsible developer/development team.
[2020-01-17T19:08:53,786][INFO ][logstash.javapipeline ][main] Starting pipeline {:pipeline_id=>"main", "pipeline.workers"=>4, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>50, "pipeline.max_inflight"=>500, :thread=>"#<Thread:0x3ade9eb9 run>"}
[2020-01-17T19:08:54,036][INFO ][logstash.inputs.s3 ][main] Registering s3 input {:bucket=>"xxxx", :region=>"us-east-1"}
[2020-01-17T19:08:56,204][INFO ][logstash.javapipeline ][main] Pipeline started {"pipeline.id"=>"main"}
[2020-01-17T19:08:56,815][INFO ][logstash.agent ] Pipelines running {:count=>1, :running_pipelines=>[:main], :non_running_pipelines=>}
[2020-01-17T19:08:59,626][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>9601}
[2020-01-17T19:09:13,201][INFO ][logstash.inputs.s3 ][main] Using the provided sincedb_path {:sincedb_path=>"/tmp/last-s3-file-s3-access-logs"}