hi,
i want to push the ELB logs from s3 to ELK for the same have i have written the logstash.conf file like this
input {
s3 {
access_key_id => "..."
secret_access_key => "..."
bucket => "..."
region => "eu-central-1"
prefix => "dxlb/AWSLogs/.../elasticloadbalancing/eu-central-1/2019/09/"
type => "elb"
}
}
filter {
if [type] == "elb" {
grok {
match => [ "message", "%{WORD:connection} %{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE:elb} %{IP:clientip}:%{INT:clientport:float} (?:(%{IP:backendip}:?:%{INT:backendport:int})|-) %{NUMBER:request_processing_time:float} %{NUMBER:backend_processing_time:float} %{N
UMBER:response_processing_time:float} (?:-|%{INT:elb_status_code:int}) (?:-|%{INT:backend_status_code:int}) %{INT:received_bytes:int} %{INT:sent_bytes:int} "%{ELB_REQUEST_LINE}" "(?:-|%{DATA:user_agent})" (?:-|%{NOTSPACE:ssl_cipher}) (?:-|%{NOTSPACE:ssl_protocol})
" ]
#match => ["message", "%{ELB_ACCESS_LOG} "%{DATA:userAgent}"( %{NOTSPACE:ssl_cipher} %{NOTSPACE:ssl_protocol})?"]
}
date {
match => [ "timestamp", "ISO8601" ]
}
geoip {
source => "clientip"
}
}
}
output {
if [type] == "elb" {
elasticsearch {
hosts => [“localhost:9200”]
index => "logstash-%{+YYYY.MM}"
user => "..."
password => "..."
}
}
}
but after applying this i getting this error. please help me out to sort this one
[2021-01-22T10:05:39,833][ERROR][org.logstash.Logstash ] java.lang.IllegalStateException: Logstash stopped processing because of an error: (SystemExit) exit
[2021-01-22T10:06:02,720][INFO ][logstash.runner ] Starting Logstash {"logstash.version"=>"7.10.2", "jruby.version"=>"jruby 9.2.13.0 (2.5.7) 2020-08-03 9a89c94bcc OpenJDK 64-Bit Server VM 11.0.8+10 on 11.0.8+10 +indy +jit [linux-x86_64]"}
[2021-01-22T10:06:05,859][ERROR][logstash.agent ] Failed to execute action {:action=>LogStash::PipelineAction::Create/pipeline_id:main, :exception=>"LogStash::ConfigurationError", :message=>"Expected one of [ \t\r\n], "#", [A-Za-z0-9_-], '"', "'", [A-Za-z_], "-", [0-9], "[", "{", "]" at line 32, column 27 (byte 1194) after output {\nif [type] == "elb" {\n elasticsearch {\n hosts => [", :backtrace=>["/usr/share/logstash/logstash-core/lib/logstash/compiler.rb:32:in compile_imperative'", "org/logstash/execution/AbstractPipelineExt.java:184:in initialize'", "org/logstash/execution/JavaBasePipelineExt.java:69:in initialize'", "/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:47:in initialize'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline_action/create.rb:52:in execute'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:365:in block in converge_state'"]}
[2021-01-22T10:06:06,196][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>9600}
[2021-01-22T10:06:11,103][INFO ][logstash.runner ] Logstash shut down.