Logstash Cloudwatch input plugin not creating the index on Kibana and Error log_group in tail

[Shahid_Chaudhary]

Hi Teach Team,

I have one ELK single node server installed on the EC2 instance.
Now i need to get logs/metrics from the Cloudwatch and other AWS manage services , did these steps.
1 - ELK stack is installed and working
2 - Create a IAM user on AWS acount with sufficient permission on the AWS services.
3 - Install the Logstash cloudwatch input plugin.
4 - Configure the Input/filter/out setting inside the conf file (/etc/logstash/conf.d/beats.conf)

"
ubuntu@ip-172-31-10-33:/etc/logstash/conf.d$ ll
-rw-r--r-- 1 root root 2166 Nov 13 21:52 beats.conf

when in check the logs of logstash tail. i am getting err of log_group .

Blockquote

sudo tail -f /var/log/logstash/logstash-plain.log
[2018-11-13T22:13:53,892][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>9600}
[2018-11-13T22:14:19,593][INFO ][logstash.runner ] Starting Logstash {"logstash.version"=>"6.4.3"}
[2018-11-13T22:14:22,182][ERROR][logstash.inputs.cloudwatch] Unknown setting 'log_group' for cloudwatch
[2018-11-13T22:14:22,258][ERROR][logstash.agent ] Failed to execute action {:action=>LogStash::PipelineAction::Create/pipeline_id:main, :exception=>"LogStash::ConfigurationError", :message=>"Something is wrong with your configuration.", :backtrace=>["/usr/share/logstash/logstash-core/lib/logstash/config/mixin.rb:86:in `config_init'", "/usr/share/logstash/logstash-core/lib/logstash/inputs/base.rb:60:in `initialize'", "org/logstash/plugins/PluginFactoryExt.java:233:in `plugin'", "org/logstash/plugins/PluginFactoryExt.java:166:in `plugin'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:71:in `plugin'", "(eval):20:in `'", "org/jruby/RubyKernel.java:994:in `eval'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:49:in `initialize'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:90:in `initialize'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline_action/create.rb:38:in `execute'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:309:in `block in converge_state'"]}
[2018-11-13T22:14:22,535][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>9600}

Blockquote

Can you Please help me to fix this issue . if the log_group issue fixed that index will reflect automaticaly on the kibana dashbard or i have to create manually. if not how can i crate the the index manually.

beats.conf

"
input {
beats {
port => "5044"
}
}

input {
cloudwatch {
namespace => "AWS/RDS"
metrics => ["CPUUtilization", "CPUCreditUsage"]
filters => { "EngineName" => "mysql" } # Only supports EngineName, DatabaseClass and DBInstanceIdentifier
region => "us-west-2"
aws_credentials_file => "/etc/aws_credentials.yaml"
}
}
input {
cloudwatch {
namespace => "AWS/EBS"
metrics => ["VolumeQueueLength"]
filters => { "tag:Monitoring" => "Yes" }
region => "us-west-2"
aws_credentials_file => "/etc/aws_credentials.yaml"
}
}
input {
cloudwatch {
log_group => "/aws/lambda/pinginternal"
aws_credentials_file => "/etc/aws_credentials.yaml"
region => "us-west-2"
type => "lambda"
}
}
filter {
grok {
match => { "message" => "%{TIMESTAMP_ISO8601}\t%{UUID:[lambda][request_id]}\t%{GREEDYDATA:message}" }
overwrite => [ "message" ]
tag_on_failure =>
}

grok {
    match => { "message" => "(?:START|END) RequestId: %{UUID:[lambda][request_id]}" }
    tag_on_failure => [ ]
}

grok {
    match => { "message" => "REPORT RequestId: %{UUID:[lambda][request_id]}\tDuration: %{BASE16FLOAT:[lambda][duration]} ms\tBilled Duration: %{BASE16FLOAT:[lambda][billed_duration]} ms \tMemory Size: %{BASE10NUM:[lambda][memory_size]} MB\tMax Memory Used: %{BASE10NUM:[lambda][memory_used]} MB" }
    tag_on_failure => [ ]
}

mutate {
    convert => {
        "[lambda][duration]" => "integer"
        "[lambda][billed_duration]" => "integer"
        "[lambda][memory_size]" => "integer"
        "[lambda][memory_used]" => "integer"
    }
}

}

output {
elasticsearch {
hosts => ["localhost:9200"]
manage_template => false
index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"

}

}
output {
stdout { codec => rubydebug }
}

"

Thanks.