We have an ELK stack app that has been down for over a month due to a credentials issue in the logstash cloudwatch plugin. The plugin is digesting data again now, but what is strange is that it is digesting logs from the beginning of time. So logs from over two years ago. Also, no data is being outputted to Elasticsearch, perhaps because that data has already been transformed and outputted previously?
My main question: is this typical behavior? I'm not very familiar with logstash and elasticsearch, but I can't imagine every time you restart logstash it starts digesting every cloudwatch log from the very first logs. Not sure if it will help, but here is the logstash conf fiel for the cloudwatch plugin:
input {
cloudwatch_logs {
access_key_id => access_here
secret_access_key => secret_here
log_group => [ "xwingui-Prod", "xwingui-Dev", "xwingui-Exp", "xwingui-Staging", "xwingui-Test", "xwingui-Jawn" ]
region => "us-west-1"
sincedb_path => "/var/lib/.sincedb"
}
}
filter {
if "Monitoring - " in [message] {
if "API" in [message] {
grok {
match => { "message" => "API Monitoring - %{GREEDYDATA:json}" }
}
mutate {
add_field => { "monitorType" => "API" }
}
} else if "RUM" in [message] {
grok {
match => { "message" => "RUM Monitoring - %{GREEDYDATA:json}" }
}
mutate {
add_field => { "monitorType" => "RUM" }
}
} else if "PikaWorker" in [message] {
grok {
match => { "message" => "PikaWorker Monitoring - %{GREEDYDATA:json}" }
}
mutate {
add_field => { "monitorType" => "PikaWorker" }
}
} else if "DataAgent" in [message] {
grok {
match => { "message" => "DataAgent Monitoring - %{GREEDYDATA:json}" }
}
mutate {
add_field => { "monitorType" => "DataAgent" }
}
} else if "Database" in [message] {
grok {
match => { "message" => "Database Monitoring - %{GREEDYDATA:json}" }
}
mutate {
add_field => { "monitorType" => "Database" }
}
}
json {
source => "json"
remove_field => "message"
}
mutate {
add_field => { "isMonitor" => True }
}
}
}
output {
elasticsearch {
hosts => [ "localhost:9200" ]
user => user_here
password => pwd_here
}
stdout {
codec => json
}
}
