usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:385] elasticsearch - retrying failed action with response code: 403 ({"type"=>"cluster_block_exception", "reason"=>"blocked by: [FORBIDDEN/12/index read-only / allow delete (api)];"})
I have 5000 files which need to be processed every five minutes, I do have 1.7 G available
logstash runs fine for 2 couples hours and it crashes with the above-mentioned error. I don't see any data in kibana and It's recurring not sure what is causing this error
below is the attached logstash.configuration
input {
file {
path =>"/home/shared/msdp/LogStashOutputFormatted/SES_VG1/.csv"
start_position => "beginning"
sincedb_path => "/dev/null"
type => "sesvg1"
}
file {
path =>"/home/shared/msdp/LogStashOutputFormatted/SES_VG1_Disk/.csv"
start_position => "beginning"
sincedb_path => "/dev/null"
type => "sesvg1disk"
}
file {
path =>"/home/shared/msdp/LogStashOutputFormatted/SES_VG2/.csv"
start_position => "beginning"
sincedb_path => "/dev/null"
type => "sesvg2"
}
file {
path =>"/home/shared/msdp/LogStashOutputFormatted/SPP_VG1/.csv"
start_position => "beginning"
sincedb_path => "/dev/null"
type => "sppvg1"
}
file {
path =>"/home/shared/msdp/LogStashOutputFormatted/SPP_VG1_Disk/.csv"
start_position => "beginning"
sincedb_path => "/dev/null"
type => "sppvg1disk"
}
file {
path =>"/home/shared/msdp/LogStashOutputFormatted/SPP_VG2/.csv"
start_position => "beginning"
sincedb_path => "/dev/null"
type => "sppvg2"
}
}
filter {
if [type] == "sesvg1disk" or [type] == "sppvg1disk"{
csv {
separator => ","
columns => ["nodeName","Filesystem","Size","Used","Avail","Utilization","MountedOn"]
}
}
else if [type] == "sppvg1"{
csv {
separator => "/"
columns => ["nodeName","APIName","Value","Share","Total"]
}
}
else if [type] == "sppvg2"{
csv {
separator => ","
columns => ["nodeName","APIName","Value","Share","Total"]
}
}
else {
csv {
separator => ","
columns => ["nodeName","APIName","Status","Share","Total"]
}
}
mutate {
convert => { "Filesystem" => "string" }
convert => { "Size" => "string" }
convert => { "Used" => "string" }
convert => { "Avail" => "string" }
convert => { "Utilization" => "integer" }
convert => { "MountedOn" => "string" }
convert => { "nodeName" => "string" }
convert => { "APIName" => "string" }
convert => { "Status" => "string" }
convert => { "Value" => "string" }
convert => { "Share" => "float" }
convert => { "Total" => "integer" }
}
}
output {
if [type] == "sesvg1" {
elasticsearch {
action => "index"
hosts => "http://localhost:9200"
index => "beatsesvg1"
}
}
else if [type] == "sesvg1disk" {
elasticsearch {
action => "index"
hosts => "http://localhost:9200"
index => "beatsesvg1disk"
}
}
else if [type] == "sppvg1" {
elasticsearch {
action => "index"
hosts => "http://localhost:9200"
index => "beatsppvg1"
}
}
else if [type] == "sppvg1disk" {
elasticsearch {
action => "index"
hosts => "http://localhost:9200"
index => "beatsppvg1disk"
}
}
else if [type] == "sppvg2" {
elasticsearch {
action => "index"
hosts => "http://localhost:9200"
index => "beatsppvg2"
}
}
else {
elasticsearch {
action => "index"
hosts => "http://localhost:9200"
index => "beatsesvg2"
}