Hello!
I am looking for some help with an older version of Logstash - v2.0.0. Disclaimer: I am not very familiar with Logstash customization / optomization and much of what I am encountering is new to me.
The service is running on RHEL 7. It starts successfully, according to the service status, but the logstash.log is getting a constant error like below. Presumably it is preventing the log generation it was giving quite a
while back.
{:timestamp=>"2018-10-02T09:30:45.451000-0500",/
:message=>"A plugin had an unrecoverable error. Will restart this plugin.\n Plugin: <LogStash::Inputs::File path=>[\"/ezproxy/audit/20*.txt\"], /
sincedb_path=>\"/var/run/logstash/ezproxy_audit_sincedb\",/
type=>\"ezproxy_audit\", /
add_field=>{\"program\"=>\"ezproxy\"}, /
codec=><LogStash::Codecs::Plain charset=>\"UTF-8\">, /
stat_interval=>1, /
discover_interval=>15, /
sincedb_write_interval=>15, /
start_position=>\"end\", /
delimiter=>\"\\n\">\n Error: No such file or directory - /var/run/logstash/ezproxy_audit_sincedb.14108.1184.174005",/
:level=>:error}
Log file location perms are set to 644 in /var/log/logstash/ .
The config file is below
/etc/logstash/conf.d/ezproxy.conf
input {
file {
path => "/ezproxy/audit/20*.txt"
sincedb_path => "/var/run/logstash/ezproxy_audit_sincedb"
#start_position => "beginning"
type => "ezproxy_audit"
add_field => { "program" => "ezproxy" }
}
file {
path => "/ezproxy/ezproxy.log"
sincedb_path => "/var/run/logstash/ezproxy_log_sincedb"
#start_position => "beginning"
type => "ezproxy_access"
add_field => { "program" => "ezproxy" }
}
}
filter {
if [type] == "ezproxy_audit" {
# replace tabs with spaces
mutate {
gsub => [ "message", "[\t]+", " " ]
}
# drop log header
if [message] == "Date/Time Event IP Username Session Other" {
drop { }
}
grok {
match => [ "message", "%{DATESTAMP:access_time}%{SPACE}(?<action>Login)\.(?<result>Success)%{SPACE}%{IP:clientip}%{SPACE}%{NOTSPACE:username}%{SPACE}%{NOTSPACE:session}",
"message", "%{DATESTAMP:access_time}%{SPACE}(?<action>Login)\.(?<result>Failure)%{SPACE}%{IP:clientip}%{SPACE}%{NOTSPACE:username}",
"message", "%{DATESTAMP:access_time}%{SPACE}(?<action>Logout)%{SPACE}%{NOTSPACE:username}%{SPACE}%{NOTSPACE:session}%{SPACE}%{NOTSPACE:other}",
"message", "%{DATESTAMP:access_time}%{SPACE}(?<action>Login)\.(?<result>Denied)%{SPACE}%{IP:clientip}%{SPACE}%{NOTSPACE:username}(%{SPACE}%{NOTSPACE:other})?",
"message", "%{DATESTAMP:access_time}%{SPACE}(?<action>System .+)",
"message", "%{DATESTAMP:access_time}%{SPACE}(?<action>UsageLimit)%{SPACE}%{IP:clientip}%{SPACE}%{NOTSPACE:username}%{SPACE}%{NOTSPACE:session}%{SPACE}%{GREEDYDATA:usage_message}",
"message", "%{DATESTAMP:access_time}%{SPACE}(?<action>UsageLimit)%{SPACE}%{NOTSPACE:username}%{SPACE}%{GREEDYDATA:usage_message}",
"message", "%{DATESTAMP:access_time}%{SPACE}%{NOTSPACE:event}%{SPACE}%{IP:clientip}%{SPACE}%{NOTSPACE:username}%{SPACE}%{NOTSPACE:session}(%{SPACE}%{NOTSPACE:other})?"
]
}
if "_grokparsefailure" not in [tags] {
if [type] == "ezproxy_audit" {
date {
# 2015-04-15 13:14:17
match => [ "access_time", "yy-MM-dd HH:mm:ss" ]
timezone => "America/Chicago"
remove_field => [ "timestamp" ]
}
mutate {
lowercase => [ "result", "event", "action" ]
add_field => [ "logsource", "%{host}" ]
remove_field => [ "host" ]
}
geoip {
source => "clientip"
}
}
}
}
else if [type] == "ezproxy_access" {
grok {
patterns_dir => "/etc/logstash/patterns"
match => [ "message", '%{IPORHOST:clientip} %{NOTSPACE:session}%{SPACE}-%{SPACE}%{NOTSPACE}%{SPACE}\[%{HTTPDATE:timestamp}\] "%{NOTSPACE:method} %{EZ_URI:uri} HTTP/%{NUMBER:httpversion}" %{NUMBER:reqstatus:int} %{NUMBER:reqbytes:int}',
"message", '%{IPORHOST:clientip}%{SPACE}-%{SPACE}-%{SPACE}-%{SPACE}\[%{HTTPDATE:timestamp}\] "%{NOTSPACE:method} %{EZ_URI:uri} HTTP/%{NUMBER:httpversion}" %{NUMBER:reqstatus:int} %{NUMBER:reqbytes:int}' ]
}
if "_grokparsefailure" not in [tags] {
mutate {
gsub => [ "timestamp", " -[0-9]+$", "" ]
}
date {
#16/Apr/2015:13:11:01 -0600
match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss" ]
timezone => "America/Chicago"
target => "timestamp"
}
grok {
match => [ "uri", "%{URIPROTO}://%{IPORHOST}(?::%{POSINT})?(?<urlpath>[^?]+)(?<urlquery>.+)?" ]
}
}
}
if "_grokparsefailure" not in [tags] {
if ! [logsource] {
mutate {
add_field => [ "logsource", "%{host}" ]
remove_field => [ "host" ]
}
}
if ! [timestamp] {
mutate {
add_field => [ "timestamp", "%{@timestamp}" ]
}
}
}
}
output {
#stdout { codec => rubydebug }
redis { host => ["10.10.60.175","10.10.60.176"] shuffle_hosts => true data_type => "list" key => "logstash" }
}
Any help is appreciated.