Logstash file plugin

Elastic Stack Logstash
1

I am using file Output plugin to store parsed events, is there any option to clear that file for every 24hr?

Below is the Example:

output
{
if ( [status] =~ "Invalid credentials" ) and "throttled" not in [tags]
{
file {
path => "/home/ec2-user/vpn_invalid.yml"
message_format => "%{username}: invalid"
}
}
}

I want to clear the data from vpn_invalid.yml file for every 24hrs.

Kindly help me.

2

You would need to write it to a time based file and the handle removal of the old one external to LS.

3

Hi Mark,

Thank you very much for quick response.

Kindly help me in creating time based file.

4

Have a read of https://www.elastic.co/guide/en/logstash/current/plugins-outputs-file.html#plugins-outputs-file-path

5

One more query regarding this, this time based file can be used in Translate filter?

6

What do you mean, can you give an example?

7

I am using above mentioned file 'vpn_invalid.yml ' in the Translate filter to match the data.

Below is the example.

if ( [vpn_status] =~ "pool returned") {
translate {
field => "username"
destination => "invalid_user"
refresh_interval => 5
dictionary_path => "/home/ec2-user/vpn_invalid.yml"
add_tag => [ "invaliduser" ]
}
}

8

You're using the same file in the output and then the filter?

9

Yes, In the output section i am using to write the username filed and in the filter part i am using to match the value.

10

Can you show the full config?

11 
input 
 {
stdin{}
 }


 
filter {
grok {
		match => { "message" => "%{MONTH:month}(?:\s|\s\s)%{MONTHDAY:day}\s(?<time> *[0-9]*[:][0-9]*[:][0-9][0-9])\s%{HOSTNAME:hostname}\s%{WORD:vpn_daemon}\:\suser\s\'%{USER:username}\' %{WORD:status}",  "%{MONTH:month}(?:\s|\s\s)%{MONTHDAY:day}\s(?<time> *[0-9]*[:][0-9]*[:][0-9][0-9])\s%{HOSTNAME:hostname}\s%{WORD:vpn_daemon}\:\suser\s\'%{USER:username}\'(?<status> \b\w+\b\s\b\w+\b\s\b\w+\b)",  "%{MONTH:month}(?:\s|\s\s)%{MONTHDAY:day}\s(?<time> *[0-9]*[:][0-9]*[:][0-9][0-9])\s%{HOSTNAME:hostname}\s%{WORD:vpn_daemon}\:\s/openvpn.auth-user.php: ERROR! Could not login to server\s%{WORD:server_name} as user %{USER:username}\:(?<status> \b\w+\b\s\b\w+\b)",  "%{MONTH:month}(?:\s|\s\s)%{MONTHDAY:day}\s(?<time> *[0-9]*[:][0-9]*[:][0-9][0-9])\s%{HOSTNAME:hostname}\s%{WORD:vpn_daemon}\:\s/openvpn.auth-user.php: ERROR! Either(?<status> \b\w+\b\s\b\w+\b\s\b\w+\b\,\s\b\w+\b\s\b\w+\b\s\b\w+\b\s\b\w+\b\s\b\w+\b)", "%{MONTH:month}(?:\s|\s\s)%{MONTHDAY:day}\s(?<time> *[0-9]*[:][0-9]*[:][0-9][0-9])\s%{HOSTNAME:hostname}\s%{WORD:vpn_daemon}\[%{INT:vpn_id}\]\:\s%{USERNAME:username}/%{IPV4:public_ip}:%{INT:publicip_port} MULTI_sva:\s(?<vpn_status> *\b\w+\b \b\w+\b) IPv4=%{IPV4:vpn_ip}, IPv6=(?<vpn_msg>.*)"   }
		add_field => [ "received_at", "%{@timestamp}" ]
		add_field => [ "received_from", "%{host}" ]
    }

geoip 
		{
           		add_tag => [ "GeoIP" ]
           		source => "public_ip"
           		add_field => ["[geoip][coordinates]","%{[geoip][longitude]}"]
           		add_field => ["[geoip][coordinates]","%{[geoip][latitude]}"]

       	}
   		mutate 
		{
      			convert => [ "[geoip][coordinates]", "float" ]
   		}

mutate {
            remove_tag => [ "_grokparsefailure" ]
        }
		
if ([vpn_daemon] !~ "openvpn")
                       { drop{} }		



if ( [status] =~ "Invalid credentials")
{
throttle {
before_count => 3
after_count => 3
period => 300
key => "%{username}"
add_tag => "throttled"
}
}


if ( [vpn_status] =~ "pool returned")  {
translate {
    field => "username"
    destination => "invalid_user"
    refresh_interval => 5
    dictionary_path => "/home/ec2-user/vpn_invalid.yml"
    add_tag => [ "invaliduser" ]
}
}
  
  
  }
 
output
{
if ( [status] =~ "Invalid credentials" ) and "throttled" not in [tags]
{
  file {
        path => "/home/ec2-user/vpn_invalid.yml"
        message_format => "%{username}: invalid"
}
}     

 }
12

You can just do;

filter {
grok {
		match => { "message" => "PATTERN1", "PATTERN2", etc etc }
}
}

Instead of having all those separate groks, just makes it cleaner and easier to read.

13

I did as you mentioned.