I am an Newbie to ELK using to stash for processing ftp logs (vsftpd, sftp). I am facing missing event issue with Logstash version: 6.1.0,
Test case: uploaded 15 files using ftp - only 7 lines logged in logstash-output.txt and http output receiving even fewer requests than 7.
The vsftpd/xferlog was updated 15 lines for 15 file upload. Am i missing anything in the configuration or anything wrong with current config?
Is there a way to track the events generated?. For this use case there should have be 15 events for type='xferlog'
logstash config
input {
file {
type => "xferlog"
path => "/var/log/vsftpd/xferlog"
id => "xfer"
start_position => "end"
sincedb_path => "/var/log/xfersincedb"
stat_interval => 0.1
}
file {
type => "vsftpd"
path => "/var/log/vsftpd.log"
start_position => "end"
id => "vsftpd"
sincedb_path => "/var/log/vsftpdsincedb"
stat_interval => 0.1
}
file {
type => "sftp"
path => "/var/log/sftp/sftp.log"
start_position => "end"
id => "sftp"
sincedb_path => "/var/log/sftpsincedb"
stat_interval => 0.1
}
}
filter {
if [type] == "xferlog" {
grok{
match => { "message" =>"(?%{SYSLOGTIMESTAMP}%{SPACE}%{YEAR})%{SPACE}%{SPACE}%{NUMBER:duration}%{SPACE}%{IP:clientIP}%{SPACE}%{NUMBER:fileSize}%{SPACE}%{UNIXPATH:filename}%{SPACE}(?\D)%{SPACE}(?C|U|T|_)%{SPACE}(?i|o|I|O)%{SPACE}(?a|g|r)%{SPACE}%{USERNAME:username}%{SPACE}%{USERNAME:serviceName}%{SPACE}(?0|1)%{SPACE}%{DATA}(?[cCiI])"}
named_captures_only => true
}
}
else if [type] == "sftp" {
grok{
match => { "message" => "%{SYSLOGTIMESTAMP:timestamp}%{SPACE}%{HOSTNAME:host}%{SPACE}%{USER:serviceName}[%{NUMBER:sessionId}]:%{SPACE}%{GREEDYDATA:data}"}
named_captures_only => true
overwrite => ["data","host"]
}
}
else if [type] == "vsftpd" {
grok {
match => {"message" =>"(?%{SYSLOGTIMESTAMP}%{SPACE}%{YEAR})%{SPACE}[pid %{NUMBER:sessionId}]%{SPACE}[%{USER:username}]%{SPACE}.*?Client%{SPACE}"%{IP:clientIP}",%{SPACE}"%{GREEDYDATA:data}""}
}
}
}
output {
file {
path => "/home/xxxx/logstash-output.txt"
flush_interval => 0
}
if [type] == "xferlog" {
http {
http_method => "post"
url => "http://localhost:xxxx/xferlog"
id => "xfercall"
automatic_retries => 5
}
}
else if [type] == "sftp" {
http {
http_method => "post"
url => "http://localhost:xxxx/sftplog"
id => "sftpcall"
automatic_retries => 5
}
}
else if [type] == "vsftpd" {
http {
http_method => "post"
url => "http://localhost:xxxx/vsftpdsessionlog"
id => "vsftpdcall"
automatic_retries => 5
}
}
}
xferlog
Fri Jan 12 15:05:45 2018 1 000.00.000.000 5105 /cbdtest1/ftpupload-1.txt b _ i r cbdtest1 ftp 0 * c
Fri Jan 12 15:05:45 2018 1 000.00.000.000 5106 /cbdtest1/ftpupload-10.txt b _ i r cbdtest1 ftp 0 * c
Fri Jan 12 15:05:45 2018 1 000.00.000.000 5106 /cbdtest1/ftpupload-11.txt b _ i r cbdtest1 ftp 0 * c
Fri Jan 12 15:05:45 2018 1 000.00.000.000 5106 /cbdtest1/ftpupload-12.txt b _ i r cbdtest1 ftp 0 * c
Fri Jan 12 15:05:45 2018 1 000.00.000.000 5106 /cbdtest1/ftpupload-13.txt b _ i r cbdtest1 ftp 0 * c
Fri Jan 12 15:05:45 2018 1 000.00.000.000 5106 /cbdtest1/ftpupload-14.txt b _ i r cbdtest1 ftp 0 * c
Fri Jan 12 15:05:45 2018 1 000.00.000.000 5106 /cbdtest1/ftpupload-15.txt b _ i r cbdtest1 ftp 0 * c
Fri Jan 12 15:05:45 2018 1 000.00.000.000 5105 /cbdtest1/ftpupload-2.txt b _ i r cbdtest1 ftp 0 * c
Fri Jan 12 15:05:45 2018 1 000.00.000.000 5105 /cbdtest1/ftpupload-3.txt b _ i r cbdtest1 ftp 0 * c
Fri Jan 12 15:05:45 2018 1 000.00.000.000 5105 /cbdtest1/ftpupload-4.txt b _ i r cbdtest1 ftp 0 * c
Fri Jan 12 15:05:45 2018 1 000.00.000.000 5105 /cbdtest1/ftpupload-5.txt b _ i r cbdtest1 ftp 0 * c
Fri Jan 12 15:05:45 2018 1 000.00.000.000 5105 /cbdtest1/ftpupload-6.txt b _ i r cbdtest1 ftp 0 * c
Fri Jan 12 15:05:45 2018 1 000.00.000.000 5105 /cbdtest1/ftpupload-7.txt b _ i r cbdtest1 ftp 0 * c
Fri Jan 12 15:05:45 2018 1 000.00.000.000 5105 /cbdtest1/ftpupload-8.txt b _ i r cbdtest1 ftp 0 * c
Fri Jan 12 15:05:45 2018 1 000.00.000.000 5105 /cbdtest1/ftpupload-9.txt b _ i r cbdtest1 ftp 0 * c