Hello!
I'm trying to setup ELK stack.
I'm importing logs from remote servers to ELK-server. Logs in JSON format with some additions from rsyslog (timestamp, remote server name, app name).
if I understand correctly, Logstash can easily parse pure JSON log with "json" filter. So I need to move away rsyslog additions.
I read documentation on Logstash filters, but still can't understand how to do it correctly.
Here is my logstash.conf:
input {
file {
path => [ "/path/to/my/log" ]
type => json
codec => json
stat_interval => 10
}
}
output {
elasticsearch {
cluster => "my_logs"
embedded => false
host => localhost
index => "my-log-%{+YYYY.MM.dd}"
}
}
Log entry is looking like this:
2015-06-15T16:26:46+03:00 remote-server-name app-name {"timestamp":1434374805391,"stats":[{"pid":"6336","cpu":"14.6","mem":"0.4","type":"worker","id":"12021","games_count":"0","clients_count":"417","tournaments_count":"320"},{"pid":"6337","cpu":"14.8","mem":"0.4","type":"worker","id":"12022","games_count":"0","clients_count":"378","tournaments_count":"285"},{"pid":"8310","cpu":"12.1","mem":"0.4","type":"worker","id":"12023","games_count":"1","clients_count":"380","tournaments_count":"307"},{"pid":"8312","cpu":"11.5","mem":"0.4","type":"worker","id":"12024","games_count":"1","clients_count":"357","tournaments_count":"281"},{"pid":"8314","cpu":"18.3","mem":"0.5","type":"worker","id":"12025","games_count":"0","clients_count":"396","tournaments_count":"318"},{"pid":"8316","cpu":"11.3","mem":"0.4","type":"worker","id":"12026","games_count":"2","clients_count":"348","tournaments_count":"284"},{"pid":"8318","cpu":"13.7","mem":"0.4","type":"worker","id":"12027","games_count":"1","clients_count":"365","tournaments_count":"289"},{"pid":"8320","cpu":"12.4","mem":"0.4","type":"worker","id":"12028","games_count":"2","clients_count":"379","tournaments_count":"306"},{"pid":"20016","cpu":"13.8","mem":"0.4","type":"worker","id":"12029","games_count":"0","clients_count":"374","tournaments_count":"302"},{"pid":"20017","cpu":"19.3","mem":"0.5","type":"worker","id":"12030","games_count":"3","clients_count":"497","tournaments_count":"380"},{"pid":"20020","cpu":"13.3","mem":"0.4","type":"worker","id":"12031","games_count":"1","clients_count":"444","tournaments_count":"359"},{"pid":"20022","cpu":"11.9","mem":"0.4","type":"worker","id":"12032","games_count":"2","clients_count":"322","tournaments_count":"258"},{"pid":"20024","cpu":"12.6","mem":"0.4","type":"worker","id":"12033","games_count":"0","clients_count":"370","tournaments_count":"281"},{"pid":"20026","cpu":"13.1","mem":"0.4","type":"worker","id":"12034","games_count":"1","clients_count":"346","tournaments_count":"276"},{"pid":"20028","cpu":"12.3","mem":"0.4","type":"worker","id":"12035","games_count":"1","clients_count":"344","tournaments_count":"273"},{"pid":"20030","cpu":"12.6","mem":"0.4","type":"worker","id":"12036","games_count":"1","clients_count":"410","tournaments_count":"307"},{"pid":"20032","cpu":"14.3","mem":"0.4","type":"worker","id":"12037","games_count":"4","clients_count":"415","tournaments_count":"323"},{"pid":"20034","cpu":"17.7","mem":"0.5","type":"worker","id":"12038","games_count":"0","clients_count":"400","tournaments_count":"315"},{"pid":"20036","cpu":"12.8","mem":"0.4","type":"worker","id":"12039","games_count":"0","clients_count":"414","tournaments_count":"330"},{"pid":"20038","cpu":"14.8","mem":"0.5","type":"worker","id":"12040","games_count":"3","clients_count":"441","tournaments_count":"360"}]}
Thank you in advance.