Hi, i am using Logstash to generate logs of IPS. But elasticsearch delete all docs except last one, i dont know why.
My Logstash config:
input {
file {
path => ["/var/log/suricata/eve.json"]
codec => "json"
type => "Suricata"
}
}
filter {
if [type] == "Suricata" {
date {
match => [ "timestamp", "ISO8601" ]
}
ruby {
code => "if event['event_type'] == 'fileinfo'; event['fileinfo']['type']=event['fileinfo']['magic'].to_s.split(',')[0]; end;"
}
}
}
output
{
elasticsearch {
action => "index"
user => "elastic"
password => "system"
hosts => ["127.0.0.1"]
document_id => "suricata"
index => "logstash-%{+YYYY.MM.DD}"
}
}
My log of suricata
{"timestamp":"2016-10-28T10:45:14.000075-0200","flow_id":904705990262178,"event_type":"flow","src_ip":"192.168.1.103","src_port":23630,"dest_ip":"64.4.23.165","dest_port":40017,"proto":"UDP","flow":{"pkts_toserver":1,"pkts_toclient":1,"bytes_toserver":73,"bytes_toclient":548,"start":"2016-10-28T10:40:12.196002-0200","end":"2016-10-28T10:40:12.383161-0200","age":0,"state":"established","reason":"timeout"}}
{"timestamp":"2016-10-28T10:45:14.000133-0200","flow_id":1410309540351611,"event_type":"flow","src_ip":"192.168.1.103","src_port":23630,"dest_ip":"189.63.100.104","dest_port":9828,"proto":"UDP","flow":{"pkts_toserver":1,"pkts_toclient":1,"bytes_toserver":161,"bytes_toclient":99,"start":"2016-10-28T10:40:13.724603-0200","end":"2016-10-28T10:40:13.745362-0200","age":0,"state":"established","reason":"timeout"}}
{"timestamp":"2016-10-28T10:45:15.000059-0200","flow_id":783012386910091,"event_type":"flow","src_ip":"192.168.1.103","src_port":57029,"dest_ip":"192.168.1.1","dest_port":53,"proto":"UDP","app_proto":"dns","flow":{"pkts_toserver":1,"pkts_toclient":1,"bytes_toserver":80,"bytes_toclient":148,"start":"2016-10-28T10:40:13.733067-0200","end":"2016-10-28T10:40:13.739907-0200","age":0,"state":"established","reason":"timeout"}}
{"timestamp":"2016-10-28T10:45:15.000129-0200","flow_id":1513588471434918,"event_type":"flow","src_ip":"192.168.1.103","src_port":23630,"dest_ip":"65.55.223.36","dest_port":40016,"proto":"UDP","flow":{"pkts_toserver":1,"pkts_toclient":1,"bytes_toserver":78,"bytes_toclient":62,"start":"2016-10-28T10:40:13.724646-0200","end":"2016-10-28T10:40:13.864608-0200","age":0,"state":"established","reason":"timeout"}}
{"timestamp":"2016-10-28T10:45:15.000155-0200","flow_id":113706174996590,"event_type":"flow","src_ip":"fe80:0000:0000:0000:2f6f:0e61:a3b4:906a","src_port":5353,"dest_ip":"ff02:0000:0000:0000:0000:0000:0000:00fb","dest_port":5353,"proto":"UDP","flow":{"pkts_toserver":7,"pkts_toclient":0,"bytes_toserver":700,"bytes_toclient":0,"start":"2016-10-28T10:44:27.024686-0200","end":"2016-10-28T10:44:43.144673-0200","age":16,"state":"new","reason":"timeout"}}
{"timestamp":"2016-10-28T10:45:15.000187-0200","flow_id":2086000237808930,"event_type":"flow","src_ip":"157.55.130.154","src_port":40002,"dest_ip":"192.168.1.103","dest_port":23630,"proto":"UDP","flow":{"pkts_toserver":1,"pkts_toclient":1,"bytes_toserver":68,"bytes_toclient":53,"start":"2016-10-28T10:40:13.724258-0200","end":"2016-10-28T10:40:13.724351-0200","age":0,"state":"established","reason":"timeout"}}
{"timestamp":"2016-10-28T10:45:15.000208-0200","flow_id":1809902534712734,"event_type":"flow","src_ip":"179.185.64.35","src_port":30818,"dest_ip":"192.168.1.103","dest_port":23630,"proto":"UDP","flow":{"pkts_toserver":3,"pkts_toclient":4,"bytes_toserver":462,"bytes_toclient":1617,"start":"2016-10-28T10:33:44.056734-0200","end":"2016-10-28T10:40:13.724586-0200","age":389,"state":"established","reason":"timeout"}}
{"timestamp":"2016-10-28T10:45:15.000229-0200","flow_id":419765544509584,"event_type":"flow","src_ip":"192.168.1.103","src_port":5353,"dest_ip":"224.0.0.251","dest_port":5353,"proto":"UDP","flow":{"pkts_toserver":7,"pkts_toclient":0,"bytes_toserver":560,"bytes_toclient":0,"start":"2016-10-28T10:44:27.024720-0200","end":"2016-10-28T10:44:43.144699-0200","age":16,"state":"new","reason":"timeout"}}
{"timestamp":"2016-10-28T10:45:15.000474-0200","flow_id":1148778261883622,"event_type":"flow","src_ip":"54.245.98.57","src_port":32099,"dest_ip":"192.168.1.103","dest_port":28432,"proto":"UDP","flow":{"pkts_toserver":3,"pkts_toclient":3,"bytes_toserver":182,"bytes_toclient":238,"start":"2016-10-28T10:44:41.502502-0200","end":"2016-10-28T10:44:44.506604-0200","age":3,"state":"new","reason":"timeout"}}
In elasticsearch i have only 1 doc, all others are removed. Someone khow what i have to do for work as append mode ?