I'm new to logstash, I'm trying to use logstash and filebeat.
'The filebeat sends the events (lines) of a log file * .txt, and logstash does the filter and cature the messages I want, however the main messages are in the first lines of code of the log, the doubt is:
How to aggregate these events with the stored dog information 1.
Question 2: How to hide the other log events to be able to use this in a database.
P; s: I already used mutate, aggretate and grok and even had success. Could you tell me what I'm missing.
example log:
XPTO,
NG,A0000
1704195438
V0.1.34
TEST
fist-pipleine
input {
beats {
port => "5043"
}
}
filter {
change "_" to "-" title archive
mutate {
gsub => ["source", "_", "-"]
}
capture name of archive 'log' 'SFCS / MAC / XXXXXXXX / DATA / HORA'
grok {
match => ["source", "%{WORD:t1}(?:[-])%{WORD:t2}(?:[-])%{WORD:mac}(?:[-])%{YEAR:year}%{MONTHNUM:month}%{MONTHDAY:day}(?:[-])%{HOUR:hour}%{MINUTE:minute}%{SECOND:second}.txt$"]
}
check first lines of archive
grok {
match => ["message", "(?.*?),$"]
match => ["message", "%{WORD:word2}(?:[,])%{WORD:word3}"]
}
grok {
match => ["name", "(?.*?),$"]
}
#Test mutate
mutate {
replace => { "%{word1}" => "%{s}"}
add_field => {"%{word1}[myNewField]" => "%{s}"}
}
add field "datetime" with data captured
mutate { add_field => ["datetime", "%{month}/%{day}/%{year} %{hour}:%{minute}"] }
mutate { remove_field => ["year", "day", "month", "hour", "minute"] }
teste aggregate
if [message] == "XPTO," {
aggregate {
task_id => "%{word1}"
code => "map['exemplo'] = 0"
map_action => "create"
}
}
if [message] == "1704195438" {
aggregate {
task_id => "%{word1}"
code => "map['exemplo'] += event.get('duration')"
map_action => "update"
}
}
mutate {
add_field => {"juntos" => "%{word1}/%{word2}/%{word3}"}
remove_field => ["word1", "word2", "word3"]
}
output {
stdout { codec => rubydebug { metadata => true } }
#stdout {
#codec => json_lines
jdbc {
connection_string => "jdbc:postgresql://myip/mydb"
username => "myuser"
password => "mypassword"
statement => [ INSERT INTO efc_log(log_recebido)VALUES ('XPTO'); ]
}
}