Hi everybody,
I´ve got a problem with a config file, which updates mappings of existing ones.
Environment:
one Debian Server with ES 7.1.1
Configs:
system_action_job.conf
input {
jdbc {
jdbc_driver_library => "/opt/jdbc/mysql-latest.jar"
jdbc_driver_class => "com.mysql.cj.jdbc.Driver"
jdbc_connection_string => "jdbc:mysql://server.sae.intra:3306/application"
jdbc_user => "user"
jdbc_password => "password"
statement => "SELECT * from system_action_job WHERE id > :sql_last_value order by id asc LIMIT 1000000"
use_column_value => true
last_run_metadata_path => "/var/lib/logstash/.system_action_job_id"
tracking_column => "id"
}
}
output{
elasticsearch {
hosts => ["localhost:9200"]
index => "system_action_log"
}
waf.conf
input {
udp {
#host => "localhost"
port => 514
type => barracuda
}
}
filter {
if [type] == "barracuda" {
# Extract Log Fields
ruby {
init => "
HEADER_FIELDS = ['cef_version','Vendor','Product','DeviceVersion','SignatureId','EventName','Severity']
#event_new = LogStash::Event.new
def store_header_field(event,field_name,field_data)
#Unescape pipes and backslash in header fields
event.set(field_name,field_data.gsub(/\\\|/, '|').gsub(/\\\\/, '\\')) unless field_data.nil?
end
"
code => "
if event.get('[message][0]') == '\"'
event.set('[message]' , event.get('[message]')[1..-2])
end
split_data = event.get('[message]').split /(?<=[^\\]\\\\)[\|]|(?<!\\)[\|]/
HEADER_FIELDS.each_with_index do |field_name, index|
store_header_field(event,field_name,split_data[index])
end
msg = split_data[HEADER_FIELDS.size..-1].join('|')
if event.get('cef_version').include? ' '
split_cef_version= event.get('cef_version').rpartition(' ')
event.set('syslog', split_cef_version[0])
event.set('cef_version',split_cef_version[2])
end
event.set('cef_version', event.get('cef_version').sub(/^CEF:/, ''))
if not msg.nil? and msg.include? '='
msg = msg.strip
# If the last KVP has no value, add an empty string, this prevents hash errors below
if msg.end_with?('=')
msg=msg + ' ' unless msg.end_with?('\=')
end
# Now parse the key value pairs into it
msg = msg.split(/ ([\w\.]+)=/)
key, value = msg.shift.split('=', 2)
event.set(key,value.gsub(/\\=/, '=').gsub(/\\\\/, '\\'))
Hash[*msg].each{ |k, v| event.set(k,v.gsub(/\\=/, '=').gsub(/\\\\/, '\\')) unless v.nil? }
end
"
remove_field => ['message']
}
# Filtering LogFields which are common to all Log Types
mutate {
convert => {"Severity" => "integer" }
}
grok {
match => {"cef_version" => ".+\:%{INT:cef_version:int}" }
overwrite => ["cef_version"]
}
grok {
match => {"DeviceReceiptTime" => "^\s*%{DATA:DeviceReceiptTime}\s*$" }
overwrite => ["DeviceReceiptTime"]
}
mutate {
gsub => ["StartTime", '\"', ""]
}
grok {
match => {"StartTime" => "^\s*%{DATA:StartTime}\s*$" }
overwrite => ["StartTime"]
}
date {
match => ["StartTime","MMM dd YYYY HH:mm:ss"]
target => "StartTime"
}
date {
match => ["DeviceReceiptTime","UNIX_MS"]
target => "DeviceReceiptTime"
}
# LogType Specific Filtering
if [LogType] == "SYS" {