Hi,
I have an XML with multiple events. Each event can contain 0 - 6 entries. I would like to make sure that Logstash will extract all parameters, add it to corresponding param field and also enrich one of the fields with those values. I managed to do all that, but quite manually by working in each of the fields. Is there a more efficient way to do it? I know that ruby might help, but i have no experience with it at all. Below logic is also vulnerable in case more params will appear in the source xml file.
Here is sample xml:
<Err>
<Msg>
<Context></Context>
<Index>1</Index>
<FullError>Parameter 1: %1, Parameter 2: %2, Parameter 3: %3</FullError>
<Params>
<Param>
<Type>60</Type>
<Value>ABC</Value>
</Param>
<Param>
<Type>10</Type>
<Value>123</Value>
</Param>
<Param>
<Type>30</Type>
<Value>ff:34:aa:12:99 Expan</Value>
</Param>
</Params>
</Msg>
</Err>
<Err>
<Msg>
<Context></Context>
<Index>2</Index>
</Msg>
</Err>
Here is the config file:
input {
file {
path => "/opt/test*.xml"
start_position => "beginning"
codec => multiline {
pattern => "</Err>"
what => "next"
negate => "true"
max_lines => 100000
}
}
}
filter {
xml {
source => "message"
target => "err"
add_field => {
"param1" => "%{[err][Msg][0][Params][0][Param][0][Value]}"
"param2" => "%{[err][Msg][0][Params][0][Param][1][Value]}"
"param3" => "%{[err][Msg][0][Params][0][Param][2][Value]}"
"param4" => "%{[err][Msg][0][Params][0][Param][3][Value]}"
"param5" => "%{[err][Msg][0][Params][0][Param][4][Value]}"
"param6" => "%{[err][Msg][0][Params][0][Param][5][Value]}"
}
}
if [param1] =~ ".*\[err\]\[Msg\]\[0\]\[Params\]\[0\]\[Param\].*" {
mutate {
remove_field => ["param1"]
}
}
if [param2] =~ ".*\[err\]\[Msg\]\[0\]\[Params\]\[0\]\[Param\].*" {
mutate {
remove_field => ["param2"]
}
}
if [param3] =~ ".*\[err\]\[Msg\]\[0\]\[Params\]\[0\]\[Param\].*" {
mutate {
remove_field => ["param3"]
}
}
if [param4] =~ ".*\[err\]\[Msg\]\[0\]\[Params\]\[0\]\[Param\].*" {
mutate {
remove_field => ["param4"]
}
}
if [param5] =~ ".*\[err\]\[Msg\]\[0\]\[Params\]\[0\]\[Param\].*" {
mutate {
remove_field => ["param5"]
}
}
if [param6] =~ ".*\[err\]\[Msg\]\[0\]\[Params\]\[0\]\[Param\].*" {
mutate {
remove_field => ["param6"]
}
}
mutate {
add_field => {
"fullerror" => "%{[err][Msg][0][FullError]}"
}
}
mutate {
gsub => [
"fullerror", "%1", "%{param1}",
"fullerror", "%2", "%{param2}",
"fullerror", "%3", "%{param3}",
"fullerror", "%4", "%{param4}",
"fullerror", "%5", "%{param5}",
"fullerror", "%6", "%{param6}"
]
}
mutate {
remove_field => ["err"]
}
}
output {
stdout {}
}
And here is the stdout{} output:
{
"@timestamp" => 2019-12-09T13:41:15.284Z,
"@version" => "1",
"path" => "/opt/test14.xml",
"fullerror" => "%{[err][Msg][0][FullError]}",
"message" => "<Err>\n<Msg>\n<Context></Context>\n<Index>2</Index>\n</Msg>\n</Err>",
"host" => "localhost",
"tags" => [
[0] "multiline"
]
}
{
"@timestamp" => 2019-12-09T13:41:15.279Z,
"@version" => "1",
"path" => "/opt/test14.xml",
"fullerror" => "Parameter 1: ABC, Parameter 2: 123, Parameter 3: ff:34:aa:12:99 Expan",
"param3" => "ff:34:aa:12:99 Expan",
"param1" => "ABC",
"param2" => "123",
"message" => "<Err>\n<Msg>\n<Context></Context>\n<Index>1</Index>\n<FullError>Parameter 1: %1, Parameter 2: %2, Parameter 3: %3</FullError>\n<Params>\n<Param>\n<Type>60</Type>\n<Value>ABC</Value>\n</Param>\n<Param>\n<Type>10</Type>\n<Value>123</Value>\n</Param>\n<Param>\n<Type>30</Type>\n<Value>ff:34:aa:12:99 Expan</Value>\n</Param>\n</Params>\n</Msg>\n</Err>",
"host" => "localhost",
"tags" => [
[0] "multiline"
]
}