Iterating over certain field extracted from xml to do few operations like remove and replace

Hi,

I have an XML with multiple events. Each event can contain 0 - 6 entries. I would like to make sure that Logstash will extract all parameters, add it to corresponding param field and also enrich one of the fields with those values. I managed to do all that, but quite manually by working in each of the fields. Is there a more efficient way to do it? I know that ruby might help, but i have no experience with it at all. Below logic is also vulnerable in case more params will appear in the source xml file.
Here is sample xml:

<Err>
<Msg>
<Context></Context>
<Index>1</Index>
<FullError>Parameter 1: %1, Parameter 2: %2, Parameter 3: %3</FullError>
<Params>
<Param>
<Type>60</Type>
<Value>ABC</Value>
</Param>
<Param>
<Type>10</Type>
<Value>123</Value>
</Param>
<Param>
<Type>30</Type>
<Value>ff:34:aa:12:99 Expan</Value>
</Param>
</Params>
</Msg>
</Err>
<Err>
<Msg>
<Context></Context>
<Index>2</Index>
</Msg>
</Err>

Here is the config file:

input {
	file {
		path => "/opt/test*.xml"
		start_position => "beginning"
		codec => multiline {
			pattern => "</Err>"
			what => "next"
			negate => "true"
			max_lines => 100000
			}
		}
}

filter {
    xml {
        source => "message"
        target => "err"
		add_field => {
			"param1" => "%{[err][Msg][0][Params][0][Param][0][Value]}"
			"param2" => "%{[err][Msg][0][Params][0][Param][1][Value]}"
			"param3" => "%{[err][Msg][0][Params][0][Param][2][Value]}"
			"param4" => "%{[err][Msg][0][Params][0][Param][3][Value]}"
			"param5" => "%{[err][Msg][0][Params][0][Param][4][Value]}"
			"param6" => "%{[err][Msg][0][Params][0][Param][5][Value]}"
		}
    }
	if [param1] =~ ".*\[err\]\[Msg\]\[0\]\[Params\]\[0\]\[Param\].*" {
		mutate {
			remove_field => ["param1"]
		}
	}
	if [param2] =~ ".*\[err\]\[Msg\]\[0\]\[Params\]\[0\]\[Param\].*" {
		mutate {
			remove_field => ["param2"]
		}
	}
	if [param3] =~ ".*\[err\]\[Msg\]\[0\]\[Params\]\[0\]\[Param\].*" {
		mutate {
			remove_field => ["param3"]
		}
	}
	if [param4] =~ ".*\[err\]\[Msg\]\[0\]\[Params\]\[0\]\[Param\].*" {
		mutate {
			remove_field => ["param4"]
		}
	}
	if [param5] =~ ".*\[err\]\[Msg\]\[0\]\[Params\]\[0\]\[Param\].*" {
		mutate {
			remove_field => ["param5"]
		}
	}
	if [param6] =~ ".*\[err\]\[Msg\]\[0\]\[Params\]\[0\]\[Param\].*" {
		mutate {
			remove_field => ["param6"]
		}
	}
	mutate {
		add_field => {
			"fullerror" => "%{[err][Msg][0][FullError]}"
		}
	}
	mutate {
		gsub => [
			"fullerror", "%1", "%{param1}",
			"fullerror", "%2", "%{param2}",
			"fullerror", "%3", "%{param3}",
			"fullerror", "%4", "%{param4}",
			"fullerror", "%5", "%{param5}",
			"fullerror", "%6", "%{param6}"
		]
	}
	mutate {
		remove_field => ["err"]
	}
}

output {
	stdout {}
}

And here is the stdout{} output:

{
    "@timestamp" => 2019-12-09T13:41:15.284Z,
      "@version" => "1",
          "path" => "/opt/test14.xml",
     "fullerror" => "%{[err][Msg][0][FullError]}",
       "message" => "<Err>\n<Msg>\n<Context></Context>\n<Index>2</Index>\n</Msg>\n</Err>",
          "host" => "localhost",
          "tags" => [
        [0] "multiline"
    ]
}
{
    "@timestamp" => 2019-12-09T13:41:15.279Z,
      "@version" => "1",
          "path" => "/opt/test14.xml",
     "fullerror" => "Parameter 1: ABC, Parameter 2: 123, Parameter 3: ff:34:aa:12:99 Expan",
        "param3" => "ff:34:aa:12:99 Expan",
        "param1" => "ABC",
        "param2" => "123",
       "message" => "<Err>\n<Msg>\n<Context></Context>\n<Index>1</Index>\n<FullError>Parameter 1: %1, Parameter 2: %2, Parameter 3: %3</FullError>\n<Params>\n<Param>\n<Type>60</Type>\n<Value>ABC</Value>\n</Param>\n<Param>\n<Type>10</Type>\n<Value>123</Value>\n</Param>\n<Param>\n<Type>30</Type>\n<Value>ff:34:aa:12:99 Expan</Value>\n</Param>\n</Params>\n</Msg>\n</Err>",
          "host" => "localhost",
          "tags" => [
        [0] "multiline"
    ]
}

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.