Xml parse in logstash

Ayaan_Shaik · February 8, 2024, 11:39am

Hi @Badger I'm Trying to parse the log file with below xml

<imm:IMM-contents xmlns:imm="http://www.saforum.org/IMMSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="SAI-AIS-IMM-XSD-A.02.13.xsd">
	<class name="OpenSafAmfConfig">
		<category>SA_CONFIG</category>
		<rdn>
			<name>amfConfig</name>
			<type>SA_STRING_T</type>
			<category>SA_CONFIG</category>
			<flag>SA_INITIALIZED</flag>
		</rdn>
		<attr>
			<name>osafAmfRestrictAutoRepairEnable</name>
			<type>SA_UINT32_T</type>
			<category>SA_CONFIG</category>
			<flag>SA_WRITABLE</flag>
		</attr>
		<attr>
			<name>osafAmfDelayNodeFailoverTimeout</name>
			<type>SA_TIME_T</type>
			<category>SA_CONFIG</category>
			<flag>SA_WRITABLE</flag>
		</attr>
		<attr>
			<name>osafAmfDelayNodeFailoverNodeWaitTimeout</name>
			<type>SA_TIME_T</type>
			<category>SA_CONFIG</category>
			<flag>SA_WRITABLE</flag>
		</attr>
	</class>

my pipeline

input {
  file {
    path => "/etc/logstash/imm.xml"
    start_position => "beginning"
    sincedb_path => "/dev/null"
    codec => multiline {
      pattern => "<class name"
      negate => true
      what => "previous"
    }
  }
}

filter
{
mutate { gsub => [ "message", "<.xml version.>", "" ] }
mutate { gsub => [ "message", "<imm:IMM-contents.>", "" ] }
xml
{
source => "message"
remove_namespaces => true
store_xml => false
#target => "theXML"
xpath => [
"/class/@name", "class_name",
"/class/category/text()", "class_category",
"/class/rdn/name/text()", "rdn_name",
"/class/rdn/type/text()", "rdn_type",
"/class/rdn/category/text()", "rdn_category",
"/class/rdn/flag/text()", "rdn_flag",
"/class/attr/name/text()", "attr_name",
"/class/attr/type/text()", "attr_type",
"/class/attr/category/text()", "attr_category",
#"/job_list/hard_request[@name]", "hard_request",
"/class/attr/flag/text()", "attr_flag",
"/class/attr/default-value/text()", "default_value"
]
}
mutate {
rename => [
"[class_name][0]", "class_name",
"[class_category][0]", "class_category",
"[rdn_name][0]", "rdn_name",
"[rdn_type][0]", "rdn_type",
"[rdn_category][0]", "rdn_category",
"[rdn_flag][0]", "rdn_flag",
"[attr_name][0]", "attr_name",
"[attr_type][0]", "attr_type",
"[attr_category][0]", "attr_category",
"[attr_flag][0]", "attr_flag",
"[default_value][0]", "default_value"
]
}
}
output
{
stdout { codec => rubydebug }
}

I want every rdn adn attr as an event or element not in a single count . instead of i want all with every element..
Please help me on this ..

Badger · February 9, 2024, 1:34am

Don't use xpath, store the xml then split { field => "[theXML][attr]" } then rename all the fields to your taste.

Ayaan_Shaik · February 9, 2024, 5:53am

Thanks, @Badger But still I'm getting error.
Can you tell me how to do it. it would be helpful for me..

Thanks,

filter
{
mutate { gsub => [ "message", "<.xml version.>", "" ] }
mutate { gsub => [ "message", "<imm:IMM-contents.>", "" ] }
xml
{
source => "message"
remove_namespaces => true
store_xml => true
target => "theXML"
split { 
    field => "[theXML][attr]",
    field => "[theXML][rdn]"
}
}
mutate {
rename => [
"[class_name][0]", "class_name",
"[class_category][0]", "class_category",
"[rdn_name][0]", "rdn_name",
"[rdn_type][0]", "rdn_type",
"[rdn_category][0]", "rdn_category",
"[rdn_flag][0]", "rdn_flag",
"[attr_name][0]", "attr_name",
"[attr_type][0]", "attr_type",
"[attr_category][0]", "attr_category",
"[attr_flag][0]", "attr_flag"
]
}
}
output
{
stdout { codec => rubydebug }
}

Above is the config file.

Rios · February 9, 2024, 7:42am

Can you replace lines like this, separated xml and 2 split:

xml
{
source => "message"
remove_namespaces => true
store_xml => true
target => "theXML"
}

split { 
    field => "[theXML][attr]"
}

split { 
    field => "[theXML][rdn]"
}

Ayaan_Shaik · February 9, 2024, 3:29pm

@Rios Thanks for the replay man.

Still I'm getting error ..
I tried with split but getting error.

Can you tell me pls how to do with split of possible can you reframe above code
Thanks..

Badger · February 9, 2024, 3:48pm

That is expected. You are consuming the file as two events (not sure why). The first does not have <attr> or <rdn> elements, so the split will fail. You can make the split conditional on the existence of the field.

Ayaan_Shaik · February 9, 2024, 4:27pm

Thanks, @Badger..
from the above xml I want 4 events rdn and 3 attr events how can I do I'm getting only 1 single event still I tried with above split I'm new to this ELK can you pls help me out ..

Ayaan_Shaik · February 10, 2024, 7:42am

@Badger can you replay please on this ..

Ayaan_Shaik · February 13, 2024, 10:40am

Hi @Rios waiting for your replay .....

Rios · February 13, 2024, 4:12pm

You can make something as Badger suggested

input {
  file {file {
    path => "/etc/logstash/imm.xml"
    start_position => "beginning"
    sincedb_path => "/dev/null"
   codec => multiline {
      pattern => "\s*<class"
      negate => true
      what => "previous"
      auto_flush_interval => 1
      multiline_tag => ""
   }
  }
}
filter {
   if ([message] =~ /^<imm/ ) {
     drop{}
   }
 
   xml {
     source => "message"
     target => "theXML"
	 force_array => false
	 remove_namespaces => true
   }
   
   split { 
      field => "[theXML][attr]"
  }
  mutate{  rename => ["[theXML][name]", "class_name",
  "[theXML][category]", "category",
  "[theXML][rdn][name]", "rnd_name",  
  "[theXML][rdn][category]", "rnd_category", 
  "[theXML][rdn][type]", "rnd_type", 
  "[theXML][rdn][flag]", "rnd_flag",
  "[theXML][attr][category]", "attr_category",
  "[theXML][attr][type]", "attr_type",
  "[theXML][attr][flag]", "attr_flag",
  "[theXML][attr][name]", "attr_name"
  ]
  }

}

output {
    stdout { codec => rubydebug{} }
}

Result:

{
     "rnd_category" => "SA_CONFIG",
         "rnd_flag" => "SA_INITIALIZED",
         "rnd_name" => "amfConfig",
         "rnd_type" => "SA_STRING_T",
         "category" => "SA_CONFIG",
    "attr_category" => "SA_CONFIG",
        "attr_flag" => "SA_WRITABLE",
        "attr_name" => "osafAmfRestrictAutoRepairEnable",
       "class_name" => "OpenSafAmfConfig",
        "attr_type" => "SA_UINT32_T"
}
{
     "rnd_category" => "SA_CONFIG",
         "rnd_flag" => "SA_INITIALIZED",
         "rnd_name" => "amfConfig",
         "rnd_type" => "SA_STRING_T",
         "category" => "SA_CONFIG",
    "attr_category" => "SA_CONFIG",
        "attr_flag" => "SA_WRITABLE",
        "attr_name" => "osafAmfDelayNodeFailoverTimeout",
       "class_name" => "OpenSafAmfConfig",
        "attr_type" => "SA_TIME_T"
}
{
     "rnd_category" => "SA_CONFIG",
         "rnd_flag" => "SA_INITIALIZED",
         "rnd_name" => "amfConfig",
         "rnd_type" => "SA_STRING_T",
         "category" => "SA_CONFIG",
    "attr_category" => "SA_CONFIG",
        "attr_flag" => "SA_WRITABLE",
        "attr_name" => "osafAmfDelayNodeFailoverNodeWaitTimeout",
       "class_name" => "OpenSafAmfConfig",
        "attr_type" => "SA_TIME_T"
}

Not sure is this what you want. If you want to more separate nodes, you have to use ruby. Maybe someone will have better idea.

system · March 12, 2024, 4:12pm

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.