Parsing measurement xml data using logstash

Elastic Stack Logstash
1 
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="MeasDataCollection.xsl"?>
<measCollecFile
xmlns="http://www.3gpp.org/ftp/specs/latest/rel-5/32_series/32401-540.zip#measCollec">
<fileHeader fileFormatVersion="32.401 V5.0"
	vendorName="Nokia"
	dnPrefix="">
	<fileSender
		localDn="SubNetwork=as1bc,ManagedElement=ces-1"
		elementType="GmscServer,Vlr" />
	<measCollec beginTime="2020-03-22T00:00:00+00:00" />
</fileHeader>
<measData>
	<managedElement
		localDn="SubNetwork=as1bc,ManagedElement=ces-1"
		userLabel=""
		vcpVersion="VM5.82.00"
		swVersion="R36.28.06.0100" />
	<measInfo>
		<granPeriod duration="PT300S" endTime="2020-03-22T00:05:00+00:00" />
		<measType p="1">VS.ADNSCacheHit</measType>
		<measType p="2">VS.ADNSCacheMiss</measType>
		<measType p="3">VS.ADNSDupeQuerySuppress</measType>
		<measType p="4">VS.ADNSCacheTTLExtended</measType>
		<measType p="5">VS.DNSQuerySent</measType>
		<measType p="6">VS.DNSQueryResponse</measType>
		<measValue measObjLdn="Service=h248, ServiceMember=Cabinet_0_Shelf_0_Card_4_Host_0_Pool_0">
			<r p="1">0</r>
			<r p="2">0</r>
			<r p="3">0</r>
			<r p="4">0</r>
			<r p="5">0</r>
			<r p="6">0</r>
		</measValue>
		<measValue measObjLdn="Service=h248, ServiceMember=Cabinet_0_Shelf_1_Card_4_Host_0_Pool_0">
			<r p="1">0</r>
			<r p="2">0</r>
			<r p="3">0</r>
			<r p="4">0</r>
			<r p="5">0</r>
			<r p="6">0</r>
		</measValue>
		<measValue measObjLdn="Service=h248, ServiceMember=Cabinet_0_Shelf_0_Card_4_Host_0_Pool_1">
			<r p="1">0</r>
			<r p="2">0</r>
			<r p="3">0</r>
			<r p="4">0</r>
			<r p="5">0</r>
			<r p="6">0</r>
		</measValue>
		<measValue measObjLdn="Service=h248, ServiceMember=Cabinet_0_Shelf_1_Card_4_Host_0_Pool_1">
			<r p="1">0</r>
			<r p="2">0</r>
			<r p="3">0</r>
			<r p="4">0</r>
			<r p="5">0</r>
			<r p="6">0</r>
		</measValue>
		<measValue measObjLdn="Service=h248, ServiceMember=Cabinet_0_Shelf_0_Card_5_Host_0_Pool_2">
			<r p="1">0</r>
			<r p="2">0</r>
			<r p="3">0</r>
			<r p="4">0</r>
			<r p="5">0</r>
			<r p="6">0</r>
		</measValue>
		<measValue measObjLdn="Service=h248, ServiceMember=Cabinet_0_Shelf_1_Card_5_Host_0_Pool_2">
			<r p="1">0</r>
			<r p="2">0</r>
			<r p="3">0</r>
			<r p="4">0</r>
			<r p="5">0</r>
			<r p="6">0</r>
		</measValue>
		<measValue measObjLdn="Service=h248, ServiceMember=Cabinet_0_Shelf_0_Card_5_Host_0_Pool_3">
			<r p="1">0</r>
			<r p="2">0</r>
			<r p="3">0</r>
			<r p="4">0</r>
			<r p="5">0</r>
			<r p="6">0</r>
		</measValue>
		<measValue measObjLdn="Service=h248, ServiceMember=Cabinet_0_Shelf_1_Card_5_Host_0_Pool_3">
			<r p="1">0</r>
			<r p="2">0</r>
			<r p="3">0</r>
			<r p="4">0</r>
			<r p="5">0</r>
			<r p="6">0</r>
		</measValue>
	</measInfo>

where VS.ADNSCacheHit in`

<measType p="1">VS.ADNSCacheHit</measType>
 represent the variable name and
    <r p="1">0</r> 
represent the value for it

so i want to have like for ServiceMember=Cabinet_0_Shelf_0_Card_4_Host_0_Pool_0 the variable VS.ADNSCacheHit = 0 , VS.ADNSCacheMiss = 0

please help

2

I suggest you read this thread.

3

i have written this conf to parse the data

    file {
        path =>"\A20200322.0000+0000-0005+0000_SubNetwork=as1bc,ManagedElement=ces-1"
		codec => multiline { pattern => "</measInfo>" negate => true what => next auto_flush_interval => 1 }
        start_position => "beginning"
        type => "xml"
    }
}

filter {
    xml { source => "message" target => "[@metadata][theXML]" force_array => false
		}
	ruby {
		code => '
			xml = event.get("[@metadata][theXML]")
			types = xml["measType"]
			values = xml["measValue"]
			a = []
			values.each { |x|
				h = {}
				h["measObjLdn"] = x["measObjLdn"]
				x["r"].each_index { |i|
					h[types[i]["content"]] = x["r"][i]["content"]
				}
				a << h
			}
				event.set("data", a)
			'
	}
}

output {
    stdout {}
}

for this data

               <?xml-stylesheet type="text/xsl" href="MeasDataCollection.xsl"?>
<measCollecFile
	xmlns="http://www.3gpp.org/ftp/specs/latest/rel-5/32_series/32401-540.zip#measCollec">
	<fileHeader fileFormatVersion="32.401 V5.0"
		vendorName="Nokia"
		dnPrefix="">
		<fileSender
			localDn="SubNetwork=as1bc,ManagedElement=ces-1"
			elementType="GmscServer,Vlr" />
		<measCollec beginTime="2020-03-22T00:00:00+00:00" />
	</fileHeader>
	<measData>
		<managedElement
			localDn="SubNetwork=as1bc,ManagedElement=ces-1"
			userLabel=""
			vcpVersion="VM5.82.00"
			swVersion="R36.28.06.0100" />
		<measInfo>
			<granPeriod duration="PT300S" endTime="2020-03-22T00:05:00+00:00" />
			<measType p="1">VS.ADNSCacheHit</measType>
			<measType p="2">VS.ADNSCacheMiss</measType>
			<measType p="3">VS.ADNSDupeQuerySuppress</measType>
			<measType p="4">VS.ADNSCacheTTLExtended</measType>
			<measType p="5">VS.DNSQuerySent</measType>
			<measType p="6">VS.DNSQueryResponse</measType>
			<measValue measObjLdn="Service=h248, ServiceMember=Cabinet_0_Shelf_0_Card_4_Host_0_Pool_0">
				<r p="1">0</r>
				<r p="2">0</r>
				<r p="3">0</r>
				<r p="4">0</r>
				<r p="5">0</r>
				<r p="6">0</r>
			</measValue>
			<measValue measObjLdn="Service=h248, ServiceMember=Cabinet_0_Shelf_1_Card_4_Host_0_Pool_0">
				<r p="1">0</r>
				<r p="2">0</r>
				<r p="3">0</r>
				<r p="4">0</r>
				<r p="5">0</r>
				<r p="6">0</r>
			</measValue>
			<measValue measObjLdn="Service=h248, ServiceMember=Cabinet_0_Shelf_0_Card_4_Host_0_Pool_1">
				<r p="1">0</r>
				<r p="2">0</r>
				<r p="3">0</r>
				<r p="4">0</r>
				<r p="5">0</r>
				<r p="6">0</r>
			</measValue>
			<measValue measObjLdn="Service=h248, ServiceMember=Cabinet_0_Shelf_1_Card_4_Host_0_Pool_1">
				<r p="1">0</r>
				<r p="2">0</r>
				<r p="3">0</r>
				<r p="4">0</r>
				<r p="5">0</r>
				<r p="6">0</r>
			</measValue>
			<measValue measObjLdn="Service=h248, ServiceMember=Cabinet_0_Shelf_0_Card_5_Host_0_Pool_2">
				<r p="1">0</r>
				<r p="2">0</r>
				<r p="3">0</r>
				<r p="4">0</r>
				<r p="5">0</r>
				<r p="6">0</r>
			</measValue>
			<measValue measObjLdn="Service=h248, ServiceMember=Cabinet_0_Shelf_1_Card_5_Host_0_Pool_2">
				<r p="1">0</r>
				<r p="2">0</r>
				<r p="3">0</r>
				<r p="4">0</r>
				<r p="5">0</r>
				<r p="6">0</r>
			</measValue>
			<measValue measObjLdn="Service=h248, ServiceMember=Cabinet_0_Shelf_0_Card_5_Host_0_Pool_3">
				<r p="1">0</r>
				<r p="2">0</r>
				<r p="3">0</r>
				<r p="4">0</r>
				<r p="5">0</r>
				<r p="6">0</r>
			</measValue>
			<measValue measObjLdn="Service=h248, ServiceMember=Cabinet_0_Shelf_1_Card_5_Host_0_Pool_3">
				<r p="1">0</r>
				<r p="2">0</r>
				<r p="3">0</r>
				<r p="4">0</r>
				<r p="5">0</r>
				<r p="6">0</r>
			</measValue>
		</measInfo>
		<measInfo>
			<granPeriod duration="PT300S" endTime="2020-03-22T00:05:00+00:00" />
			<measType p="1">VS.aveCpuUsage</measType>
			<measType p="2">VS.peakCpuUsage</measType>
			<measValue measObjLdn="Cabinet=0, Shelf=0, Card=4, Host=0">
				<r p="1">1.21</r>
				<r p="2">2.38</r>
			</measValue>
			<measValue measObjLdn="Cabinet=0, Shelf=1, Card=4, Host=0">
				<r p="1">1.46</r>
				<r p="2">2.42</r>
			</measValue>
			<measValue measObjLdn="Cabinet=0, Shelf=0, Card=5, Host=0">
				<r p="1">1.14</r>
				<r p="2">1.88</r>
			</measValue>
			<measValue measObjLdn="Cabinet=0, Shelf=1, Card=5, Host=0">
				<r p="1">1.17</r>
				<r p="2">2.06</r>
			</measValue>
		</measInfo>
	</measData>
	<fileFooter>
		<measCollec endTime="2020-03-22T00:05:00+00:00" />
	</fileFooter>
</measCollecFile>

getting output as

[2020-04-20T17:37:35,023][WARN ][logstash.filters.xml     ][main] Error parsing xml with XmlSimple {:source=>"message", :value=>"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<?xml-stylesheet type=\"text/xsl\" href=\"MeasDataCollection.xsl\"?>\n<measCollecFile\n\txmlns=\"http://www.3gpp.org/ftp/specs/latest/rel-5/32_series/32401-540.zip#measCollec\">\n\t<fileHeader fileFormatVersion=\"32.401 V5.0\"\n\t\tvendorName=\"Nokia\"\n\t\tdnPrefix=\"\">\n\t\t<fileSender\n\t\t\tlocalDn=\"SubNetwork=as1bc,ManagedElement=ces-1\"\n\t\t\telementType=\"GmscServer,Vlr\" />\n\t\t<measCollec beginTime=\"2020-03-22T00:00:00+00:00\" />\n\t</fileHeader>\n\t<measData>\n\t\t<managedElement\n\t\t\tlocalDn=\"SubNetwork=as1bc,ManagedElement=ces-1\"\n\t\t\tuserLabel=\"\"\n\t\t\tvcpVersion=\"VM5.82.00\"\n\t\t\tswVersion=\"R36.28.06.0100\" />\n\t\t<measInfo>\n\t\t\t<granPeriod duration=\"PT300S\" endTime=\"2020-03-22T00:05:00+00:00\" />\n\t\t\t<measType p=\"1\">VS.ADNSCacheHit</measType>\n\t\t\t<measType p=\"2\">VS.ADNSCacheMiss</measType>\n\t\t\t<measType p=\"3\">VS.ADNSDupeQuerySuppress</measType>\n\t\t\t<measType p=\"4\">VS.ADNSCacheTTLExtended</measType>\n\t\t\t<measType p=\"5\">VS.DNSQuerySent</measType>\n\t\t\t<measType p=\"6\">VS.DNSQueryResponse</measType>\n\t\t\t<measValue measObjLdn=\"Service=h248, ServiceMember=Cabinet_0_Shelf_0_Card_4_Host_0_Pool_0\">\n\t\t\t\t<r p=\"1\">0</r>\n\t\t\t\t<r p=\"2\">0</r>\n\t\t\t\t<r p=\"3\">0</r>\n\t\t\t\t<r p=\"4\">0</r>\n\t\t\t\t<r p=\"5\">0</r>\n\t\t\t\t<r p=\"6\">0</r>\n\t\t\t</measValue>\n\t\t\t<measValue measObjLdn=\"Service=h248, ServiceMember=Cabinet_0_Shelf_1_Card_5_Host_0_Pool_3\">\n\t\t\t\t<r p=\"1\">0</r>\n\t\t\t\t<r p=\"2\">0</r>\n\t\t\t\t<r p=\"3\">0</r>\n\t\t\t\t<r p=\"4\">0</r>\n\t\t\t\t<r p=\"5\">0</r>\n\t\t\t\t<r p=\"6\">0</r>\n\t\t\t</measValue>\n\t\t</measInfo>", :exception=>#<REXML::ParseException: No close tag for /measCollecFile/measData
Line: 91
Position: 2811
Last 80 unconsumed characters:
>, :backtrace=>["uri:classloader:/META-INF/jruby.home/lib/ruby/stdlib/rexml/parsers/treeparser.rb:28:in `parse'", "uri:classloader:/META-INF/jruby.home/lib/ruby/stdlib/rexml/document.rb:288:in `build'", "uri:classloader:/META-INF/jruby.home/lib/ruby/stdlib/rexml/logstash-7.6.1/logstash-core/lib/logstash/java_pipeline.rb:262:in `block in start_workers'"]}
[2020-04-20T17:37:35,255][ERROR][logstash.filters.ruby    ][main] Ruby exception occurred: undefined method `[]' for nil:NilClass
logstash-7.6.1/vendor/bundle/jruby/2.5.0/gems/awesome_print-1.7.0/lib/awesome_print/formatters/base_formatter.rb:31: warning: constant ::Fixnum is deprecated
{
          "host" => "DESKTOP-AJVSOQJ",
          "tags" => [
        [0] "multiline",
        [1] "_xmlparsefailure",
        [2] "_rubyexception"
    ],
      "@version" => "1",
          "path" => "A20200322.0000+0000-0005+0000_SubNetwork=as1bc,ManagedElement=ces-1",
          "type" => "xml",
    "@timestamp" => 2020-04-20T15:37:34.113Z,
       "message" => "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<?xml-stylesheet type=\"text/xsl\" href=\"MeasDataCollection.xsl\"?>\n<measCollecFile\n\txmlns=\"http://www.3gpp.org/ftp/specs/latest/rel-5/32_series/32401-540.zip#measCollec\">\n\t<fileHeader fileFormatVersion=\"32.401 V5.0\"\n\t\tvendorName=\"Nokia\"\n\t\tdnPrefix=\"\">\n\t\t<fileSender\n\t\t\tlocalDn=\"SubNetwork=as1bc,ManagedElement=ces-1\"\n\t\t\telementType=\"GmscServer,Vlr\" />\n\t\t<measCollec beginTime=\"2020-03-22T00:00:00+00:00\" />\n\t</fileHeader>\n\t<measData>\n\t\t<managedElement\n\t\t\tlocalDn=\"SubNetwork=as1bc,ManagedElement=ces-1\"\n\t\t\tuserLabel=\"\"\n\t\t\tvcpVersion=\"VM5.82.00\"\n\t\t\tswVersion=\"R36.28.06.0100\" />\n\t\t<measInfo>\n\t\t\t<granPeriod duration=\"PT300S\" endTime=\"2020-03-22T00:05:00+00:00\" />\n\t\t\t<measType p=\"1\">VS.ADNSCacheHit</measType>\n\t\t\t<measType p=\"2\">VS.ADNSCacheMiss</measType>\n\t\t\t<measType p=\"3\">VS.ADNSDupeQuerySuppress</measType>\n\t\t\t<measType p=\"4\">VS.ADNSCacheTTLExtended</measType>\n\t\t\t<measType p=\"5\">VS.DNSQuerySent</measType>\n\t\t\t<measType p=\"6\">VS.DNSQueryResponse</measType>\n\t\t\t<measValue measObjLdn=\"Service=h248, ServiceMember=Cabinet_0_Shelf_1_Card_5_Host_0_Pool_3\">\n\t\t\t\t<r p=\"1\">0</r>\n\t\t\t\t<r p=\"2\">0</r>\n\t\t\t\t<r p=\"3\">0</r>\n\t\t\t\t<r p=\"4\">0</r>\n\t\t\t\t<r p=\"5\">0</r>\n\t\t\t\t<r p=\"6\">0</r>\n\t\t\t</measValue>\n\t\t</measInfo>"
}
{
          "host" => "DESKTOP-AJVSOQJ",
          "tags" => [
        [0] "multiline"
    ],
      "@version" => "1",
          "path" => "A20200322.0000+0000-0005+0000_SubNetwork=as1bc,ManagedElement=ces-1",
          "type" => "xml",
    "@timestamp" => 2020-04-20T15:37:34.182Z,
       "message" => "\t\t<measInfo>\n\t\t\t<granPeriod duration=\"PT300S\" endTime=\"2020-03-22T00:05:00+00:00\" />\n\t\t\t<measType p=\"1\">VS.aveCpuUsage</measType>\n\t\t\t<measType p=\"2\">VS.peakCpuUsage</measType>\n\t\t\t<measValue measObjLdn=\"Cabinet=0, Shelf=0, Card=4, Host=0\">\n\t\t\t\t<r p=\"1\">1.21</r>\n\t\t\t\t<r p=\"2\">2.38</r>\n\t\t\t</measValue>\n\t\t\t<measValue measObjLdn=\"Cabinet=0, Shelf=1, Card=4, Host=0\">\n\t\t\t\t<r p=\"1\">1.46</r>\n\t\t\t\t<r p=\"2\">2.42</r>\n\t\t\t</measValue>\n\t\t\t<measValue measObjLdn=\"Cabinet=0, Shelf=0, Card=5, Host=0\">\n\t\t\t\t<r p=\"1\">1.14</r>\n\t\t\t\t<r p=\"2\">1.88</r>\n\t\t\t</measValue>\n\t\t\t<measValue measObjLdn=\"Cabinet=0, Shelf=1, Card=5, Host=0\">\n\t\t\t\t<r p=\"1\">1.17</r>\n\t\t\t\t<r p=\"2\">2.06</r>\n\t\t\t</measValue>\n\t\t</measInfo>",
          "data" => [
        [0] {
             "VS.aveCpuUsage" => "1.21",
            "VS.peakCpuUsage" => "2.38",
                 "measObjLdn" => "Cabinet=0, Shelf=0, Card=4, Host=0"
        },
        [1] {
             "VS.aveCpuUsage" => "1.46",
            "VS.peakCpuUsage" => "2.42",
                 "measObjLdn" => "Cabinet=0, Shelf=1, Card=4, Host=0"
        },
        [2] {
             "VS.aveCpuUsage" => "1.14",
            "VS.peakCpuUsage" => "1.88",
                 "measObjLdn" => "Cabinet=0, Shelf=0, Card=5, Host=0"
        },
        [3] {
             "VS.aveCpuUsage" => "1.17",
            "VS.peakCpuUsage" => "2.06",
                 "measObjLdn" => "Cabinet=0, Shelf=1, Card=5, Host=0"
        }
    ]
}
[2020-04-20T17:37:35,938][WARN ][logstash.filters.xml     ][main] Error parsing xml with XmlSimple {:source=>"message", :value=>"\t</measData>\n\t<fileFooter>\n\t\t<measCollec endTime=\"2020-03-22T00:05:00+00:00\" />\n\t</fileFooter>\n</measCollecFile>", :exception=>#<REXML::ParseException: Missing end tag for '' (got "measData")
Line: 1
Position: 13
Last 80 unconsumed characters:
[2020-04-20T17:37:35,992][ERROR][logstash.filters.ruby    ][main] Ruby exception occurred: undefined method `[]' for nil:NilClass
{
          "host" => "DESKTOP-AJVSOQJ",
          "tags" => [
        [0] "multiline",
        [1] "_xmlparsefailure",
        [2] "_rubyexception"
    ],
      "@version" => "1",
          "path" => "A20200322.0000+0000-0005+0000_SubNetwork=as1bc,ManagedElement=ces-1",
          "type" => "xml",
    "@timestamp" => 2020-04-20T15:37:35.594Z,
       "message" => "\t</measData>\n\t<fileFooter>\n\t\t<measCollec endTime=\"2020-03-22T00:05:00+00:00\" />\n\t</fileFooter>\n</measCollecFile>"
}

log stash is only converting one of the tags

1)can you help me with removing the starting part of the xml , i do went through the the mutuate stuff

mutate { gsub => [ "message", "^<\?xml[^
	]+
	", ""] }

but couldnot understad it .as i want to remove more stuff than the xml tag.

  1. i want to add values localDn="SubNetwork=as1bc,ManagedElement=ces-1 in each array parsed ?
4

Because you are using input file codec multiline you need to remove the bit before your pattern so try adding below to your filter to drop that line (which is then the begining of the document untill the 1st match of patten).

if "<?xml version" in [message] { drop { }}

5

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.