Using Xpath Dynamically

Elastic Stack Logstash
1

I want to use xpath in logstash with dynamic way

My log file:

<q1:InvoiceQueryOpenRequest id="id1" xsi:type="q1:InvoiceQueryOpenRequest" xmlns:q1="java:com.turkcelltech.collgw.model.invoice">
            <bankId xsi:type="xsd:int">23</bankId>
            <compId xsi:type="xsd:int">533</compId>
            <curr xsi:type="xsd:string">949</curr>
            <custId xsi:nil="true"/>
            <invCount xsi:type="xsd:int">5</invCount>
            <msgDate xsi:nil="true"/>
            <msisdn xsi:type="xsd:long">1234567</msisdn>
            <orig xsi:nil="true"/>
            <period xsi:type="xsd:string">201901</period>
            <procDate xsi:nil="true"/>
            <procTime xsi:nil="true"/>
            <sessionId xsi:type="xsd:string">********************</sessionId>
            <stan xsi:type="xsd:long">0</stan>
        </q1:InvoiceQueryOpenRequest>

My config filter:

xpath =>{ "//q1:InvoiceQueryOpenRequest/*/text()" => "nvoiceQueryOpenRequest" }

2

What output do you get with output { stdout { codec => rubydebug } }?

3

I'm seeing REXML::UndefinedNamespaceException: Undefined prefix xsi found
and a tag with _xmlparsefailure.

4

My full config

input {
	file { 
		 path => "C:\Users\Desktop\xml\20190118.log" 
		 type => "test-xml" 
		 start_position => "beginning" 
		 codec => multiline { 
			 pattern => "^" 
			 negate => true 
			 what => "previous" 
		 } 
	 }
}
filter {
	
	mutate { gsub => [ "message", "^[^<]+<", "<" ] }
    xml {
        source => "message"
        target => "theXML"
        store_xml => true
		xpath =>{ 
"//q1:InvoiceQueryOpenRequest/*/text()" => "nvoiceQueryOpenRequest" 
}
    }
	mutate {
      remove_field => ["message",
					   "[theXML]"
					  ]
    }
	
}

output {
	file
	{
		path => "C:\Users\Desktop\xml2\20190118.json"
	}
}
5

I tried this config but got a xml parse failure.

input {
  generator {
    message => '<q1:InvoiceQueryOpenRequest id="id1" xsi:type="q1:InvoiceQueryOpenRequest" xmlns:q1="java:com.turkcelltech.collgw.model.invoice"><bankId xsi:type="xsd:int">23</bankId><compId xsi:type="xsd:int">533</compId><curr xsi:type="xsd:string">949</curr><custId xsi:nil="true"/><invCount xsi:type="xsd:int">5</invCount><msgDate xsi:nil="true"/><msisdn xsi:type="xsd:long">1234567</msisdn><orig xsi:nil="true"/><period xsi:type="xsd:string">201901</period><procDate xsi:nil="true"/><procTime xsi:nil="true"/><sessionId xsi:type="xsd:string">***</sessionId><stan xsi:type="xsd:long">0</stan></q1:InvoiceQueryOpenRequest>'
    count => 1
  }
}
filter {
  xml {
    namespaces => {
      "xsi" => "http://www.w3.org/2001/XMLSchema-instance"
      "xsd" => "http://www.w3.org/2001/XMLSchema"
      "xmlns" => "http://www.w3.org/2001/XMLSchema"
    }
    remove_namespaces => true
    # store_xml => true
    target => "[log_event]"
    source => "[message]"
    force_array => false
    # force_content => false
   # xpath =>{ "//q1:InvoiceQueryOpenRequest/*/text()" => "nvoiceQueryOpenRequest" }
  }
}

output {
  stdout { codec => rubydebug }
}

Please post the unedited contents of the file (redact sensitive or private info).

6

The config I wrote is correct and having a JSON output but the problem is using XPath. I want to use XPath to get everything within "InvoiceQueryOpenRequest"

The org xml is:

2019-01-18 14:03:07,666 - Request - *************** - http://****************************************** - getOpenInvoices - 
<?xml version="1.0" encoding="utf-8"?>
<soap:Envelope xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/" xmlns:soapenc="http://schemas.xmlsoap.org/soap/encoding/" xmlns:tns="http://******************************" xmlns:types="http://**************************************" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema">
    <soap:Body soap:encodingStyle="http://schemas.xmlsoap.org/soap/encoding/">
        <tns:getOpenInvoices>
            <invoiceQueryOpenRequest href="#id1" />
        </tns:getOpenInvoices>
        <q1:InvoiceQueryOpenRequest id="id1" xsi:type="q1:InvoiceQueryOpenRequest" xmlns:q1="***************************">
            <bankId xsi:type="xsd:int">23</bankId>
            <compId xsi:type="xsd:int">533</compId>
            <curr xsi:type="xsd:string">949</curr>
            <custId xsi:nil="true" />
            <invCount xsi:type="xsd:int">5</invCount>
            <msgDate xsi:nil="true" />
            <msisdn xsi:type="xsd:long">123456</msisdn>
            <orig xsi:nil="true" />
            <period xsi:type="xsd:string">201901</period>
            <procDate xsi:nil="true" />
            <procTime xsi:nil="true" />
            <sessionId xsi:type="xsd:string">**********************</sessionId>
            <stan xsi:type="xsd:long">0</stan>
        </q1:InvoiceQueryOpenRequest>
    </soap:Body>
</soap:Envelope>
7

Ok. I stripped the namespace stuff - now I can parse.

Without xpath, I have:

input {
  generator {
    message => '<InvoiceQueryOpenRequest id="id1"><bankId>23</bankId><compId>533</compId><curr>949</curr><custId/><invCount>5</invCount><msgDate/><msisdn>1234567</msisdn><orig/><period>201901</period><procDate/><procTime/><sessionId>****</sessionId><stan>0</stan></InvoiceQueryOpenRequest>'
    count => 1
  }
}
filter {
  xml {
    remove_namespaces => true
    store_xml => true
    source => "[message]"
    target => "[log_event]"
    force_array => false
    # force_content => false
    # xpath =>{ "//InvoiceQueryOpenRequest/*/text()" => "nvoiceQueryOpenRequest" }
  }
}

output {
  stdout { codec => rubydebug }
}

Gives:

{
          "host" => "Elastics-MacBook-Pro.local",
      "sequence" => 0,
       "message" => "<InvoiceQueryOpenRequest id=\"id1\"><bankId>23</bankId><compId>533</compId><curr>949</curr><custId/><invCount>5</invCount><msgDate/><msisdn>1234567</msisdn><orig/><period>201901</period><procDate/><procTime/><sessionId>****</sessionId><stan>0</stan></InvoiceQueryOpenRequest>",
      "@version" => "1",
    "@timestamp" => 2019-02-05T11:52:18.941Z,
     "log_event" => {
           "compId" => "533",
           "custId" => nil,
           "period" => "201901",
         "procTime" => nil,
        "sessionId" => "****",
             "curr" => "949",
         "invCount" => "5",
             "stan" => "0",
          "msgDate" => nil,
               "id" => "id1",
           "msisdn" => "1234567",
             "orig" => nil,
           "bankId" => "23",
         "procDate" => nil
    }
}

With xpath, I have:

input {
  generator {
    message => '<InvoiceQueryOpenRequest id="id1"><bankId>23</bankId><compId>533</compId><curr>949</curr><custId/><invCount>5</invCount><msgDate/><msisdn>1234567</msisdn><orig/><period>201901</period><procDate/><procTime/><sessionId>****</sessionId><stan>0</stan></InvoiceQueryOpenRequest>'
    count => 1
  }
}
filter {
  xml {
    remove_namespaces => true
    # store_xml => true
    store_xml => false
    source => "[message]"
    # target => "[log_event]"
    force_array => false
    # force_content => false
    xpath =>{ "//InvoiceQueryOpenRequest/*/text()" => "nvoiceQueryOpenRequest" }
  }
}

output {
  stdout { codec => rubydebug }
}

Gives:

{
                  "@version" => "1",
                      "host" => "Elastics-MacBook-Pro.local",
                  "sequence" => 0,
                "@timestamp" => 2019-02-05T11:54:28.911Z,
    "nvoiceQueryOpenRequest" => [
        [0] "23",
        [1] "533",
        [2] "949",
        [3] "5",
        [4] "1234567",
        [5] "201901",
        [6] "****",
        [7] "0"
    ],
                   "message" => "<InvoiceQueryOpenRequest id=\"id1\"><bankId>23</bankId><compId>533</compId><curr>949</curr><custId/><invCount>5</invCount><msgDate/><msisdn>1234567</msisdn><orig/><period>201901</period><procDate/><procTime/><sessionId>****</sessionId><stan>0</stan></InvoiceQueryOpenRequest>"
}

Show us the shape that you want to see in your document then we can see if an xpath statement can be created.

8

Output will be JSON file like this:

{
  
      "q1:InvoiceQueryOpenRequest": {
        "bankId": {
          "#text": "23"
        },
        "compId": {
          "#text": "533"
        },
        "curr": {
          "#text": "949"
        },
        "custId": { 
        "-xsi:nil": "true"
        },
        "invCount": {
          "#text": "5"
        },
        "msgDate": {
        "-xsi:nil": "true"
        },
        "msisdn": {
          "#text": "5338409600"
        },
        "orig": { 
        "-xsi:nil": "true"
        },
        "period": {
          "#text": "201901"
        },
        "procDate": { 
         "-xsi:nil": "true" 
         },
        "procTime": { 
         "-xsi:nil": "true" 
        },
        "sessionId": {

          "#text": "sdahaF9nB/8foQJv7o4iJA=="
        },
        "stan": {

          "#text": "0"
        }
}
9

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.