Usage of grok filter

Elastic Stack Logstash
1

Tying to use grok to add regex part of the XML as a field.

    	grok {
    		patterns_dir => ["tns:.+?>"]
    		match => { "message" => "methodName:"} }
    }

My config:

  input {
    	file { 
    		 path => "C:\Users\Desktop\xml\20190207.log" 
    		 type => "test-xml" 
    		 start_position => "beginning" 
    		 codec => multiline { 
    			 pattern => "^" 
    			 negate => true 
    			 what => "previous" 
    		 } 
    	 }
    }
    filter {
    	mutate {
    			gsub => ["message", "xsi:\w+=(?<grp1>\"|')\w+(:\w+)?\k<grp1>\s*", ""]
    			gsub => ["message", 'href="#\w+"', ""]
    			gsub => ["message", "<soap:Body.+?>|</soap:Body>", ""]
    	}
    	grok {
    		patterns_dir => ["tns:.+?>"]
    		match => { "message" => "methodName:"} }
    }

    filter {
    	if( [message] =~ "^.*- Request -.*<"){
    		mutate { gsub => [ "message", "^[^<]+<", "<" ] }
    			xml {
    				remove_namespaces => true
    				store_xml => true
    				source => "message"
    				target => "Request"
    				force_array => false
    			}
    			mutate {
    		  remove_field => ["message",
    						   "[Request][Header]",
    						   "[Request][xmlns:xsi]",
    						   "[Request][xmlns:soap]",
    						   "[Request][xmlns:tns]",
    						   "[Request][xmlns:types]",
    						   "[Request][xmlns:xsd]",
    						   "[Request][xmlns:env]",
    						   "[Request][xmlns:soapenc]",
    						   "[Request][soap:encodingStyle]"
    						  ]
    		}
    	}
    	else if ( [message] =~ "^.*- Response -.*<"){
    		mutate { gsub => [ "message", "^[^<]+<", "<" ] }
    		xml {
    			remove_namespaces => true
    			store_xml => true
    			source => "message"
    			target => "Response"
    			force_array => false
    		}
    		mutate {
    		  remove_field => ["message",
    						   "[Response][Header]",
    						   "[Response][xmlns:xsi]",
    						   "[Response][xmlns:soap]",
    						   "[Response][xmlns:tns]",
    						   "[Response][xmlns:types]",
    						   "[Response][xmlns:xsd]",
    						   "[Response][xmlns:env]",
    						   "[Response][xmlns:soapenc]",
    						   "[Response][soap:encodingStyle]"
    						  ]
    		}
    	}
    }

    output {
    	elasticsearch {
      		hosts => ["******.*********.com:9200"]
      		index => "testing-%{+yyyy.MM.dd}"
      	}
    }

XML File:

2019-01-18 14:03:07,666 - Request - ****** - ******************************************- getOpenInvoices - 
<?xml version="1.0" encoding="utf-8"?>
<soap:Envelope xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/">
    <soap:Body soap:encodingStyle="http://schemas.xmlsoap.org/soap/encoding/">
        <tns:getOpenInvoices>
            <invoiceQueryOpenRequest href="#id1" />
        </tns:getOpenInvoices>
        <q1:InvoiceQueryOpenRequest id="id1" xsi:type="q1:InvoiceQueryOpenRequest" xmlns:q1="java:com.turkcelltech.collgw.model.invoice">
            <bankId xsi:type="xsd:int">23</bankId>
            <compId xsi:type="xsd:int">533</compId>
            <curr xsi:type="xsd:string">949</curr>
            <custId xsi:nil="true" />
            <invCount xsi:type="xsd:int">5</invCount>
            <msgDate xsi:nil="true" />
            <msisdn xsi:type="xsd:long">************</msisdn>
            <orig xsi:nil="true" />
            <period xsi:type="xsd:string">201901</period>
            <procDate xsi:nil="true" />
            <procTime xsi:nil="true" />
            <sessionId xsi:type="xsd:string">**********</sessionId>
            <stan xsi:type="xsd:long">0</stan>
        </q1:InvoiceQueryOpenRequest>
    </soap:Body>
</soap:Envelope>
2

patterns_dir takes an array of directory names. Logstash will then read files from those directories expecting them to defined patterns.

3

I Just want to use "tns:.+?" regex to add that to a new field just stuck there

4

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.