Filebeat to Logstash, Logstash to Elasticsearch

I am using two different servers and my Filebeat server sends logs to the Logstash server without changing anything. Then Logstash server parse incoming files and forward it to the elastic. However, in elastic logs i can see that _xmlparsefailure.
Normally without using those servers and try to send logs to the elastic using logstash codes on my local machine there will be no parse failures

Elastic:

Logstash:

input {
	beats {
		port => "5044"
	 }
}
filter {
		if [message] =~ /^\s*$/ {
		  drop { }
		}
		mutate {
				gsub => ["message", "^\t*\r?\n\t*", ""]
				gsub => ["message", "xsi:\w+=(?<grp1>\"|')\w+(:\w+)?\k<grp1>\s*", ""]
				gsub => ["message", 'href="#\w+"', ""]
				
		}
		if( [message] =~ "^.*kktcell.*<"){
			mutate {
				gsub => ["message", "<soap:Body.+?>|</soap:Body>", ""]
				gsub => ["message", "<env:Body.+?>|</env:Body>", ""]
			}
		}
		else if ( [message] =~ "^.*88.255.27.59.*<"){
			mutate {
				gsub => ["message", "<soap:Body>|</soap:Body>", ""]
			}
		}
		if ( [message] =~ "^.*77.92.16.129.*<"){
			mutate {
				gsub => ["message", "<soap:Body>|</soap:Body>", ""]
			}
		}
		grok {
			match => { "message" => "(?<methodName>((?<=\s-\s)\w+(?= - <\?xml)))" }
		}
}
filter {
	if( [message] =~ "^.*- Request -.*<"){
		mutate { gsub => [ "message", "^[^<]+<", "<" ] }
			xml {
				remove_namespaces => true
				store_xml => true
				source => "message"
				target => "Properties"
				force_array => false
			}
			mutate {
		  add_field => { "messageType" => "Request" }
		}
	}
	else if ( [message] =~ "^.*- Response -.*<"){
		mutate { gsub => [ "message", "^[^<]+<", "<" ] }
		xml {
			remove_namespaces => true
			store_xml => true
			source => "message"
			target => "Response"
			force_array => false
		}
		mutate {
		  add_field => { "messageType" => "Response" }
		}
	}
	mutate {
	  remove_field => ["message",
			   "[Properties][Header]",
			   "[Properties][xmlns:xsi]",
			   "[Properties][xmlns:soap]",
			   "[Properties][xmlns:tns]",
			   "[Properties][xmlns:types]",
			   "[Properties][xmlns:xsd]",
			   "[Properties][xmlns:env]",
			   "[Properties][xmlns:soapenc]",
			   "[Properties][soap:encodingStyle]",
			   "[Properties][Array][Item]"
			  ]
		}
}

output {
	elasticsearch {
  		hosts => ["****.********.com:9200"]
  		index => "*********-%{+yyyy.MM.dd}"
  	}
}

Filebeat:

filebeat.prospectors:

- type: log

  enabled: true

  paths:
    - C:\Logs\Kurumlar\*.log

multiline.pattern: '^\['
  multiline.negate: true
  multiline.match: after

filebeat.config.modules:
  path: ${path.config}/modules.d/*.yml

  reload.enabled: true

  reload.period: 10s


setup.template.settings:
  index.number_of_shards: 3

output.logstash:
  hosts: ["192.168.***.***:5044"]

Example Log:

2019-01-17 17:35:31,597 - Request - *****.***** - http://*****.kktcell.com:****/*****GWws/*****GWBank - payInvoice - 
<?xml version="1.0" encoding="utf-8"?>
<soap:Envelope xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/" xmlns:soapenc="http://schemas.xmlsoap.org/soap/encoding/" xmlns:tns="http://my.turkcell.com.tr:7002/KKTCCollGWws/KKTCCollGW" xmlns:types="http://my.turkcell.com.tr:7002/KKTCCollGWws/KKTCCollGW/encodedTypes" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema">
    <soap:Body soap:encodingStyle="http://schemas.xmlsoap.org/soap/encoding/">
        <tns:payInvoice>
            <invoicePaymentRequest href="#id1" />
        </tns:payInvoice>
        <q1:InvoicePaymentRequest id="id1" xsi:type="q1:InvoicePaymentRequest" xmlns:q1="java:com.turkcelltech.collgw.model.invoice">
            <accDate xsi:type="xsd:string">20190117</accDate>
            <bankId xsi:type="xsd:int">23</bankId>
            <captDate xsi:type="xsd:string">****</captDate>
            <compId xsi:type="xsd:int">****</compId>
            <curr xsi:type="xsd:string">949</curr>
            <invCount xsi:type="xsd:int">1</invCount>
            <invoiceList href="#id2" />
            <msgDate xsi:nil="true" />
            <msisdn xsi:type="xsd:long">*******</msisdn>
            <opType xsi:type="xsd:int">10</opType>
            <orig href="#id3" />
            <paymSource xsi:type="xsd:int">4</paymSource>
            <procDate xsi:type="xsd:string">0117</procDate>
            <sessionId xsi:type="xsd:string">**********</sessionId>
            <stan xsi:type="xsd:long">35852</stan>
        </q1:InvoicePaymentRequest>
        <soapenc:Array id="id2" xmlns:q2="java:com.******.collgw.model.invoice" soapenc:arrayType="q2:InvoiceDetail[1]">
            <Item href="#id4" />
        </soapenc:Array>
        <q3:OriginatorId id="id3" xsi:type="q3:OriginatorId" xmlns:q3="java:com.******.collgw.model.common">
            <branch xsi:type="xsd:string">11</branch>
            <city xsi:type="xsd:int">0</city>
            <teller xsi:type="xsd:string">45</teller>
            <user xsi:type="xsd:string">****, E</user>
        </q3:OriginatorId>
        <q4:InvoiceDetail id="id4" xsi:type="q4:InvoiceDetail" xmlns:q4="java:com.*******.collgw.model.invoice">
            <controlId xsi:type="xsd:string">******</controlId>
            <duedate xsi:type="xsd:string">20190117</duedate>
            <invAmount xsi:type="xsd:double">0</invAmount>
            <invNo xsi:type="xsd:long">******</invNo>
            <invType xsi:type="xsd:int">0</invType>
            <nameSurname xsi:type="xsd:string">Su*** Al******</nameSurname>
            <orderNo xsi:type="xsd:int">0</orderNo>
            <period xsi:type="xsd:string">******</period>
            <prevMsisdn xsi:type="xsd:long">*****</prevMsisdn>
            <remark xsi:type="xsd:string">******</remark>
        </q4:InvoiceDetail>
    </soap:Body>
</soap:Envelope>

I was not able to reproduce your issue, but you need to be aware of a couple of things, which maybe your issue:

  1. When using your filebeat configuration to read this sample log that you have attached make sure that the sample log file ends with an empty line otherwise the last line won't end with a newline character which means Filebeat won't read it thus producing a malformed xml (since it won't be closed by </soap:Envelope>).
  2. Your multiline configuration seems wrong. To fix it:
    i. Make sure that the first configuration setting multiline.pattern: '^\[' is indented properly (i.e. it should start with two spaces). The configuration you have attached is not indented which would prevent Filebeat from starting.
    ii. The pattern '^\[' does not seems right now. The pattern that delimiters your log is of a date so try with something like the following instead: multiline.pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}\s

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.