I am using two different servers and my Filebeat server sends logs to the Logstash server without changing anything. Then Logstash server parse incoming files and forward it to the elastic. However, in elastic logs i can see that _xmlparsefailure.
Normally without using those servers and try to send logs to the elastic using logstash codes on my local machine there will be no parse failures
Elastic:
Logstash:
input {
beats {
port => "5044"
}
}
filter {
if [message] =~ /^\s*$/ {
drop { }
}
mutate {
gsub => ["message", "^\t*\r?\n\t*", ""]
gsub => ["message", "xsi:\w+=(?<grp1>\"|')\w+(:\w+)?\k<grp1>\s*", ""]
gsub => ["message", 'href="#\w+"', ""]
}
if( [message] =~ "^.*kktcell.*<"){
mutate {
gsub => ["message", "<soap:Body.+?>|</soap:Body>", ""]
gsub => ["message", "<env:Body.+?>|</env:Body>", ""]
}
}
else if ( [message] =~ "^.*88.255.27.59.*<"){
mutate {
gsub => ["message", "<soap:Body>|</soap:Body>", ""]
}
}
if ( [message] =~ "^.*77.92.16.129.*<"){
mutate {
gsub => ["message", "<soap:Body>|</soap:Body>", ""]
}
}
grok {
match => { "message" => "(?<methodName>((?<=\s-\s)\w+(?= - <\?xml)))" }
}
}
filter {
if( [message] =~ "^.*- Request -.*<"){
mutate { gsub => [ "message", "^[^<]+<", "<" ] }
xml {
remove_namespaces => true
store_xml => true
source => "message"
target => "Properties"
force_array => false
}
mutate {
add_field => { "messageType" => "Request" }
}
}
else if ( [message] =~ "^.*- Response -.*<"){
mutate { gsub => [ "message", "^[^<]+<", "<" ] }
xml {
remove_namespaces => true
store_xml => true
source => "message"
target => "Response"
force_array => false
}
mutate {
add_field => { "messageType" => "Response" }
}
}
mutate {
remove_field => ["message",
"[Properties][Header]",
"[Properties][xmlns:xsi]",
"[Properties][xmlns:soap]",
"[Properties][xmlns:tns]",
"[Properties][xmlns:types]",
"[Properties][xmlns:xsd]",
"[Properties][xmlns:env]",
"[Properties][xmlns:soapenc]",
"[Properties][soap:encodingStyle]",
"[Properties][Array][Item]"
]
}
}
output {
elasticsearch {
hosts => ["****.********.com:9200"]
index => "*********-%{+yyyy.MM.dd}"
}
}
Filebeat:
filebeat.prospectors:
- type: log
enabled: true
paths:
- C:\Logs\Kurumlar\*.log
multiline.pattern: '^\['
multiline.negate: true
multiline.match: after
filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: true
reload.period: 10s
setup.template.settings:
index.number_of_shards: 3
output.logstash:
hosts: ["192.168.***.***:5044"]
Example Log:
2019-01-17 17:35:31,597 - Request - *****.***** - http://*****.kktcell.com:****/*****GWws/*****GWBank - payInvoice -
<?xml version="1.0" encoding="utf-8"?>
<soap:Envelope xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/" xmlns:soapenc="http://schemas.xmlsoap.org/soap/encoding/" xmlns:tns="http://my.turkcell.com.tr:7002/KKTCCollGWws/KKTCCollGW" xmlns:types="http://my.turkcell.com.tr:7002/KKTCCollGWws/KKTCCollGW/encodedTypes" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema">
<soap:Body soap:encodingStyle="http://schemas.xmlsoap.org/soap/encoding/">
<tns:payInvoice>
<invoicePaymentRequest href="#id1" />
</tns:payInvoice>
<q1:InvoicePaymentRequest id="id1" xsi:type="q1:InvoicePaymentRequest" xmlns:q1="java:com.turkcelltech.collgw.model.invoice">
<accDate xsi:type="xsd:string">20190117</accDate>
<bankId xsi:type="xsd:int">23</bankId>
<captDate xsi:type="xsd:string">****</captDate>
<compId xsi:type="xsd:int">****</compId>
<curr xsi:type="xsd:string">949</curr>
<invCount xsi:type="xsd:int">1</invCount>
<invoiceList href="#id2" />
<msgDate xsi:nil="true" />
<msisdn xsi:type="xsd:long">*******</msisdn>
<opType xsi:type="xsd:int">10</opType>
<orig href="#id3" />
<paymSource xsi:type="xsd:int">4</paymSource>
<procDate xsi:type="xsd:string">0117</procDate>
<sessionId xsi:type="xsd:string">**********</sessionId>
<stan xsi:type="xsd:long">35852</stan>
</q1:InvoicePaymentRequest>
<soapenc:Array id="id2" xmlns:q2="java:com.******.collgw.model.invoice" soapenc:arrayType="q2:InvoiceDetail[1]">
<Item href="#id4" />
</soapenc:Array>
<q3:OriginatorId id="id3" xsi:type="q3:OriginatorId" xmlns:q3="java:com.******.collgw.model.common">
<branch xsi:type="xsd:string">11</branch>
<city xsi:type="xsd:int">0</city>
<teller xsi:type="xsd:string">45</teller>
<user xsi:type="xsd:string">****, E</user>
</q3:OriginatorId>
<q4:InvoiceDetail id="id4" xsi:type="q4:InvoiceDetail" xmlns:q4="java:com.*******.collgw.model.invoice">
<controlId xsi:type="xsd:string">******</controlId>
<duedate xsi:type="xsd:string">20190117</duedate>
<invAmount xsi:type="xsd:double">0</invAmount>
<invNo xsi:type="xsd:long">******</invNo>
<invType xsi:type="xsd:int">0</invType>
<nameSurname xsi:type="xsd:string">Su*** Al******</nameSurname>
<orderNo xsi:type="xsd:int">0</orderNo>
<period xsi:type="xsd:string">******</period>
<prevMsisdn xsi:type="xsd:long">*****</prevMsisdn>
<remark xsi:type="xsd:string">******</remark>
</q4:InvoiceDetail>
</soap:Body>
</soap:Envelope>