Parsing log in logstash with format xml and json embebed

Oscar_Lopez · August 17, 2023, 4:14pm

hello everyone
Hello everyone, at this moment I am trying to ingest some logs in elasticsearch with logstash with the following structure:

<ns0:MessageID xmlns:ns0="http://www.ZZZ.com/namespaces/tnt/plugins/jms">ID:XXXX-XXXX.32186428E5BA21DA7E:664761</ns0:MessageID>

<tns2:Response xmlns:tns2="http://www.example.org/JsonResource">
tns2:statusCode200</tns2:statusCode>
tns2:reasonPhraseOK</tns2:reasonPhrase>
tns2:JsonString{"messages":[{"to":{"phoneNumber":"577777777777"},"status":{"groupId":1,"groupName":"PENDING","id":7,"name":"PENDING_ENROUTE","description":"Message sent to next instance"},"messageId":"f8508e60-fc06-4fd7-a563-5b6d68453353"}]}</tns2:JsonString>
</tns2:Response>

<tns14:ws-integration-management-inputProcessType xmlns:tns3="http://www.example.org/InputMessageSchema" xmlns:tns14="http://www.example.org/ws-external-notification" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:tib="http://www.ZZZ.com/bw/xslt/custom-functions">
tns14:uuide836613b-4970-467a-b35a-fe8c598326bc</tns14:uuid>
tns14:Resourceadvanced</tns14:Resource>
tns14:MethodPOST</tns14:Method>
tns14:Versionv1</tns14:Version>
<tns14:PostData />
tns14:Headers
tns14:Nameuuid</tns14:Name>
tns14:Valuee836613b-4970-467a-b35a-fe8c598326bc</tns14:Value>
</tns14:Headers>
tns14:Headers
tns14:NameX-dynaTrace</tns14:Name>
tns14:ValueFW4;1001013543;19;1308049219;1252479;4;1301609612;535;2152;2h01;3h4df73f43;4h131c7f;5h01;6he1f5eca126367eb6ed7394993b5e0892;7hb67bbb4986015ac9</tns14:Value>
</tns14:Headers>
tns14:Headers
tns14:Nametracestate</tns14:Name>
tns14:Value4d94fc8c-3baa4127@dt=fw4;13;4df73f43;131c7f;4;0;0;217;815a;2h01;3h4df73f43;4h131c7f;5h01;7hb67bbb4986015ac9</tns14:Value>
</tns14:Headers>
tns14:Headers
tns14:Nametraceparent</tns14:Name>
tns14:Value00-e1f5eca126367eb6ed7394993b5e0892-b67bbb4986015ac9-01</tns14:Value>
</tns14:Headers>
tns14:ws-integration-management-reqElement
tns14:partnerIdchatbot-18</tns14:partnerId>
<tns14:documentType />
<tns14:documentNumber />
tns14:messageState4</tns14:messageState>
tns14:channel3</tns14:channel>
<tns14:segmentation xsi:nil="true" />
tns14:cellPhone7777777777</tns14:cellPhone>
tns14:sourceCOL_NotificationWhatsapp</tns14:source>
tns14:messageDescription{"messages":[{"to":{"phoneNumber":"577777777777"},"status":{"groupId":1,"groupName":"PENDING","id":7,"name":"PENDING_ENROUTE","description":"Message sent to next instance"},"messageId":"f8508e60-fc06-4fd7-a563-5b6d68453353"}]}</tns14:messageDescription>
<tns14:messageId />
</tns14:ws-integration-management-reqElement>
</tns14:ws-integration-management-inputProcessType>

<tns3:Request xmlns:tns3="http://www.example.org/InputMessageSchema">
tns3:uuide836613b-4970-467a-b35a-fe8c598326bc</tns3:uuid>
tns3:PartnerIdchatbot-18</tns3:PartnerId>
tns3:Resourceadvanced</tns3:Resource>
tns3:PostData{"scenarioKey":"3972323B9F14B4F87DF50569D647E4AB","destinations":[{"to":{"phoneNumber":"577777777777"}}],"whatsApp":{"text":"1. Te diagnosticaron una enfermedad \n:white_check_mark:2. Murió el asegurado \n:white_check_mark:3. Tienes una incapacidad \n:white_check_mark:4. Te robaron de forma violenta \n"}}</tns3:PostData>
tns3:MethodPOST</tns3:Method>
tns3:Versionv1</tns3:Version>
<tns3:Headers xmlns:tns2="http://www.example.org/InputMessageSchema" xmlns:ns="http://YYY.com.co/Schemas/GenerateUUID/v1.xsd" xmlns:tns4="http://xmlns.example.com/RestAPIHTTPS/parameters" xmlns:tns1="http://tns.ZZZ.com/bw/REST">
tns3:Nameuuid</tns3:Name>
tns3:Valuee836613b-4970-467a-b35a-fe8c598326bc</tns3:Value>
</tns3:Headers>
<tns3:Headers xmlns:tns2="http://www.example.org/InputMessageSchema" xmlns:ns="http://YYY.com.co/Schemas/GenerateUUID/v1.xsd" xmlns:tns4="http://xmlns.example.com/RestAPIHTTPS/parameters" xmlns:tns1="http://tns.ZZZ.com/bw/REST">
tns3:NameX-dynaTrace</tns3:Name>
tns3:ValueFW4;1001013543;19;1308049219;1252479;4;1301609612;535;2152;2h01;3h4df73f43;4h131c7f;5h01;6he1f5eca126367eb6ed7394993b5e0892;7hb67bbb4986015ac9</tns3:Value>
</tns3:Headers>
<tns3:Headers xmlns:tns2="http://www.example.org/InputMessageSchema" xmlns:ns="http://BNPParibasCardif.com.co/Schemas/GenerateUUID/v1.xsd" xmlns:tns4="http://xmlns.example.com/RestAPIHTTPS/parameters" xmlns:tns1="http://tns.tibco.com/bw/REST">
tns3:Nametracestate</tns3:Name>
tns3:Value4d94fc8c-3baa4127@dt=fw4;13;4df73f43;131c7f;4;0;0;217;815a;2h01;3h4df73f43;4h131c7f;5h01;7hb67bbb4986015ac9</tns3:Value>
</tns3:Headers>
<tns3:Headers xmlns:tns2="http://www.example.org/InputMessageSchema" xmlns:ns="http://YYY.com.co/Schemas/GenerateUUID/v1.xsd" xmlns:tns4="http://xmlns.example.com/RestAPIHTTPS/parameters" xmlns:tns1="http://tns.tibco.com/bw/REST">
tns3:Nametraceparent</tns3:Name>
tns3:Value00-e1f5eca126367eb6ed7394993b5e0892-b67bbb4986015ac9-01</tns3:Value>
</tns3:Headers>
</tns3:Request>

<tns2:Request xmlns:tns2="http://www.example.org/InputMessageSchema">
tns2:uuide836613b-4970-467a-b35a-fe8c598326bc</tns2:uuid>
tns2:PartnerIdchatbot-18</tns2:PartnerId>
tns2:Resourceadvanced</tns2:Resource>
tns2:PostData{"scenarioKey":"3972323B9F14B4F87DF50569D647E4AB","destinations":[{"to":{"phoneNumber":"577777777777"}}],"whatsApp":{"text":"1. Te diagnosticaron una enfermedad \n:white_check_mark:2. Murió el asegurado \n:white_check_mark:3. Tienes una incapacidad \n:white_check_mark:4. Te robaron de forma violenta \n"}}</tns2:PostData>
tns2:MethodPOST</tns2:Method>
tns2:Versionv1</tns2:Version>
<tns3:Headers xmlns:ns="http://YYY.com.co/Schemas/GenerateUUID/v1.xsd" xmlns:tns3="http://www.example.org/InputMessageSchema" xmlns:tns4="http://xmlns.example.com/RestAPIHTTPS/parameters" xmlns:tns1="http://tns.tibco.com/bw/REST">
tns3:Nameuuid</tns3:Name>
tns3:Valuee836613b-4970-467a-b35a-fe8c598326bc</tns3:Value>
</tns3:Headers>
<tns3:Headers xmlns:ns="http://YYY.com.co/Schemas/GenerateUUID/v1.xsd" xmlns:tns3="http://www.example.org/InputMessageSchema" xmlns:tns4="http://xmlns.example.com/RestAPIHTTPS/parameters" xmlns:tns1="http://tns.ZZZ.com/bw/REST">
tns3:NameX-dynaTrace</tns3:Name>
tns3:ValueFW4;1001013543;19;1308049219;1252479;4;1301609612;535;2152;2h01;3h4df73f43;4h131c7f;5h01;6he1f5eca126367eb6ed7394993b5e0892;7hb67bbb4986015ac9</tns3:Value>
</tns3:Headers>
<tns3:Headers xmlns:ns="http://YYY.com.co/Schemas/GenerateUUID/v1.xsd" xmlns:tns3="http://www.example.org/InputMessageSchema" xmlns:tns4="http://xmlns.example.com/RestAPIHTTPS/parameters" xmlns:tns1="http://tns.tibco.com/bw/REST">
tns3:Nametracestate</tns3:Name>
tns3:Value4d94fc8c-3baa4127@dt=fw4;13;4df73f43;131c7f;4;0;0;217;815a;2h01;3h4df73f43;4h131c7f;5h01;7hb67bbb4986015ac9</tns3:Value>
</tns3:Headers>
<tns3:Headers xmlns:ns="http://YYY.com.co/Schemas/GenerateUUID/v1.xsd" xmlns:tns3="http://www.example.org/InputMessageSchema" xmlns:tns4="http://xmlns.example.com/RestAPIHTTPS/parameters" xmlns:tns1="http://tns.XXX.com/bw/REST">
tns3:Nametraceparent</tns3:Name>
tns3:Value00-e1f5eca126367eb6ed7394993b5e0892-b67bbb4986015ac9-01</tns3:Value>
</tns3:Headers>
</tns2:Request>

I am using the following configuration file with logstash, I am able to ingest data and place the information of the xml structure in the field called xml

input{
file{
type => "info"
path => "C:/elastic/logstash-8.9.0-windows-x86_64/logstash-8.9.0/logingesta/info_generate_xml_10/"
codec => multiline {
pattern => "^%{TIMESTAMP_ISO8601} "
negate => true
what => "previous"
auto_flush_interval => 10
}
start_position => "beginning"
sincedb_path => "C:/elastic/logs/null"
}
}

The filter part of this file is commented out to indicate that it is

optional.

filter {

grok {

# match => { "message" => "%{TIMESTAMP_ISO8601:timestamp} %{UUID:uuid} %{DATA:correlationId} ?>%{GREEDYDATA:xml}" }

}

xml {

# source => "xml"
# store_xml => true
# target => "parsed_xml"
# remove_namespaces => true

}

mutate {

# add_field => {
  # "json_messageDescription" => "%{[parsed_xml][tns14:messageDescription][0]}"
# }

}

ruby {

# code => '
  # event.set("parsed_json_messageDescription", JSON.parse(event.get("json_messageDescription")))
# '

}

filter {
grok {
match => { "message" => "%{TIMESTAMP_ISO8601:timestamp} %{UUID:uuid} %{DATA:correlationId} ?>%{GREEDYDATA:xml}" }
}

xml {
source => "xml"
store_xml => true
target => "parsed_xml"
remove_namespaces => true
}

if [parsed_xml][tns2][Request][0][PostData][0] {
mutate {
add_field => {
"json_data" => "%{[parsed_xml][tns2][Request][0][PostData][0]}"
}
}
} else if [parsed_xml][tns14][ws-integration-management-reqElement][0][messageDescription][0] {
mutate {
add_field => {
"json_data" => "%{[parsed_xml][tns14][ws-integration-management-reqElement][0][messageDescription][0]}"
}
}
}

json {
source => "json_data"
target => "parsed_json_data"
}
}
output {
stdout { codec => rubydebug }
elasticsearch {
action => "index"
hosts => "https://127.0.0.1:9200"
user => "elastic"
password => "+uv1=3DU-UHQw8VD3cR+"
data_stream => "false"
ssl => true
cacert => "C:\elastic\elasticsearch-8.9.0-windows-x86_64\elasticsearch-8.9.0\config\certs\http_ca.crt"
index => "xmlwhatsapp"
}
}
*
I have not been able to parse the information that comes in JSON format internally in the XML field. Please can you help me, the nodes are not being taken into account in the conditional.

leandrojmp · August 17, 2023, 4:21pm

Hello,

Can you please edit your post and share the xml and configuration using the preformatted text option? It is pretty hard to understand without proper formatting.

system · September 14, 2023, 4:21pm

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.