hello everyone
Hello everyone, at this moment I am trying to ingest some logs in elasticsearch with logstash with the following structure:
<ns0:MessageID xmlns:ns0="http://www.ZZZ.com/namespaces/tnt/plugins/jms">ID:XXXX-XXXX.32186428E5BA21DA7E:664761</ns0:MessageID>
<tns2:Response xmlns:tns2="http://www.example.org/JsonResource">
tns2:statusCode200</tns2:statusCode>
tns2:reasonPhraseOK</tns2:reasonPhrase>
tns2:JsonString{"messages":[{"to":{"phoneNumber":"577777777777"},"status":{"groupId":1,"groupName":"PENDING","id":7,"name":"PENDING_ENROUTE","description":"Message sent to next instance"},"messageId":"f8508e60-fc06-4fd7-a563-5b6d68453353"}]}</tns2:JsonString>
</tns2:Response>
<tns14:ws-integration-management-inputProcessType xmlns:tns3="http://www.example.org/InputMessageSchema" xmlns:tns14="http://www.example.org/ws-external-notification" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:tib="http://www.ZZZ.com/bw/xslt/custom-functions">
tns14:uuide836613b-4970-467a-b35a-fe8c598326bc</tns14:uuid>
tns14:Resourceadvanced</tns14:Resource>
tns14:MethodPOST</tns14:Method>
tns14:Versionv1</tns14:Version>
<tns14:PostData />
tns14:Headers
tns14:Nameuuid</tns14:Name>
tns14:Valuee836613b-4970-467a-b35a-fe8c598326bc</tns14:Value>
</tns14:Headers>
tns14:Headers
tns14:NameX-dynaTrace</tns14:Name>
tns14:ValueFW4;1001013543;19;1308049219;1252479;4;1301609612;535;2152;2h01;3h4df73f43;4h131c7f;5h01;6he1f5eca126367eb6ed7394993b5e0892;7hb67bbb4986015ac9</tns14:Value>
</tns14:Headers>
tns14:Headers
tns14:Nametracestate</tns14:Name>
tns14:Value4d94fc8c-3baa4127@dt=fw4;13;4df73f43;131c7f;4;0;0;217;815a;2h01;3h4df73f43;4h131c7f;5h01;7hb67bbb4986015ac9</tns14:Value>
</tns14:Headers>
tns14:Headers
tns14:Nametraceparent</tns14:Name>
tns14:Value00-e1f5eca126367eb6ed7394993b5e0892-b67bbb4986015ac9-01</tns14:Value>
</tns14:Headers>
tns14:ws-integration-management-reqElement
tns14:partnerIdchatbot-18</tns14:partnerId>
<tns14:documentType />
<tns14:documentNumber />
tns14:messageState4</tns14:messageState>
tns14:channel3</tns14:channel>
<tns14:segmentation xsi:nil="true" />
tns14:cellPhone7777777777</tns14:cellPhone>
tns14:sourceCOL_NotificationWhatsapp</tns14:source>
tns14:messageDescription{"messages":[{"to":{"phoneNumber":"577777777777"},"status":{"groupId":1,"groupName":"PENDING","id":7,"name":"PENDING_ENROUTE","description":"Message sent to next instance"},"messageId":"f8508e60-fc06-4fd7-a563-5b6d68453353"}]}</tns14:messageDescription>
<tns14:messageId />
</tns14:ws-integration-management-reqElement>
</tns14:ws-integration-management-inputProcessType>
<tns3:Request xmlns:tns3="http://www.example.org/InputMessageSchema">
tns3:uuide836613b-4970-467a-b35a-fe8c598326bc</tns3:uuid>
tns3:PartnerIdchatbot-18</tns3:PartnerId>
tns3:Resourceadvanced</tns3:Resource>
tns3:PostData{"scenarioKey":"3972323B9F14B4F87DF50569D647E4AB","destinations":[{"to":{"phoneNumber":"577777777777"}}],"whatsApp":{"text":"1. Te diagnosticaron una enfermedad \n:white_check_mark:2. Murió el asegurado \n:white_check_mark:3. Tienes una incapacidad \n:white_check_mark:4. Te robaron de forma violenta \n"}}</tns3:PostData>
tns3:MethodPOST</tns3:Method>
tns3:Versionv1</tns3:Version>
<tns3:Headers xmlns:tns2="http://www.example.org/InputMessageSchema" xmlns:ns="http://YYY.com.co/Schemas/GenerateUUID/v1.xsd" xmlns:tns4="http://xmlns.example.com/RestAPIHTTPS/parameters" xmlns:tns1="http://tns.ZZZ.com/bw/REST">
tns3:Nameuuid</tns3:Name>
tns3:Valuee836613b-4970-467a-b35a-fe8c598326bc</tns3:Value>
</tns3:Headers>
<tns3:Headers xmlns:tns2="http://www.example.org/InputMessageSchema" xmlns:ns="http://YYY.com.co/Schemas/GenerateUUID/v1.xsd" xmlns:tns4="http://xmlns.example.com/RestAPIHTTPS/parameters" xmlns:tns1="http://tns.ZZZ.com/bw/REST">
tns3:NameX-dynaTrace</tns3:Name>
tns3:ValueFW4;1001013543;19;1308049219;1252479;4;1301609612;535;2152;2h01;3h4df73f43;4h131c7f;5h01;6he1f5eca126367eb6ed7394993b5e0892;7hb67bbb4986015ac9</tns3:Value>
</tns3:Headers>
<tns3:Headers xmlns:tns2="http://www.example.org/InputMessageSchema" xmlns:ns="http://BNPParibasCardif.com.co/Schemas/GenerateUUID/v1.xsd" xmlns:tns4="http://xmlns.example.com/RestAPIHTTPS/parameters" xmlns:tns1="http://tns.tibco.com/bw/REST">
tns3:Nametracestate</tns3:Name>
tns3:Value4d94fc8c-3baa4127@dt=fw4;13;4df73f43;131c7f;4;0;0;217;815a;2h01;3h4df73f43;4h131c7f;5h01;7hb67bbb4986015ac9</tns3:Value>
</tns3:Headers>
<tns3:Headers xmlns:tns2="http://www.example.org/InputMessageSchema" xmlns:ns="http://YYY.com.co/Schemas/GenerateUUID/v1.xsd" xmlns:tns4="http://xmlns.example.com/RestAPIHTTPS/parameters" xmlns:tns1="http://tns.tibco.com/bw/REST">
tns3:Nametraceparent</tns3:Name>
tns3:Value00-e1f5eca126367eb6ed7394993b5e0892-b67bbb4986015ac9-01</tns3:Value>
</tns3:Headers>
</tns3:Request>
<tns2:Request xmlns:tns2="http://www.example.org/InputMessageSchema">
tns2:uuide836613b-4970-467a-b35a-fe8c598326bc</tns2:uuid>
tns2:PartnerIdchatbot-18</tns2:PartnerId>
tns2:Resourceadvanced</tns2:Resource>
tns2:PostData{"scenarioKey":"3972323B9F14B4F87DF50569D647E4AB","destinations":[{"to":{"phoneNumber":"577777777777"}}],"whatsApp":{"text":"1. Te diagnosticaron una enfermedad \n:white_check_mark:2. Murió el asegurado \n:white_check_mark:3. Tienes una incapacidad \n:white_check_mark:4. Te robaron de forma violenta \n"}}</tns2:PostData>
tns2:MethodPOST</tns2:Method>
tns2:Versionv1</tns2:Version>
<tns3:Headers xmlns:ns="http://YYY.com.co/Schemas/GenerateUUID/v1.xsd" xmlns:tns3="http://www.example.org/InputMessageSchema" xmlns:tns4="http://xmlns.example.com/RestAPIHTTPS/parameters" xmlns:tns1="http://tns.tibco.com/bw/REST">
tns3:Nameuuid</tns3:Name>
tns3:Valuee836613b-4970-467a-b35a-fe8c598326bc</tns3:Value>
</tns3:Headers>
<tns3:Headers xmlns:ns="http://YYY.com.co/Schemas/GenerateUUID/v1.xsd" xmlns:tns3="http://www.example.org/InputMessageSchema" xmlns:tns4="http://xmlns.example.com/RestAPIHTTPS/parameters" xmlns:tns1="http://tns.ZZZ.com/bw/REST">
tns3:NameX-dynaTrace</tns3:Name>
tns3:ValueFW4;1001013543;19;1308049219;1252479;4;1301609612;535;2152;2h01;3h4df73f43;4h131c7f;5h01;6he1f5eca126367eb6ed7394993b5e0892;7hb67bbb4986015ac9</tns3:Value>
</tns3:Headers>
<tns3:Headers xmlns:ns="http://YYY.com.co/Schemas/GenerateUUID/v1.xsd" xmlns:tns3="http://www.example.org/InputMessageSchema" xmlns:tns4="http://xmlns.example.com/RestAPIHTTPS/parameters" xmlns:tns1="http://tns.tibco.com/bw/REST">
tns3:Nametracestate</tns3:Name>
tns3:Value4d94fc8c-3baa4127@dt=fw4;13;4df73f43;131c7f;4;0;0;217;815a;2h01;3h4df73f43;4h131c7f;5h01;7hb67bbb4986015ac9</tns3:Value>
</tns3:Headers>
<tns3:Headers xmlns:ns="http://YYY.com.co/Schemas/GenerateUUID/v1.xsd" xmlns:tns3="http://www.example.org/InputMessageSchema" xmlns:tns4="http://xmlns.example.com/RestAPIHTTPS/parameters" xmlns:tns1="http://tns.XXX.com/bw/REST">
tns3:Nametraceparent</tns3:Name>
tns3:Value00-e1f5eca126367eb6ed7394993b5e0892-b67bbb4986015ac9-01</tns3:Value>
</tns3:Headers>
</tns2:Request>
I am using the following configuration file with logstash, I am able to ingest data and place the information of the xml structure in the field called xml
input{
file{
type => "info"
path => "C:/elastic/logstash-8.9.0-windows-x86_64/logstash-8.9.0/logingesta/info_generate_xml_10/"
codec => multiline {
pattern => "^%{TIMESTAMP_ISO8601} "
negate => true
what => "previous"
auto_flush_interval => 10
}
start_position => "beginning"
sincedb_path => "C:/elastic/logs/null"
}
}
The filter part of this file is commented out to indicate that it is
optional.
filter {
grok {
# match => { "message" => "%{TIMESTAMP_ISO8601:timestamp} %{UUID:uuid} %{DATA:correlationId} ?>%{GREEDYDATA:xml}" }
}
xml {
# source => "xml"
# store_xml => true
# target => "parsed_xml"
# remove_namespaces => true
}
mutate {
# add_field => {
# "json_messageDescription" => "%{[parsed_xml][tns14:messageDescription][0]}"
# }
}
ruby {
# code => '
# event.set("parsed_json_messageDescription", JSON.parse(event.get("json_messageDescription")))
# '
}
}
filter {
grok {
match => { "message" => "%{TIMESTAMP_ISO8601:timestamp} %{UUID:uuid} %{DATA:correlationId} ?>%{GREEDYDATA:xml}" }
}
xml {
source => "xml"
store_xml => true
target => "parsed_xml"
remove_namespaces => true
}
if [parsed_xml][tns2][Request][0][PostData][0] {
mutate {
add_field => {
"json_data" => "%{[parsed_xml][tns2][Request][0][PostData][0]}"
}
}
} else if [parsed_xml][tns14][ws-integration-management-reqElement][0][messageDescription][0] {
mutate {
add_field => {
"json_data" => "%{[parsed_xml][tns14][ws-integration-management-reqElement][0][messageDescription][0]}"
}
}
}
json {
source => "json_data"
target => "parsed_json_data"
}
}
output {
stdout { codec => rubydebug }
elasticsearch {
action => "index"
hosts => "https://127.0.0.1:9200"
user => "elastic"
password => "+uv1=3DU-UHQw8VD3cR+"
data_stream => "false"
ssl => true
cacert => "C:\elastic\elasticsearch-8.9.0-windows-x86_64\elasticsearch-8.9.0\config\certs\http_ca.crt"
index => "xmlwhatsapp"
}
}
*
I have not been able to parse the information that comes in JSON format internally in the XML field. Please can you help me, the nodes are not being taken into account in the conditional.