XML to Elasticsearch through Logstash

Elastic Stack Logstash
1

I'm trying to parse in XML source which is a few 800 lines, and looks like this :

  <?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>
<m2m:cin xmlns:m2m=\"http://www.onem2m.org/xml/protocols\"
		 xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\">
	<ty>4</ty>
	<ri>CI00000000001244656716</ri>
	<rn>CI00000000001244656716</rn>
	<pi>CT00000000000000046769</pi>
	<ct>2018-02-05T15:06:30+09:00</ct>
	<lt>2018-02-05T15:06:30+09:00</lt>
	<ppt>
		<gwl>36.83115, 127.11185, 76</gwl>
		<geui>0017b2fffe0ad93e</geui>
	</ppt>
	<sr>/0240771000000168/v1_0/remoteCSE-00000168000c05c016104807/container-LoRa//subscription-SS00000000000000261472</sr>
	<et>2018-02-06T15:06:30+09:00</et>
	<st>11785</st>
	<cr>RC00000000000000050648</cr>
	<cnf>LoRa/Sensor</cnf>
	<cs>76</cs>
	<con>010400003039499602d2499602d203e703e70000000003e703e70000000000000000fff51234</con>
</m2m:cin>

And My Logstash Configuration looks like this:

input {
	http{
		
	}
}
filter {
	xml{
		remove_namespaces => true
		store_xml => false
		source => "message"
		xpath => ["/m2m:cin/con/text()", "parsedCon"]
	}
}

output {
	elasticsearch {
		index => "parse"
		hosts => "localhost:9200"
		document_type => "doc"
	}
}

My expects is like this:
parsedCon : 010400003039499602d2499602d203e703e70000000003e703e70000000000000000fff51234

But result is like this :

{
  "took": 3,
  "timed_out": false,
  "_shards": {
    "total": 5,
    "successful": 5,
    "skipped": 0,
    "failed": 0
  },
  "hits": {
    "total": 1,
    "max_score": 1,
    "hits": [
      {
        "_index": "parse",
        "_type": "doc",
        "_id": "Nk5fk2IBMu4GeP3SYb9S",
        "_score": 1,
        "_source": {
          "@timestamp": "2018-04-05T01:15:23.818Z",
          "host": "0:0:0:0:0:0:0:1",
          "tags": [
            "_xmlparsefailure"
          ],
          "message": """
<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>
<m2m:cin xmlns:m2m=\"http://www.onem2m.org/xml/protocols\"
		 xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\">
	<ty>4</ty>
	<ri>CI00000000001244656716</ri>
	<rn>CI00000000001244656716</rn>
	<pi>CT00000000000000046769</pi>
	<ct>2018-02-05T15:06:30+09:00</ct>
	<lt>2018-02-05T15:06:30+09:00</lt>
	<ppt>
		<gwl>36.83115, 127.11185, 76</gwl>
		<geui>0017b2fffe0ad93e</geui>
	</ppt>
	<sr>/0240771000000168/v1_0/remoteCSE-00000168000c05c016104807/container-LoRa//subscription-SS00000000000000261472</sr>
	<et>2018-02-06T15:06:30+09:00</et>
	<st>11785</st>
	<cr>RC00000000000000050648</cr>
	<cnf>LoRa/Sensor</cnf>
	<cs>76</cs>
	<con>010400003039499602d2499602d203e703e70000000003e703e70000000000000000fff51234</con>
</m2m:cin>
""",
          "@version": "1",
          "headers": {
            "http_postman_token": "4b48bdec-2335-4301-a51f-4ef7f63f01a8",
            "http_connection": "keep-alive",
            "http_user_agent": "PostmanRuntime/7.1.1",
            "http_host": "localhost:8080",
            "http_accept_encoding": "gzip, deflate",
            "request_path": "/",
            "http_version": "HTTP/1.1",
            "request_method": "PUT",
            "content_length": "809",
            "content_type": "application/xml",
            "http_accept": "*/*",
            "http_cache_control": "no-cache",
            "request_uri": "/"
          }
        }
      }
    ]
  }
}

There is a "tags" which tells "_xmlparsefailure".. Why xmlparsefailure??

2

Does the XML come in pretty printed? If so, you'll need to use the multiline codec on your input to place each event on a single line.

3

There is a "tags" which tells "_xmlparsefailure".. Why xmlparsefailure??

The Logstash log will contain details about the problem.

4

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.