XML parsing using logstash


(Ramanna ) #1

Hi,

I am working on Nessus logs ex: https://gist.github.com/najmisyahir/7a1d2c17257ced61257e3033cd55d090

But i am getting results in array for each fields, please help me to convert to json records.

FYI: i am using logstash 6.0.1 on ubuntu 16

logstash.conf -

input {
  file {
    path => "/sampledata/Basic_Scan_Cluster_y7zzst.nessus"
    sincedb_path => "/dev/null"
    start_position => "beginning"
    codec => multiline {
      pattern => "<Report |</NessusClientData_v2>"
      auto_flush_interval => 1
      negate => "true"
      what => "previous"
      max_lines => 1000000000
      max_bytes => "50 MiB"

    }
      tags => "nessus"
      type => "nessus"
  }
}

filter {
  ##interpret the message as XML
    if [type] == "nessus" {
        xml {
            source => "message"
            store_xml => "false"
            force_array => "false"
            
            xpath => ["/Report/ReportHost/@name", host_ip]
            xpath => ["/Report/ReportHost/ReportItem/@pluginName", plugin_name]
            xpath => ["/Report/ReportHost/ReportItem/@pluginID", plugin_id]
            xpath => ["/Report/ReportHost/ReportItem/@severity", risk_score]
            xpath => ["/Report/ReportHost/ReportItem/@port", port]
            xpath => ["/Report/ReportHost/ReportItem/@svc_name", svc_name]
            xpath => ["/Report/ReportHost/ReportItem/@protocol", protocol]
            xpath => ["/Report/ReportHost/ReportItem/@pluginFamily", plugin_family]
            xpath => ["/Report/ReportHost/ReportItem/description/text()", description]
            xpath => ["/Report/ReportHost/ReportItem/risk_factor/text()", risk_factor]
            xpath => ["/Report/ReportHost/ReportItem/see_also/text()", see_also]
            xpath => ["/Report/ReportHost/ReportItem/solution/text()", solution]
            xpath => ["/Report/ReportHost/ReportItem/synopsis/text()", synopsis]
            xpath => ["/Report/ReportHost/ReportItem/plugin_output/text()", plugin_output]
            xpath => ["/Report/ReportHost/HostProperties/tag[@name='HOST_START']/text()", report_host_start]
            xpath => ["/Report/ReportHost/HostProperties/tag[@name='HOST_END']/text()", report_host_end]
        }
        mutate {
          remove_field => ["message"]
          convert => {
              "risk_score" => "integer"
          }
        }

        date {
            match => ["report_host_start", "EEE MMM dd HH:mm:ss yyyy"]
            target => "report_host_start"
            locale => "en_US"
        }
        date {
            match => ["report_host_end", "EEE MMM dd HH:mm:ss yyyy"]
            target => "report_host_end"
            locale => "en_US"
        }
 
    }
}

output {
     elasticsearch { 
        hosts => ["localhost:9200"]
        index => "nessus-data-%{+YYYY.MM.dd}"
      }
      stdout { codec => rubydebug }
}

#2

This might help: Split ES event


(Ramanna ) #3

Thanks @Jenni, i did it.


(system) #4

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.