Parsing Dynatrace dashboard reports in XML format

Elastic Stack Logstash
7

Here is the final version of my script handling the cases where
measurement.count == 0 and measurement.count == 1

def register(params)

end

def filter(event)
    theEvents = []
    xml =  event.get("[@metadata][theXML]")
    recordTimestamp = event.get("record_timestamp")

    xml["data"]["chartdashlet"].each { |dashlet|
        dashlet["measures"]["measure"].each { |y|
            #logger.trace("measure_name is: #{y["measure"]}")
            measurement = y["measurement"]
           
            anEvent = Hash[ "inputtype", "dynatrace_dashboard_report", "dashboard", xml["name"], "chartdashlet", dashlet["name"], "measure", y["measure"], "measure_agg_type", y["aggregation"] ]
            anEvent["record_timestamp"] = recordTimestamp
            
            if measurement.nil? #handles when there is no measurement under a measure
                anEvent["value"] = 0
            elsif y["aggregation"] == "Average"
                #do if more than 1 measurement
                if measurement.kind_of?(Array)
                    sum = 0.0
                    count = 0
                    measurement.each { |z|
                        sum += z["sum"].to_f
                        count += z["count"].to_i
                    }
                    anEvent["value"] = sum/count
                else #do when there is only 1 record
                    anEvent["value"] = measurement["avg"]
                end

            elsif y["aggregation"] == "Count"
                if measurement.kind_of?(Array)
                    count = 0
                    measurement.each { |z|
                        count += z["count"].to_i
                    }
                    anEvent["value"] = count
                else #handles when there is only 1 record
                   anEvent["value"] = measurement["count"]
                end
            end

            theEvents << LogStash::Event.new(anEvent)
        }
    }
    theEvents
end

For other file based exceptions, logstash adds _rubyexception tag to the input object and my strategy is to directly push these records to my index in order to keep track of exception counts.

For others who may need this kind of event processing, please note that all the input fields (except the ones existing at the time i instantiate a new event in Ruby) are discarded in case of successfull ruby script execution since i produce 5 events out of a single input event, but when it fails, all the input fields are output with an extra _rubyexception tag:

        {
          "@timestamp": "2019-11-15T11:00:59.138Z",
          "input": {
            "type": "log"
          },
          "inputtype": "dynatrace_dashboard_report",
          "tags": [
            "beats_input_codec_plain_applied",
            **"_rubyexception"**
          ],
          "@version": "1",
          "record_timestamp": "2019-11-15T07:14:00.000Z",
          "log": {
            "file": {
              "path": "/home/xxx/input-filebeat/dynatrace_048_(SGT)_BANKA_SIGORTACILIĞI_20191115101400.out"
            },
            "offset": 0,
            "flags": [
              "multiline"
            ]
          },
          "from_millis": "1573802040000",
          "agent": {
            "type": "filebeat",
            "id": "8dcd2d3e-be27-4cc7-ae38-14a7a16874fb",
            "version": "7.4.2",
            "hostname": "somehost",
            "ephemeral_id": "430243c8-e494-407b-b1c0-57155a4cbc45"
          }
        }

the final conf:

 filter {
  xml {
    source => "message"
    target => "[@metadata][theXML]"
    force_array => false
    remove_field => [ "message", "host", "ecs" ]
    add_field => { "inputtype" => "dynatrace_dashboard_report" }
    xpath => [ "/dashboardreport/source/filters/filter/text()" , "[@metadata][report_filter_text]" ]
  }
  dissect {
    mapping => { "[@metadata][report_filter_text]" => "%{?filter_type}?%{from_millis}:%{?end_millis}" }
  }
  date {
    match => [ "from_millis", "UNIX_MS" ]
    target => "record_timestamp"
  }
  ruby {
    path => "/home/xxx/code/dynatrace.rb"

  }
}
output {
  stdout {
    codec => rubydebug { metadata => true }
  }
  if [inputtype] == "dynatrace_dashboard_report" {
    file {
      path => "/home/xxx/debug-logstash1.out"
      codec => "json"
    }
  }
}