Calculate time difference between two log lines in a file

Elastic Stack Logstash
2

I cannot see what you did wrong there, but I would do it in a different way, using aggregate.

    dissect { mapping => { "message" => "%{[@metadata][timestamp]} %{+[@metadata][timestamp]} %{event} - %{[@metadata][restOfLine]}" } }
    date { match => [ "[@metadata][timestamp]", "YYYY-MM-dd HH:mm:ss" ] }
    kv { source => "[@metadata][restOfLine]" target => "[@metadata][keys]" field_split => "," value_split => ":" remove_char_key => " " }
    aggregate {
        task_id => "%{[@metadata][keys][jobid]}"
        code => '
            keys = event.get("[@metadata][keys]")
            if keys
                keys.each { |k, v|
                    map[k] = v
                }
            end

            e = event.get("event")
            map["time#{e}"] = event.get("@timestamp")

            event.cancel
        '
        push_map_as_event_on_timeout => true
        timeout => 3
        timeout_code => '
            event.set("duration", event.get("timeFinished").to_f - event.get("timeStarted").to_f)
        '
    }

will produce

    "@version" => "1",
       "owner" => "/DC=EU/DC=EGI/C=GR/O=Robots/O=Greek Research and Technology Network/CN=Robot:argo-egi@grnet.gr",
      "lrmsid" => "66918",
       "queue" => "parallel1",
 "timeStarted" => 2019-07-15T13:26:40.000Z,
  "@timestamp" => 2019-10-15T15:22:13.905Z,
        "name" => "org.nordugrid.ARC-CE-result-ops",
"timeFinished" => 2019-07-15T13:31:10.000Z,
    "unixuser" => "45001:45000",
        "lrms" => "SLURM",
       "jobid" => "V3gNDm1MD7unkvVVEmSjiJLoABFKDmABFKDmtdGKDmABFKDmN8Erlm",
    "duration" => 270.0

Obviously you can move the keys under [JOB][job_Details] using string interpolation in the keys.each loop as I do in storing the timestamps.

1 Like