Sum field values from nested json and put them in new field using logstash ruby plugin

This is my input stored in .log file

{"@timestamp":"2020-02-12T17:53:54.305Z","beat":{"hostname":"xx.xx","name":"xx.xx","version":"5.2.1"},"input_type":"log","log_dc":"xx","log_dc_type":"rg","message":"{\"server_name\":\"test1\",\"remote_address\":\"xx.xx\",\"forwarded_for\":\"public, xx.xx\",\"remote_user\":\"id1\",\"timestamp_start\":\"1581530032201\",\"timestamp_finish\":\"1581530033511\",\"time_start\":\"12/Feb/2020:17:53:52 +0000\",\"time_finish\":\"12/Feb/2020:17:53:53 +0000\",\"request_method\":\"PUT\",\"request_uri\":\"/test123\",\"protocol\":\"HTTP/1.1\",\"status\":200,\"response_length\":\"0\",\"request_length\":\"165\",\"user_agent\":\"curl/7.47.0\",\"request_latency\":\"1310\",\"request_id\":\"xx-aa\",\"request_type\":\"REST.PUT.CONTAINER\",\"interface_type\":\"xx\",\"requested_location_constraint\":\"usa\",\"storage_account_id\":\"aa-bb\",\"storage_location_id\":\"cc-dd\",\"container_name\":\"test123\",\"container_id\":\"1234\",\"forwarded\":\"proto=https\",\"is_secure\":true,\"https\":{\"protocol\":\"TLSv1.2\",\"cipher_suite\":\"SHA\"},\"encryption_method\":\"abc\",\"key_crn_location\":\"usa\",\"container_region\":\"us\",\"cloud_iam\":{\"subject_adjn_id\":\"id1\",\"subject_type\":\"user\",\"decision\":\"PERMIT\",\"is_stale\":false,\"is_throttling\":false,\"op_code\":\"testop\",\"crn\":\"123\",\"pdp_elapsed_time\":\"116\",\"pdp_request_tx_id\":\"51234\",\"num_attempts\":1,\"subject\":\"gtb@hotmail.com\",\"is_failed\":false},\"sse_kms\":{\"ghost\":{\"result\":\"success\",\"code\":200,\"message\":\"OK\",\"endpoints\":{\"public_endpoint\":\"endpoint1\"},\"ghost_endpoint\":\"iax\",\"ghost_instance\":\"kub1\",\"latency\":\"294\",\"num_attempts\":1,\"transaction_id\":\"xx-aa\"},\"sit\":{\"result\":\"success\",\"latency\":\"380\"},\"kms\":{\"registration\":{\"result\":\"success\",\"code\":201,\"message\":\"OK\",\"time_start\":\"2020-02-12T17:53:52.996Z\",\"latency\":\"227\",\"num_attempts\":1,\"endpoint\":\"endpoint1/api/v2/keys/\",\"correlation_id\":\"xx-aa\",\"crk_crn_status\":\"registered_mutable\",\"request_method\":\"POST\"},\"wrap\":{\"result\":\"success\",\"code\":200,\"message\":\"OK\",\"time_start\":\"2020-02-12T17:53:53.224Z\",\"latency\":\"252\",\"num_attempts\":1,\"endpoint\":\"endpoint1/api/v2/keys/\",\"correlation_id\":\"kk-xxa\",\"crk_crn_status\":\"registered_mutable\",\"request_method\":\"POST\"}}},\"remote_user_subject\":\"gtb@hotmail.com\",\"account_d\":\"asdf\",\"principals\":{\"identity\":\"assdf\"},\"type\":\"http\",\"format\":1}","offset":21664278,"source":"/test.log","type":"testlogstopic"}

{"@timestamp":"2020-02-12T17:53:54.290Z","beat":{"hostname":"sdfdg.addf","name":"sdfdg.addf","version":"5.2.1"},"input_type":"log","log_dc":"xx","log_dc_type":"rg","message":"{\"server_name\":\"test1\",\"remote_address\":\"xx.xx\",\"forwarded_for\":\"public, xx.xx\",\"remote_user\":\"id1\",\"timestamp_start\":\"1581530033582\",\"timestamp_finish\":\"1581530034128\",\"time_start\":\"12/Feb/2020:17:53:53 +0000\",\"time_finish\":\"12/Feb/2020:17:53:54 +0000\",\"request_method\":\"PUT\",\"request_uri\":\"/test123/testob\",\"protocol\":\"HTTP/1.1\",\"status\":200,\"response_length\":\"0\",\"request_length\":\"8\",\"user_agent\":\"curl/7.47.0\",\"request_latency\":\"546\",\"request_id\":\"df.dfg\",\"request_type\":\"REST.PUT.OBJECT\",\"interface_type\":\"xx\",\"stat\":{\"client_wait\":0.021,\"storage_wait\":0.539,\"digest\":0.007,\"commit\":18.955,\"turn_around_time\":19.655,\"total_transfer\":19.534,\"pre_transfer\":525.722,\"post_transfer\":0.074},\"object_length\":\"8\",\"version_name\":\"dfdg-shjf2304\",\"version_transient\":true,\"delete_marker\":false,\"last_modified\":\"2020-02-12T17:53:54.109Z\",\"last_changed\":\"2020-02-12T17:53:54.109Z\",\"object_name\":\"testob\",\"storage_account_id\":\"aa-bb\",\"storage_location_id\":\"cc-dd\",\"container_name\":\"test123\",\"container_id\":\"1234\",\"forwarded\":\"proto=https\",\"is_secure\":true,\"https\":{\"protocol\":\"TLSv1.2\",\"cipher_suite\":\"SHA\"},\"encryption_method\":\"abc\",\"cloud_iam\":{\"subject_adjn_id\":\"id1\",\"subject_type\":\"user\",\"decision\":\"PERMIT\",\"is_stale\":false,\"is_throttling\":false,\"op_code\":\"ops1\",\"crn\":\"123\",\"pdp_elapsed_time\":\"192\",\"pdp_request_tx_id\":\"sdf23\",\"num_attempts\":1,\"subject\":\"gtb@hotmail.com\",\"is_failed\":false},\"sse_kms\":{\"sit\":{\"result\":\"success\",\"latency\":\"178\"},\"kms\":{\"unwrap\":{\"result\":\"success\",\"code\":200,\"message\":\"OK\",\"is_cached\":false,\"time_start\":\"2020-02-12T17:53:53.958Z\",\"latency\":\"150\",\"num_attempts\":1,\"endpoint\":\"endpoint1/api/v2/keys/\",\"correlation_id\":\"df.dfg\",\"request_method\":\"POST\"}}},\"remote_user_subject\":\"gtb@hotmail.com\",\"account_d\":\"asdf\",\"principals\":{\"identity\":\"assdf\"},\"type\":\"http\",\"format\":1}","offset":146222031,"source":"/test.log","type":"testlogstopic"}

input.conf

input {
    file {
        path => "/var/log/gang_input1.log"
        start_position => beginning
        codec => "json"
    }
}

filter.conf

filter {

    json {
      source => "message"
    }
    mutate {
      convert => {"request_latency" => "float"}
      convert => {"response_length" => "float"}
      convert => {"object_length" => "float"}
      convert => {"request_length" => "float"}
      convert => {"timestamp_start" => "integer"}
      convert => {"timestamp_finish" => "integer"}
      convert => {"client_wait" => "float"}
      convert => {"turn_around_time" => "float"}
      convert => {"total_transfer" => "float"}
      convert => {"pre_transfer" => "float"}
      convert => {"post_transfer" => "float"}
      convert => {"commit" => "float"}
      convert => {"digest" => "float"}
      convert => {"container_bytes_used" => "float"}
      convert => {"container_object_count" => "float"}
      convert => {"partial_storage_account_bytes_used" => "float"}
      convert => {"partial_storage_account_object_count" => "float"}
      convert => {"storage_wait" => "float"}
      convert => {"first_byte_latency" => "float"}
      convert => {"object_count" => "float"}
      convert => {"format" => "float"}
      convert => {"[cloud_iam][pdp_elapsed_time]" => "float"}
      convert => {"[sse_kp][cache_latency]" => "float"}
      convert => {"[sse_kp][service_instance_token_iam_latency]" => "float"}
      convert => {"[sse_kp][kp_latency]" => "float"}
      convert => {"[sse_kms][ghost][latency]" => "float"}
      convert => {"[sse_kms][kms][latency]" => "float"}
      convert => {"[sse_kms][sit][latency]" => "float"}
      convert => {"[sse_kms][kms][registration][latency]" => "float"}
      convert => {"[sse_kms][kms][deregistration][latency]" => "float"}
      convert => {"[sse_kms][kms][wrap][latency]" => "float"}
      convert => {"[sse_kms][sit][unwrap][latency]" => "float"}
  

      rename => { "status" => "access_status" }
    }
 ruby {
   code => "event.set('total_latency', event.get('[sse_kms][kms][registration][latency]'+'[sse_kms][kms][deregistration][latency]'+'[sse_kms][kms][wrap][latency]'+'[sse_kms][kms][unwrap][latency]'))"
  }
}

output.conf

output {
  file {
    path => "/var/log/gang_output1.log"
    codec => "json"
  }
}

I am trying to add the values of [sse_kms][kms][registration][latency]"+"[sse_kms][kms][deregistration][latency]"+"[sse_kms][kms][wrap][latency]"+"[sse_kms][kms][unwrap][latency] and put the result in new field total_latency but haven't had any luck. Some logs will be few latency fields and some will have all of them but I think logstash is smart enough to ignore if field is not present. Please advise on how to accomplish this. Currently I am getting total_latency=>nil as output

I would try

code => "event.set('total_latency', event.get('[sse_kms][kms][registration][latency]').to_f + event.get('[sse_kms][kms][deregistration][latency]').to_f + event.get('[sse_kms][kms][wrap][latency]').to_f + event.get('[sse_kms][kms][unwrap][latency]').to_f)"

Thanks @Badger. That seems to be working. Although I am little confused why we should do event.get('[sse_kms][kms][unwrap][latency]').to_f) when the above mutate converts them to float ? Also should I again explicitly convert the total_latency field as float using any mutate block at the end because I need this to be a number to build dashboards . Really appreciate your input

If all of the fields are present then to to_f would not be needed, but if the first field is missing you need to to_f to convert the nil to 0.0

I do not think you need to convert total_latency to float, it should be one already.

@Gangadhar_Mahadevan I think another option would be

code => "event.set('total_latency', 0.0 + event.get('[sse_kms][kms][registration][latency]') + event.get('[sse_kms][kms][deregistration][latency]') + event.get('[sse_kms][kms][wrap][latency]') + event.get('[sse_kms][kms][unwrap][latency]'))"

AFAIK the 0.0 would force ruby to use the + operator of float, which I believe would autoconvert nil to 0.0 if a field was missing. The LHS of an operator determines which operator gets used. The operators for float, integer, etc. are all different, because (in effect) they cast the RHS to the type of the LHS. I apologize for the hand-waving.

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.