This is my input stored in .log file
{"@timestamp":"2020-02-12T17:53:54.305Z","beat":{"hostname":"xx.xx","name":"xx.xx","version":"5.2.1"},"input_type":"log","log_dc":"xx","log_dc_type":"rg","message":"{\"server_name\":\"test1\",\"remote_address\":\"xx.xx\",\"forwarded_for\":\"public, xx.xx\",\"remote_user\":\"id1\",\"timestamp_start\":\"1581530032201\",\"timestamp_finish\":\"1581530033511\",\"time_start\":\"12/Feb/2020:17:53:52 +0000\",\"time_finish\":\"12/Feb/2020:17:53:53 +0000\",\"request_method\":\"PUT\",\"request_uri\":\"/test123\",\"protocol\":\"HTTP/1.1\",\"status\":200,\"response_length\":\"0\",\"request_length\":\"165\",\"user_agent\":\"curl/7.47.0\",\"request_latency\":\"1310\",\"request_id\":\"xx-aa\",\"request_type\":\"REST.PUT.CONTAINER\",\"interface_type\":\"xx\",\"requested_location_constraint\":\"usa\",\"storage_account_id\":\"aa-bb\",\"storage_location_id\":\"cc-dd\",\"container_name\":\"test123\",\"container_id\":\"1234\",\"forwarded\":\"proto=https\",\"is_secure\":true,\"https\":{\"protocol\":\"TLSv1.2\",\"cipher_suite\":\"SHA\"},\"encryption_method\":\"abc\",\"key_crn_location\":\"usa\",\"container_region\":\"us\",\"cloud_iam\":{\"subject_adjn_id\":\"id1\",\"subject_type\":\"user\",\"decision\":\"PERMIT\",\"is_stale\":false,\"is_throttling\":false,\"op_code\":\"testop\",\"crn\":\"123\",\"pdp_elapsed_time\":\"116\",\"pdp_request_tx_id\":\"51234\",\"num_attempts\":1,\"subject\":\"gtb@hotmail.com\",\"is_failed\":false},\"sse_kms\":{\"ghost\":{\"result\":\"success\",\"code\":200,\"message\":\"OK\",\"endpoints\":{\"public_endpoint\":\"endpoint1\"},\"ghost_endpoint\":\"iax\",\"ghost_instance\":\"kub1\",\"latency\":\"294\",\"num_attempts\":1,\"transaction_id\":\"xx-aa\"},\"sit\":{\"result\":\"success\",\"latency\":\"380\"},\"kms\":{\"registration\":{\"result\":\"success\",\"code\":201,\"message\":\"OK\",\"time_start\":\"2020-02-12T17:53:52.996Z\",\"latency\":\"227\",\"num_attempts\":1,\"endpoint\":\"endpoint1/api/v2/keys/\",\"correlation_id\":\"xx-aa\",\"crk_crn_status\":\"registered_mutable\",\"request_method\":\"POST\"},\"wrap\":{\"result\":\"success\",\"code\":200,\"message\":\"OK\",\"time_start\":\"2020-02-12T17:53:53.224Z\",\"latency\":\"252\",\"num_attempts\":1,\"endpoint\":\"endpoint1/api/v2/keys/\",\"correlation_id\":\"kk-xxa\",\"crk_crn_status\":\"registered_mutable\",\"request_method\":\"POST\"}}},\"remote_user_subject\":\"gtb@hotmail.com\",\"account_d\":\"asdf\",\"principals\":{\"identity\":\"assdf\"},\"type\":\"http\",\"format\":1}","offset":21664278,"source":"/test.log","type":"testlogstopic"}
{"@timestamp":"2020-02-12T17:53:54.290Z","beat":{"hostname":"sdfdg.addf","name":"sdfdg.addf","version":"5.2.1"},"input_type":"log","log_dc":"xx","log_dc_type":"rg","message":"{\"server_name\":\"test1\",\"remote_address\":\"xx.xx\",\"forwarded_for\":\"public, xx.xx\",\"remote_user\":\"id1\",\"timestamp_start\":\"1581530033582\",\"timestamp_finish\":\"1581530034128\",\"time_start\":\"12/Feb/2020:17:53:53 +0000\",\"time_finish\":\"12/Feb/2020:17:53:54 +0000\",\"request_method\":\"PUT\",\"request_uri\":\"/test123/testob\",\"protocol\":\"HTTP/1.1\",\"status\":200,\"response_length\":\"0\",\"request_length\":\"8\",\"user_agent\":\"curl/7.47.0\",\"request_latency\":\"546\",\"request_id\":\"df.dfg\",\"request_type\":\"REST.PUT.OBJECT\",\"interface_type\":\"xx\",\"stat\":{\"client_wait\":0.021,\"storage_wait\":0.539,\"digest\":0.007,\"commit\":18.955,\"turn_around_time\":19.655,\"total_transfer\":19.534,\"pre_transfer\":525.722,\"post_transfer\":0.074},\"object_length\":\"8\",\"version_name\":\"dfdg-shjf2304\",\"version_transient\":true,\"delete_marker\":false,\"last_modified\":\"2020-02-12T17:53:54.109Z\",\"last_changed\":\"2020-02-12T17:53:54.109Z\",\"object_name\":\"testob\",\"storage_account_id\":\"aa-bb\",\"storage_location_id\":\"cc-dd\",\"container_name\":\"test123\",\"container_id\":\"1234\",\"forwarded\":\"proto=https\",\"is_secure\":true,\"https\":{\"protocol\":\"TLSv1.2\",\"cipher_suite\":\"SHA\"},\"encryption_method\":\"abc\",\"cloud_iam\":{\"subject_adjn_id\":\"id1\",\"subject_type\":\"user\",\"decision\":\"PERMIT\",\"is_stale\":false,\"is_throttling\":false,\"op_code\":\"ops1\",\"crn\":\"123\",\"pdp_elapsed_time\":\"192\",\"pdp_request_tx_id\":\"sdf23\",\"num_attempts\":1,\"subject\":\"gtb@hotmail.com\",\"is_failed\":false},\"sse_kms\":{\"sit\":{\"result\":\"success\",\"latency\":\"178\"},\"kms\":{\"unwrap\":{\"result\":\"success\",\"code\":200,\"message\":\"OK\",\"is_cached\":false,\"time_start\":\"2020-02-12T17:53:53.958Z\",\"latency\":\"150\",\"num_attempts\":1,\"endpoint\":\"endpoint1/api/v2/keys/\",\"correlation_id\":\"df.dfg\",\"request_method\":\"POST\"}}},\"remote_user_subject\":\"gtb@hotmail.com\",\"account_d\":\"asdf\",\"principals\":{\"identity\":\"assdf\"},\"type\":\"http\",\"format\":1}","offset":146222031,"source":"/test.log","type":"testlogstopic"}
input.conf
input {
file {
path => "/var/log/gang_input1.log"
start_position => beginning
codec => "json"
}
}
filter.conf
filter {
json {
source => "message"
}
mutate {
convert => {"request_latency" => "float"}
convert => {"response_length" => "float"}
convert => {"object_length" => "float"}
convert => {"request_length" => "float"}
convert => {"timestamp_start" => "integer"}
convert => {"timestamp_finish" => "integer"}
convert => {"client_wait" => "float"}
convert => {"turn_around_time" => "float"}
convert => {"total_transfer" => "float"}
convert => {"pre_transfer" => "float"}
convert => {"post_transfer" => "float"}
convert => {"commit" => "float"}
convert => {"digest" => "float"}
convert => {"container_bytes_used" => "float"}
convert => {"container_object_count" => "float"}
convert => {"partial_storage_account_bytes_used" => "float"}
convert => {"partial_storage_account_object_count" => "float"}
convert => {"storage_wait" => "float"}
convert => {"first_byte_latency" => "float"}
convert => {"object_count" => "float"}
convert => {"format" => "float"}
convert => {"[cloud_iam][pdp_elapsed_time]" => "float"}
convert => {"[sse_kp][cache_latency]" => "float"}
convert => {"[sse_kp][service_instance_token_iam_latency]" => "float"}
convert => {"[sse_kp][kp_latency]" => "float"}
convert => {"[sse_kms][ghost][latency]" => "float"}
convert => {"[sse_kms][kms][latency]" => "float"}
convert => {"[sse_kms][sit][latency]" => "float"}
convert => {"[sse_kms][kms][registration][latency]" => "float"}
convert => {"[sse_kms][kms][deregistration][latency]" => "float"}
convert => {"[sse_kms][kms][wrap][latency]" => "float"}
convert => {"[sse_kms][sit][unwrap][latency]" => "float"}
rename => { "status" => "access_status" }
}
ruby {
code => "event.set('total_latency', event.get('[sse_kms][kms][registration][latency]'+'[sse_kms][kms][deregistration][latency]'+'[sse_kms][kms][wrap][latency]'+'[sse_kms][kms][unwrap][latency]'))"
}
}
output.conf
output {
file {
path => "/var/log/gang_output1.log"
codec => "json"
}
}
I am trying to add the values of [sse_kms][kms][registration][latency]"+"[sse_kms][kms][deregistration][latency]"+"[sse_kms][kms][wrap][latency]"+"[sse_kms][kms][unwrap][latency]
and put the result in new field total_latency
but haven't had any luck. Some logs will be few latency fields and some will have all of them but I think logstash is smart enough to ignore if field is not present. Please advise on how to accomplish this. Currently I am getting total_latency=>nil
as output