Create json object with logstash

Elastic Stack Logstash
1

Hi

I face some trouble with JSON logs. I have a big JSON message comming in my Logstash and I made some parse in my file to create a new clean message. But now I want to create nested JSON field as I can have an array of nested json in my input.

here is my example:

this is my incomming message:

{
"timestamp": 1543400113822,
"correlationId": "b16afe5bf51593fa597b0faf",
"processInfo": {
	"hostname": "host1",
	"domainId": "75c0688d-ffab-458a-8744-f01b154d27f0",
	"groupId": "group-13",
	"groupName": "group23",
	"serviceId": "instance-57",
	"serviceName": "Manager",
	"version": "v7.5-Internal"
},
"circuitPath": [{
		"policy": "API OAuth 2.0 Security Device",
		"execTime": 2,
		"filters": [{
				"name": "<anonymous>",
				"type": "ValidateOAuthAccessTokenFilter",
				"class": "com.vordel.circuit.oauth.provider.ValidateOAuthAccessTokenFilter",
				"status": "Fail",
				"filterTime": 1543400113821,
				"execTime": 2
			}
		]
	}, {
		"policy": "API Broker",
		"execTime": 0,
		"filters": [{
				"name": "Set service context",
				"type": "ApiServiceContextFilter",
				"class": "com.vordel.coreapireg.runtime.broker.ApiServiceContextFilter",
				"status": "Pass",
				"filterTime": 1543400113821,
				"execTime": 0
			}, {
				"name": "Authentication Failure",
				"type": "ApiShuntFilter",
				"class": "com.vordel.coreapireg.runtime.broker.ApiShuntFilter",
				"status": "Fail",
				"filterTime": 1543400113821,
				"execTime": 0
			}
		]
	}
]
}

This is my logstash configuration

	json {
					source => "message"
					target => "json"
			}

			date {
					match  => ["[json][timestamp]", "UNIX_MS"]
					target => "@timestamp"
			}
				mutate {
					add_field => {
							"api_correlationId" => "%{[json][correlationId]}"
							"api_hostname" => "%{[json][processInfo][hostname]}"
							"api_instance" => "%{[json][processInfo][groupName]}"
							"api_service_name" => "%{[json][processInfo][serviceName]}"
					}
			}

if ("circuitPath" in [json]) {
					split {
							field => "[json][circuitPath]"
					}

					mutate {
							add_field => {
							
#my goal here is to recreate the filters objects with only the following field in a nested object
				
									"[api_policy_name]" => "%{[json][circuitPath][policy]}"
									"[api_policy_Exec_Time]" => "%{[json][circuitPath][execTime]}"
									
									
									"[api_filter_name]" => "%{[json][circuitPath][N][filters][N][name]}"
									"[api_filter_type]" => "%{[json][circuitPath][N][filters][N][type]}"
									"[api_filter_status]" => "%{[json][circuitPath][N][filters][N][status]}"
									"[api_filter_exec_time]" => "%{[json][circuitPath][N][filters][N][execTime]}"

							}
					}
					mutate {
						remove_field => [ "message" ]
					}
			}

So at the end my json should look like this:

{
api_correlationId
api_hostname
api_instance
api_service_name
circuit: [{
    api_policy_name:"API OAuth 2.0 Security Device"
    api_policy_Exec_Time:"2"
     [{
        api_filter_name:"anonymous"
        api_filter_type:"ValidateOAuthAccessTokenFilter"
        api_filter_status:"Fail"
        api_filter_exec_time:"2"
     }]
},
{
    api_policy_name:"API Broker"
    api_policy_Exec_Time:"0"
     [{
         api_filter_name:"Set service context"
         api_filter_type:"ApiServiceContextFilter"
         api_filter_status:"Pass"
         api_filter_exec_time:"0"
      },
     {
         api_filter_name:"Authentication Failure"
         api_filter_type:"ApiShuntFilter"
         api_filter_status:"Fail"
         api_filter_exec_time:"0"
   }]
}]
}

I can't find a way of doing it.

2

And what is the result you are currently getting?

3

Here is the configuration I finaly used

if ("circuitPath" in [json]) {
					split {
							field => "[json][circuitPath]"
					}

					mutate {
							add_field => {
							
									"[api_policy_name]" => "%{[json][circuitPath][policy]}"
									"[api_policy_Exec_Time]" => "%{[json][circuitPath][execTime]}"
							}
					}
					
					mutate {
							add_field => {
									#search for here		
									"[api_filter_name]" => "%{[json][circuitPath][filters][0][name]}"
									"[api_filter_type]" => "%{[json][circuitPath][filters][0][type]}"
									"[api_filter_status]" => "%{[json][circuitPath][filters][0][status]}"
									"[api_filter_exec_time]" => "%{[json][circuitPath][filters][0][execTime]}"

							}
					}
			}

With this log in input :

 {
"timestamp": 1543487516616,
"correlationId": "1cc0ff5b3284091b4c191146",
"processInfo": {
	"hostname": "host1",
	"domainId": "75c0688d-ffab-458a-8744-f01b154d27f0",
	"groupId": "group-13",
	"groupName": "Manager1",
	"serviceId": "instance-57",
	"serviceName": "ManagerLow",
	"version": "v7.5.3-Internal"
},
"circuitPath": [{
		"policy": "API Broker",
		"execTime": 0,
		"filters": [{
				"name": "Not Found",
				"type": "ApiShuntFilter",
				"class": "com.vordel.coreapireg.runtime.broker.ApiShuntFilter",
				"status": "Fail",
				"filterTime": 1543487516614,
				"execTime": 0
			}, {
				"name": "Not Found",
				"type": "ApiShuntFailureFilter",
				"class": "com.vordel.coreapireg.runtime.broker.ApiShuntFailureFilter",
				"status": "Pass",
				"filterTime": 1543487516615,
				"execTime": 0
			}
		]
	}
]
 }

I got:

{
"timestamp": 1543487516616,
"api_correlationId": "1cc0ff5b3284091b4c191146",
"host": "host1",
"api_hostname": "Manager1",
"api_instance": "ManagerLow",
"api_policy_Exec_Time": "API Broker",
"api_filter_exec_time": 0,
"api_filter_name": "Not Found",
"api_filter_type": "ApiShuntFilter",
"api_filter_status": "Fail",
"api_filter_exec_time": 0	
}

So I can only take my first element

4

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.