Parsing json objects with dynamic key

[nmeaux]

Hello,

I am trying to parse a nested json which have dynamic keys.
I am just able for now to extract data from the first json object, i dont understand what is going wrong in my ruby filter.
Also in the output the whole input json is shown, i just want to have the result of the ruby filter parsing.
Logstash version : 6.2.4
Thanks in advance for any help.

input {
	file {
		path => ["/tmp/test.json"]
		start_position => "beginning"
		codec => "json"
		sincedb_path => "/dev/null"
	}
}
filter {
	mutate {
		remove_field => [ "@version", "@timestamp", "host", "path", "version" ]
	}
	ruby {
		code => "
			event.to_hash.clone.each_value{|v|
				if v.is_a? Hash
					v.each_pair{|k,v|
						event.set(k,v)
					}
				end
			}
		"
	}
}

output {
	file {
		path => "/tmp/test-result.json"
		write_behavior => "overwrite"
	}
	
}

input file /tmp/test.json ( with json pretty format ) :
{
    "10": {
        "latitude": "483751298",
        "longitude": "193654098"
    },
    "11": {
        "latitude": "583751298",
        "longitude": "293654098"
    },
    "12": {
        "latitude": "583751298",
        "longitude": "393654098"
    },
    "version": "1"
}

output file /tmp/test-result.json ( with json pretty format ) :
{
    "10": {
        "latitude": "483751298",
        "longitude": "193654098"
    },
    "11": {
        "latitude": "583751298",
        "longitude": "293654098"
    },
    "12": {
        "latitude": "583751298",
        "longitude": "393654098"
    },
    "latitude": "483751298",
    "longitude": "193654098"
}

Regards,

Nicolas

[magnusbaeck (Magnus Bäck)]

What's the expected result? Do you want latitude and longitude to be arrays?

[nmeaux]

Hello,
And thanks for you reply.
In fact i was trying to manipule an input json and this kind of structure in particular to be able to insert data into elasticsearch, so remove some elements, transforming it into array in order to apply the split filter.
And i did finally found a solution, maybe not perfect, but it work :

input {
        file {
                path => ["/tmp/test.json"]
                start_position => "beginning"
                sincedb_path => "/dev/null"
                codec => "json"
        }
}
filter {
        json {
                source => "message"
        }
        mutate {
                remove_field => [ "tags", "@version", "@timestamp", "host", "path", "version" ]
        }
        ruby {
                code => "
                        jobs_json = []
                        attribs = %w(account user_id)
                        event.to_hash.each do |id,jobs|
                                jobs.delete_if{|k,v| !attribs.include?(k.to_s)}
                                event.remove(id);
                                jobs_json << jobs
                        end
                        event.set('[jobs]',jobs_json);
                "
        }
        split {
                field => "jobs"
        }
        mutate {
                remove_field => [ "@version", "@timestamp" ]
        }
}
output {
        file {
                path => "/tmp/test-result.json"
                #write_behavior => "overwrite"
        }
}

With a slightly different input :
cat /tmp/test.json
{"14760970":{"account":"vd23s5n","job_id":14760970,"job_state":"PENDING","user_id":4113},"14782989":{"account":"12ssa","job_id":14782989,"job_state":"RUNNING","user_id":713},"14760971":{"account":"asdf5a","job_id":14760971,"job_state":"COMPLETE","user_id":4113},"14760972":{"account":"vd23s5n","job_id":14760972,"job_state":"PENDING","user_id":513}}

I can open this output :
cat /tmp/test-result.json
{"jobs":{"user_id":713,"account":"12ssa"}}
{"jobs":{"user_id":4113,"account":"vd23s5n"}}
{"jobs":{"user_id":513,"account":"vd23s5n"}}
{"jobs":{"user_id":4113,"account":"asdf5a"}}

rgds

Nicolas