How can I parse nested JSON strings to JSON objects?

Elastic Stack Logstash
1

Could some possibly provide some guidance on how to parse out an array of JSON strings into individual JSON objects with Logstash json parser??? My logstash filter look as follows...

	filter {
		grok {
				add_tag => [ "valid" ]
				pattern_definitions => {
				DIMENSION => "\{\"Name\"\:\"%{WORD}\"\,\"Value\"\:%{DATA}\}"
			}
				match => {
				"message" => "\{\"MetricName\"\:\"(?<MetricName>rtt)\"\,\"Timestamp\"\:\"%{TIMESTAMP_ISO8601:Timestamp}\"\,\"Unit\"\:\"%{WORD:Unit}\"\,\"StatisticValues\"\:\{\"SampleCount\"\:%{NUMBER:[StatisticValues][SampleCount]:int}\,\"Sum\"\:%{NUMBER:[StatisticValues][Sum]:float}\,\"Minimum\"\:%{NUMBER:[StatisticValues][Minimum]:float}\,\"Maximum\"\:%{NUMBER:[StatisticValues][Maximum]:float}\}\,\"Dimensions\"\:\[%{DIMENSION:Dimensions}\,%{DIMENSION:Dimensions}\,%{DIMENSION:Dimensions}\,%{DIMENSION:Dimensions}"
						}
			remove_field => ["message"]
			}
		if "valid" not in [tags] {
			drop {}
			}
	}

This returns the following output which includes a number of values nested under the "Dimensions" object that I'm trying to expand out......

           "@version" => "1",
    "StatisticValues" => {
                "Sum" => 529.0240000000001,
        "SampleCount" => 23,
            "Maximum" => 59.368,
            "Minimum" => 13.476
    },
         "Dimensions" => [
        [0] "{\"Name\":\"instance\",\"Value\":\"i-123456789\"}",
        [1] "{\"Name\":\"session\",\"Value\":\"1234cv-adsf-123123-12312-dfasddfsdf\"}",
        [2] "{\"Name\":\"connection\",\"Value\":\"1\"}",
        [3] "{\"Name\":\"channel\",\"Value\":\"main\"}"
    ],
          "Timestamp" => "2021-10-28T09:34:27Z",
               "Unit" => "Milliseconds",
         "@timestamp" => 2021-10-28T16:04:54.981Z,
               "tags" => [],
         "MetricName" => "rtt"

Ideally I'm trying to filter to achieve the following output....

        "@version" => "1",
		"StatisticValues" => {
        "Sum" => 529.0240000000001,
        "SampleCount" => 23,
        "Maximum" => 59.368,
        "Minimum" => 13.476
		"instance"=> "i-123456789",
		"session" => 1234cv-adsf-123123-12312-dfasddfsdf,
		"connection" => 1,
		"channel" => main
        "Timestamp" => "2021-10-28T09:34:27Z",
        "Unit" => "Milliseconds",
        "@timestamp" => 2021-10-28T16:04:54.981Z,
        "MetricName" => "rtt"

Any help/guidance is much appreciated.

2

Look at the answer I wrote to your question a couple of days ago. Specifically the x["Dimensions"].each { |y| loop.

3

Hi Badger,

I did in fact test that today in conjunction with the existing configuration but the same results are returned. When I review the logs there are some Ruby exceptions logged, specifically....

[2021-10-28T17:55:53,638][WARN ][logstash.filters.split   ] Only String and Array types are splittable. field:[@metadata][result] is of type = NilClass
[2021-10-28T17:55:53,638][WARN ][logstash.filters.split   ] Only String and Array types are splittable. field:[@metadata][result] is of type = NilClass
[2021-10-28T17:55:53,648][WARN ][logstash.filters.split   ] Only String and Array types are splittable. field:[@metadata][result] is of type = NilClass
[2021-10-28T17:55:53,648][WARN ][logstash.filters.split   ] Only String and Array types are splittable. field:[@metadata][result] is of type = NilClass
[2021-10-28T17:55:53,648][WARN ][logstash.filters.split   ] Only String and Array types are splittable. field:[@metadata][result] is of type = NilClass
[2021-10-28T17:55:53,653][ERROR][logstash.filters.ruby    ] Ruby exception occurred: undefined method `each' for nil:NilClass {:class=>"NoMethodError", :backtrace=>["(ruby filter code):2:in `block in filter_method'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-filter-ruby-3.1.7/lib/logstash/filters/ruby.rb:93:in `inline_script'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-filter-ruby-3.1.7/lib/logstash/filters/ruby.rb:86:in `filter'", "/usr/share/logstash/logstash-core/lib/logstash/filters/base.rb:143:in `do_filter'", "/usr/share/logstash/logstash-core/lib/logstash/filters/base.rb:162:in `block in multi_filter'", "org/jruby/RubyArray.java:1792:in `each'", "/usr/share/logstash/logstash-core/lib/logstash/filters/base.rb:159:in `multi_filter'", "org/logstash/config/ir/compiler/AbstractFilterDelegatorExt.java:115:in `multi_filter'", "(eval):265:in `block in filter_func'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:358:in `filter_batch'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:337:in `worker_loop'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:304:in `block in start_workers'"]}
[2021-10-28T17:55:53,657][ERROR][logstash.filters.ruby    ] Ruby exception occurred: undefined method `each' for nil:NilClass {:class=>"NoMethodError", :backtrace=>["(ruby filter code):2:in `block in filter_method'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-filter-ruby-3.1.7/lib/logstash/filters/ruby.rb:93:in `inline_script'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-filter-ruby-3.1.7/lib/logstash/filters/ruby.rb:86:in `filter'", "/usr/share/logstash/logstash-core/lib/logstash/filters/base.rb:143:in `do_filter'", "/usr/share/logstash/logstash-core/lib/logstash/filters/base.rb:162:in `block in multi_filter'", "org/jruby/RubyArray.java:1792:in `each'", "/usr/share/logstash/logstash-core/lib/logstash/filters/base.rb:159:in `multi_filter'", "org/logstash/config/ir/compiler/AbstractFilterDelegatorExt.java:115:in `multi_filter'", "(eval):265:in `block in filter_func'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:358:in `filter_batch'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:337:in `worker_loop'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:304:in `block in start_workers'"]}
[2021-10-28T17:55:53,657][ERROR][logstash.filters.ruby    ] Ruby exception occurred: undefined method `each' for nil:NilClass {:class=>"NoMethodError", :backtrace=>["(ruby filter code):2:in `block in filter_method'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-filter-ruby-3.1.7/lib/logstash/filters/ruby.rb:93:in `inline_script'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-filter-ruby-3.1.7/lib/logstash/filters/ruby.rb:86:in `filter'", "/usr/share/logstash/logstash-core/lib/logstash/filters/base.rb:143:in `do_filter'", "/usr/share/logstash/logstash-core/lib/logstash/filters/base.rb:162:in `block in multi_filter'", "org/jruby/RubyArray.java:1792:in `each'", "/usr/share/logstash/logstash-core/lib/logstash/filters/base.rb:159:in `multi_filter'", "org/logstash/config/ir/compiler/AbstractFilterDelegatorExt.java:115:in `multi_filter'", "(eval):265:in `block in filter_func'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:358:in `filter_batch'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:337:in `worker_loop'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:304:in `block in start_workers'"]}
[2021-10-28T17:55:53,659][ERROR][logstash.filters.ruby    ] Ruby exception occurred: undefined method `each' for nil:NilClass {:class=>"NoMethodError", :backtrace=>["(ruby filter code):2:in `block in filter_method'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-filter-ruby-3.1.7/lib/logstash/filters/ruby.rb:93:in `inline_script'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-filter-ruby-3.1.7/lib/logstash/filters/ruby.rb:86:in `filter'", "/usr/share/logstash/logstash-core/lib/logstash/filters/base.rb:143:in `do_filter'", "/usr/share/logstash/logstash-core/lib/logstash/filters/base.rb:162:in `block in multi_filter'", "org/jruby/RubyArray.java:1792:in `each'", "/usr/share/logstash/logstash-core/lib/logstash/filters/base.rb:159:in `multi_filter'", "org/logstash/config/ir/compiler/AbstractFilterDelegatorExt.java:115:in `multi_filter'", "(eval):265:in `block in filter_func'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:358:in `filter_batch'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:337:in `worker_loop'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:304:in `block in start_workers'"]}
[
4

My filter looks as follows...

filter {

    grok {
                        add_tag => [ "valid" ]
                        pattern_definitions => {
                        DIMENSION => "\{\"Name\"\:\"%{WORD}\"\,\"Value\"\:%{DATA}\}"
        }
                        match => {
                        "message" => "\{\"MetricName\"\:\"(?<MetricName>rtt)\"\,\"Timestamp\"\:\"%{TIMESTAMP_ISO8601:Timestamp}\"\,\"Unit\"\:\"%{WORD:Unit}\"\,\"StatisticValues\"\:\{\"SampleCount\"\:%{NUMBER:[StatisticValues][SampleCount]:int}\,\"Sum\"\:%{NUMBER:[StatisticValues][Sum]:float}\,\"Minimum\"\:%{NUMBER:[StatisticValues][Minimum]:float}\,\"Maximum\"\:%{NUMBER:[StatisticValues][Maximum]:float}\}\,\"Dimensions\"\:\[%{DIMENSION:Dimensions}\,%{DIMENSION:Dimensions}\,%{DIMENSION:Dimensions}\,%{DIMENSION:Dimensions}"
                                        }
        remove_field => ["message"]
                }
        if "valid" not in [tags] {
                drop {}
                }
        mutate {
        remove_tag => [ "valid" ]
                }



 json { source => "message" target => "[@metadata][data]" remove_field => [ "message" ] }
                        ruby {
                                        code => '
                                        d = event.get("[@metadata][data]")
                                        if d.is_a? Array
                                                newD = []
                                                d.each { |x|
                                                                item = {}
                                                                item["Timestamp"] = x["Timestamp"]
                                                                item[x["MetricName"]] = x["Value"]

                                                                if x["StatisticValues"]
                                                                item["StatisticValues"] = x["StatisticValues"]
                                                                end

                                                                if x["Dimensions"]
                                                                        x["Dimensions"].each { |y|
                                                                        item[y["Name"]] = y["Value"]
                                                                                                                }
                                                                        end
                                                                        newD << item
                                                        }
                                                event.set("[@metadata][result]", newD)
                                        end
                                        '
                                }
                        split { field => "[@metadata][result]" }
                        ruby { code => 'event.get("[@metadata][result]").each { |k, v| event.set(k,v) }' }
                        date { match => [ "Timestamp", "ISO8601" ] }
5

As I said before

The code I provided is just an outline of an approach, not production ready code. It would need a lot of error checking.

For example

split { field => "[@metadata][result]" }
ruby { code => 'event.get("[@metadata][result]").each { |k, v| event.set(k,v) }' }

would need to be surrounded with

if [@metadata][result] {
}

Additionally, I assumed that the data always looked like your example JSON. If there are variations you will need to write additional code to handle them.

6

Hi Badger, I tried that (included the "if [@metadata][result]" condition) but still no success, the results are still the same.

Capture
Capture983×817 152 KB
.... not sure there is any further guidance you can provide but if there was it would be great to hear it.

7

My configuration looks as follows ....

input {
  file{
        path => "/tmp/parsetest/test2.json"
        start_position => "beginning"
        sincedb_path => "/dev/null"
        }
}

filter {
    grok {
                        add_tag => [ "valid" ]
                        pattern_definitions => {
                        DIMENSION => "\{\"Name\"\:\"%{WORD}\"\,\"Value\"\:%{DATA}\}"
        }
                        match => {
                        "message" => "\{\"MetricName\"\:\"(?<MetricName>rtt)\"\,\"Timestamp\"\:\"%{TIMESTAMP_ISO8601:Timestamp}\"\,\"Unit\"\:\"%{WORD:Unit}\"\,\"StatisticValues\"\:\{\"SampleCount\"\:%{NUMBER:[StatisticValues][SampleCount]:int}\,\"Sum\"\:%{NUMBER:[StatisticValues][Sum]:float}\,\"Minimum\"\:%{NUMBER:[StatisticValues][Minimum]:float}\,\"Maximum\"\:%{NUMBER:[StatisticValues][Maximum]:float}\}\,\"Dimensions\"\:\[%{DIMENSION:Dimensions}\,%{DIMENSION:Dimensions}\,%{DIMENSION:Dimensions}\,%{DIMENSION:Dimensions}"
                                        }
        remove_field => ["message"]
                }
        if "valid" not in [tags] {
                drop {}
                }
        mutate {
        remove_tag => [ "valid" ]
                }

 json { source => "message" target => "[@metadata][data]" remove_field => [ "message" ] }
                        ruby {
                                        code => '
                                        d = event.get("[@metadata][data]")
                                        if d.is_a? Array
                                                newD = []
                                                d.each { |x|
                                                                item = {}
                                                                item["Timestamp"] = x["Timestamp"]
                                                                item[x["MetricName"]] = x["Value"]

                                                                if x["StatisticValues"]
                                                                item["StatisticValues"] = x["StatisticValues"]
                                                                end

                                                                if x["Dimensions"]
                                                                        x["Dimensions"].each { |y|
                                                                        item[y["Name"]] = y["Value"]
                                                                                                                }
                                                                        end
                                                                        newD << item
                                                        }
                                                event.set("[@metadata][result]", newD)
                                        end
                                        '
                                }
						if [@metadata][result] {
							split { field => "[@metadata][result]" }
							ruby { code => 'event.get("[@metadata][result]").each { |k, v| event.set(k,v) }' }
							date { match => [ "Timestamp", "ISO8601" ] }
								}
	
output {
  stdout { codec => rubydebug }
}
8

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.