### Author:  [@dudwell](https://discuss.elastic.co/u/dudwell)
#### Post date: [October 28, 2021, 4:24pm UTC](https://discuss.elastic.co/t/how-can-i-parse-nested-json-strings-to-json-objects/287944/1 "2021-10-28T16:24:40Z")
Could some possibly provide some guidance on how to parse out an array of JSON strings into individual JSON objects with Logstash json parser??? My logstash filter look as follows...
```auto
filter {
grok {
add_tag => ["valid"]
pattern_definitions => {
DIMENSION => "\{\"Name\"\:\"%{WORD}\"\,\"Value\"\:%{DATA}\}"
}
match => {
"message" => "\{\"MetricName\"\:\"(?rtt)\"\,\"Timestamp\"\:\"%{TIMESTAMP_ISO8601:Timestamp}\"\,\"Unit\"\:\"%{WORD:Unit}\"\,\"StatisticValues\"\:\{\"SampleCount\"\:%{NUMBER:[StatisticValues][SampleCount]:int}\,\"Sum\"\:%{NUMBER:[StatisticValues][Sum]:float}\,\"Minimum\"\:%{NUMBER:[StatisticValues][Minimum]:float}\,\"Maximum\"\:%{NUMBER:[StatisticValues][Maximum]:float}\}\,\"Dimensions\"\:\[%{DIMENSION:Dimensions}\,%{DIMENSION:Dimensions}\,%{DIMENSION:Dimensions}\,%{DIMENSION:Dimensions}"
}
remove_field => ["message"]
}
if "valid" not in [tags] {
drop {}
}
}
```
This returns the following output which includes a number of values nested under the "Dimensions" object that I'm trying to expand out......
```auto
"@version" => "1",
"StatisticValues" => {
"Sum" => 529.0240000000001,
"SampleCount" => 23,
"Maximum" => 59.368,
"Minimum" => 13.476
},
"Dimensions" => [
[0] "{\"Name\":\"instance\",\"Value\":\"i-123456789\"}",
[1] "{\"Name\":\"session\",\"Value\":\"1234cv-adsf-123123-12312-dfasddfsdf\"}",
[2] "{\"Name\":\"connection\",\"Value\":\"1\"}",
[3] "{\"Name\":\"channel\",\"Value\":\"main\"}"
],
"Timestamp" => "2021-10-28T09:34:27Z",
"Unit" => "Milliseconds",
"@timestamp" => 2021-10-28T16:04:54.981Z,
"tags" => [],
"MetricName" => "rtt"
```
Ideally I'm trying to filter to achieve the following output....
```auto
"@version" => "1",
"StatisticValues" => {
"Sum" => 529.0240000000001,
"SampleCount" => 23,
"Maximum" => 59.368,
"Minimum" => 13.476
"instance"=> "i-123456789",
"session" => 1234cv-adsf-123123-12312-dfasddfsdf,
"connection" => 1,
"channel" => main
"Timestamp" => "2021-10-28T09:34:27Z",
"Unit" => "Milliseconds",
"@timestamp" => 2021-10-28T16:04:54.981Z,
"MetricName" => "rtt"
```
Any help/guidance is much appreciated.
---
### Author:  [@Badger](https://discuss.elastic.co/u/Badger)
#### Post date: [October 28, 2021, 5:39pm UTC](https://discuss.elastic.co/t/how-can-i-parse-nested-json-strings-to-json-objects/287944/2 "2021-10-28T17:39:10Z")
Look at the [answer](https://discuss.elastic.co/t/can-anyone-help-on-nested-json-parsing-with-logstash/287661/2) I wrote to your question a couple of days ago. Specifically the `x["Dimensions"].each { |y|` loop.
---
### Author:  [@dudwell](https://discuss.elastic.co/u/dudwell)
#### Post date: [October 28, 2021, 5:58pm UTC](https://discuss.elastic.co/t/how-can-i-parse-nested-json-strings-to-json-objects/287944/3 "2021-10-28T17:58:20Z")
Hi Badger,
I did in fact test that today in conjunction with the existing configuration but the same results are returned. When I review the logs there are some Ruby exceptions logged, specifically....
```auto
[2021-10-28T17:55:53,638][WARN][logstash.filters.split] Only String and Array types are splittable. field:[@metadata][result] is of type = NilClass
[2021-10-28T17:55:53,638][WARN][logstash.filters.split] Only String and Array types are splittable. field:[@metadata][result] is of type = NilClass
[2021-10-28T17:55:53,648][WARN][logstash.filters.split] Only String and Array types are splittable. field:[@metadata][result] is of type = NilClass
[2021-10-28T17:55:53,648][WARN][logstash.filters.split] Only String and Array types are splittable. field:[@metadata][result] is of type = NilClass
[2021-10-28T17:55:53,648][WARN][logstash.filters.split] Only String and Array types are splittable. field:[@metadata][result] is of type = NilClass
[2021-10-28T17:55:53,653][ERROR][logstash.filters.ruby] Ruby exception occurred: undefined method `each' for nil:NilClass {:class=>"NoMethodError", :backtrace=>["(ruby filter code):2:in `block in filter_method'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-filter-ruby-3.1.7/lib/logstash/filters/ruby.rb:93:in `inline_script'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-filter-ruby-3.1.7/lib/logstash/filters/ruby.rb:86:in `filter'", "/usr/share/logstash/logstash-core/lib/logstash/filters/base.rb:143:in `do_filter'", "/usr/share/logstash/logstash-core/lib/logstash/filters/base.rb:162:in `block in multi_filter'", "org/jruby/RubyArray.java:1792:in `each'", "/usr/share/logstash/logstash-core/lib/logstash/filters/base.rb:159:in `multi_filter'", "org/logstash/config/ir/compiler/AbstractFilterDelegatorExt.java:115:in `multi_filter'", "(eval):265:in `block in filter_func'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:358:in `filter_batch'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:337:in `worker_loop'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:304:in `block in start_workers'"]}
[2021-10-28T17:55:53,657][ERROR][logstash.filters.ruby] Ruby exception occurred: undefined method `each' for nil:NilClass {:class=>"NoMethodError", :backtrace=>["(ruby filter code):2:in `block in filter_method'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-filter-ruby-3.1.7/lib/logstash/filters/ruby.rb:93:in `inline_script'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-filter-ruby-3.1.7/lib/logstash/filters/ruby.rb:86:in `filter'", "/usr/share/logstash/logstash-core/lib/logstash/filters/base.rb:143:in `do_filter'", "/usr/share/logstash/logstash-core/lib/logstash/filters/base.rb:162:in `block in multi_filter'", "org/jruby/RubyArray.java:1792:in `each'", "/usr/share/logstash/logstash-core/lib/logstash/filters/base.rb:159:in `multi_filter'", "org/logstash/config/ir/compiler/AbstractFilterDelegatorExt.java:115:in `multi_filter'", "(eval):265:in `block in filter_func'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:358:in `filter_batch'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:337:in `worker_loop'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:304:in `block in start_workers'"]}
[2021-10-28T17:55:53,657][ERROR][logstash.filters.ruby] Ruby exception occurred: undefined method `each' for nil:NilClass {:class=>"NoMethodError", :backtrace=>["(ruby filter code):2:in `block in filter_method'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-filter-ruby-3.1.7/lib/logstash/filters/ruby.rb:93:in `inline_script'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-filter-ruby-3.1.7/lib/logstash/filters/ruby.rb:86:in `filter'", "/usr/share/logstash/logstash-core/lib/logstash/filters/base.rb:143:in `do_filter'", "/usr/share/logstash/logstash-core/lib/logstash/filters/base.rb:162:in `block in multi_filter'", "org/jruby/RubyArray.java:1792:in `each'", "/usr/share/logstash/logstash-core/lib/logstash/filters/base.rb:159:in `multi_filter'", "org/logstash/config/ir/compiler/AbstractFilterDelegatorExt.java:115:in `multi_filter'", "(eval):265:in `block in filter_func'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:358:in `filter_batch'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:337:in `worker_loop'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:304:in `block in start_workers'"]}
[2021-10-28T17:55:53,659][ERROR][logstash.filters.ruby] Ruby exception occurred: undefined method `each' for nil:NilClass {:class=>"NoMethodError", :backtrace=>["(ruby filter code):2:in `block in filter_method'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-filter-ruby-3.1.7/lib/logstash/filters/ruby.rb:93:in `inline_script'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-filter-ruby-3.1.7/lib/logstash/filters/ruby.rb:86:in `filter'", "/usr/share/logstash/logstash-core/lib/logstash/filters/base.rb:143:in `do_filter'", "/usr/share/logstash/logstash-core/lib/logstash/filters/base.rb:162:in `block in multi_filter'", "org/jruby/RubyArray.java:1792:in `each'", "/usr/share/logstash/logstash-core/lib/logstash/filters/base.rb:159:in `multi_filter'", "org/logstash/config/ir/compiler/AbstractFilterDelegatorExt.java:115:in `multi_filter'", "(eval):265:in `block in filter_func'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:358:in `filter_batch'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:337:in `worker_loop'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:304:in `block in start_workers'"]}
[
```
---
### Author:  [@dudwell](https://discuss.elastic.co/u/dudwell)
#### Post date: [October 28, 2021, 6:03pm UTC](https://discuss.elastic.co/t/how-can-i-parse-nested-json-strings-to-json-objects/287944/4 "2021-10-28T18:03:05Z")
My filter looks as follows...
```auto
filter {
grok {
add_tag => ["valid"]
pattern_definitions => {
DIMENSION => "\{\"Name\"\:\"%{WORD}\"\,\"Value\"\:%{DATA}\}"
}
match => {
"message" => "\{\"MetricName\"\:\"(?rtt)\"\,\"Timestamp\"\:\"%{TIMESTAMP_ISO8601:Timestamp}\"\,\"Unit\"\:\"%{WORD:Unit}\"\,\"StatisticValues\"\:\{\"SampleCount\"\:%{NUMBER:[StatisticValues][SampleCount]:int}\,\"Sum\"\:%{NUMBER:[StatisticValues][Sum]:float}\,\"Minimum\"\:%{NUMBER:[StatisticValues][Minimum]:float}\,\"Maximum\"\:%{NUMBER:[StatisticValues][Maximum]:float}\}\,\"Dimensions\"\:\[%{DIMENSION:Dimensions}\,%{DIMENSION:Dimensions}\,%{DIMENSION:Dimensions}\,%{DIMENSION:Dimensions}"
}
remove_field => ["message"]
}
if "valid" not in [tags] {
drop {}
}
mutate {
remove_tag => ["valid"]
}
json { source => "message" target => "[@metadata][data]" remove_field => ["message"] }
ruby {
code => '
d = event.get("[@metadata][data]")
if d.is_a? Array
newD = []
d.each { |x|
item = {}
item["Timestamp"] = x["Timestamp"]
item[x["MetricName"]] = x["Value"]
if x["StatisticValues"]
item["StatisticValues"] = x["StatisticValues"]
end
if x["Dimensions"]
x["Dimensions"].each { |y|
item[y["Name"]] = y["Value"]
}
end
newD << item
}
event.set("[@metadata][result]", newD)
end
'
}
split { field => "[@metadata][result]" }
ruby { code => 'event.get("[@metadata][result]").each { |k, v| event.set(k,v) }' }
date { match => ["Timestamp", "ISO8601"] }
```
---
### Author:  [@Badger](https://discuss.elastic.co/u/Badger)
#### Post date: [October 28, 2021, 6:04pm UTC](https://discuss.elastic.co/t/how-can-i-parse-nested-json-strings-to-json-objects/287944/5 "2021-10-28T18:04:18Z")
As I said before
> The code I provided is just an outline of an approach, not production ready code. It would need a lot of error checking.
For example
```
split { field => "[@metadata][result]" }
ruby { code => 'event.get("[@metadata][result]").each { |k, v| event.set(k,v) }' }
```
would need to be surrounded with
```
if [@metadata][result] {
}
```
Additionally, I assumed that the data always looked like your example JSON. If there are variations you will need to write additional code to handle them.
---
### Author:  [@dudwell](https://discuss.elastic.co/u/dudwell)
#### Post date: [October 31, 2021, 2:49pm UTC](https://discuss.elastic.co/t/how-can-i-parse-nested-json-strings-to-json-objects/287944/6 "2021-10-31T14:49:28Z")
Hi Badger, I tried that (included the "if [@metadata][result]" condition) but still no success, the results are still the same.
.... not sure there is any further guidance you can provide but if there was it would be great to hear it.
---
### Author:  [@dudwell](https://discuss.elastic.co/u/dudwell)
#### Post date: [October 31, 2021, 2:50pm UTC](https://discuss.elastic.co/t/how-can-i-parse-nested-json-strings-to-json-objects/287944/7 "2021-10-31T14:50:37Z")
My configuration looks as follows ....
```auto
input {
file{
path => "/tmp/parsetest/test2.json"
start_position => "beginning"
sincedb_path => "/dev/null"
}
}
filter {
grok {
add_tag => ["valid"]
pattern_definitions => {
DIMENSION => "\{\"Name\"\:\"%{WORD}\"\,\"Value\"\:%{DATA}\}"
}
match => {
"message" => "\{\"MetricName\"\:\"(?rtt)\"\,\"Timestamp\"\:\"%{TIMESTAMP_ISO8601:Timestamp}\"\,\"Unit\"\:\"%{WORD:Unit}\"\,\"StatisticValues\"\:\{\"SampleCount\"\:%{NUMBER:[StatisticValues][SampleCount]:int}\,\"Sum\"\:%{NUMBER:[StatisticValues][Sum]:float}\,\"Minimum\"\:%{NUMBER:[StatisticValues][Minimum]:float}\,\"Maximum\"\:%{NUMBER:[StatisticValues][Maximum]:float}\}\,\"Dimensions\"\:\[%{DIMENSION:Dimensions}\,%{DIMENSION:Dimensions}\,%{DIMENSION:Dimensions}\,%{DIMENSION:Dimensions}"
}
remove_field => ["message"]
}
if "valid" not in [tags] {
drop {}
}
mutate {
remove_tag => ["valid"]
}
json { source => "message" target => "[@metadata][data]" remove_field => ["message"] }
ruby {
code => '
d = event.get("[@metadata][data]")
if d.is_a? Array
newD = []
d.each { |x|
item = {}
item["Timestamp"] = x["Timestamp"]
item[x["MetricName"]] = x["Value"]
if x["StatisticValues"]
item["StatisticValues"] = x["StatisticValues"]
end
if x["Dimensions"]
x["Dimensions"].each { |y|
item[y["Name"]] = y["Value"]
}
end
newD << item
}
event.set("[@metadata][result]", newD)
end
'
}
if [@metadata][result] {
split { field => "[@metadata][result]" }
ruby { code => 'event.get("[@metadata][result]").each { |k, v| event.set(k,v) }' }
date { match => ["Timestamp", "ISO8601"] }
}
output {
stdout { codec => rubydebug }
}
```
---
### Author:  [@system](https://discuss.elastic.co/u/system)
#### Post date: [November 28, 2021, 2:51pm UTC](https://discuss.elastic.co/t/how-can-i-parse-nested-json-strings-to-json-objects/287944/8 "2021-11-28T14:51:35Z")
This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.