Aggregate multiple nested (recursive) logstash

Elastic Stack Logstash
10

You could try using

    aggregate {
        task_id => "%{id}"
        code => '
            map["properties"] ||= {}
            map["properties"]["id"] = event.get("id")
            map["properties"]["description"] = event.get("description")

            map["properties"]["detail_list"] ||= []
            map["properties"]["Detail"] ||= {}
            id_2 = event.get("id_2")
            if (id_2 != nil)
                if !( map["properties"]["detail_list"].include? id_2 )
                    map["properties"]["detail_list"] << id_2
                    map["properties"]["Detail"][id_2] = {
                        "id_2" => event.get("id_2"),
                        "cod" => event.get("cod_2"),
                        "descr" => event.get("descr_2")
                    }
                end
            end

            map["properties"]["Detail"][id_2]["sub_detail_list"] ||= []
            map["properties"]["Detail"][id_2]["subDetail"] ||= []
            id_3 = event.get("id_3")
            if (id_3 != nil)
                if !( map["properties"]["Detail"][id_2]["sub_detail_list"].include? id_3 )
                    map["properties"]["Detail"][id_2]["sub_detail_list"] << id_3
                    map["properties"]["Detail"][id_2]["subDetail"] << {
                    "id_3" => event.get("id_3"),
                    "cod" => event.get("cod_3"),
                    "descr" => event.get("descr_3")
                    }
                end
            end

            event.cancel()
        '

I use single quotes around the code and double quotes inside it so that I can add debugging statements like

puts "#{id_2} #{id_3}"

Which is what told me inserting sub_detail should not be inside the if (id_2 != nil) test. That code will get you an event like

"properties" => {
         "Detail" => {
        "1" => {
            "sub_detail_list" => [
                [0] "1"
            ],
                      "descr" => "Detail A",
                        "cod" => "A",
                  "subDetail" => [
                [0] {
                    "descr" => "Detail X1",
                     "id_3" => "1",
                      "cod" => "X1"
                }
            ],
                       "id_2" => "1"
        },
        "2" => {
            "sub_detail_list" => [
                [0] "1",
                [1] "2",
                [2] "3"
            ],
                      "descr" => "Detail B",
                        "cod" => "B",
                  "subDetail" => [
                [0] {
                    "descr" => "Detail X1",
                     "id_3" => "1",
                      "cod" => "X1"
                },
                [1] {
                    "descr" => "Detail X2",
                     "id_3" => "2",
                      "cod" => "X2"
                },
                [2] {
                    "descr" => "Detail X3",
                     "id_3" => "3",
                      "cod" => "X3"
                }
            ],
                       "id_2" => "2"
        }
    },
    "description" => "text1",
    "detail_list" => [
        [0] "1",
        [1] "2"
    ],
             "id" => "1"
}

That may not be exactly what you want but should get you much closer.

1 Like