Getting Json Formated Data to elastic in multiple Docs

Elastic Stack Logstash
1

Hello,

certainly this has been asked many times before but after hours of working on this topic I was unable to solve my problem.

I have Json data that looks like this in ruby debug, I want to ghav each of those randomly choosen connection Identifiers to become a single entry in elastic with all attributes attached to this elastic object ( sorry Im trying to get the wording right bit will most certainly fail :wink: ), can anyone push me in the right direction?:

{
    "@timestamp" => 2020-04-07T15:08:33.479Z,
    "12210d17-8467-4cc7-9e5f-ce70725a7b44" => {
                  "identifier" => "12210d17-8467-4cc7-9e5f-ce70725a7b44",
                   "startDate" => 1586271140009,
                 "connectable" => true,
        "connectionIdentifier" => "1",
                    "username" => "xxx",
                  "remoteHost" => "149.242.248.XX"
    },
    "b2bcac55-b4e6-404d-8c93-78a059de9978" => {
                  "identifier" => "b2bcac55-b4e6-404d-8c93-78a059de9978",
                   "startDate" => 1586269346913,
                 "connectable" => true,
        "connectionIdentifier" => "13",
                    "username" => "xxx",
                  "remoteHost" => "149.242.248.XX"
    },
    "f85bca9e-518b-4040-94b8-c69a9e780961" => {
                  "identifier" => "f85bca9e-518b-4040-94b8-c69a9e780961",
                   "startDate" => 1586267209033,
                 "connectable" => true,
        "connectionIdentifier" => "5181",
                    "username" => "xxx",
                  "remoteHost" => "149.242.248.XX"
    },
    "1c5a6980-64b3-426b-8d30-20d9513e059f" => {
                  "identifier" => "1c5a6980-64b3-426b-8d30-20d9513e059f",
                   "startDate" => 1586261548607,
                 "connectable" => true,
        "connectionIdentifier" => "532",
                    "username" => "xxx",
                  "remoteHost" => "149.242.248.XX"
    },
    "@version" => "1",
    "e822956d-40b5-4c24-962e-a07a5b0e0ac9" => {
                  "identifier" => "e822956d-40b5-4c24-962e-a07a5b0e0ac9",
                   "startDate" => 1586263078302,
                 "connectable" => true,
        "connectionIdentifier" => "4622",
                    "username" => "xxx",
                  "remoteHost" => "149.242.248.XX"
    },
    "ee461fb0-0932-4b2f-a2d1-d05485920404" => {
                  "identifier" => "ee461fb0-0932-4b2f-a2d1-d05485920404",
                   "startDate" => 1586271711558,
                 "connectable" => true,
        "connectionIdentifier" => "2933",
                    "username" => "xxx",
                  "remoteHost" => "149.242.248.XX"
    },
    "command" => "/opt/get_history.sh",
    "ddfe1895-8ba3-4f91-99d3-cbd10ab90b5a" => {
                  "identifier" => "ddfe1895-8ba3-4f91-99d3-cbd10ab90b5a",
                   "startDate" => 1586270958993,
                 "connectable" => true,
        "connectionIdentifier" => "4453",
                    "username" => "xxx",
                  "remoteHost" => "149.242.248.XX"
    },
    "19b7f430-61c5-481a-9c3a-4923f542d8fc" => {
                  "identifier" => "19b7f430-61c5-481a-9c3a-4923f542d8fc",
                   "startDate" => 1586257524378,
                 "connectable" => true,
        "connectionIdentifier" => "3246",
                    "username" => "xxx",
                  "remoteHost" => "149.242.248.XX"
    },
    "9b2dd1a6-8fde-4f3a-aea7-e9480a3a9d30" => {
                  "identifier" => "9b2dd1a6-8fde-4f3a-aea7-e9480a3a9d30",
                   "startDate" => 1586268862890,
                 "connectable" => true,
        "connectionIdentifier" => "2265",
                    "username" => "xxx",
                  "remoteHost" => "149.242.248.XX"
    },
    "9ad05e55-dd2f-4f63-a5d2-692b0fe96ca0" => {
                  "identifier" => "9ad05e55-dd2f-4f63-a5d2-692b0fe96ca0",
                   "startDate" => 1586268084592,
                 "connectable" => true,
        "connectionIdentifier" => "5405",
                    "username" => "xxx",
                  "remoteHost" => "149.242.248.XX"
    },
    "host" => "66c51c3d15fc",
    "f5bbc6de-32b0-4647-b7f8-337d708884c9" => {
                  "identifier" => "f5bbc6de-32b0-4647-b7f8-337d708884c9",
                   "startDate" => 1586270758153,
                 "connectable" => true,
        "connectionIdentifier" => "3393",
                    "username" => "xxx",
                  "remoteHost" => "149.242.248.XX"
    },
    "0be8f744-d4e3-4ba1-8053-2acb3c8e8ff0" => {
                  "identifier" => "0be8f744-d4e3-4ba1-8053-2acb3c8e8ff0",
                   "startDate" => 1586236423739,
                 "connectable" => true,
        "connectionIdentifier" => "2873",
                    "username" => "xxx",
                  "remoteHost" => "149.242.248.XX"
    },
    "507c213a-9362-4565-8a01-0033edc5b38e" => {
                  "identifier" => "507c213a-9362-4565-8a01-0033edc5b38e",
                   "startDate" => 1586264411463,
                 "connectable" => true,
        "connectionIdentifier" => "3091",
                    "username" => "xxx",
                  "remoteHost" => "149.242.248.XX"
    },
    "df79596d-60bf-41d0-959e-a9f8d9f5c138" => {
                  "identifier" => "df79596d-60bf-41d0-959e-a9f8d9f5c138",
                   "startDate" => 1586271680146,
                 "connectable" => true,
        "connectionIdentifier" => "4051",
                    "username" => "xxx",
                  "remoteHost" => "149.242.248.XX"
    },
    "2cd356a9-5bc9-4c4a-a7ab-f77c9a5a5097" => {
                  "identifier" => "2cd356a9-5bc9-4c4a-a7ab-f77c9a5a5097",
                   "startDate" => 1586271991684,
                 "connectable" => true,
        "connectionIdentifier" => "4864",
                    "username" => "xxx",
                  "remoteHost" => "149.242.248.XX"
    },
    "bf958868-d8e7-4d82-8297-2fdb3e1121f9" => {
                  "identifier" => "bf958868-d8e7-4d82-8297-2fdb3e1121f9",
                   "startDate" => 1586244261862,
                 "connectable" => true,
        "connectionIdentifier" => "1650",
                    "username" => "xxx",
                  "remoteHost" => "149.242.248.XX"
    },
    "9a67ee72-b41a-49ed-95fa-e66c937ca6cb" => {
                  "identifier" => "9a67ee72-b41a-49ed-95fa-e66c937ca6cb",
                   "startDate" => 1586268488567,
                 "connectable" => true,
        "connectionIdentifier" => "4783",
                    "username" => "xxx",
                  "remoteHost" => "149.242.248.XX"
    },
    "bd20687b-ed3f-4aca-8cf1-360c1e54aeb7" => {
                  "identifier" => "bd20687b-ed3f-4aca-8cf1-360c1e54aeb7",
                   "startDate" => 1586263966578,
                 "connectable" => true,
        "connectionIdentifier" => "5744",
                    "username" => "xxx",
                  "remoteHost" => "149.242.248.XX"
    }
}
2

I would use a ruby filter to do that. I haven't tested it, but something like

ruby {
    code => '
        timestamp = event.get("@timestamp")
        version = event.get("@version")
        host = event.get("host")
        command = event.get("command")

        a = []
        event.to_hash.each { |k, v|
            unless [ "@timestamp", "@version", "command", "host" ].include? (k)
                v["@timestamp"] = timestamp
                v["@version"] = version
                v["command"] = command
                v["host"] = host
                a << v
            end
        }
        event.set("stuff", a)
    '
}
split { field => "stuff" }
3

Hi, awesome.

This worked out og the box.

My each loop looks like this now:

event.to_hash.each { |k, v|
unless [ "@timestamp", "@version", "command", "host" ].include? (k)
v["@timestamp"] = timestamp
v["@version"] = version
v["command"] = command
v["host"] = host
a << v
event.remove(k)
end

I wanted to get rid of the original json structure and tried this. And all I have in elastic are connection objects now. Does this seem to make sense or am I making a mistake in thinking here. At least for me it looks right :slight_smile:

Thanks you

4

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.