Extract some words in a keyword field

Elastic Stack Logstash
1

Hi,

I have a field (keyword) with the following value (this value can be in different order and different values):

{"sessRules":{"SetLTEQoS":{"authSessAmbr":{"uplink":"4200 Mbps","downlink":"4200 Mbps"},"authDefQos":{"arp":{"priorityLevel":1,"preemptCap":"MAY_PREEMPT","preemptVuln":"NOT_PREEMPTABLE"},"5qi":8},"sessRuleId":"SetLTEQoS"}},"pccRules":{"Unica_Whatsapp_Messaging_Free_Rd":{"pccRuleId":"Unica_Whatsapp_Messaging_Free_Rd"},"5G_Openet_52":{"pccRuleId":"5G_Openet_52"},"Skype_Teams_Group_Rd":{"pccRuleId":"Skype_Teams_Group_Rd"},"Waze_Free_Rd":{"pccRuleId":"Waze_Free_Rd"},"Twitter_Free_Rd":{"pccRuleId":"Twitter_Free_Rd"},"Unica_Facebook_SP":{"pccRuleId":"Unica_Facebook_SP"},"Unica_Whatsapp_VoIP_P2P_Rd":{"pccRuleId":"Unica_Whatsapp_VoIP_P2P_Rd"},"Webex_Group_Rd":{"pccRuleId":"Webex_Group_Rd"},"Instagram_Free_Rd":{"pccRuleId":"Instagram_Free_Rd"}},"online":true,"policyCtrlReqTriggers":["RES_MO_RE","UE_IP_CH","PS_DA_OFF","DEF_QOS_CH","SE_AMBR_CH","NUM_OF_PACKET_FILTER","RAT_TY_CH"],"suppFeat":"2"}

I need to create a new field that contains the values after "pccRuleId". bellow is an example of value i need in the new field.

Unica_Whatsapp_Messaging_Free_Rd,5G_Openet_52,Skype_Teams_Group_Rd,Waze_Free_Rd,Twitter_Free_Rd,Unica_Facebook_SP,Unica_Whatsapp_VoIP_P2P_Rd,Webex_Group_Rd,Instagram_Free_Rd

How can i do it in logstash's filter ? (remebering the values after pccRuleId word can change)

2

I would do that using a ruby filter. It is unclear whether you want an array or a string for all the rules, so choose either a or b from this

    ruby {
        code => '
            begin
                a = []
                b = ""
                event.get("pccRules").each { |k, v|
                    a << v["pccRuleId"]
                    b += v["pccRuleId"] + ","
                }
                if b != ""
                    event.set("rules1", b.chomp())
                end
                if a != []
                    event.set("rules2", a)
                end
            rescue
            end
        '
    }
3

Hello Baden,

I've tried it but it doesn't work

My output:

> {
>                   "ip_src" => "172.118.6.119",
>          "http2_data_data" => "{\"sessRules\":{\"SetLTEQoS\":{\"authSessAmbr\":{\"uplink\":\"4200 Mbps\",\"downlink\":\"4200 Mbps\"},\"authDefQos\":{\"arp\":{\"priorityLevel\":1,\"preemptCap\":\"MAY_PREEMPT\",\"preemptVuln\":\"NOT_PREEMPTABLE\"},\"5qi\":8},\"sessRuleId\":\"SetLTEQoS\"}},\"pccRules\":{\"Netflix_Free_Rd\":{\"pccRuleId\":\"Netflix_Free_Rd\"},\"Youtube_Free_Rd\":{\"pccRuleId\":\"Youtube_Free_Rd\"},\"Twitter_Free_Rd\":{\"pccRuleId\":\"Twitter_Free_Rd\"},\"Unica_Facebook_SP\":{\"pccRuleId\":\"Unica_Facebook_SP\"},\"TikTok_Free_Rd\":{\"pccRuleId\":\"TikTok_Free_Rd\"},\"Disney_Free_Rd\":{\"pccRuleId\":\"Disney_Free_Rd\"},\"HBO_Free_Rd\":{\"pccRuleId\":\"HBO_Free_Rd\"},\"5G_Openet_217\":{\"pccRuleId\":\"5G_Openet_217\"},\"Waze_Free_Rd2\":{\"pccRuleId\":\"Waze_Free_Rd2\"},\"Claro_Video_Free_Rd\":{\"pccRuleId\":\"Claro_Video_Free_Rd\"},\"Net_Now_Rd\":{\"pccRuleId\":\"Net_Now_Rd\"},\"Discovery_Free_Rd\":{\"pccRuleId\":\"Discovery_Free_Rd\"},\"GloboP_Free_Rd\":{\"pccRuleId\":\"GloboP_Free_Rd\"},\"Instagram_Free_Rd\":{\"pccRuleId\":\"Instagram_Free_Rd\"},\"Plt_Free_Rd\":{\"pccRuleId\":\"Plt_Free_Rd\"}},\"online\":true,\"policyCtrlReqTriggers\":[\"RES_MO_RE\",\"UE_IP_CH\",\"PS_DA_OFF\",\"DEF_QOS_CH\",\"SE_AMBR_CH\",\"NUM_OF_PACKET_FILTER\",\"RAT_TY_CH\"],\"suppFeat\":\"2\"}",
>                     "host" => "gateway-proxy-7749547bcb-l6ghn",
>                     "tags" => [
>         [0] "openet_pcf5gsa_http2"
>     ],
>         "frame_time_epoch" => "1693598747.756821000",
>                 "@version" => "1",
>                  "message" => "Sep  1, 2023 20:05:47.756821000 UTC|1693598747.756821000|172.118.6.119|8080|100.64.0.161|25044|||50501,50501||7b227365737352756c6573223a7b225365744c5445516f53223a7b226175746853657373416d6272223a7b2275706c696e6b223a2234323030204d627073222c22646f776e6c696e6b223a2234323030204d627073227d2c2261757468446566516f73223a7b22617270223a7b227072696f726974794c6576656c223a312c22707265656d7074436170223a224d41595f505245454d5054222c22707265656d707456756c6e223a224e4f545f505245454d505441424c45227d2c22357169223a387d2c227365737352756c654964223a225365744c5445516f53227d7d2c2270636352756c6573223a7b224e6574666c69785f467265655f5264223a7b2270636352756c654964223a224e6574666c69785f467265655f5264227d2c22596f75747562655f467265655f5264223a7b2270636352756c654964223a22596f75747562655f467265655f5264227d2c22547769747465725f467265655f5264223a7b2270636352756c654964223a22547769747465725f467265655f5264227d2c22556e6963615f46616365626f6f6b5f5350223a7b2270636352756c654964223a22556e6963615f46616365626f6f6b5f5350227d2c2254696b546f6b5f467265655f5264223a7b2270636352756c654964223a2254696b546f6b5f467265655f5264227d2c224469736e65795f467265655f5264223a7b2270636352756c654964223a224469736e65795f467265655f5264227d2c2248424f5f467265655f5264223a7b2270636352756c654964223a2248424f5f467265655f5264227d2c2235475f4f70656e65745f323137223a7b2270636352756c654964223a2235475f4f70656e65745f323137227d2c2257617a655f467265655f526432223a7b2270636352756c654964223a2257617a655f467265655f526432227d2c22436c61726f5f566964656f5f467265655f5264223a7b2270636352756c654964223a22436c61726f5f566964656f5f467265655f5264227d2c224e65745f4e6f775f5264223a7b2270636352756c654964223a224e65745f4e6f775f5264227d2c22446973636f766572795f467265655f5264223a7b2270636352756c654964223a22446973636f766572795f467265655f5264227d2c22476c6f626f505f467265655f5264223a7b2270636352756c654964223a22476c6f626f505f467265655f5264227d2c22496e7374616772616d5f467265655f5264223a7b2270636352756c654964223a22496e7374616772616d5f467265655f5264227d2c22506c745f467265655f5264223a7b2270636352756c654964223a22506c745f467265655f5264227d7d2c226f6e6c696e65223a747275652c22706f6c6963794374726c5265715472696767657273223a5b225245535f4d4f5f5245222c2255455f49505f4348222c2250535f44415f4f4646222c224445465f514f535f4348222c2253455f414d42525f4348222c224e554d5f4f465f5041434b45545f46494c544552222c225241545f54595f4348225d2c227375707046656174223a2232227d",
>                     "path" => "/test/trafego_http2/processado/gateway-proxy-7749547bcb-l6ghn_01092023-200501.csv",
>           "http2_streamid" => "50501,50501",
>                   "ip_dst" => "100.64.0.161",
>              "tcp_dstport" => "25044",
>               "frame_time" => "Sep  1, 2023 20:05:47.756821000 UTC",
>              "tcp_srcport" => "8080",
>     "tcp_analysis_ack_rtt" => nil,
>     "http2_headers_status" => nil,
>       "http2_headers_path" => nil,
>               "@timestamp" => 2023-09-01T20:05:47.756Z
> }

Here my logstash's filter:

filter {
    csv {
        separator => "|"
        columns => [ "frame_time", "frame_time_epoch", "ip_src", "tcp_srcport", "ip_dst", "tcp_dstport", "tcp_analysis_ack_rtt", "http2_headers_path", "http2_streamid", "http2_headers_status", "http2_data_data" ]
    }

    if [frame_time_epoch] {
        date {
            match => [ "frame_time_epoch","UNIX" ]
            target => "@timestamp"
            timezone => "America/Sao_Paulo"
        }
    }


    if [http2_data_data] {
        mutate {
                split => { "http2_data_data" => "," }
        }
        ruby {
            code => '
                    teste = event.get("[http2_data_data]")
                    teste.each_with_index { | item, index |
                        event.set("[http2_data_data][#{index}]", item.split.pack("H*"))
                    }
            '
        }
        mutate { join => ["http2_data_data", ","] }
    }


    ruby {
        code => '
            begin
                a = []
                b = ""
                event.get("http2_data_data").each { |k, v|
                    a << v["pccRuleId"]
                    b += v["pccRuleId"] + ","
                }
                if b != ""
                    event.set("rules1", b.chomp())
                end
                if a != []
                    event.set("rules2", a)
                end
            rescue
            end
        '
    }

}

i am doing something wrong

4

You need a json filter to parse http2_data_data.

5

It worked perfectly !! Thanks Badger !!

    json {
       source => "http2_data_data"
       target => "http2_data_data_json"
    }

    ruby {
     code => '
        begin
            a = []
            b = ""
            event.get("[http2_data_data_json][pccRules]").each { |k, v|
                a << v["pccRuleId"]
                b += v["pccRuleId"] + ","
            }
            if b != ""
                event.set("rules1", b.chomp())
            end
            if a != []
                event.set("rules2", a)
            end
        rescue
        end
    '
  }
6

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.