Delete the last field in an array on Logstash field

Elastic Stack Logstash
1

Hi,
Here is my logstash configuration file :

 input {
    file {
            path           => "/usr/share/logstash/logs"
            codec           => "cef"
            start_position => "beginning"
            sincedb_path => "/dev/null"
    }
}

 filter {
    mutate {
            split => { "deviceCustomString3" => "\r\n" }
    }
 }

output {
stdout{}
}

And an log extract :

{
     "deviceCustomString4Label" => "attack_type",
     "deviceCustomString2Label" => "http_class_name",
          "deviceCustomString4" => "Buffer Overflow",
                   "sourcePort" => "39651",
                   "requestUrl" => "/api/v1/auth/login",
                "sourceAddress" => "x.x.x.x",
          "deviceCustomString2" => "/Common/xxxxx",
                         "name" => "Illegal URL length",
            "devicePostReferer" => "Referer: https://xxxxx",
                     "@version" => "1",
                         "path" => "/usr/share/logstash/logs",
          "devicePostUserAgent" => "Accept: application/json, text/plain, */*",
          "deviceCustomNumber1" => "201",
     "deviceCustomString5Label" => "x_forwarded_for_header_value",
                "deviceVersion" => "x.x.x",
                "deviceProduct" => "xxx",
                   "cefVersion" => "xxx:xxx:0",
                "deviceAddress" => "x.x.x.x",
          "deviceCustomString1" => "/Common/TEMPLATE",
          "deviceCustomString6" => "FR",
     "deviceCustomIPv6Address2" => "",
"deviceCustomIPv6Address4Label" => "ip_address_intelligence",
            "deviceReceiptTime" => "Nov 07 2018 03:51:11",
               "sourceUserName" => "N/A",
          "deviceCustomNumber2" => "3",
           "destinationAddress" => "x.x.x.x",
     "deviceCustomString6Label" => "geo_location",
            "deviceCustomDate1" => "Nov 06 2018 16:58:52",
"deviceCustomIPv6Address3Label" => "destination_address",
     "deviceCustomIPv6Address4" => "N/A",
     "deviceCustomNumber2Label" => "violation_rating",
           "deviceEventClassId" => "Illegal URL length",
                 "sourceUserId" => "7f4f66179a45f7ca",
                         "host" => "xxxxx",
                "requestMethod" => "POST",
                   "@timestamp" => 2018-11-07T15:23:12.590Z,
          "deviceCustomNumber3" => "0",
               "devicePostHost" => "User-Agent: Mozilla/5.0 (Android 7.0; Mobile; rv:62.0) Gecko/62.0 Firefox/62.0",
     "deviceCustomNumber1Label" => "response_code",
                 "deviceAction" => "alerted",
             "deviceExternalId" => "2",
     "deviceCustomIPv6Address1" => "",
     "deviceCustomString1Label" => "policy_name",
       "deviceCustomDate1Label" => "policy_apply_date",
                      "message" => "N/A",
                         "type" => "xxx",
     "deviceCustomString3Label" => "full_request",
"deviceCustomIPv6Address1Label" => "device_address",
                     "severity" => "4",
          "applicationProtocol" => "HTTPS",
          "deviceCustomString3" => [
    [ 0] "POST /api/v1/auth/login HTTP/1.1",
    [ 1] "Host: xxxx",
    [ 2] "User-Agent: Mozilla/5.0 (Android 7.0; Mobile; rv:62.0) Gecko/62.0 Firefox/62.0",
    [ 3] "Accept: application/json, text/plain, */*",
    [ 4] "Accept-Language: fr-FR,fr;q=0.8,en-US;q=0.5,en;q=0.3",
    [ 5] "Accept-Encoding: gzip, deflate, br",
    [ 6] "Content-Type: text/plain",
    [ 7] "Referer: xxxxx",
    [ 8] "Content-Length: 61",
    [ 9] "Cookie: xxxxxxxxx,
    [10] "DNT: 1",
    [11] "Connection: keep-alive",
    [12] "X-Forwarded-For: x.x.x.x",
    [13] "",
    [14] "{\"login\":\"xxxx \",\"password\":\"xxxx\"}"
],

For the field "deviceCustomString3" there are many fields (0 to 14), but the length can change, and I would like to add a filter to delete just the last field (the login and password one).

I think in ruby it is possible, but i'm a bit newbie a this language.
Anyone can help me ?

2

Have you tried using the mutate filter with the gsub function? It lets you replace text in a field that matches a regex. I'm not the greatest with regex but I image something like /\[14\].*\n/ would remove entry 14.

3

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.