Remove Specific Field matching pattern

Elastic Stack Logstash
1

Hello I am trying to remove specific fields in logstash before it goes to elasticssearch, I tried below config.
with drop option.

if  "[response][body][entries][values]" == '^n1D.*' {
        drop { }
      }

I have also tried with Ruby filter but nothing works.

ruby {
    code => "
    event.to_hash.keys.each { |k|
    if k.start_with?('n1D')
      event.remove(k)
    end
    }
   "
   }

OUTPUT

"response" => {
        "body" => {
            "entries" => {
                "values" => {
                    "n1D_lastCheck" => nil,
                    "n1D ModifiedCheck" => nil
                }
            }
        }
    },

Fields are still showing in logs.
Please Help.

2 
if  "[response][body][entries][values]" == '^n1D.*' { drop { }  }

This will not work because it is checking if the field value is exactly equal to that string, it is not a regexp match (which would be =~ instead of ==) and it is testing the field value, not the field name.

ruby {
    code => "
        event.to_hash.keys.each { |k|
            if k.start_with?('n1D')
                event.remove(k)
            end
        }
    "
}

This does not work because it only tests the top-level fields (e.g [response] in your example). If you just need to do this for that one field you could try

ruby {
    code => '
        v = event.get("[response][body][entries][values]")
        if v.is_a? Hash
            v.to_hash.keys.each { |k|
                if k.start_with?("n1D")
                    event.remove(k)
                end
            }
        end
   '
}

If you need to recursively process all fields of an event then see this thread.

3

Thanks for replying. I tried your ruby code. but seems like I am doing something wrong.

Ruby Script

def register(params)
    @field = params['keys']
end

def removeKeys(object, name, keys, event)
#puts "removeKeys called for #{name}"
    if object
        if object.kind_of?(Hash) and object != {}
            object.each { |k, v| removeKeys(v, "#{name}[#{k}]", keys, event) }
        elsif object.kind_of?(Array) and object != []
            object.each_index { |i|
                removeKeys(object[i], "#{name}[#{i}]", keys, event)
            }
        else
            lastElement = name.gsub(/^.*\[/, "").gsub(/\]$/, "")
            if keys.include? lastElement
#puts "removing #{name}"
            event.remove(name)
            end
        end
    end
end

def filter(event)
    event.to_hash.each { |k, v|
            removeKeys(v, "[#{k}]", @field, event)
    }
    [event]
end

logstash config

http {
        verb => "GET"
        url => "http://10.2.2.15:8008/api/neow/v1.0/entry/Infra%20Change/CRQ00000000001"
        headers => {
            "Authorization" => "%{[@metadata][request][token]}"
        }
        request_timeout => 9
        body_format => "json"
        target_body => "[response][body]"
        target_headers => "[response][headers]"
        
           
    }

    split {
        field => "[response][body][entries]"
    }


    ruby {
        path => "/etc/logstash/ruby-script/removeKeys.rb"
        script_params => { keys => "n1D" }
    }





{
           "agent" => {
        "name" => "logstash"
    },
      "@timestamp" => 2023-03-22T18:32:29.946004276Z,
         "service" => {
        "environment" => "QA",
               "name" => "main",
               "type" => "change system"
    },
        "response" => {
        "body" => {
             "entries" => {
                   "values" => {
                                          "n1D_ChgCoord_Site" => nil,
                                "n1D Copy Change Environment" => nil,
                                    "n1D Approval Phase Name" => nil,
                                  "ChangeRequestStatusString" => "11",
                                          "n1D Report Locale" => nil,
                                    "n1D_RelationAction_Copy" => nil,
                                       "SLA Breach Exception" => nil,
                                                "AllCIsInTab" => "Yes",
                                      "UC Res Result Seconds" => 0,
                                  "Show For Process_Assignee" => nil,
                                   "Change ID" => "CRQ000000600004",
                                                    "Site ID" => nil,
                                               "Active Tasks" => nil,
                                               "n1D_HelpText" => nil,
                                                "BTSEnd Time" => nil,
                                             "n1d_Permission" => "Infrastructure Change Master",
                                             "Override Group" => nil,
                                                  "AllowNext" => nil,
                                 "n1D Relationship Selection" => nil,
                                     "zTmpRSSOAuthentication" => nil,
                                      "Support Organization2" => "User Service Desk",
                                          "n1D_RefreshAprTab" => nil,
                                                   "n1D Site" => nil,
                                           "Show For Process" => nil,
                                           "n1D Notify CA/CM" => nil,
                                        "n1D_Task Event Code" => nil,
                                            "Active Approval" => nil,
                                                "n1D FINCost" => nil,
                                             "n1D_DateTime01" => nil,
                                        "Return Code_Manager" => nil,
                                             "n1D_DateTime02" => nil,
                                          "n1D_BTSWorkdayTag" => nil,
                                          "n1D_TmpRejectFlag" => nil,
                                        "n1D_Approval_Status" => nil,
                                   "n1D_Time_to_Plan_Seconds" => nil,
                             "OLA Res Business Hour Seconds2" => 0,
                                "n1D_Time_to_Approve_Seconds" => nil,
                           "n1D_IsRequestedForInfoSetByWFlow" => nil,
                                 "n1D_QuestionDialogViewName" => nil,
                                           "n1D_VendorAccess" => nil,
                                            "zTmpCheckCHGDPN" => nil,
                                "n1D_AbydosAddSRD_SchemaName" => nil,
                                          "NewBroadcastCount" => nil,
                                   "SLA Resp Target Duration" => nil,
                                "n1D Prev Reported to Vendor" => nil,
                                    "n1D Permission Group ID" => nil,
                                  "n1D Update CMDB Assc Flag" => nil,
                                     "n1D_QuestionsPageCount" => nil,
                                           "n1D_TypeSelector" => nil,

Thanks

4

You could try

    ruby {
        init => '
            def doSomething(object, name, event)
#puts "Working on #{name}"
#Removed "if object" test since we need to process null valued fields
                    if object.kind_of?(Hash) and object != {}
                        object.each { |k, v| doSomething(v, "#{name}[#{k}]", event) }
                    elsif object.kind_of?(Array) and object != []
                        object.each_index { |i| doSomething(object[i], "#{name}[#{i}]", event) }
                    else
                        lastElement = name.gsub(/^.*\[/, "").gsub(/\]$/, "")
                        if lastElement =~ /^n1D/
                            event.remove(name)
                        end
                end
            end
        '
        code => '
            event.to_hash.each { |k, v|
                doSomething(v, "[#{k}]", event)
            }
        '
    }
5

Awesome, it really works,

Salute.

6

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.