Change Nil values to set default value

[rubhamra (heyanB)]

Logstash is dropping fields which has "nil" values, but I don't want those fields to be drop, but at least we can set it to default values if the field is nil, else it has it's original value.

I tried with this code found on the discussion form, but couldn't work.

Logstash Config

    ruby {
        code => "
            hash = event.to_hash
            hash.each do |k,v|
                if v == nil
                    event[k] = 0
                end
            end
        "
}

Logstash OUTPUT

"response" => {
        "body" => {
                "entries" => {
                       "values" => {
                             "ChangeRequestStatusString" => "11",
                                     "ConditionDisplay1" => nil,
                                     "ConditionDisplay2" => nil,
                                     "ConditionDisplay3" => nil,
                                  "SLA Breach Exception" => nil,
                                     "ConditionDisplay4" => nil,
                                     "ConditionDisplay5" => nil,
                                        "Middle Initial" => "B",
                                 "UC Res Result Seconds" => 0,
                                           "AllCIsInTab" => "Yes",
                             "Show For Process_Assignee" => nil,
                              "Infrastructure Change ID" => "CRQ000000600000",
                                               "Site ID" => nil,
                                            "Release_ID" => nil,
                                          "Active Tasks" => nil,
                                           "BTSEnd Time" => nil,
                                        "Permission" => "Infrastructure Master",
                                              "SFA_Date" => "2022-05-10T15:02:02.000+0000",
                                    "Performance Rating" => 3,
                                            "InstanceId" => "IDGAA5V0frw3re3eSQQMWS",
                            "UC Acknowledgment End Date" => nil,
                                        "Override Group" => nil,
                                             "AllowNext" => nil,
                                       "Actual End Date" => "2022-05-14T02:30:00.000+0000",
                                "zTmpRSSOAuthentication" => nil,
                                 "Support Organization2" => "User Service Desk",
                                      "Show For Process" => nil,
                                       "FindApproversOn" => nil,
                                       "Active Approval" => nil,
                                "OLA Res Target Seconds" => 0,
                                   "Return Code_Manager" => nil,
                                          "SRInstanceID" => nil,
                                    "Requested End Date" => nil,
                                       "View Accessible" => nil,
                                  "zTmpIntAutoStartTime" => nil,
                        "OLA Res Business Hour Seconds2" => 0,
                                 "StartStop_EffortLogId" => nil,
                                       "zTmpCheckCHGDPN" => nil,
                              "SLA Resp Target Duration" => nil,
                                     "NewBroadcastCount" => nil,
                                             "ChgImpGrp" => nil,
                                         "AllowApproval" => nil,
                                          "Corporate ID" => "4333333",
                                    "StartStop_Assignee" => nil,
                                              "Work Log" => nil,
                                       "z2AF_Work Log01" => nil,
                                       "z2AF_Work Log02" => nil,
                                        "zD_TADOccurred" => nil,
                                       "z2AF_Work Log03" => nil,

Please help ,
Thank You

[leandrojmp (Leandro Pereira)]

@rubhamra

Logstash is dropping fields which has "nil" values, but I don't want those fields to be drop

Can you provide more context? Logstash does not drop field unless you have a mutate with the remove_field telling it to remove the field.

If the value is nil, it will output as nil.

Also, your code will only work for top level fields, it is a similar issue as your previous question.

You probably could rework that code to do what you want.

[rubhamra (heyanB)]

It's an API Call but most of the fields are empty, but I can see the fields in Logstash output, but it's not writing to the index.

it's my complete logstash config.

input {
	exec {
        id => "change_id"
        command => "curl --location 'http://10.2.2.15:8008/api/jwt/login' --header 'authString: a' --header 'Content-Type: application/x-www-form-urlencoded' --header 'Cookie: citrix_ns_id=r32re23e32e-ricS91uTQpY-O0-gN_oW8cnD3CPqAlumaUiGWWdtOw==kbgUZA==Rh5zXPy7_LpJU9uwdlIIalR10AQ=; citrix_ns_id_.fds.com_%2F_wat=AAAAAAUGsLUhuaPL_YJya8hbcov9D-mU4rM-WA8eWZMN4K_wqh-u4kFLJTYA4X0gjrh4SQaMUKUq8TeB3wpmXYyQmRDmV6Xe5Wh6AFXMnKJbqteWoMOaT7zJbZ3KyUklI5U7ImM=&AAAAAAU4vLkpdx3qGPqZAvuWeJUERgJUaa9fgyzMGd_00ObS9sT7wtP51jDMcZCZKUOkR1KKuIwx20EMB0iTf115ayK5&; NSC_jutn-vbu-sftubqj.dhj.dpn=ffffffff09f61dae45525d5f4f58455e445a4a422978' --data-urlencode 'username=admin' --data-urlencode 'password=4er23re32e32'"
	    interval => 60000000
		#codec => "json"
		type => "restapi"
        
        add_field => {
            "[service][environment]" => "QA"
            "[service][type]" => "System"
            "[service][name]" => "Main"
            "[event][dataset]" => "restapi"
            "[agent][name]" => "logstash"
            "[metricset][type]" => "restapi"
            "[metricset][name]" => "change_mgmt"
        }
    }
}

filter {

  



    mutate {
        rename => {
            "message" => "[@metadata][request][token]"
        }
        capitalize => [ "[host][name]" ]
        #copy => {
        #    "@timestamp" => "[event][start]"
        #}
        add_field => {
            "[event][start]" => "%{+yyyy-MM-dd}T%{+HH:mm:ss.SSS}Z"
        }
    }



    http {
        verb => "GET"
        url => "http://10.2.2.15:8008/api/arsys/v1.0/entry/Infrastructure%20Change/CRQ0000000000"
        headers => {
            "Authorization" => "%{[@metadata][request][token]}"
        }
        request_timeout => 9
        body_format => "json"
        target_body => "[response][body]"
        target_headers => "[response][headers]"
        
           
    }

    split {
        field => "[response][body][entries]"
    }


    ruby {
        init => '
            def doSomething(object, name, event)
#puts "Working on #{name}"
#Removed "if object" test since we need to process null valued fields
                    if object.kind_of?(Hash) and object != {}
                        object.each { |k, v| doSomething(v, "#{name}[#{k}]", event) }
                    elsif object.kind_of?(Array) and object != []
                        object.each_index { |i| doSomething(object[i], "#{name}[#{i}]", event) }
                    else
                        lastElement = name.gsub(/^.*\[/, "").gsub(/\]$/, "")
                        if lastElement =~ /^n1D/ or lastElement =~ /^n1G/
                            event.remove(name)
                        end
                end
            end
        '
        code => '
            event.to_hash.each { |k, v|
                doSomething(v, "[#{k}]", event)
            }
        '
}


ruby {
        code => "
            hash = event.to_hash
            hash.each do |k,v|
                if v == nil
                    event[k] = 0
                end
            end
        "
}

    

    mutate {
        rename => { "[response][body][entries][_links][self][0]" => "[response][body][entries][values][url]" }
    }

    mutate { remove_field => [ "process" , "[response][headers]", "@metadata", "[response][body][_links]","[[response][body][entries][_links]]", "[response][body][entries][values][Request ID]" ] }

    mutate {
        add_field => {
            "[event][end]" => "%{+yyyy-MM-dd}T%{+HH:mm:ss.SSS}Z"
        }
    }
}

output {		
   elasticsearch {
        id => "restapi-main"
        hosts => ["https://localhost:9200"]
        #ssl => true
        #cacert => "/usr/share/logstash/certs/http_ca.crt"
        cacert => '/etc/logstash/config/certs/ca.crt'
        user => "admin"
        password => "rferfewrw"
	    index => "restapi-data-%{+YYYY-MM-dd}"
	    document_id => "%{[response][body][entries][values][Request_ID]}"
        action => "index"
	}
    stdout {
        codec => rubydebug {
            metadata => true
        }
    }
}

I don't want Logstash to ignore those empty fields, I tried with that ruby code, cannot be able to work it.

[rubhamra (heyanB)]

I tried to understand ruby gsub to make have better understanding. and tried to remove null values but still it's not working properly. it's removing some of the fields but not properly.

ruby {
        init => '
            def doSomething(object, name, event)
#puts "Working on #{name}"
#Removed "if object" test since we need to process null valued fields
                    if object.kind_of?(Hash) and object != {}
                        object.each { |k, v| doSomething(v, "#{name}[#{k}]", event) }
                    elsif object.kind_of?(Array) and object != []
                        object.each_index { |i| doSomething(object[i], "#{name}[#{i}]", event) }
                    else
                        lastElement = name.gsub(/^.*\[/, "").gsub(/\]$/, "")
                        if lastElement =~ /^z1D/ or lastElement =~ /^.*=> nil/
                            event.remove(name)
                            event.set(lastElement, "NA")
                            
                        end
                end
            end
        '
        code => '
            event.to_hash.each { |k, v|
                doSomething(v, "[#{k}]", event)
            }
        '
}

[WARN ] 2023-03-24 19:20:32.624 [[main]>worker1] elasticsearch - Could not index event to {"type"=>"illegal_argument_exception", "reason"=>"Limit of total fields [1000] has been exceeded while adding new fields [97]"}}}}
{
                                    {
                                    "n1D Urgency" => "NA",
                              "n1D_ChgCoord_Site" => "NA",
                    "n1D Copy Change Environment" => "NA",
                        "n1D Approval Phase Name" => "NA",
                             "n1D_SequenceChange" => "NA",
                               "n1D CHG Rules ID" => "NA",
                              "n1D Report Locale" => "NA",
                        "n1D_RelationAction_Copy" => "NA",
                        "n1D Lock Down Requester" => "NA",
                    "n1D Impacts Approval Status" => "NA",
                                   "n1D_HelpText" => "NA",
                         "n1D CostClassification" => "NA",
                                 "n1D_ChangeFlag" => "NA",
                       "n1D_Broadcast Auto Popup" => "NA",
                                    "n1D_DPNSkip" => "NA",
                                 "n1D Task Notes" => "NA",
                     "n1D Relationship Selection" => "NA",
                              "n1D_RefreshAprTab" => "NA",
                                       "n1D Site" => "NA",
                                           "tags" => [
        
    ],
                               "n1D Notify CA/CM" => "NA",
                            "n1D_Task Event Code" => "NA",
                                    "n1D FINCost" => "NA",
                       "n1D CAB Manager Login ID" => "NA",
                                "n1D_CI_Priority" => "NA",
                                     "n1D Region" => "NA",
                                 "n1D_DateTime01" => "NA",
                                 "n1D_DateTime02" => "NA",
                                  "n1D_ClassName" => "NA",
                                 "n1D Site Group" => "NA",
                              "n1D_BTSWorkdayTag" => "NA",
                              "n1D Support Staff" => "NA",
                               "n1D CostRecCount" => "NA",
                              "n1D_TmpRejectFlag" => "NA",
                            "n1D_Approval_Status" => "NA",
                             "n1D Auto Generate?" => "NA",
                       "n1D_Time_to_Plan_Seconds" => "NA",
                    "n1D_Time_to_Approve_Seconds" => "NA",
                       "n1D_InclusionSpportGroup" => "NA",
               "n1D_IsRequestedForInfoSetByWFlow" => "NA",
                     "n1D_QuestionDialogViewName" => "NA",
                               "n1D_VendorAccess" => "NA",
                                    "n1D_TmpGUID" => "NA",
                      "n1D Predefined Process ID" => "NA",
                    "n1D_AbydosAddSRD_SchemaName" => "NA",
              "n1D Copy Infrastructure Change ID" => "NA",
                    "n1D Prev Reported to Vendor" => "NA",
                        "n1D Permission Group ID" => "NA",

Please suggest