How to remove document I logstash pipilnie that has no usefull fields

Elastic Stack Logstash
1

Hello,

I use the logstash SNMP plugin to query my device. After that, I do some split and field renaming as below.

input {
    snmp {
        interval => 5
        hosts => [
            {host => "udp:ip/port" community => "secret"}
        ]
        tables => [
            {
               name => "interfaces"
               columns => [
                        "1.3.6.1.2.1.2.1",
                        "1.3.6.1.2.1.2.2.1.1",
                        "1.3.6.1.2.1.2.2.1.3",
                        "1.3.6.1.2.1.2.2.1.4",
                        "1.3.6.1.2.1.2.2.1.5",
                        "1.3.6.1.2.1.2.2.1.6",
                        "1.3.6.1.2.1.2.2.1.7",
                        "1.3.6.1.2.1.2.2.1.8",
                        "1.3.6.1.2.1.2.2.1.9",
                        "1.3.6.1.2.1.2.2.1.10",
                        "1.3.6.1.2.1.2.2.1.11",
                        "1.3.6.1.2.1.2.2.1.12",
                        "1.3.6.1.2.1.2.2.1.13",
                        "1.3.6.1.2.1.2.2.1.14",
                        "1.3.6.1.2.1.2.2.1.15",
                        "1.3.6.1.2.1.2.2.1.16",
                        "1.3.6.1.2.1.2.2.1.17",
                        "1.3.6.1.2.1.2.2.1.18",
                        "1.3.6.1.2.1.2.2.1.19",
                        "1.3.6.1.2.1.2.2.1.20",
                        "1.3.6.1.2.1.31.1.1.1.1",
                        "1.3.6.1.2.1.2.1"
                        ]
            }

        ]
        add_field => { "host" => "%{[@metadata][host_address]}"}
       tags => ["snmp", "interface"]
    }
}

filter{
        split {
                field => "interfaces"
        }


        mutate {
                remove_field => [ "[interfaces]" ]
                # CPU

                # INTERFACE
                rename => { "[interfaces][iso.org.dod.internet.mgmt.mib-2.interfaces.ifTable.ifEntry.ifOutOctets]" => "cisco.device.system.interface.ifOutOctets"}
                rename => { "[interfaces][iso.org.dod.internet.mgmt.mib-2.interfaces.ifTable.ifEntry.ifType]" => "cisco.device.system.interface.ifType"}
                rename => { "[interfaces][iso.org.dod.internet.mgmt.mib-2.interfaces.ifTable.ifEntry.ifSpeed]" => "cisco.device.system.interface.ifSpeed"}
                rename => { "[interfaces][iso.org.dod.internet.mgmt.mib-2.ifMIB.ifMIBObjects.ifXTable.ifXEntry.ifName]" => "cisco.device.system.interface.ifName"}
                rename => { "[interfaces][iso.org.dod.internet.mgmt.mib-2.interfaces.ifTable.ifEntry.ifOutUcastPkts]" => "cisco.device.system.interface.ifOutUcastPkts"}
                rename => { "[interfaces][iso.org.dod.internet.mgmt.mib-2.interfaces.ifTable.ifEntry.ifInNUcastPkts]" => "cisco.device.system.interface.ifInNUcastPkts"}
                rename => { "[interfaces][iso.org.dod.internet.mgmt.mib-2.interfaces.ifTable.ifEntry.ifInUcastPkts]" => "cisco.device.system.interface.ifInUcastPkts"}
                rename => { "[interfaces][iso.org.dod.internet.mgmt.mib-2.interfaces.ifTable.ifEntry.ifOutNUcastPkts]" => "cisco.device.system.interface.ifOutNUcastPkts"}
                rename => { "[interfaces][iso.org.dod.internet.mgmt.mib-2.interfaces.ifTable.ifEntry.ifInDiscards]" => "cisco.device.system.interface.ifInDiscards"}
                rename => { "[interfaces][iso.org.dod.internet.mgmt.mib-2.interfaces.ifTable.ifEntry.ifOutErrors]" => "cisco.device.system.interface.ifOutErrors"}
                rename => { "[interfaces][iso.org.dod.internet.mgmt.mib-2.interfaces.ifTable.ifEntry.ifOperStatus]" => "cisco.device.system.interface.ifOperStatus"}
                rename => { "[interfaces][iso.org.dod.internet.mgmt.mib-2.interfaces.ifTable.ifEntry.ifIndex]" => "cisco.device.system.interface.ifIndex"}
                rename => { "[interfaces][iso.org.dod.internet.mgmt.mib-2.interfaces.ifTable.ifEntry.ifInErrors]" => "cisco.device.system.interface.ifInErrors"}
                rename => { "[interfaces][iso.org.dod.internet.mgmt.mib-2.interfaces.ifTable.ifEntry.ifOutDiscards]" => "cisco.device.system.interface.ifOutDiscards"}
                rename => { "[interfaces][iso.org.dod.internet.mgmt.mib-2.interfaces.ifTable.ifEntry.ifInOctets]" => "cisco.device.system.interface.ifInOctets"}
                rename => { "[interfaces][iso.org.dod.internet.mgmt.mib-2.interfaces.ifTable.ifEntry.ifPhysAddress]" => "cisco.device.system.interface.ifPhysAddress"}
                rename => { "[interfaces][iso.org.dod.internet.mgmt.mib-2.interfaces.ifTable.ifEntry.ifLastChange]" => "cisco.device.system.interface.ifLastChange"}
                rename => { "[interfaces][iso.org.dod.internet.mgmt.mib-2.interfaces.ifTable.ifEntry.ifMtu]" => "cisco.device.system.interface.ifMtu"}

                # HOST
                rename => { "host" => "host.ip"}
                add_field => { "host.hostname" => "Hostname" }
        }
        # Adding status explanation
        if 1 == [cisco.device.system.interface.ifOperStatus] {
          mutate { add_field => {  "cisco.device.system.interface.ifOperStatusText" => "up" }}
        } else if 2 == [cisco.device.system.interface.ifOperStatus] {
          mutate { add_field => {  "cisco.device.system.interface.ifOperStatusText" => "down" }}
        } else if 3 == [cisco.device.system.interface.ifOperStatus] {
          mutate { add_field => {  "cisco.device.system.interface.ifOperStatusText" => "testing" }}
        } else if 4 == [cisco.device.system.interface.ifOperStatus] {
          mutate { add_field => {  "cisco.device.system.interface.ifOperStatusText" => "unknown" }}
        } else if 5 == [cisco.device.system.interface.ifOperStatus] {
          mutate { add_field => {  "cisco.device.system.interface.ifOperStatusText" => "notPresent" }}
        } else if 6 == [cisco.device.system.interface.ifOperStatus] {
          mutate { add_field => {  "cisco.device.system.interface.ifOperStatusText" => "lowerLayerDown" }}
        }

}

output {
# file {
#   path => "/etc/logstash/conf.d/tests/snmp.txt"
# }
 stdout { codec => rubydebug }
}

The output looks like below.

[...]
{
                                          "@version" => "1",
         "cisco.device.system.interface.ifOutOctets" => 0,
    "cisco.device.system.interface.ifOperStatusText" => "down",
                                           "host.ip" => "ip",
      "cisco.device.system.interface.ifOutUcastPkts" => 0,
          "cisco.device.system.interface.ifInOctets" => 0,
          "cisco.device.system.interface.ifInErrors" => 0,
        "cisco.device.system.interface.ifInDiscards" => 0,
       "cisco.device.system.interface.ifOutDiscards" => 0,
      "cisco.device.system.interface.ifInNUcastPkts" => 0,
                                              "tags" => [
        [0] "snmp",
        [1] "interface"
    ],
        "cisco.device.system.interface.ifOperStatus" => 2,
                                     "host.hostname" => "Hostname",
               "cisco.device.system.interface.ifMtu" => 1500,
       "cisco.device.system.interface.ifPhysAddress" => "00:00:00:00:00:00",
              "cisco.device.system.interface.ifType" => 6,
              "cisco.device.system.interface.ifName" => "management",
             "cisco.device.system.interface.ifSpeed" => 100000000,
       "cisco.device.system.interface.ifInUcastPkts" => 128,
     "cisco.device.system.interface.ifOutNUcastPkts" => 0,
        "cisco.device.system.interface.ifLastChange" => 400,
                                        "@timestamp" => 2020-10-25T17:54:17.809Z,
         "cisco.device.system.interface.ifOutErrors" => 0,
             "cisco.device.system.interface.ifIndex" => 7
}
{
    "host.hostname" => "Hostname",
       "@timestamp" => 2020-10-25T17:54:17.809Z,
          "host.ip" => "Ip",
             "tags" => [
        [0] "snmp",
        [1] "interface"
    ],
         "@version" => "1"
}

I want to instruct logstash to delete this last document hance it does not contains any useful information.

How can I do It and where does it come from?

2

I would guess there is an empty entry in the interfaces array that the snmp input emits. I would remove that using ruby before doing the split.

3

So I notice that an empty document is a leftover of a document that at the beginning had an interfaces field. I removed the after a split because they are no longer necessary. How would you delete this document using ruby?

4

In ruby you would call event.cancel to delete an entire event. There is an example here.

5 
 ruby {
                code => '
                        if event.get("interfaces")
                        event.cancel
                    end
                '
        }

This is deleting the whole event. Can I somehow narrow this just to a document containing that field?

6

Yes, you can write arbitrary code to evaluate the fields of the event to see whether or not you want to retain it.

7

Can you pass me some resource to learn from? I have never coded in ruby nor in ruby in logstash.

8

Perhaps you could use a conditional and the drop filter to drop the event. https://www.elastic.co/guide/en/logstash/current/plugins-filters-drop.html

9

In my configuration file, I had this above oid and the additional document was related to this. It was some leftover form testing solution show basically the deleting it and the split function was my solution.

10

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.