SNMP Polling Very slow boot up time

I'm looking at using Logstash to poll several hosts for SNMP data migrating from an old python
script in favor of ELK.

The pipeline is really incredibly simple. read data, convert to message format (ruby), send to google pubsub.

The message processing time is great but the boot up time before logstash is able to process data is around 20+ minutes. I was wondering if that's expected or if I'm doing something wrong. I have about 200 hosts i'm monitoring, with the config broken down into a config file for each host.

Example below:

input {

    
    snmp {
        # query interval
        interval => 30
        tables => [
            
            { "name" => "oid.TIMETRA-SAP-MIB::sapBaseStatsEgressQchipDroppedInProfOctets" "columns" => [ "1.3.6.1.4.1.6527.3.1.2.4.3.6.1.16" ] } ,
            
            { "name" => "oid.TIMETRA-SAP-MIB::sapBaseStatsEgressQchipDroppedInProfPackets" "columns" => [ "1.3.6.1.4.1.6527.3.1.2.4.3.6.1.15" ] } ,
            
            { "name" => "oid.TIMETRA-SAP-MIB::sapBaseStatsEgressQchipDroppedOutProfOctets" "columns" => [ "1.3.6.1.4.1.6527.3.1.2.4.3.6.1.18" ] } ,
            
            { "name" => "oid.TIMETRA-SAP-MIB::sapBaseStatsEgressQchipDroppedOutProfPackets" "columns" => [ "1.3.6.1.4.1.6527.3.1.2.4.3.6.1.17" ] } ,
            
            { "name" => "oid.TIMETRA-SAP-MIB::sapBaseStatsEgressQchipForwardedInProfOctets" "columns" => [ "1.3.6.1.4.1.6527.3.1.2.4.3.6.1.20" ] } ,
            
            { "name" => "oid.TIMETRA-SAP-MIB::sapBaseStatsEgressQchipForwardedInProfPackets" "columns" => [ "1.3.6.1.4.1.6527.3.1.2.4.3.6.1.19" ] } ,
            
            { "name" => "oid.TIMETRA-SAP-MIB::sapBaseStatsEgressQchipForwardedOutProfOctets" "columns" => [ "1.3.6.1.4.1.6527.3.1.2.4.3.6.1.22" ] } ,
            
            { "name" => "oid.TIMETRA-SAP-MIB::sapBaseStatsEgressQchipForwardedOutProfPackets" "columns" => [ "1.3.6.1.4.1.6527.3.1.2.4.3.6.1.21" ] } ,
            
            { "name" => "oid.TIMETRA-SAP-MIB::sapBaseStatsIngressQchipDroppedHiPrioOctets" "columns" => [ "1.3.6.1.4.1.6527.3.1.2.4.3.6.1.8" ] } ,
            
            { "name" => "oid.TIMETRA-SAP-MIB::sapBaseStatsIngressQchipDroppedHiPrioPackets" "columns" => [ "1.3.6.1.4.1.6527.3.1.2.4.3.6.1.7" ] } ,
            
            { "name" => "oid.TIMETRA-SAP-MIB::sapBaseStatsIngressQchipDroppedLoPrioOctets" "columns" => [ "1.3.6.1.4.1.6527.3.1.2.4.3.6.1.10" ] } ,
            
            { "name" => "oid.TIMETRA-SAP-MIB::sapBaseStatsIngressQchipDroppedLoPrioPackets" "columns" => [ "1.3.6.1.4.1.6527.3.1.2.4.3.6.1.9" ] } ,
            
            { "name" => "oid.TIMETRA-SAP-MIB::sapBaseStatsIngressQchipForwardedInProfOctets" "columns" => [ "1.3.6.1.4.1.6527.3.1.2.4.3.6.1.12" ] } ,
            
            { "name" => "oid.TIMETRA-SAP-MIB::sapBaseStatsIngressQchipForwardedInProfPackets" "columns" => [ "1.3.6.1.4.1.6527.3.1.2.4.3.6.1.11" ] } ,
            
            { "name" => "oid.TIMETRA-SAP-MIB::sapBaseStatsIngressQchipForwardedOutProfOctets" "columns" => [ "1.3.6.1.4.1.6527.3.1.2.4.3.6.1.14" ] } ,
            
            { "name" => "oid.TIMETRA-SAP-MIB::sapBaseStatsIngressQchipForwardedOutProfPackets" "columns" => [ "1.3.6.1.4.1.6527.3.1.2.4.3.6.1.13" ] } ,
            
            { "name" => "oid.IF-MIB::ifHCInOctets" "columns" => [ "1.3.6.1.2.1.31.1.1.1.6" ] } ,
            
            { "name" => "oid.IF-MIB::ifHCOutOctets" "columns" => [ "1.3.6.1.2.1.31.1.1.1.10" ] } 
            
        ]

        # List of hosts to poll
        hosts => [{host => "udp:example-host/161" community => "public" version => "2c"  retries => 2  timeout => 1000 } ]
        add_field => {
            "[@metadata][projectId]" => 1
            "[@metadata][collectionId]" => 1
        }
    }
## the snmp{} section would repeat for additional intervals.  This particular host has 30 s interval, 300s, 600s and 3600s

I can group the configs by grouping of oIds and send an array of hosts instead. Is there an optimal way of configuring logstash where the bootup time isn't so long?

Also any known limits beyond physical memory/cpu ?

Am I reading this right? You have 4 snmp inputs for a single host (so that you can use different intervals), and 200 hosts, so a total of 800 snmp inputs. And the startup time is around 1200 seconds. I believe each input runs in its own thread so it would probably improve if each input could process more than one host.

Thanks for your answer @Badger. It took some time to adjust the configuration but now I'm getting an expected error when I have more then 1 host setup.

logstash_1  | [2020-08-18T16:33:31,303][ERROR][logstash.inputs.snmp     ][main][b3a79bf1dba072a1da231cabed21723413181cbdac22fdf837bfbac87468244f] error invoking table operation on OID: oid.ALPHA-RESOURCE-MIB::componentListConfiguredName.36, ignoring {:exception=>#<LogStash::SnmpClientError: error sending snmp table request to target : >, :backtrace=>["/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-snmp-1.2.2/lib/logstash/inputs/snmp/base_client.rb:119:in `block in table'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-snmp-1.2.2/lib/logstash/inputs/snmp/base_client.rb:114:in `table'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-snmp-1.2.2/lib/logstash/inputs/snmp.rb:189:in `block in run'", "org/jruby/RubyArray.java:1809:in `each'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-snmp-1.2.2/lib/logstash/inputs/snmp.rb:187:in `block in run'", "org/jruby/RubyArray.java:1809:in `each'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-snmp-1.2.2/lib/logstash/inputs/snmp.rb:167:in `run'", "/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:345:in `inputworker'", "/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:336:in `block in start_input'"]}

This is my configuration file with some strings/hosts changed.

input {


    snmp {
        interval => 30

        tables => [

            { "name" => "oid.ALPHA-RESOURCE-MIB::componentListConfiguredName.34" "columns" => [ "1.3.6.1.4.1.7309.5.2.1.2.1.3.34" ] } ,

            { "name" => "oid.ALPHA-RESOURCE-MIB::componentListConfiguredName.35" "columns" => [ "1.3.6.1.4.1.7309.5.2.1.2.1.3.35" ] } ,

            { "name" => "oid.ALPHA-RESOURCE-MIB::componentListConfiguredName.36" "columns" => [ "1.3.6.1.4.1.7309.5.2.1.2.1.3.36" ] } ,

            { "name" => "oid.ALPHA-RESOURCE-MIB::componentListSystemPointer.35" "columns" => [ "1.3.6.1.4.1.7309.5.2.1.2.1.7.35" ] } ,

            { "name" => "oid.ALPHA-RESOURCE-MIB::componentListSystemPointer.36" "columns" => [ "1.3.6.1.4.1.7309.5.2.1.2.1.7.36" ] }

        ]

        # List of hosts to poll
        hosts => [ 
              {host => "udp:Hostname1/161" community => "public" version => "2c"  retries => 2  timeout => 1000 }, 
              {host => "udp:hostname2/161" community => "public" version => "2c"  retries => 2  timeout => 1000 } 
        ]

        add_field => {
            "[@metadata][projectId]" => 1
            "[@metadata][collectionId]" => 1
        }
    }

}

On the other hand this is a working configuration:

input {


    snmp {
        interval => 600

        tables => [

            { "name" => "oid.ALPHA-RESOURCE-MIB::componentListConfiguredName.34" "columns" => [ "1.3.6.1.4.1.7309.5.2.1.2.1.3.34" ] } ,

            { "name" => "oid.ALPHA-RESOURCE-MIB::componentListConfiguredName.35" "columns" => [ "1.3.6.1.4.1.7309.5.2.1.2.1.3.35" ] } ,

            { "name" => "oid.ALPHA-RESOURCE-MIB::componentListConfiguredName.36" "columns" => [ "1.3.6.1.4.1.7309.5.2.1.2.1.3.36" ] } ,

            { "name" => "oid.ALPHA-RESOURCE-MIB::componentListSystemPointer.35" "columns" => [ "1.3.6.1.4.1.7309.5.2.1.2.1.7.35" ] } ,

            { "name" => "oid.ALPHA-RESOURCE-MIB::componentListSystemPointer.36" "columns" => [ "1.3.6.1.4.1.7309.5.2.1.2.1.7.36" ] }

        ]

        hosts => [{host => "udp:HOSTNAME1/161" community => "public" version => "2c"  retries => 2  timeout => 1000 } ]

        add_field => {
            "[@metadata][projectId]" => 1
            "[@metadata][collectionId]" => 1
        }
    }


}

I tried making the configuration all one line for hosts it didn't seem to make any difference. Any thoughts?

UPDATE:: I misspoke, it seems to be an issue with some of the MIBs i was using, not the config. The Alpha PDU oids are not working for some reason.

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.