Logstash - Losing packets

HI all,
I am ingesting syslogs through logstash and usually I have roughly 9k e/s. However, recently, I have seen it drop down to 5k e/s and fluctuate between 5k e/s - 9k e/s.
Can someone help me diagnose why this is happening?

Things to note:

  • Logstash 7.4.0
  • CPU is usually around 80%
  • Free memory is 5.9GB (I have 32GB allocated)

Node pipeline stats:

  "host" : "mgnt-logstash01",
  "version" : "7.4.0",
  "http_address" : "127.0.0.1:9600",
  "id" : "X",
  "name" : "mgnt-logstash-01",
  "ephemeral_id" : "X",
  "status" : "green",
  "snapshot" : false,
  "pipeline" : {
    "workers" : 8,
    "batch_size" : 10000,
    "batch_delay" : 50
  },
  "pipelines" : {
    "Infra_Syslog" : {
      "events" : {
        "out" : 4470962057,
        "duration_in_millis" : 3003174588,
        "in" : 4470972521,
        "queue_push_duration_in_millis" : 550976911,
        "filtered" : 4470962057
      },
      "plugins" : {
        "inputs" : [ {
          "id" : "670fc938f41298303b0b754578dc7e15d9497b2fa9ee1017810c3b097c0267d0",
          "queue_size" : 2000,
          "workers" : 2,
          "events" : {
            "out" : 4458943645,
            "queue_push_duration_in_millis" : 534397844
          },
          "name" : "udp"
        }, {
          "id" : "099f3eea267c36717652cb4341616d9bf93cba1e16a7af1d6e822de155583088",
          "events" : {
            "out" : 12028878,
            "queue_push_duration_in_millis" : 16579067
          },
          "name" : "tcp"
        } ],
        "codecs" : [ {
          "id" : "plain_6966be23-4ca7-4a68-b98e-73a2b9758e74",
          "decode" : {
            "out" : 0,
            "duration_in_millis" : 0,
            "writes_in" : 0
          },
          "encode" : {
            "duration_in_millis" : 0,
            "writes_in" : 0
          },
          "name" : "plain"
        }, {
          "id" : "json_b185f81d-56a8-41f8-9915-786c3da49090",
          "decode" : {
            "out" : 0,
            "duration_in_millis" : 0,
            "writes_in" : 0
          },
          "encode" : {
            "duration_in_millis" : 53343934,
            "writes_in" : 4470962911
          },
          "name" : "json"
        }, {
          "id" : "line_315db2b2-595d-4945-a11e-a738eb24c5d2",
          "decode" : {
            "out" : 12028878,
            "duration_in_millis" : 17364669,
            "writes_in" : 1793739
          },
          "encode" : {
            "duration_in_millis" : 0,
            "writes_in" : 0
          },
          "name" : "line"
        }, {
          "id" : "plain_77d2b923-a026-433b-8c5c-ec1fbc8abbf3",
          "decode" : {
            "out" : 4458943649,
            "duration_in_millis" : 644170181,
            "writes_in" : 4458943650
          },
          "encode" : {
            "duration_in_millis" : 0,
            "writes_in" : 0
          },
          "name" : "plain"
        }, {
          "id" : "plain_611b47de-ed9e-42b7-917f-a92ce818831c",
          "decode" : {
            "out" : 0,
            "duration_in_millis" : 0,
            "writes_in" : 0
          },
          "encode" : {
            "duration_in_millis" : 0,
            "writes_in" : 0
          },
          "name" : "plain"
        } ],
        "filters" : [ {
          "id" : "934a771838c9bcd1f436e379c0696a6f32058b1ebcdf7b5daf750bfca5bca9c2",
          "matches" : 1,
          "events" : {
            "out" : 1,
            "in" : 1,
            "duration_in_millis" : 12045
          },
          "patterns_per_field" : {
            "message" : 1
          },
          "failures" : 0,
          "name" : "grok"
        }, {
          "id" : "bdd0fd6f8cc37fa6072af652986607049fd404854a3dd569a7c97c740136cd2e",
          "matches" : 440989731,
          "events" : {
            "out" : 4470969057,
            "in" : 4470969057,
            "duration_in_millis" : 145737941
          },
          "patterns_per_field" : {
            "message" : 1
          },
          "failures" : 4029979326,
          "name" : "grok"
        }, {
          "id" : "6f598b9bc4ad7e3b20796a6850b249d9d35df1f9c679eabbc532a92a6a5e58dc",
          "matches" : 15277710,
          "events" : {
            "out" : 37213132,
            "in" : 37213132,
            "duration_in_millis" : 2429139
          },
          "failures" : 21935422,
          "name" : "dissect"
        }, {
          "id" : "217ad07965724e24768bf54001a67f85d30f0da9674c971ac56a2293ba453a83",
          "events" : {
            "out" : 255838401,
            "in" : 255838668,
            "duration_in_millis" : 104272893
          },
          "name" : "kv"
        }, {
          "id" : "04c5fb0683e49f7995aef5e2d24b1c10428510c2c4e2fe845e1551786f72992f",
          "events" : {
            "out" : 13205943,
            "in" : 13205943,
            "duration_in_millis" : 1148899
          },
          "failures" : 13205943,
          "name" : "dissect"
        }, {
          "id" : "0e8db4ac0424af114bfb2c67e3faaeefad27736a2470cdb58433cdb9a63567c3",
          "matches" : 0,
          "events" : {
            "out" : 4029977656,
            "in" : 4029979326,
            "duration_in_millis" : 97451945
          },
          "patterns_per_field" : {
            "message" : 1
          },
          "failures" : 4029978667,
          "name" : "grok"
        }, {
          "id" : "87b758b9304a426b4ded4d65f53c6dca5d7aa83bd60271cb99ec04aa7bf93706",
          "matches" : 357601,
          "events" : {
            "out" : 37570738,
            "in" : 37570738,
            "duration_in_millis" : 5072079
          },
          "failures" : 37213137,
          "name" : "dissect"
        }, {
          "id" : "061f1ab7f6a108e7a8da89af0a046d60c0892fd2ebb37b2e0a20646856313ec8",
          "events" : {
            "out" : 1,
            "in" : 1,
            "duration_in_millis" : 3942
          },
          "name" : "kv"
        }, {
          "id" : "fa9e17418c9cf8154457b5fd2ea5ab49db134494cbb38d622fe4411cafd221fc",
          "events" : {
            "out" : 4470964056,
            "in" : 4470964056,
            "duration_in_millis" : 58656898
          },
          "name" : "mutate"
        }, {
          "id" : "1279866a7ad11feddd713b1a607a98f0347c11310219d0aa129be66738f43d25",
          "matches" : 8729470,
          "events" : {
            "out" : 21935412,
            "in" : 21935422,
            "duration_in_millis" : 1287181
          },
          "failures" : 13205950,
          "name" : "dissect"
        }, {
          "id" : "7e2652d3a5abfd42e54eff6fdfc6c6d2eefebf891707098ea72aadf51f1d2001",
          "events" : {
            "out" : 37213132,
            "in" : 37213132,
            "duration_in_millis" : 4043519
          },
          "failures" : 37213132,
          "name" : "dissect"
        }, {
          "id" : "46a5277c0331012bd8182f90533d18a8e5fbbf2f4a0574b404240fc795e657d1",
          "events" : {
            "out" : 255838401,
            "in" : 255838401,
            "duration_in_millis" : 2839473
          },
          "name" : "mutate"
        }, {
          "id" : "f34120474514182b3339279327f0f27c9ef227d9098327dfe286765d8904273f",
          "matches" : 8,
          "events" : {
            "out" : 13205943,
            "in" : 13205943,
            "duration_in_millis" : 1121878
          },
          "failures" : 13205935,
          "name" : "dissect"
        }, {
          "id" : "0680d729de3501ab850aa52d20a5fbf9cd9766fbfa3cdbe4fb9302e67b1b412c",
          "matches" : 3736568229,
          "events" : {
            "out" : 4029977656,
            "in" : 4029977656,
            "duration_in_millis" : 203891648
          },
          "patterns_per_field" : {
            "message" : 1
          },
          "failures" : 293409427,
          "name" : "grok"
        }, {
          "id" : "6de2ec8bca57acc80224a79b9e63f8c0de83b1f2632453c40da90fef94591c04",
          "matches" : 255838668,
          "events" : {
            "out" : 293409427,
            "in" : 293409427,
            "duration_in_millis" : 9612380
          },
          "failures" : 37570759,
          "name" : "dissect"
        }, {
          "id" : "9a1159c054348705b1c00b8d6e42f443b45ac969884d8ea3108c52787e7085a7",
          "events" : {
            "out" : 4470964056,
            "in" : 4470964056,
            "duration_in_millis" : 222271525
          },
          "name" : "prune"
        }, {
          "id" : "218524cdbe33380671f5a459f90796e4d8e034e54b82a7a764046f10d304b07d",
          "events" : {
            "out" : 1,
            "in" : 1,
            "duration_in_millis" : 3336
          },
          "name" : "mutate"
        } ],
        "outputs" : [ {
          "id" : "c6ca9ff474e0f2c0432cc17124dde6dc03fcd6753be2bd0b6e75ba16f95d16dc",
          "events" : {
            "out" : 4470962057,
            "in" : 4470963057,
            "duration_in_millis" : 296835858
          },
          "name" : "udp"
        }, {
          "id" : "96f6a07550c3c42f5a5a411db143c2dc1a881d648590563009a56a7d4862d9b0",
          "bulk_requests" : {
            "successes" : 1,
            "responses" : {
              "200" : 1
            }
          },
          "events" : {
            "out" : 1,
            "in" : 1,
            "duration_in_millis" : 27067
          },
          "documents" : {
            "successes" : 1
          },
          "name" : "elasticsearch"
        }, {
          "id" : "9e7d9879fd913c7d24371735728a31c4edadbd0e641fedde20bf8ddcbc08ea3a",
          "bulk_requests" : {
            "successes" : 4547103,
            "responses" : {
              "200" : 4547103
            }
          },
          "events" : {
            "out" : 4470963056,
            "in" : 4470964056,
            "duration_in_millis" : 1657533658
          },
          "documents" : {
            "successes" : 4470963056
          },
          "name" : "elasticsearch"
        } ]
      },
      "reloads" : {
        "successes" : 0,
        "last_error" : null,
        "last_failure_timestamp" : null,
        "failures" : 18,
        "last_success_timestamp" : null
      },
      "queue" : {
        "type" : "persisted",
        "events_count" : 19,
        "queue_size_in_bytes" : 19699803,
        "max_queue_size_in_bytes" : 966367641600
      },
      "dead_letter_queue" : {
        "queue_size_in_bytes" : 5
      },
      "hash" : "45c4be097fd6462a583ae16b44bb741698f4774ef2e6abe6c6d2cf87bf033d1e",
      "ephemeral_id" : "07dd2c6e-98a2-43a1-bdba-c665e913e6f7"
    },
    ".monitoring-logstash" : {
      "events" : null,
      "plugins" : {
        "inputs" : [ ],
        "codecs" : [ ],
        "filters" : [ ],
        "outputs" : [ ]
      },
      "reloads" : {
        "successes" : 0,
        "last_error" : null,
        "last_failure_timestamp" : null,
        "failures" : 0,
        "last_success_timestamp" : null
      },
      "queue" : null
    }
  }

Has anyone experienced this before?

Any suggestions are greatly appreciated

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.