HI all,
I am ingesting syslogs through logstash and usually I have roughly 9k e/s. However, recently, I have seen it drop down to 5k e/s and fluctuate between 5k e/s - 9k e/s.
Can someone help me diagnose why this is happening?
Things to note:
- Logstash 7.4.0
- CPU is usually around 80%
- Free memory is 5.9GB (I have 32GB allocated)
Node pipeline stats:
"host" : "mgnt-logstash01", "version" : "7.4.0", "http_address" : "127.0.0.1:9600", "id" : "X", "name" : "mgnt-logstash-01", "ephemeral_id" : "X", "status" : "green", "snapshot" : false, "pipeline" : { "workers" : 8, "batch_size" : 10000, "batch_delay" : 50 }, "pipelines" : { "Infra_Syslog" : { "events" : { "out" : 4470962057, "duration_in_millis" : 3003174588, "in" : 4470972521, "queue_push_duration_in_millis" : 550976911, "filtered" : 4470962057 }, "plugins" : { "inputs" : [ { "id" : "670fc938f41298303b0b754578dc7e15d9497b2fa9ee1017810c3b097c0267d0", "queue_size" : 2000, "workers" : 2, "events" : { "out" : 4458943645, "queue_push_duration_in_millis" : 534397844 }, "name" : "udp" }, { "id" : "099f3eea267c36717652cb4341616d9bf93cba1e16a7af1d6e822de155583088", "events" : { "out" : 12028878, "queue_push_duration_in_millis" : 16579067 }, "name" : "tcp" } ], "codecs" : [ { "id" : "plain_6966be23-4ca7-4a68-b98e-73a2b9758e74", "decode" : { "out" : 0, "duration_in_millis" : 0, "writes_in" : 0 }, "encode" : { "duration_in_millis" : 0, "writes_in" : 0 }, "name" : "plain" }, { "id" : "json_b185f81d-56a8-41f8-9915-786c3da49090", "decode" : { "out" : 0, "duration_in_millis" : 0, "writes_in" : 0 }, "encode" : { "duration_in_millis" : 53343934, "writes_in" : 4470962911 }, "name" : "json" }, { "id" : "line_315db2b2-595d-4945-a11e-a738eb24c5d2", "decode" : { "out" : 12028878, "duration_in_millis" : 17364669, "writes_in" : 1793739 }, "encode" : { "duration_in_millis" : 0, "writes_in" : 0 }, "name" : "line" }, { "id" : "plain_77d2b923-a026-433b-8c5c-ec1fbc8abbf3", "decode" : { "out" : 4458943649, "duration_in_millis" : 644170181, "writes_in" : 4458943650 }, "encode" : { "duration_in_millis" : 0, "writes_in" : 0 }, "name" : "plain" }, { "id" : "plain_611b47de-ed9e-42b7-917f-a92ce818831c", "decode" : { "out" : 0, "duration_in_millis" : 0, "writes_in" : 0 }, "encode" : { "duration_in_millis" : 0, "writes_in" : 0 }, "name" : "plain" } ], "filters" : [ { "id" : "934a771838c9bcd1f436e379c0696a6f32058b1ebcdf7b5daf750bfca5bca9c2", "matches" : 1, "events" : { "out" : 1, "in" : 1, "duration_in_millis" : 12045 }, "patterns_per_field" : { "message" : 1 }, "failures" : 0, "name" : "grok" }, { "id" : "bdd0fd6f8cc37fa6072af652986607049fd404854a3dd569a7c97c740136cd2e", "matches" : 440989731, "events" : { "out" : 4470969057, "in" : 4470969057, "duration_in_millis" : 145737941 }, "patterns_per_field" : { "message" : 1 }, "failures" : 4029979326, "name" : "grok" }, { "id" : "6f598b9bc4ad7e3b20796a6850b249d9d35df1f9c679eabbc532a92a6a5e58dc", "matches" : 15277710, "events" : { "out" : 37213132, "in" : 37213132, "duration_in_millis" : 2429139 }, "failures" : 21935422, "name" : "dissect" }, { "id" : "217ad07965724e24768bf54001a67f85d30f0da9674c971ac56a2293ba453a83", "events" : { "out" : 255838401, "in" : 255838668, "duration_in_millis" : 104272893 }, "name" : "kv" }, { "id" : "04c5fb0683e49f7995aef5e2d24b1c10428510c2c4e2fe845e1551786f72992f", "events" : { "out" : 13205943, "in" : 13205943, "duration_in_millis" : 1148899 }, "failures" : 13205943, "name" : "dissect" }, { "id" : "0e8db4ac0424af114bfb2c67e3faaeefad27736a2470cdb58433cdb9a63567c3", "matches" : 0, "events" : { "out" : 4029977656, "in" : 4029979326, "duration_in_millis" : 97451945 }, "patterns_per_field" : { "message" : 1 }, "failures" : 4029978667, "name" : "grok" }, { "id" : "87b758b9304a426b4ded4d65f53c6dca5d7aa83bd60271cb99ec04aa7bf93706", "matches" : 357601, "events" : { "out" : 37570738, "in" : 37570738, "duration_in_millis" : 5072079 }, "failures" : 37213137, "name" : "dissect" }, { "id" : "061f1ab7f6a108e7a8da89af0a046d60c0892fd2ebb37b2e0a20646856313ec8", "events" : { "out" : 1, "in" : 1, "duration_in_millis" : 3942 }, "name" : "kv" }, { "id" : "fa9e17418c9cf8154457b5fd2ea5ab49db134494cbb38d622fe4411cafd221fc", "events" : { "out" : 4470964056, "in" : 4470964056, "duration_in_millis" : 58656898 }, "name" : "mutate" }, { "id" : "1279866a7ad11feddd713b1a607a98f0347c11310219d0aa129be66738f43d25", "matches" : 8729470, "events" : { "out" : 21935412, "in" : 21935422, "duration_in_millis" : 1287181 }, "failures" : 13205950, "name" : "dissect" }, { "id" : "7e2652d3a5abfd42e54eff6fdfc6c6d2eefebf891707098ea72aadf51f1d2001", "events" : { "out" : 37213132, "in" : 37213132, "duration_in_millis" : 4043519 }, "failures" : 37213132, "name" : "dissect" }, { "id" : "46a5277c0331012bd8182f90533d18a8e5fbbf2f4a0574b404240fc795e657d1", "events" : { "out" : 255838401, "in" : 255838401, "duration_in_millis" : 2839473 }, "name" : "mutate" }, { "id" : "f34120474514182b3339279327f0f27c9ef227d9098327dfe286765d8904273f", "matches" : 8, "events" : { "out" : 13205943, "in" : 13205943, "duration_in_millis" : 1121878 }, "failures" : 13205935, "name" : "dissect" }, { "id" : "0680d729de3501ab850aa52d20a5fbf9cd9766fbfa3cdbe4fb9302e67b1b412c", "matches" : 3736568229, "events" : { "out" : 4029977656, "in" : 4029977656, "duration_in_millis" : 203891648 }, "patterns_per_field" : { "message" : 1 }, "failures" : 293409427, "name" : "grok" }, { "id" : "6de2ec8bca57acc80224a79b9e63f8c0de83b1f2632453c40da90fef94591c04", "matches" : 255838668, "events" : { "out" : 293409427, "in" : 293409427, "duration_in_millis" : 9612380 }, "failures" : 37570759, "name" : "dissect" }, { "id" : "9a1159c054348705b1c00b8d6e42f443b45ac969884d8ea3108c52787e7085a7", "events" : { "out" : 4470964056, "in" : 4470964056, "duration_in_millis" : 222271525 }, "name" : "prune" }, { "id" : "218524cdbe33380671f5a459f90796e4d8e034e54b82a7a764046f10d304b07d", "events" : { "out" : 1, "in" : 1, "duration_in_millis" : 3336 }, "name" : "mutate" } ], "outputs" : [ { "id" : "c6ca9ff474e0f2c0432cc17124dde6dc03fcd6753be2bd0b6e75ba16f95d16dc", "events" : { "out" : 4470962057, "in" : 4470963057, "duration_in_millis" : 296835858 }, "name" : "udp" }, { "id" : "96f6a07550c3c42f5a5a411db143c2dc1a881d648590563009a56a7d4862d9b0", "bulk_requests" : { "successes" : 1, "responses" : { "200" : 1 } }, "events" : { "out" : 1, "in" : 1, "duration_in_millis" : 27067 }, "documents" : { "successes" : 1 }, "name" : "elasticsearch" }, { "id" : "9e7d9879fd913c7d24371735728a31c4edadbd0e641fedde20bf8ddcbc08ea3a", "bulk_requests" : { "successes" : 4547103, "responses" : { "200" : 4547103 } }, "events" : { "out" : 4470963056, "in" : 4470964056, "duration_in_millis" : 1657533658 }, "documents" : { "successes" : 4470963056 }, "name" : "elasticsearch" } ] }, "reloads" : { "successes" : 0, "last_error" : null, "last_failure_timestamp" : null, "failures" : 18, "last_success_timestamp" : null }, "queue" : { "type" : "persisted", "events_count" : 19, "queue_size_in_bytes" : 19699803, "max_queue_size_in_bytes" : 966367641600 }, "dead_letter_queue" : { "queue_size_in_bytes" : 5 }, "hash" : "45c4be097fd6462a583ae16b44bb741698f4774ef2e6abe6c6d2cf87bf033d1e", "ephemeral_id" : "07dd2c6e-98a2-43a1-bdba-c665e913e6f7" }, ".monitoring-logstash" : { "events" : null, "plugins" : { "inputs" : [ ], "codecs" : [ ], "filters" : [ ], "outputs" : [ ] }, "reloads" : { "successes" : 0, "last_error" : null, "last_failure_timestamp" : null, "failures" : 0, "last_success_timestamp" : null }, "queue" : null } }