~All of our Logstash instances gets tons of warnings on shutdown.
Warning seems to tell us that something is blocked: "block in start_workers"
Our Logstash applications run with pipeline.yml:
pipeline.yml example
- pipeline.id: xferlog-pipeline
path.config: "/<internal-path>/xferlog.conf"
- pipeline.id: syslog-pipeline
path.config: "/<internal-path>/syslog.conf"
Stacktrace
[2024-11-04T15:01:33,487][WARN ][org.logstash.execution.ShutdownWatcherExt] {"inflight_count"=>0, "stalling_threads_info"=>{"other"=>[{"thread_id"=>6533, "name"=>"[xferlog-pipeline]<file", "current_call"=>"
[...]/vendor/bundle/jruby/3.1.0/gems/logstash-input-file-4.4.5/lib/filewatch/watch.rb:55:in `sleep'"}, {"thread_id"=>6534, "name"=>"[xferlog-pipeline]<file", "current_call"=>"
[...]/vendor/bundle/jruby/3.1.0/gems/logstash-input-file-4.4.5/lib/filewatch/watch.rb:55:in `sleep'"}, {"thread_id"=>6535, "name"=>"[xferlog-pipeline]<file", "current_call"=>"
[...]/vendor/bundle/jruby/3.1.0/gems/logstash-input-file-4.4.5/lib/filewatch/watch.rb:55:in `sleep'"}, {"thread_id"=>6536, "name"=>"[xferlog-pipeline]<file", "current_call"=>"
[...]/vendor/bundle/jruby/3.1.0/gems/logstash-input-file-4.4.5/lib/filewatch/watch.rb:55:in `sleep'"}, {"thread_id"=>6537, "name"=>"[xferlog-pipeline]<file", "current_call"=>"
[...]/vendor/bundle/jruby/3.1.0/gems/logstash-input-file-4.4.5/lib/filewatch/watch.rb:55:in `sleep'"}, {"thread_id"=>6538, "name"=>"[xferlog-pipeline]<file", "current_call"=>"
[...]/vendor/bundle/jruby/3.1.0/gems/logstash-input-file-4.4.5/lib/filewatch/watch.rb:55:in `sleep'"}, {"thread_id"=>6539, "name"=>"[xferlog-pipeline]<file", "current_call"=>"
[...]/vendor/bundle/jruby/3.1.0/gems/logstash-input-file-4.4.5/lib/filewatch/watch.rb:55:in `sleep'"}, {"thread_id"=>6482, "name"=>"[xferlog-pipeline]-pipeline-manager", "current_call"=>"
[...]/vendor/bundle/jruby/3.1.0/gems/thwait-0.2.0/lib/thwait.rb:112:in `pop'"}],
["LogStash::Filters::GeoIP", {"add_tag"=>["geoip"], "source"=>"[ip]", "id"=>"8307b8bfd76f245feab6f6123316261140e43c8778f089da8e980049e7d9eaff", "target"=>"geoip"}]=>[{"thread_id"=>6509, "name"=>"[xferlog-pipeline]>worker0", "current_call"=>"
[...]/logstash-core/lib/logstash/java_pipeline.rb:304:in `block in start_workers'"}, {"thread_id"=>6510, "name"=>"[xferlog-pipeline]>worker1", "current_call"=>"
[...]/logstash-core/lib/logstash/java_pipeline.rb:304:in `block in start_workers'"}, {"thread_id"=>6511, "name"=>"[xferlog-pipeline]>worker2", "current_call"=>"
[...]/logstash-core/lib/logstash/java_pipeline.rb:304:in `block in start_workers'"}, {"thread_id"=>6512, "name"=>"[xferlog-pipeline]>worker3", "current_call"=>"
[...]/logstash-core/lib/logstash/java_pipeline.rb:304:in `block in start_workers'"}, {"thread_id"=>6513, "name"=>"[xferlog-pipeline]>worker4", "current_call"=>"
[...]/logstash-core/lib/logstash/java_pipeline.rb:304:in `block in start_workers'"}, {"thread_id"=>6514, "name"=>"[xferlog-pipeline]>worker5", "current_call"=>"
[...]/logstash-core/lib/logstash/java_pipeline.rb:304:in `block in start_workers'"}, {"thread_id"=>6515, "name"=>"[xferlog-pipeline]>worker6", "current_call"=>"
[...]/logstash-core/lib/logstash/java_pipeline.rb:304:in `block in start_workers'"}, {"thread_id"=>6516, "name"=>"[xferlog-pipeline]>worker7", "current_call"=>"
[...]/logstash-core/lib/logstash/java_pipeline.rb:304:in `block in start_workers'"}, {"thread_id"=>6517, "name"=>"[xferlog-pipeline]>worker8", "current_call"=>"
[...]/logstash-core/lib/logstash/java_pipeline.rb:304:in `block in start_workers'"}, {"thread_id"=>6518, "name"=>"[xferlog-pipeline]>worker9", "current_call"=>"
[...]/logstash-core/lib/logstash/java_pipeline.rb:304:in `block in start_workers'"}, {"thread_id"=>6519, "name"=>"[xferlog-pipeline]>worker10", "current_call"=>"
[...]/logstash-core/lib/logstash/java_pipeline.rb:304:in `block in start_workers'"}, {"thread_id"=>6520, "name"=>"[xferlog-pipeline]>worker11", "current_call"=>"
[...]/logstash-core/lib/logstash/java_pipeline.rb:304:in `block in start_workers'"}, {"thread_id"=>6521, "name"=>"[xferlog-pipeline]>worker12", "current_call"=>"
[...]/logstash-core/lib/logstash/java_pipeline.rb:304:in `block in start_workers'"}, {"thread_id"=>6522, "name"=>"[xferlog-pipeline]>worker13", "current_call"=>"
[...]/logstash-core/lib/logstash/java_pipeline.rb:304:in `block in start_workers'"}, {"thread_id"=>6523, "name"=>"[xferlog-pipeline]>worker14", "current_call"=>"
[...]/logstash-core/lib/logstash/java_pipeline.rb:304:in `block in start_workers'"}, {"thread_id"=>6524, "name"=>"[xferlog-pipeline]>worker15", "current_call"=>"
[...]/logstash-core/lib/logstash/java_pipeline.rb:304:in `block in start_workers'"}, {"thread_id"=>6525, "name"=>"[xferlog-pipeline]>worker16", "current_call"=>"
[...]/logstash-core/lib/logstash/java_pipeline.rb:304:in `block in start_workers'"}, {"thread_id"=>6526, "name"=>"[xferlog-pipeline]>worker17", "current_call"=>"
[...]/logstash-core/lib/logstash/java_pipeline.rb:304:in `block in start_workers'"}, {"thread_id"=>6527, "name"=>"[xferlog-pipeline]>worker18", "current_call"=>"
[...]/logstash-core/lib/logstash/java_pipeline.rb:304:in `block in start_workers'"}, {"thread_id"=>6528, "name"=>"[xferlog-pipeline]>worker19", "current_call"=>"
[...]/logstash-core/lib/logstash/java_pipeline.rb:304:in `block in start_workers'"}, {"thread_id"=>6529, "name"=>"[xferlog-pipeline]>worker20", "current_call"=>"
[...]/logstash-core/lib/logstash/java_pipeline.rb:304:in `block in start_workers'"}, {"thread_id"=>6530, "name"=>"[xferlog-pipeline]>worker21", "current_call"=>"
[...]/logstash-core/lib/logstash/java_pipeline.rb:304:in `block in start_workers'"}, {"thread_id"=>6531, "name"=>"[xferlog-pipeline]>worker22", "current_call"=>"
[...]/logstash-core/lib/logstash/java_pipeline.rb:304:in `block in start_workers'"}, {"thread_id"=>6532, "name"=>"[xferlog-pipeline]>worker23", "current_call"=>"
[...]/logstash-core/lib/logstash/java_pipeline.rb:304:in `block in start_workers'"}]}}
....
[2024-11-04T15:01:33,488][ERROR][org.logstash.execution.ShutdownWatcherExt] The shutdown process appears to be stalled due to busy or blocked plugins. Check the logs for more information.
...
[2024-11-04T15:07:54,463][INFO ][logstash.javapipeline ][xferlog-pipeline] Pipeline terminated {"pipeline.id"=>"xferlog-pipeline"}
[2024-11-04T15:07:54,537][INFO ][logstash.pipelinesregistry] Removed pipeline from
registry successfully {:pipeline_id=>:"xferlog-pipeline"}
...
[2024-11-04T15:10:20,108][INFO ][logstash.javapipeline ][syslog-pipeline] Pipeline terminated {"pipeline.id"=>"syslog-pipeline"}
[2024-11-04T15:10:20,368][INFO ][logstash.pipelinesregistry] Removed pipeline from registry successfully {:pipeline_id=>:"syslog-pipeline"}
[2024-11-04T15:10:20,378][INFO ][logstash.runner ] Logstash shut down.
logstash.yml
# Settings file in YAML
#
# Settings can be specified either in hierarchical form, e.g.:
#
# pipeline:
# batch:
# size: 125
# delay: 5
#
# Or as flat keys:
#
# pipeline.batch.size: 125
# pipeline.batch.delay: 5
#
# ------------ Node identity ------------
#
# Use a descriptive name for the node:
#
# node.name: test
#
# If omitted the node name will default to the machine's host name
#
# ------------ Data path ------------------
#
# Which directory should be used by logstash and its plugins
# for any persistent needs. Defaults to LOGSTASH_HOME/data
#
# path.data:
#
# ------------ Pipeline Settings --------------
#
# The ID of the pipeline.
#
# pipeline.id: main
#
# Set the number of workers that will, in parallel, execute the filters+outputs
# stage of the pipeline.
#
# This defaults to the number of the host's CPU cores.
#
# pipeline.workers: 2
#
# How many events to retrieve from inputs before sending to filters+workers
#
# pipeline.batch.size: 125
#
# How long to wait in milliseconds while polling for the next event
# before dispatching an undersized batch to filters+outputs
#
# pipeline.batch.delay: 50
#
# Force Logstash to exit during shutdown even if there are still inflight
# events in memory. By default, logstash will refuse to quit until all
# received events have been pushed to the outputs.
#
# WARNING: Enabling this can lead to data loss during shutdown
#
# pipeline.unsafe_shutdown: false
#
# Set the pipeline event ordering. Options are "auto" (the default), "true" or "false".
# "auto" automatically enables ordering if the 'pipeline.workers' setting
# is also set to '1', and disables otherwise.
# "true" enforces ordering on the pipeline and prevent logstash from starting
# if there are multiple workers.
# "false" disables any extra processing necessary for preserving ordering.
#
# pipeline.ordered: auto
#
# Sets the pipeline's default value for `ecs_compatibility`, a setting that is
# available to plugins that implement an ECS Compatibility mode for use with
# the Elastic Common Schema.
# Possible values are:
# - disabled
# - v1
# - v8 (default)
# Pipelines defined before Logstash 8 operated without ECS in mind. To ensure a
# migrated pipeline continues to operate as it did before your upgrade, opt-OUT
# of ECS for the individual pipeline in its `pipelines.yml` definition. Setting
# it here will set the default for _all_ pipelines, including new ones.
#
# pipeline.ecs_compatibility: v8
#
# ------------ Pipeline Configuration Settings --------------
#
# Where to fetch the pipeline configuration for the main pipeline
#
# path.config:
#
# Pipeline configuration string for the main pipeline
#
# config.string:
#
# At startup, test if the configuration is valid and exit (dry run)
#
# config.test_and_exit: false
#
# Periodically check if the configuration has changed and reload the pipeline
# This can also be triggered manually through the SIGHUP signal
#
# config.reload.automatic: false
#
# How often to check if the pipeline configuration has changed (in seconds)
# Note that the unit value (s) is required. Values without a qualifier (e.g. 60)
# are treated as nanoseconds.
# Setting the interval this way is not recommended and might change in later versions.
#
# config.reload.interval: 3s
#
# Show fully compiled configuration as debug log message
# NOTE: --log.level must be 'debug'
#
# config.debug: false
#
# When enabled, process escaped characters such as \n and \" in strings in the
# pipeline configuration files.
#
# config.support_escapes: false
#
# ------------ API Settings -------------
# Define settings related to the HTTP API here.
#
# The HTTP API is enabled by default. It can be disabled, but features that rely
# on it will not work as intended.
#
# api.enabled: true
#
# By default, the HTTP API is not secured and is therefore bound to only the
# host's loopback interface, ensuring that it is not accessible to the rest of
# the network.
# When secured with SSL and Basic Auth, the API is bound to _all_ interfaces
# unless configured otherwise.
#
api.http.host: 192.168.230.120
#
# The HTTP API web server will listen on an available port from the given range.
# Values can be specified as a single port (e.g., `9600`), or an inclusive range
# of ports (e.g., `9600-9700`).
#
# api.http.port: 9600-9700
#
# The HTTP API includes a customizable "environment" value in its response,
# which can be configured here.
#
# api.environment: "production"
#
# The HTTP API can be secured with SSL (TLS). To do so, you will need to provide
# the path to a password-protected keystore in p12 or jks format, along with credentials.
#
# api.ssl.enabled: false
# api.ssl.keystore.path: /path/to/keystore.jks
# api.ssl.keystore.password: "y0uRp4$$w0rD"
#
# The availability of SSL/TLS protocols depends on the JVM version. Certain protocols are
# disabled by default and need to be enabled manually by changing `jdk.tls.disabledAlgorithms`
# in the $JDK_HOME/conf/security/java.security configuration file.
#
# api.ssl.supported_protocols: [TLSv1.2,TLSv1.3]
#
# The HTTP API can be configured to require authentication. Acceptable values are
# - `none`: no auth is required (default)
# - `basic`: clients must authenticate with HTTP Basic auth, as configured
# with `api.auth.basic.*` options below
# api.auth.type: none
#
# When configured with `api.auth.type` `basic`, you must provide the credentials
# that requests will be validated against. Usage of Environment or Keystore
# variable replacements is encouraged (such as the value `"${HTTP_PASS}"`, which
# resolves to the value stored in the keystore's `HTTP_PASS` variable if present
# or the same variable from the environment)
#
# api.auth.basic.username: "logstash-user"
# api.auth.basic.password: "s3cUreP4$$w0rD"
#
# When setting `api.auth.basic.password`, the password should meet
# the default password policy requirements.
# The default password policy requires non-empty minimum 8 char string that
# includes a digit, upper case letter and lower case letter.
# Policy mode sets Logstash to WARN or ERROR when HTTP authentication password doesn't
# meet the password policy requirements.
# The default is WARN. Setting to ERROR enforces stronger passwords (recommended).
#
# api.auth.basic.password_policy.mode: WARN
#
# ------------ Module Settings ---------------
# Define modules here. Modules definitions must be defined as an array.
# The simple way to see this is to prepend each `name` with a `-`, and keep
# all associated variables under the `name` they are associated with, and
# above the next, like this:
#
# modules:
# - name: MODULE_NAME
# var.PLUGINTYPE1.PLUGINNAME1.KEY1: VALUE
# var.PLUGINTYPE1.PLUGINNAME1.KEY2: VALUE
# var.PLUGINTYPE2.PLUGINNAME1.KEY1: VALUE
# var.PLUGINTYPE3.PLUGINNAME3.KEY1: VALUE
#
# Module variable names must be in the format of
#
# var.PLUGIN_TYPE.PLUGIN_NAME.KEY
#
# modules:
#
# ------------ Cloud Settings ---------------
# Define Elastic Cloud settings here.
# Format of cloud.id is a base64 value e.g. dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRub3RhcmVhbCRpZGVudGlmaWVy
# and it may have an label prefix e.g. staging:dXMtZ...
# This will overwrite 'var.elasticsearch.hosts' and 'var.kibana.host'
# cloud.id: <identifier>
#
# Format of cloud.auth is: <user>:<pass>
# This is optional
# If supplied this will overwrite 'var.elasticsearch.username' and 'var.elasticsearch.password'
# If supplied this will overwrite 'var.kibana.username' and 'var.kibana.password'
# cloud.auth: elastic:<password>
#
# ------------ Queuing Settings --------------
#
# Internal queuing model, "memory" for legacy in-memory based queuing and
# "persisted" for disk-based acked queueing. Defaults is memory
#
# queue.type: memory
#
# If `queue.type: persisted`, the directory path where the pipeline data files will be stored.
# Each pipeline will group its PQ files in a subdirectory matching its `pipeline.id`.
# Default is path.data/queue.
#
# path.queue:
#
# If using queue.type: persisted, the page data files size. The queue data consists of
# append-only data files separated into pages. Default is 64mb
#
# queue.page_capacity: 64mb
#
# If using queue.type: persisted, the maximum number of unread events in the queue.
# Default is 0 (unlimited)
#
# queue.max_events: 0
#
# If using queue.type: persisted, the total capacity of the queue in number of bytes.
# If you would like more unacked events to be buffered in Logstash, you can increase the
# capacity using this setting. Please make sure your disk drive has capacity greater than
# the size specified here. If both max_bytes and max_events are specified, Logstash will pick
# whichever criteria is reached first
# Default is 1024mb or 1gb
#
# queue.max_bytes: 1024mb
#
# If using queue.type: persisted, the maximum number of acked events before forcing a checkpoint
# Default is 1024, 0 for unlimited
#
# queue.checkpoint.acks: 1024
#
# If using queue.type: persisted, the maximum number of written events before forcing a checkpoint
# Default is 1024, 0 for unlimited
#
# queue.checkpoint.writes: 1024
#
# If using queue.type: persisted, the interval in milliseconds when a checkpoint is forced on the head page
# Default is 1000, 0 for no periodic checkpoint.
#
# queue.checkpoint.interval: 1000
#
# ------------ Dead-Letter Queue Settings --------------
# Flag to turn on dead-letter queue.
#
# dead_letter_queue.enable: false
# If using dead_letter_queue.enable: true, the maximum size of each dead letter queue. Entries
# will be dropped if they would increase the size of the dead letter queue beyond this setting.
# Default is 1024mb
# dead_letter_queue.max_bytes: 1024mb
# If using dead_letter_queue.enable: true, the interval in milliseconds where if no further events eligible for the DLQ
# have been created, a dead letter queue file will be written. A low value here will mean that more, smaller, queue files
# may be written, while a larger value will introduce more latency between items being "written" to the dead letter queue, and
# being available to be read by the dead_letter_queue input when items are written infrequently.
# Default is 5000.
#
# dead_letter_queue.flush_interval: 5000
# If using dead_letter_queue.enable: true, controls which entries should be dropped to avoid exceeding the size limit.
# Set the value to `drop_newer` (default) to stop accepting new events that would push the DLQ size over the limit.
# Set the value to `drop_older` to remove queue pages containing the oldest events to make space for new ones.
#
# dead_letter_queue.storage_policy: drop_newer
# If using dead_letter_queue.enable: true, the interval that events have to be considered valid. After the interval has
# expired the events could be automatically deleted from the DLQ.
# The interval could be expressed in days, hours, minutes or seconds, using as postfix notation like 5d,
# to represent a five days interval.
# The available units are respectively d, h, m, s for day, hours, minutes and seconds.
# If not specified then the DLQ doesn't use any age policy for cleaning events.
#
# dead_letter_queue.retain.age: 1d
# If using dead_letter_queue.enable: true, the directory path where the data files will be stored.
# Default is path.data/dead_letter_queue
#
# path.dead_letter_queue:
#
# ------------ Debugging Settings --------------
#
# Options for log.level:
# * fatal
# * error
# * warn
# * info (default)
# * debug
# * trace
#
# log.level: info
# path.logs:
# Lgq 240116
path.logs: /data2/appelk/logs
#
# ------------ Other Settings --------------
#
# Allow or block running Logstash as superuser (default: true)
# allow_superuser: false
#
# Where to find custom plugins
# path.plugins: []
#
# Flag to output log lines of each pipeline in its separate log file. Each log filename contains the pipeline.name
# Default is false
# pipeline.separate_logs: false
#
# ------------ X-Pack Settings (not applicable for OSS build)--------------
#
# X-Pack Monitoring
# https://www.elastic.co/guide/en/logstash/current/monitoring-logstash.html
#xpack.monitoring.enabled: false
#xpack.monitoring.elasticsearch.username: logstash_system
#xpack.monitoring.elasticsearch.password: password
#xpack.monitoring.elasticsearch.proxy: ["http://proxy:port"]
#xpack.monitoring.elasticsearch.hosts: ["https://es1:9200", "https://es2:9200"]
# an alternative to hosts + username/password settings is to use cloud_id/cloud_auth
#xpack.monitoring.elasticsearch.cloud_id: monitoring_cluster_id:xxxxxxxxxx
#xpack.monitoring.elasticsearch.cloud_auth: logstash_system:password
# another authentication alternative is to use an Elasticsearch API key
#xpack.monitoring.elasticsearch.api_key: "id:api_key"
#xpack.monitoring.elasticsearch.ssl.certificate_authority: "/path/to/ca.crt"
#xpack.monitoring.elasticsearch.ssl.ca_trusted_fingerprint: xxxxxxxxxx
#xpack.monitoring.elasticsearch.ssl.truststore.path: path/to/file
#xpack.monitoring.elasticsearch.ssl.truststore.password: password
# use either keystore.path/keystore.password or certificate/key configurations
#xpack.monitoring.elasticsearch.ssl.keystore.path: /path/to/file
#xpack.monitoring.elasticsearch.ssl.keystore.password: password
#xpack.monitoring.elasticsearch.ssl.certificate: /path/to/file
#xpack.monitoring.elasticsearch.ssl.key: /path/to/key
#xpack.monitoring.elasticsearch.ssl.verification_mode: full
#xpack.monitoring.elasticsearch.ssl.cipher_suites: []
#xpack.monitoring.elasticsearch.sniffing: false
#xpack.monitoring.collection.interval: 10s
#xpack.monitoring.collection.pipeline.details.enabled: true
#
# X-Pack Management
# https://www.elastic.co/guide/en/logstash/current/logstash-centralized-pipeline-management.html
#xpack.management.enabled: false
#xpack.management.pipeline.id: ["main", "apache_logs"]
#xpack.management.elasticsearch.username: logstash_admin_user
#xpack.management.elasticsearch.password: password
#xpack.management.elasticsearch.proxy: ["http://proxy:port"]
#xpack.management.elasticsearch.hosts: ["https://es1:9200", "https://es2:9200"]
# an alternative to hosts + username/password settings is to use cloud_id/cloud_auth
#xpack.management.elasticsearch.cloud_id: management_cluster_id:xxxxxxxxxx
#xpack.management.elasticsearch.cloud_auth: logstash_admin_user:password
# another authentication alternative is to use an Elasticsearch API key
#xpack.management.elasticsearch.api_key: "id:api_key"
#xpack.management.elasticsearch.ssl.ca_trusted_fingerprint: xxxxxxxxxx
#xpack.management.elasticsearch.ssl.certificate_authority: "/path/to/ca.crt"
#xpack.management.elasticsearch.ssl.truststore.path: /path/to/file
#xpack.management.elasticsearch.ssl.truststore.password: password
# use either keystore.path/keystore.password or certificate/key configurations
#xpack.management.elasticsearch.ssl.keystore.path: /path/to/file
#xpack.management.elasticsearch.ssl.keystore.password: password
#xpack.management.elasticsearch.ssl.certificate: /path/to/file
#xpack.management.elasticsearch.ssl.key: /path/to/certificate_key_file
#xpack.management.elasticsearch.ssl.cipher_suites: []
#xpack.management.elasticsearch.ssl.verification_mode: full
#xpack.management.elasticsearch.sniffing: false
#xpack.management.logstash.poll_interval: 5s
# X-Pack GeoIP plugin
# https://www.elastic.co/guide/en/logstash/current/plugins-filters-geoip.html#plugins-filters-geoip-manage_update
#xpack.geoip.download.endpoint: "https://geoip.elastic.co/v1/database"
#xpack.geoip.downloader.enabled: true
.conf example
input {
file {
type => "boron"
path => "/<internal-path>/logs/syslog"
start_position => "beginning"
stat_interval => "600 second"
}
file {
type => "ddpftp02"
path => "/<internal-path>/logs/syslog"
start_position => "beginning"
stat_interval => "600 second"
}
file {
type => "ddpftp03"
path => "/<internal-path>/logs/syslog"
start_position => "beginning"
stat_interval => "600 second"
}
file {
type => "thpftp02"
path => "/<internal-path>/logs/syslog"
start_position => "beginning"
stat_interval => "600 second"
}
file {
type => "thpftp03"
path => "/<internal-path>/logs/syslog"
start_position => "beginning"
stat_interval => "600 second"
}
file {
type => "thpftp04"
path => "/<internal-path>/logs/syslog"
start_position => "beginning"
stat_interval => "600 second"
}
file {
type => "thpftp05"
path => "/<internal-path>/logs/syslog"
start_position => "beginning"
stat_interval => "600 second"
}
}
filter {
if "USER" in [message] {
fingerprint {
target => "[@metadata][fingerprint]"
method => "SHA256"
source => ["message"]
}
if "boron" in [message] {
grok {
match => {"message" => "%{MONTH:[@metadata][month]}\s+%{MONTHDAY:[@metadata][monthday]}\s+%{TIME:[@metadata][time]}\s+%{DATA:server}\s+%{DATA}\[%{NUMBER:pid:int}\]\s+%{DATA}\(%{DATA}\[%{IP:ip}\]\):\s%{DATA}\s%{DATA:user}:\s+%{GREEDYDATA:status}"}
add_tag => ["log"]
overwrite => ["message"]
}
mutate { add_field => {"logTime" => "%{+YYYY}-%{[@metadata][month]}-%{[@metadata][monthday]};%{[@metadata][time]}.000" }}
} else {
grok {
match => {"message" => "%{TIMESTAMP_ISO8601:logTime}\s+%{DATA:server}\s+%{DATA}\[%{NUMBER:pid:int}\]\s+%{DATA}\(%{DATA}\[%{IP:ip}\]\):\s%{DATA}\s%{DATA:user}:\s+%{GREEDYDATA:status}"}
add_tag => ["log"]
overwrite => ["message"]
}
}
if "_grokparsefailure" in [tags] {
drop { }
}
date {
match => ["logTime", "YYYY-MMM-dd;HH:mm:ss.SSS", "ISO8601"]
timezone => "Europe/Oslo"
target => ["logTime"]
}
if "192.168." not in [message] {
geoip {
source => "[ip]"
target => "geoip"
add_tag => ["geoip"]
}
}
} else {
drop { }
}
}
output {
elasticsearch {
hosts => ["https://*****:9200"]
index => "ftp-sys-log-%{+YYYY.MM.dd}"
user => "logstash_internal"
password => "*******"
ssl_certificate_authorities => "/<internal-path>/****.pem"
document_id => "%{[@metadata][fingerprint]}"
}
}
At first I thought it was always related to LogStash::Filters::GeoIP, but I see different warnings all the time, such as: LogStash::Filters::Drop, LogStash::Filters::Grok.
Anyone know what is causing this, or how we can investigate this further? While this is only a warning, a shutdown can take 10+ minutes, which causes issues for our weekly restart job.
Have we given the pipelines too few workers/resources so there is a big queue when we try to shutdown?