OS
DISTRIB_ID=Ubuntu
DISTRIB_RELEASE=18.04
DISTRIB_CODENAME=bionic
DISTRIB_DESCRIPTION="Ubuntu 18.04.1 LTS"
NAME="Ubuntu"
VERSION="18.04.1 LTS (Bionic Beaver)"
ID=ubuntu
ID_LIKE=debian
PRETTY_NAME="Ubuntu 18.04.1 LTS"
VERSION_ID="18.04"
HOME_URL="https://www.ubuntu.com/"
SUPPORT_URL="https://help.ubuntu.com/"
BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/"
PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy"
VERSION_CODENAME=bionic
UBUNTU_CODENAME=bionic
# free -h
total used free shared buff/cache available
Mem: 31G 17G 10G 1.1M 2.8G 13G
Swap: 0B 0B 0B
ELK
#logstash --version
logstash 6.4.2
#elasticsearch --version
Version: 6.4.2, Build: default/deb/04711c2/2018-09-26T13:34:09.098244Z, JVM: 1.8.0_181
#kibana --version
6.4.2
pipeline.yml
- pipeline.id: dnslog
path.config: "/etc/logstash/dnslog/conf.d/*.conf"
queue.drain: true
queue.type: persisted
queue.max_bytes: 100mb
pipeline.workers: 5
- pipeline.id: elastiflow
path.config: "/etc/logstash/elastiflow/conf.d/*.conf"
queue.type: memory
pipeline.workers: 4
logstash.yml
# grep -v -E "#|^$" /etc/logstash/logstash.yml
path.data: /var/lib/logstash
log.level: info
path.logs: /var/log/logstash
xpack.monitoring.enabled: true
xpack.monitoring.elasticsearch.url: ["http://localhost:9200"]
xpack.monitoring.elasticsearch.sniffing: false
xpack.monitoring.collection.pipeline.details.enabled: true
elasticsearch.yml
# grep -v -E "#|^$" /etc/elasticsearch/elasticsearch.yml
node.name: elasti
path.data: /var/lib/elasticsearch
path.logs: /var/log/elasticsearch
bootstrap.memory_lock: false
bootstrap.system_call_filter: false
network.host: localhost
http.port: 9200
thread_pool.bulk.queue_size: 1000
thread_pool.write.queue_size: 1000
discovery.type: single-node
logger.org.elasticsearch.transport: debug
logstash-plain.log
[2018-10-11T18:55:02,625][WARN ][logstash.outputs.elasticsearch] Marking url as dead. Last error: [LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError] Elasticsearch Unreachable: [http://elastic:xxxxxx@127.0.0.1:9200/][Manticore::SocketTimeout] Read timed out {:url=>http://elastic:xxxxxx@127.0.0.1:9200/, :error_message=>"Elasticsearch Unreachable: [http://elastic:xxxxxx@127.0.0.1:9200/][Manticore::SocketTimeout] Read timed out", :error_class=>"LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError"}
[2018-10-11T18:55:02,626][ERROR][logstash.outputs.elasticsearch] Attempted to send a bulk request to elasticsearch' but Elasticsearch appears to be unreachable or down! {:error_message=>"Elasticsearch Unreachable: [http://elastic:xxxxxx@127.0.0.1:9200/][Manticore::SocketTimeout] Read timed out", :class=>"LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError", :will_retry_in_seconds=>64}
[2018-10-11T18:55:02,628][WARN ][logstash.outputs.elasticsearch] Marking url as dead. Last error: [LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError] Elasticsearch Unreachable: [http://elastic:xxxxxx@127.0.0.1:9200/][Manticore::SocketTimeout] Read timed out {:url=>http://elastic:xxxxxx@127.0.0.1:9200/, :error_message=>"Elasticsearch Unreachable: [http://elastic:xxxxxx@127.0.0.1:9200/][Manticore::SocketTimeout] Read timed out", :error_class=>"LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError"}
[2018-10-11T18:55:02,629][ERROR][logstash.outputs.elasticsearch] Attempted to send a bulk request to elasticsearch' but Elasticsearch appears to be unreachable or down! {:error_message=>"Elasticsearch Unreachable: [http://elastic:xxxxxx@127.0.0.1:9200/][Manticore::SocketTimeout] Read timed out", :class=>"LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError", :will_retry_in_seconds=>64}
[2018-10-11T18:55:02,633][WARN ][logstash.outputs.elasticsearch] Marking url as dead. Last error: [LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError] Elasticsearch Unreachable: [http://elastic:xxxxxx@127.0.0.1:9200/][Manticore::SocketTimeout] Read timed out {:url=>http://elastic:xxxxxx@127.0.0.1:9200/, :error_message=>"Elasticsearch Unreachable: [http://elastic:xxxxxx@127.0.0.1:9200/][Manticore::SocketTimeout] Read timed out", :error_class=>"LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError"}
[2018-10-11T18:55:02,633][ERROR][logstash.outputs.elasticsearch] Attempted to send a bulk request to elasticsearch' but Elasticsearch appears to be unreachable or down! {:error_message=>"Elasticsearch Unreachable: [http://elastic:xxxxxx@127.0.0.1:9200/][Manticore::SocketTimeout] Read timed out", :class=>"LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError", :will_retry_in_seconds=>64}
[2018-10-11T18:55:05,703][INFO ][logstash.outputs.elasticsearch] Running health check to see if an Elasticsearch connection is working {:healthcheck_url=>http://elastic:xxxxxx@127.0.0.1:9200/, :path=>"/"}
[2018-10-11T18:55:05,708][WARN ][logstash.outputs.elasticsearch] Restored connection to ES instance {:url=>"http://elastic:xxxxxx@127.0.0.1:9200/"}
[2018-10-11T18:55:06,769][INFO ][logstash.outputs.elasticsearch] Running health check to see if an Elasticsearch connection is working {:healthcheck_url=>http://elastic:xxxxxx@127.0.0.1:9200/, :path=>"/"}
[2018-10-11T18:55:06,812][WARN ][logstash.outputs.elasticsearch] Restored connection to ES instance {:url=>"http://elastic:xxxxxx@127.0.0.1:9200/"}
elasticsearch.log
[2018-10-11T18:54:41,321][ERROR][o.e.a.b.TransportBulkAction] [elasti] failed to execute pipeline for a bulk request
org.elasticsearch.common.util.concurrent.EsRejectedExecutionException: rejected execution of org.elasticsearch.ingest.PipelineExecutionService$1@1c51d8d1 on EsThreadPoolExecutor[name = elasti/write, queue capacity = 10000, org.elasticsea
rch.common.util.concurrent.EsThreadPoolExecutor@6d685369[Running, pool size = 8, active threads = 8, queued tasks = 10000, completed tasks = 31658]]
[2018-10-11T18:54:41,321][ERROR][o.e.a.b.TransportBulkAction] [elasti] failed to execute pipeline for a bulk request
org.elasticsearch.common.util.concurrent.EsRejectedExecutionException: rejected execution of org.elasticsearch.ingest.PipelineExecutionService$1@68875ff7 on EsThreadPoolExecutor[name = elasti/write, queue capacity = 1000, org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor@6d685369[Running, pool size = 8, active threads = 8, queued tasks = 1000, completed tasks = 31658]]
I'm experiencing the same issue when i enable 2 pipelines.
It's ok if only 1 pipeline running.
Anyone know how to troubleshoot or fine tuning the setup feel free to comment.