I have done Deadletter queue configration in one file and given to that as volume for the docker service , everything we have dockerized it and here are my files
persistent-queue and dlq config
pipeline.batch.size: 125
pipeline.batch.delay: 50
queue.type: persisted
path.queue: path.data/queue
queue.checkpoint.acks: 0
queue.checkpoint.writes: 0
queue.checkpoint.interval: 0
dead_letter_queue.enable: true
path.data: /usr/share/logstash/data
path.dead_letter_queue: "/usr/share/logstash/data/dead_letter_queue"
path.config: "/usr/share/logstash/pipeline/dead_letter_queue.conf"
dead_letter_queue.flush_interval: 5000
dead_letter_queue.retain.age: 2d
Dlq.conf
input {
dead_letter_queue {
path => "/usr/share/logstash/data/dead_letter_queue"
commit_offsets => true
pipeline_id => "main"
clean_consumed => true
}
}
output {
stdout {
codec => rubydebug {
metadata => true # This makes the logs to appear in console
}
}
opensearch {
hosts => ["${LOGSTASH_OPENSEARCH_HOST}"]
ssl => true
ssl_certificate_verification => false
user => "${LOGSTASH_OPENSEARCH_USER}"
password => "${LOGSTASH_OPENSEARCH_PASSWORD}"
index => "arraysofcars"
document_id => "%{id}"
ecs_compatibility => disabled
}
}
yml file
version: '3'
services:
db:
container_name: mysql_container
image: mysql:5.7
restart: always
environment:
MYSQL_ROOT_PASSWORD: "root"
MYSQL_USER: "yash"
MYSQL_PASSWORD: "password"
MYSQL_DATABASE: "testDb"
ports:
- "3306:3306"
volumes:
- mysqldata:/var/lib/mysql
networks:
- share-network
elasticsearch:
image: amazon/opendistro-for-elasticsearch:1.13.2
container_name: elasticsearch
environment:
- bootstrap.memory_lock=true
- cluster.name=odfe-cluster
- discovery.type=single-node
- XPACK_MONITORING_ENABLED=true
- "ES_JAVA_OPTS=-Xms512m -Xmx512m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 65536 # maximum number of open files for the Elasticsearch user, set to at least 65536 on modern systems
hard: 65536
volumes:
- odfe-data5:/usr/share/elasticsearch/data
ports:
- 9200:9200
- 9600:9600 # required for Performance Analyzer
networks:
- share-network
kibana:
image: opensearchproject/opensearch-dashboards:latest
container_name: odfe-kibana5
ports:
- 5601:5601
environment:
OPENSEARCH_HOSTS: '["https://elasticsearch:9200"]'
DISABLE_SECURITY_DASHBOARDS_PLUGIN : "true"
volumes:
- ./kibana.yml:/usr/share/kibana/config/kibana.yml
networks:
- share-network
logstash:
# image: napses/mvp-logstash:latest
build:
context: .
dockerfile: ./Dockerfile
container_name: mvp_logstash
environment:
- LOGSTASH_JDBC_URL=jdbc:mysql://db:3306/testDb?useSSL=false
- LOGSTASH_JDBC_DRIVER=com.mysql.cj.jdbc.Driver
- LOGSTASH_JDBC_USERNAME=yash
- LOGSTASH_JDBC_PASSWORD=password
- LOGSTASH_OPENSEARCH_HOST=https://elasticsearch:9200
- LOGSTASH_OPENSEARCH_USER=admin
- LOGSTASH_OPENSEARCH_PASSWORD=admin
- XPACK_SECURITY_ENABLED=false
- XPACK_REPORTING_ENABLED=false
- XPACK_MONITORING_ENABLED=false
volumes:
- ./pipelines/mysql_pipelines.conf:/usr/share/logstash/pipeline/logstash.conf
- ./pipelines/dead_letter_queue.conf:/usr/share/logstash/pipeline/dead_letter_queue.conf
- ./pq-dlq.yml:/usr/share/pq-dlq/config/pq-dlq.yml
ports:
- 9300:9300
- 5044:5044
networks:
- share-network
depends_on:
- elasticsearch
- kibana
- db
volumes:
odfe-data5:
mysqldata: {}
networks:
share-network:
driver: bridge
and i am getting this errror
Pipeline_id:main
mvp_logstash | Plugin: <LogStash::Inputs::DeadLetterQueue pipeline_id=>"main", clean_consumed=>true, path=>"/usr/share/logstash/data/dead_letter_queue", id=>"103d70998afc3a81ca902851533b56b646779ccdf856eed2189371fb6d44003a", commit_offsets=>true, enable_metric=>true, codec=><LogStash::Codecs::Plain id=>"plain_71995489-b604-4ba1-b411-9adf60d53a11", enable_metric=>true, charset=>"UTF-8">>
mvp_logstash | Error: DLQ sub-path /usr/share/logstash/data/dead_letter_queue/main does not exist
mvp_logstash | Exception: Java::JavaNioFile::NoSuchFileException
mvp_logstash | Stack: org.logstash.input.DeadLetterQueueInputPlugin.lazyInitQueueReader(org/logstash/input/DeadLetterQueueInputPlugin.java:81)
)