Pleasant good day, for my docker compose ELK set up I have created a custom image for Logstash for the purposes of creating a directory to bind mount the logs into as well as writing my own logstash.conf pipeline. The logs I am sending are bind mounted logs from my wordpress apache server, existing on the project's parent directory (logs > wordpress). I am able to view the logs being written to my logstash container when I navigate to the path described in the custom image, however there isn't a sincedb file created for the specified input path file plugin nor am I seeing the logs in my kibana dashboard when I navigate to: http://localhost:5601/app/management/data/index_management/indices
Your assistance in this matter would be greatly appreciated, please see below my docker compose file, logstash.conf and logstashDockerfile:
Docker compose:
version: "3.9"
services:
wordpress:
container_name: wordpress
build:
context: .
dockerfile: wp.Dockerfile
restart: always
stdin_open: true
tty: true
user: 1000:1000
environment:
WORDPRESS_DB_HOST: mariadb
WORDPRESS_DB_USER: db_user
WORDPRESS_DB_PASSWORD: db_user_pass
WORDPRESS_DB_NAME: db_name
#WORDPRESS_DEBUG: 1
volumes:
- ./wordpress:/var/www/html
- ./logs/wordpress:/var/log/apache2
- wp_custom_logs:/var/log/apache2.custom.log
mariadb:
container_name: mariadb
image: mariadb
restart: always
environment:
MYSQL_DATABASE: db_name
MYSQL_USER: db_user
MYSQL_PASSWORD: db_user_pass
MYSQL_RANDOM_ROOT_PASSWORD: root_pass
volumes:
- mariadb_data:/var/lib/mysql
nginx:
container_name: nginx
build:
context: .
dockerfile: nginx.Dockerfile
restart: unless-stopped
ports:
- 8080:80
- 443:443
volumes:
- ./certs:/etc/nginx/certs/self-signed
- nginx_data:/var/www/html
- ./logs/nginx:/var/log/nginx
depends_on:
- wordpress
phpmyadmin:
container_name: phpmyadmin
depends_on:
- mariadb
# image: phpmyadmin/phpmyadmin
build:
context: .
dockerfile: pma.Dockerfile
restart: unless-stopped
environment:
PMA_HOST: mariadb
PMA_PORT: 3306
#PMA_ARBITRARY: 1
PMA_USER: db_user
PMA_PASSWORD: db_user_pass
PMA_ROOT_PASSWORD: root_pass
PHP_UPLOAD_MAX_FILESIZE: 1G
PHP_MAX_INPUT_VARS: 1G
ports:
- 8085:80
redis:
container_name: redis
# image: redis:latest
build:
context: .
dockerfile: redis.Dockerfile
restart: unless-stopped
#command: redis-server --maxmemory 1024mb --maxmemory-policy allkeys-lru --requirepass db_user_pass --appendonly yes --bind redis
#command: sh -c "./init.sh"
command: redis-server --loglevel warning --maxmemory 2048mb --maxmemory-policy allkeys-lru --requirepass db_user_pass --appendonly yes --bind redis
environment:
TZ: "America/Port_of_Spain"
volumes:
- /etc/localtime:/etc/localtime:ro
- dataredis:/data:rw
mailhog:
container_name: mailhog
image: mailhog/mailhog
ports:
- "8025:8025"
- "1025:1025"
command: ["-storage=maildir", "-maildir-path=/maildir"]
volumes:
- ./mailhog/data:/maildir
wpcli:
container_name: wpcli
depends_on:
- wordpress
image: wordpress:cli
user: 1000:1000
command: tail -f /dev/null
volumes:
- ./wordpress:/var/www/html
environment:
WORDPRESS_DB_HOST: mariadb
WORDPRESS_DB_USER: db_user
WORDPRESS_DB_PASSWORD: db_user_pass
WORDPRESS_DB_NAME: db_name
# Minimal ELK setup (without certs)
elasticsearch:
container_name: elasticsearch
image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
environment:
[
"ES_JAVA_OPTS=-Xms2g -Xmx2g",
"bootstrap.memory_lock=true",
"discovery.type=single-node",
"xpack.security.enabled=false",
"xpack.security.enrollment.enabled=false",
]
ports:
- 9200:9200
networks:
- elastic
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 65536
hard: 65536
healthcheck:
test: ["CMD-SHELL", "curl --fail http://localhost:9200 || exit 1"]
interval: 30s
timeout: 10s
retries: 5
kibana:
image: docker.elastic.co/kibana/kibana:${STACK_VERSION}
container_name: kibana
environment:
XPACK_APM_SERVICEMAPENABLED: "true"
XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY: d1a66dfd-c4d3-4a0a-8290-2abcb83ab3aa
LOGGING_ROOT_LEVEL: error
ports:
- 5601:5601
networks:
- elastic
logstash:
container_name: logstash
build:
context: .
dockerfile: logstash.Dockerfile
args:
- STACK_VERSION=${STACK_VERSION}
volumes:
- ./logs/wordpress:/var/log/wordpress
depends_on:
elasticsearch:
condition: service_healthy
environment:
- "ELASTICSEARCH_HOSTS=http://elasticsearch:9200"
#- "LOG_LEVEL=debug"
networks:
- elastic
networks:
elastic:
driver: bridge
volumes:
db_data:
wordpress:
mariadb_data:
nginx_data:
dataredis:
mailhog:
wp_custom_logs:
Logstash.conf
input{
file{
path => ["/var/log/wordpress/custom.log"]
}
}
output{
stdout { codec => rubydebug }
elasticsearch{
hosts => ["http://elasticsearch:9200"]
}
}
logstashDockerfile
ARG STACK_VERSION
FROM docker.elastic.co/logstash/logstash:${STACK_VERSION}
USER root
RUN mkdir /var/log/wordpress
RUN rm -f /usr/share/logstash/pipeline/logstash.conf
COPY ./logstash.conf /usr/share/logstash/pipeline/logstash.conf
USER 1000
Note: Stack version is 8.12.0