Hi everyone,
My first post, so sorry if I made some mistakes.
I'm trying to use metricbeat to monitor Logstash.
I have elastic, kibana, metricbeat and logstash running in docker containers with 2 docker-compose files (1 for Elastic, kibana & metricbeat, 1 for the logstash processes).
I succeeded to configure for elasticsearch & kibana, but not for logstash.
I don't have a host for logstash, only file logs.
Here is my docker-compose file :
version: "3.7"
services:
es01:
image: docker.elastic.co/elasticsearch/elasticsearch:7.11.1
container_name: ${ENV}_es01
restart: always
environment:
- cluster.name=${ENV}_docker-cluster
- node.name=${ENV}_es_node_01
- discovery.seed_hosts=es01
- cluster.initial_master_nodes=${ENV}_es_node_01
- bootstrap.memory_lock=true
- ELASTIC_PASSWORD=${DOCKER_ES_PWD}
- xpack.security.enabled=true
- xpack.security.http.ssl.enabled=true
- xpack.security.http.ssl.verification_mode=certificate
- xpack.security.http.ssl.key=${DOCKER_CERTS_DIR}/es01/es01.key
- xpack.security.http.ssl.certificate_authorities=${DOCKER_CERTS_DIR}/ca/ca.crt
- xpack.security.http.ssl.certificate=${DOCKER_CERTS_DIR}/es01/es01.crt
- xpack.security.transport.ssl.enabled=true
- xpack.security.transport.ssl.verification_mode=certificate
- xpack.security.transport.ssl.certificate_authorities=${DOCKER_CERTS_DIR}/ca/ca.crt
- xpack.security.transport.ssl.certificate=${DOCKER_CERTS_DIR}/es01/es01.crt
- xpack.security.transport.ssl.key=${DOCKER_CERTS_DIR}/es01/es01.key
- network.host=0.0.0.0 # Boostrap checks carried out, accessible from all IP addresses on the host machine
- transport.host=0.0.0.0
ports:
- 9200:9200
healthcheck:
test: curl --cacert ${DOCKER_CERTS_DIR}/ca/ca.crt -s https://localhost:9200 >/dev/null; if [[ $$? == 52 ]]; then echo 0; else echo 1; fi
interval: 30s
timeout: 10s
retries: 5
volumes:
- esData:/usr/share/elasticsearch/data
- ./elasticsearch_jvm/:/usr/share/elasticsearch/config/jvm.options.d/
- ./logs/:/usr/share/elasticsearch/logs/
- certs:${DOCKER_CERTS_DIR}
ulimits:
memlock:
soft: -1
hard: -1
networks:
- elk_network
kibana:
build:
context: ./kibana
args:
- KIBANA_VERSION=7.11.1
container_name: ${ENV}_kibana
env_file:
- .env
restart: always
environment:
- ELASTICSEARCH_URL="https://es01:9200"
- ELASTICSEARCH_HOSTS="https://es01:9200"
- SERVER_HOST=0.0.0.0
- ELASTICSEARCH_USERNAME=elastic # Default user - do not change
- ELASTICSEARCH_PASSWORD=${DOCKER_ES_PWD}
- ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=${DOCKER_CERTS_DIR}/ca/ca.crt
- SERVER_SSL_ENABLED=true
- SERVER_SSL_KEY=${DOCKER_CERTS_DIR}/kibana/kibana.key
- SERVER_SSL_CERTIFICATE=${DOCKER_CERTS_DIR}/kibana/kibana.crt
ports:
- 5601:5601
volumes:
- certs:${DOCKER_CERTS_DIR}
networks:
- elk_network
depends_on:
- es01
metricbeat:
build:
context: ./metricbeat
args:
- METRICBEAT_VERSION=${DOCKER_METRICBEAT_VERSION}
container_name: ${ENV}_metricbeat
env_file:
- ./metricbeat/metricbeat.local
image: hint-enabled-metricbeat:7.11.1
user: root
environment:
- ELASTICSEARCH_HOSTS=${ELASTICSEARCH_HOST}
volumes:
- metricbeatData:/usr/share/metricbeat/data
- /var/run/docker.sock:/var/run/docker.sock
networks:
- elk_network
depends_on:
- es01
networks:
elk_network:
name: ${ENV}_elk_network
volumes:
certs:
driver: local
driver_opts:
type: 'none'
o: 'bind'
device: "${DOCKER_DATA_PATH_CERTS}"
esData:
driver: local
driver_opts:
type: 'none'
o: 'bind'
device: "${DOCKER_DATA_PATH_ES}"
metricbeatData:
driver: local
driver_opts:
type: 'none'
o: 'bind'
device: "${DOCKER_DATA_PATH_METRIC}"
My docker-compose file for logstash containers :
version: '3.7'
services:
logstash:
image: logstash-project_01:latest
container_name: ${ENV}_logstash_project_01
restart: always
environment:
- "LS_JAVA_OPTS=-Xms1g -Xmx2g"
volumes:
- ${DOCKER_DATA_LOGSTASH_PATH}/.logstash_jdbc_last_run:${LOGSTASH_DOCKER_PATH}/.logstash_jdbc_last_run:rw
And my metric conf file :
metricbeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false
#============================== ES =====================================
output.elasticsearch:
hosts: ['${ELASTICSEARCH_HOST}:9200']
username: '${ELASTICSEARCH_USERNAME}'
password: '${ELASTICSEARCH_PASSWORD}'
#============================== Modules =====================================
# enabled modules for monitoring (e.g. elasticsearch-xpack)
metricbeat.modules:
- module: elasticsearch
xpack.enabled: true
period: 10s
hosts: '${METRICBEAT_ES_MODULES}'
username: '${ELASTICSEARCH_USERNAME}'
password: '${ELASTICSEARCH_PASSWORD}'
- module: kibana
xpack.enabled: true
period: 10s
hosts: '${KIBANA_HOST}'
username: '${ELASTICSEARCH_USERNAME}'
password: '${ELASTICSEARCH_PASSWORD}'
#basepath: ""
- module: logstash
xpack.enabled: true
period: 10s
#hosts: ["unix:///var/run/docker.sock"] -> not working
#hosts:["file:///var/lib/docker/containers/${data.docker.container.id}/*.log"] -> not working
metricbeat.autodiscover:
providers:
- type: docker
hints.enabled: true
I don't know what to fill for "module: logstash / hosts".
Thanks a lot for your help everyone.