How use metricbeat for logstash logs files in docker container

Hi everyone,

My first post, so sorry if I made some mistakes.

I'm trying to use metricbeat to monitor Logstash.
I have elastic, kibana, metricbeat and logstash running in docker containers with 2 docker-compose files (1 for Elastic, kibana & metricbeat, 1 for the logstash processes).
I succeeded to configure for elasticsearch & kibana, but not for logstash.
I don't have a host for logstash, only file logs.

Here is my docker-compose file :

version: "3.7"

services:
  es01:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.11.1
    container_name: ${ENV}_es01
    restart: always
    environment:
      - cluster.name=${ENV}_docker-cluster
      - node.name=${ENV}_es_node_01
      - discovery.seed_hosts=es01
      - cluster.initial_master_nodes=${ENV}_es_node_01
      - bootstrap.memory_lock=true
      - ELASTIC_PASSWORD=${DOCKER_ES_PWD}
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=true
      - xpack.security.http.ssl.verification_mode=certificate
      - xpack.security.http.ssl.key=${DOCKER_CERTS_DIR}/es01/es01.key
      - xpack.security.http.ssl.certificate_authorities=${DOCKER_CERTS_DIR}/ca/ca.crt
      - xpack.security.http.ssl.certificate=${DOCKER_CERTS_DIR}/es01/es01.crt
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.certificate_authorities=${DOCKER_CERTS_DIR}/ca/ca.crt
      - xpack.security.transport.ssl.certificate=${DOCKER_CERTS_DIR}/es01/es01.crt
      - xpack.security.transport.ssl.key=${DOCKER_CERTS_DIR}/es01/es01.key
      - network.host=0.0.0.0  # Boostrap checks carried out, accessible from all IP addresses on the host machine
      - transport.host=0.0.0.0
    ports:
      - 9200:9200
    healthcheck:
      test: curl --cacert ${DOCKER_CERTS_DIR}/ca/ca.crt -s https://localhost:9200 >/dev/null; if [[ $$? == 52 ]]; then echo 0; else echo 1; fi
      interval: 30s
      timeout: 10s
      retries: 5
    volumes:
      - esData:/usr/share/elasticsearch/data
      - ./elasticsearch_jvm/:/usr/share/elasticsearch/config/jvm.options.d/
      - ./logs/:/usr/share/elasticsearch/logs/
      - certs:${DOCKER_CERTS_DIR}
    ulimits:
      memlock:
        soft: -1
        hard: -1
    networks:
      - elk_network

kibana:
    build:
      context: ./kibana
      args:
        - KIBANA_VERSION=7.11.1
    container_name: ${ENV}_kibana
    env_file:
        - .env
    restart: always
    environment:
      - ELASTICSEARCH_URL="https://es01:9200"
      - ELASTICSEARCH_HOSTS="https://es01:9200"
      - SERVER_HOST=0.0.0.0
      - ELASTICSEARCH_USERNAME=elastic # Default user - do not change
      - ELASTICSEARCH_PASSWORD=${DOCKER_ES_PWD}
      - ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=${DOCKER_CERTS_DIR}/ca/ca.crt
      - SERVER_SSL_ENABLED=true
      - SERVER_SSL_KEY=${DOCKER_CERTS_DIR}/kibana/kibana.key
      - SERVER_SSL_CERTIFICATE=${DOCKER_CERTS_DIR}/kibana/kibana.crt
    ports:
      - 5601:5601
    volumes:
      - certs:${DOCKER_CERTS_DIR}
    networks:
      - elk_network
    depends_on:
      - es01

metricbeat:
    build:
      context: ./metricbeat
      args:
        - METRICBEAT_VERSION=${DOCKER_METRICBEAT_VERSION}
    container_name: ${ENV}_metricbeat
    env_file:
      - ./metricbeat/metricbeat.local
    image: hint-enabled-metricbeat:7.11.1
    user: root
    environment:
      - ELASTICSEARCH_HOSTS=${ELASTICSEARCH_HOST}
    volumes:
      - metricbeatData:/usr/share/metricbeat/data
      - /var/run/docker.sock:/var/run/docker.sock
    networks:
      - elk_network
    depends_on:
      - es01

networks:
  elk_network:
    name: ${ENV}_elk_network

volumes:
  certs:
    driver: local
    driver_opts:
      type: 'none'
      o: 'bind'
      device: "${DOCKER_DATA_PATH_CERTS}"

  esData:
    driver: local
    driver_opts:
      type: 'none'
      o: 'bind'
      device: "${DOCKER_DATA_PATH_ES}"

  metricbeatData:
    driver: local
    driver_opts:
      type: 'none'
      o: 'bind'
      device: "${DOCKER_DATA_PATH_METRIC}"

My docker-compose file for logstash containers :

version:  '3.7'

services:
  logstash:
    image: logstash-project_01:latest
    container_name: ${ENV}_logstash_project_01
    restart: always
    environment:
      - "LS_JAVA_OPTS=-Xms1g -Xmx2g"
    volumes:
      - ${DOCKER_DATA_LOGSTASH_PATH}/.logstash_jdbc_last_run:${LOGSTASH_DOCKER_PATH}/.logstash_jdbc_last_run:rw

And my metric conf file :

metricbeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  reload.enabled: false

#============================== ES =====================================
output.elasticsearch:
  hosts: ['${ELASTICSEARCH_HOST}:9200']
  username: '${ELASTICSEARCH_USERNAME}'
  password: '${ELASTICSEARCH_PASSWORD}'

#============================== Modules =====================================
# enabled modules for monitoring (e.g. elasticsearch-xpack)
metricbeat.modules:
  - module: elasticsearch
    xpack.enabled: true
    period: 10s
    hosts: '${METRICBEAT_ES_MODULES}'
    username: '${ELASTICSEARCH_USERNAME}'
    password: '${ELASTICSEARCH_PASSWORD}'

  - module: kibana
    xpack.enabled: true
    period: 10s
    hosts: '${KIBANA_HOST}'
    username: '${ELASTICSEARCH_USERNAME}'
    password: '${ELASTICSEARCH_PASSWORD}'
    #basepath: ""

  - module: logstash
    xpack.enabled: true
    period: 10s
    #hosts: ["unix:///var/run/docker.sock"] -> not working
    #hosts:["file:///var/lib/docker/containers/${data.docker.container.id}/*.log"] -> not working

metricbeat.autodiscover:
  providers:
     - type: docker
       hints.enabled: true

I don't know what to fill for "module: logstash / hosts".
Thanks a lot for your help everyone.

By default, logstash listens on port 9600, you can maybe connect in metricbeat container (docker exec -it <metricbeat_container> /bin/bash) and try a ping to logstash container first then curl http://logstash_container:9600 (your docker-compose for logstash does not have a network hence it's not clear if logstash and metricbeat can communicate at all)

Unrelated note xms and xmx should be same value per doc

Thanks

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.