How use metricbeat for logstash logs files in docker container

Hi everyone,

My first post, so sorry if I made some mistakes.

I'm trying to use metricbeat to monitor Logstash.
I have elastic, kibana, metricbeat and logstash running in docker containers with 2 docker-compose files (1 for Elastic, kibana & metricbeat, 1 for the logstash processes).
I succeeded to configure for elasticsearch & kibana, but not for logstash.
I don't have a host for logstash, only file logs.

Here is my docker-compose file :

version: "3.7"

services:
  es01:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.11.1
    container_name: ${ENV}_es01
    restart: always
    environment:
      - cluster.name=${ENV}_docker-cluster
      - node.name=${ENV}_es_node_01
      - discovery.seed_hosts=es01
      - cluster.initial_master_nodes=${ENV}_es_node_01
      - bootstrap.memory_lock=true
      - ELASTIC_PASSWORD=${DOCKER_ES_PWD}
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=true
      - xpack.security.http.ssl.verification_mode=certificate
      - xpack.security.http.ssl.key=${DOCKER_CERTS_DIR}/es01/es01.key
      - xpack.security.http.ssl.certificate_authorities=${DOCKER_CERTS_DIR}/ca/ca.crt
      - xpack.security.http.ssl.certificate=${DOCKER_CERTS_DIR}/es01/es01.crt
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.certificate_authorities=${DOCKER_CERTS_DIR}/ca/ca.crt
      - xpack.security.transport.ssl.certificate=${DOCKER_CERTS_DIR}/es01/es01.crt
      - xpack.security.transport.ssl.key=${DOCKER_CERTS_DIR}/es01/es01.key
      - network.host=0.0.0.0  # Boostrap checks carried out, accessible from all IP addresses on the host machine
      - transport.host=0.0.0.0
    ports:
      - 9200:9200
    healthcheck:
      test: curl --cacert ${DOCKER_CERTS_DIR}/ca/ca.crt -s https://localhost:9200 >/dev/null; if [[ $$? == 52 ]]; then echo 0; else echo 1; fi
      interval: 30s
      timeout: 10s
      retries: 5
    volumes:
      - esData:/usr/share/elasticsearch/data
      - ./elasticsearch_jvm/:/usr/share/elasticsearch/config/jvm.options.d/
      - ./logs/:/usr/share/elasticsearch/logs/
      - certs:${DOCKER_CERTS_DIR}
    ulimits:
      memlock:
        soft: -1
        hard: -1
    networks:
      - elk_network

kibana:
    build:
      context: ./kibana
      args:
        - KIBANA_VERSION=7.11.1
    container_name: ${ENV}_kibana
    env_file:
        - .env
    restart: always
    environment:
      - ELASTICSEARCH_URL="https://es01:9200"
      - ELASTICSEARCH_HOSTS="https://es01:9200"
      - SERVER_HOST=0.0.0.0
      - ELASTICSEARCH_USERNAME=elastic # Default user - do not change
      - ELASTICSEARCH_PASSWORD=${DOCKER_ES_PWD}
      - ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=${DOCKER_CERTS_DIR}/ca/ca.crt
      - SERVER_SSL_ENABLED=true
      - SERVER_SSL_KEY=${DOCKER_CERTS_DIR}/kibana/kibana.key
      - SERVER_SSL_CERTIFICATE=${DOCKER_CERTS_DIR}/kibana/kibana.crt
    ports:
      - 5601:5601
    volumes:
      - certs:${DOCKER_CERTS_DIR}
    networks:
      - elk_network
    depends_on:
      - es01

metricbeat:
    build:
      context: ./metricbeat
      args:
        - METRICBEAT_VERSION=${DOCKER_METRICBEAT_VERSION}
    container_name: ${ENV}_metricbeat
    env_file:
      - ./metricbeat/metricbeat.local
    image: hint-enabled-metricbeat:7.11.1
    user: root
    environment:
      - ELASTICSEARCH_HOSTS=${ELASTICSEARCH_HOST}
    volumes:
      - metricbeatData:/usr/share/metricbeat/data
      - /var/run/docker.sock:/var/run/docker.sock
    networks:
      - elk_network
    depends_on:
      - es01

networks:
  elk_network:
    name: ${ENV}_elk_network

volumes:
  certs:
    driver: local
    driver_opts:
      type: 'none'
      o: 'bind'
      device: "${DOCKER_DATA_PATH_CERTS}"

  esData:
    driver: local
    driver_opts:
      type: 'none'
      o: 'bind'
      device: "${DOCKER_DATA_PATH_ES}"

  metricbeatData:
    driver: local
    driver_opts:
      type: 'none'
      o: 'bind'
      device: "${DOCKER_DATA_PATH_METRIC}"

My docker-compose file for logstash containers :

version:  '3.7'

services:
  logstash:
    image: logstash-project_01:latest
    container_name: ${ENV}_logstash_project_01
    restart: always
    environment:
      - "LS_JAVA_OPTS=-Xms1g -Xmx2g"
    volumes:
      - ${DOCKER_DATA_LOGSTASH_PATH}/.logstash_jdbc_last_run:${LOGSTASH_DOCKER_PATH}/.logstash_jdbc_last_run:rw

And my metric conf file :

metricbeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  reload.enabled: false

#============================== ES =====================================
output.elasticsearch:
  hosts: ['${ELASTICSEARCH_HOST}:9200']
  username: '${ELASTICSEARCH_USERNAME}'
  password: '${ELASTICSEARCH_PASSWORD}'

#============================== Modules =====================================
# enabled modules for monitoring (e.g. elasticsearch-xpack)
metricbeat.modules:
  - module: elasticsearch
    xpack.enabled: true
    period: 10s
    hosts: '${METRICBEAT_ES_MODULES}'
    username: '${ELASTICSEARCH_USERNAME}'
    password: '${ELASTICSEARCH_PASSWORD}'

  - module: kibana
    xpack.enabled: true
    period: 10s
    hosts: '${KIBANA_HOST}'
    username: '${ELASTICSEARCH_USERNAME}'
    password: '${ELASTICSEARCH_PASSWORD}'
    #basepath: ""

  - module: logstash
    xpack.enabled: true
    period: 10s
    #hosts: ["unix:///var/run/docker.sock"] -> not working
    #hosts:["file:///var/lib/docker/containers/${data.docker.container.id}/*.log"] -> not working

metricbeat.autodiscover:
  providers:
     - type: docker
       hints.enabled: true

I don't know what to fill for "module: logstash / hosts".
Thanks a lot for your help everyone.

By default, logstash listens on port 9600, you can maybe connect in metricbeat container (docker exec -it <metricbeat_container> /bin/bash) and try a ping to logstash container first then curl http://logstash_container:9600 (your docker-compose for logstash does not have a network hence it's not clear if logstash and metricbeat can communicate at all)

Unrelated note xms and xmx should be same value per doc

Thanks