I am trying to leverage the docker auto discovery provider. I followed every tutorial I could find regarding filebeat and I still cannot get Kibana to display the logs. I am trying to get this working with the Elastic Search Containers. I have three ES nodes and one kibana node all running inside docker. I have the filebeat container running as recommended with correct volume mounts and user. It appears data is making it to the Elastic Search cluster because the filebeat index has documents in it. I don't see any log messages in kibana though when I go to logs or discover. Am I missing something in my configuration with filebeat?
I did not notice anything in the filebeat logs to indicate any errors. I see messages like below:
filebeat | 2021-04-11T13:36:25.953Z INFO log/harvester.go:302 Harvester started for file: /var/lib/docker/containers/c4e7e39e3f1feb5ec7f6249e5d6b9e1d48455e02f11529888fd01fdef6f24970/c4e7e39e3f1feb5ec7f6249e5d6b9e1d48455e02f11529888fd01fdef6f24970-json.log
filebeat | 2021-04-11T13:36:25.975Z INFO [autodiscover] autodiscover/autodiscover.go:113 Starting autodiscover manager
filebeat | 2021-04-11T13:36:26.140Z INFO log/input.go:157 Configured paths: [/var/lib/docker/containers/7cef67c42a0593caca0448f1a534115465b0892670a61547c1aa11c77b8c6d33/*-json.log]
filebeat | 2021-04-11T13:36:26.140Z INFO log/input.go:157 Configured paths: [/var/lib/docker/containers/7cef67c42a0593caca0448f1a534115465b0892670a61547c1aa11c77b8c6d33/*-json.log]
filebeat | 2021-04-11T13:36:26.140Z INFO log/input.go:157 Configured paths: [/var/lib/docker/containers/7cef67c42a0593caca0448f1a534115465b0892670a61547c1aa11c77b8c6d33/*-json.log]
filebeat logs
filebeat | 2021-04-11T13:36:27.020Z INFO [index-management] idxmgmt/std.go:261 Auto ILM enable success.
filebeat | 2021-04-11T13:36:27.021Z INFO [index-management.ilm] ilm/std.go:139 do not generate ilm policy: exists=true, overwrite=false
filebeat | 2021-04-11T13:36:27.021Z INFO [index-management] idxmgmt/std.go:274 ILM policy successfully loaded.
filebeat | 2021-04-11T13:36:27.021Z INFO [index-management] idxmgmt/std.go:407 Set setup.template.name to '{filebeat-7.12.0 {now/d}-000001}' as ILM is enabled.
filebeat | 2021-04-11T13:36:27.021Z INFO [index-management] idxmgmt/std.go:412 Set setup.template.pattern to 'filebeat-7.12.0-*' as ILM is enabled.
filebeat | 2021-04-11T13:36:27.021Z INFO [index-management] idxmgmt/std.go:446 Set settings.index.lifecycle.rollover_alias in template to {filebeat-7.12.0 {now/d}-000001} as ILM is enabled.
filebeat | 2021-04-11T13:36:27.021Z INFO [index-management] idxmgmt/std.go:450 Set settings.index.lifecycle.name in template to {filebeat {"policy":{"phases":{"hot":{"actions":{"rollover":{"max_age":"30d","max_size":"50gb"}}}}}}} as ILM is enabled.
filebeat | 2021-04-11T13:36:27.027Z INFO template/load.go:97 Template filebeat-7.12.0 already exists and will not be overwritten.
filebeat | 2021-04-11T13:36:27.027Z INFO [index-management] idxmgmt/std.go:298 Loaded index template.
filebeat | 2021-04-11T13:36:27.029Z INFO [index-management] idxmgmt/std.go:309 Write alias successfully generated.
filebeat | 2021-04-11T13:36:27.049Z INFO [publisher_pipeline_output] pipeline/output.go:151 Connection to backoff(elasticsearch(https://es01:9200)) established
filebeat.yml
filebeat.config:
modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false
filebeat.autodiscover:
providers:
- type: docker
hints.enabled: true
filebeat.modules:
- module: system
- module: elasticsearch
- module: kibana
processors:
- add_docker_metadata: ~
setup.kibana:
host: "${CONTAINER_KIBANA}:${PORT_KIBANA}"
username: "${ELASTIC_KIBANA_SYSTEM_USER}"
password: "${ELASTIC_KIBANA_SYSTEM_PASSWORD}"
protocol: "https"
ssl.enabled: true
ssl.verification_mode: "certificate"
ssl.certificate_authorities:
- '${CERTS_DIR}/ca/ca.crt'
ssl.certificate: "${CERTS_DIR}/${CONTAINER_FILEBEAT}/${CONTAINER_FILEBEAT}.crt"
ssl.key: "${CERTS_DIR}/${CONTAINER_FILEBEAT}/${CONTAINER_FILEBEAT}.key"
output.elasticsearch:
hosts: ["${CONTAINER_ELASTIC_SEARCH}:${PORT_ELASTIC_SEARCH}"]
username: "${ELASTIC_BEATS_SYSTEM_USER}"
password: "${ELASTIC_BEATS_SYSTEM_PASSWORD}"
protocol: "https"
ssl.certificate: "${CERTS_DIR}/${CONTAINER_FILEBEAT}/${CONTAINER_FILEBEAT}.crt"
ssl.key: "${CERTS_DIR}/${CONTAINER_FILEBEAT}/${CONTAINER_FILEBEAT}.key"
ssl.certificate_authorities:
- "${CERTS_DIR}/ca/ca.crt"
ssl.verification_mode: "certificate"
ssl.enabled: true
docker-compose.yml
es01:
image: ${IMAGE_ELASTIC_SEARCH:?IMAGE_ELASTIC_SEARCH}:${ELASTIC_VERSION:?ELASTIC_VERSION}
container_name: ${CONTAINER_ELASTIC_SEARCH:?CONTAINER_ELASTIC_SEARCH}01
labels:
co.elastic.logs/fileset.stdout: access
co.elastic.logs/fileset.stderr: error
co.elastic.logs/module: elasticsearch
environment:
- node.name=${CONTAINER_ELASTIC_SEARCH}01
- cluster.name=${ELASTIC_CLUSTER_NAME:?ELASTIC_CLUSTER_NAME}
- discovery.seed_hosts=${CONTAINER_ELASTIC_SEARCH}02,${CONTAINER_ELASTIC_SEARCH}03
- cluster.initial_master_nodes=${CONTAINER_ELASTIC_SEARCH}01,${CONTAINER_ELASTIC_SEARCH}02,${CONTAINER_ELASTIC_SEARCH}03
- bootstrap.memory_lock=true
- ELASTIC_PASSWORD=$ELASTIC_BOOTSTRAP_PASSWORD
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
- xpack.license.self_generated.type=basic
- xpack.security.enabled=true
- xpack.security.http.ssl.enabled=true
- xpack.security.http.ssl.key=${ELASTIC_CERTS_DIR}/${CONTAINER_ELASTIC_SEARCH}01/${CONTAINER_ELASTIC_SEARCH}01.key
- xpack.security.http.ssl.certificate_authorities=${ELASTIC_CERTS_DIR}/ca/ca.crt
- xpack.security.http.ssl.certificate=${ELASTIC_CERTS_DIR}/${CONTAINER_ELASTIC_SEARCH}01/${CONTAINER_ELASTIC_SEARCH}01.crt
- xpack.security.transport.ssl.enabled=true
- xpack.security.transport.ssl.verification_mode=certificate
- xpack.security.transport.ssl.certificate_authorities=${ELASTIC_CERTS_DIR}/ca/ca.crt
- xpack.security.transport.ssl.certificate=${ELASTIC_CERTS_DIR}/${CONTAINER_ELASTIC_SEARCH}01/${CONTAINER_ELASTIC_SEARCH}01.crt
- xpack.security.transport.ssl.key=${ELASTIC_CERTS_DIR}/${CONTAINER_ELASTIC_SEARCH}01/${CONTAINER_ELASTIC_SEARCH}01.key
ulimits:
memlock:
soft: -1
hard: -1
volumes:
- "${PATH_ROOT}/conf/elk/99-sysctl.conf:/etc/sysctl.d/99-sysctl.conf"
- data01:/usr/share/elasticsearch/data
- elastic-certs:${ELASTIC_CERTS_DIR}
- "${PATH_ROOT}/logs/elk/elastic/:/usr/share/elasticsearch/logs/"
ports:
- 9200:9200
networks:
dotmusic_net:
ipv4_address: ${DOCKER_IP_ELASTIC_1:?DOCKER_IP_ELASTIC_1}
healthcheck:
test: curl --cacert ${ELASTIC_CERTS_DIR}/ca/ca.crt -s https://localhost:9200 >/dev/null; if [[ $$? == 52 ]]; then echo 0; else echo 1; fi
interval: 30s
timeout: 10s
retries: 5
kibana:
image: ${IMAGE_KIBANA:?IMAGE_KIBANA}
container_name: ${CONTAINER_KIBANA}
environment:
SERVER_NAME: ${KIBANA_SERVER_NAME}
ELASTIC_SEARCH_URL: https://${CONTAINER_ELASTIC_SEARCH}01:9200
ELASTICSEARCH_HOSTS: '["https://${CONTAINER_ELASTIC_SEARCH}01:9200","https://${CONTAINER_ELASTIC_SEARCH}02:9200","https://${CONTAINER_ELASTIC_SEARCH}03:9200"]'
SERVER_SSL_ENABLED: "true"
SERVER_SSL_CERTIFICATE: ${ELASTIC_CERTS_DIR}/${CONTAINER_KIBANA}/${CONTAINER_KIBANA}.crt
SERVER_SSL_KEY: ${ELASTIC_CERTS_DIR}/${CONTAINER_KIBANA}/${CONTAINER_KIBANA}.key
ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES: '["${ELASTIC_CERTS_DIR}/ca/ca.crt"]'
ELASTICSEARCH_USERNAME: ${ELASTIC_KIBANA_SYSTEM_USER:?ELASTIC_KIBANA_SYSTEM_USER}
ELASTICSEARCH_PASSWORD: ${ELASTIC_KIBANA_SYSTEM_PASSWORD:?ELASTIC_KIBANA_SYSTEM_PASSWORD}
XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY: ${KIBANA_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY:?KIBANA_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY}
labels:
co.elastic.logs/enabled: "true"
co.elastic.logs/module: "kibana"
co.elastic.logs/fileset.stdout: access
co.elastic.logs/fileset.stderr: error
ports:
- 5601:5601
volumes:
- elastic-certs:${ELASTIC_CERTS_DIR}
networks:
dotmusic_net:
ipv4_address: ${DOCKER_IP_KIBANA:?DOCKER_IP_KIBANA}
filebeat:
image: ${IMAGE_FILEBEAT:?IMAGE_FILEBEAT}:${FILEBEAT_VERSION:?FILEBEAT_VERSION}
container_name: ${CONTAINER_FILEBEAT:?CONTAINER_FILEBEAT}
user: root
volumes:
- "${PATH_ROOT}/conf/elk/filebeat.docker.yml:/usr/share/filebeat/filebeat.yml:ro"
- "/var/run/docker.sock:/var/run/docker.sock:ro"
- elastic-certs:${ELASTIC_CERTS_DIR}
- "/var/lib/docker/containers:/var/lib/docker/containers:ro"
depends_on:
- ${CONTAINER_ELASTIC_SEARCH}01
- ${CONTAINER_KIBANA}
environment:
CONTAINER_ELASTIC_SEARCH: ${CONTAINER_ELASTIC_SEARCH}01
PORT_ELASTIC_SEARCH: ${PORT_ELASTIC_SEARCH}
CONTAINER_KIBANA: ${CONTAINER_KIBANA}
PORT_KIBANA: ${PORT_KIBANA}
CERTS_DIR: ${ELASTIC_CERTS_DIR}
CONTAINER_FILEBEAT: ${CONTAINER_FILEBEAT}
ELASTIC_BEATS_SYSTEM_USER: ${ELASTIC_USER}
ELASTIC_BEATS_SYSTEM_PASSWORD: ${ELASTIC_PASSWORD}
ELASTIC_KIBANA_SYSTEM_USER: ${ELASTIC_USER}
ELASTIC_KIBANA_SYSTEM_PASSWORD: ${ELASTIC_PASSWORD}
networks:
dotmusic_net:
ipv4_address: ${DOCKER_IP_FILEBEAT:?DOCKER_IP_FILEBEAT}