Docker Elasticsearch Kibana Issues with Elastic-Agent sending data

I want to use docker to host Elasticsearch and Kibana. I have that working with xpac. I will include my yml files below. I want to enable the elastic-agent on my host computer to ingest logs. This is just a small dev environment so I have single node working. IF any of my ymls are incorrect please tell me what to fix them with. I enabled the elastic integration for endpoint security. I feel like I followed your user guide, but it's hard to follow. I really appreciate the help. I can't the agent to send any data in.

My agent fleet screen shows:

Green check - Elasticsearch security. Set xpack.security.enabled to true .
Green Check - API key service. Set xpack.security.authc.api_key.enabled to true
Green check - Kibana security. Set xpack.security.enabled to true
Red X - Kibana encryption key. Set xpack.encryptedSavedObjects.encryptionKey 

.env file

COMPOSE_PROJECT_NAME=es 
CERTS_DIR=/usr/share/elasticsearch/config/certificates 
VERSION=7.10.0

instances.yml

instances:
  - name: es01
    dns:
      - es01 
      - localhost
    ip:
      - 127.0.0.1
  - name: 'kib01'
    dns: 
      - kib01
      - localhost

create-certs.yml

version: '2.2'

services:
  create_certs:
    image: docker.elastic.co/elasticsearch/elasticsearch:${VERSION}
    container_name: create_certs
    command: >
      bash -c '
        yum install -y -q -e 0 unzip;
        if [[ ! -f /certs/bundle.zip ]]; then
          bin/elasticsearch-certutil cert --silent --pem --in config/certificates/instances.yml -out /certs/bundle.zip;
          unzip /certs/bundle.zip -d /certs; 
        fi;
        chown -R 1000:0 /certs
      '
    working_dir: /usr/share/elasticsearch
    volumes: 
      - certs:/certs
      - .:/usr/share/elasticsearch/config/certificates
    networks:
      - elastic        

volumes: 
  certs:
    driver: local

networks:
  elastic:
    driver: bridge

elastic-docker-tls.yml

version: '2.2'

services:
  es01:
    image: docker.elastic.co/elasticsearch/elasticsearch:${VERSION}
    container_name: es01
    environment:
      - node.name=es01
      - discovery.type=single-node
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"      
      - xpack.license.self_generated.type=trial # <1>
      - xpack.security.enabled=true      
      - xpack.security.http.ssl.enabled=true # <2>
      - xpack.security.http.ssl.key=$CERTS_DIR/es01/es01.key
      - xpack.security.http.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt
      - xpack.security.http.ssl.certificate=$CERTS_DIR/es01/es01.crt
      - xpack.security.transport.ssl.enabled=true # <3>
      - xpack.security.transport.ssl.verification_mode=certificate # <4>
      - xpack.security.transport.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt
      - xpack.security.transport.ssl.certificate=$CERTS_DIR/es01/es01.crt
      - xpack.security.transport.ssl.key=$CERTS_DIR/es01/es01.key
      - xpack.security.authc.api_key.enabled=true
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes: 
      - data01:/usr/share/elasticsearch/data
      - certs:$CERTS_DIR
    ports:
      - 9200:9200
      - 9300:9300
    networks:
      - elastic
      
    healthcheck:
      test: curl --cacert $CERTS_DIR/ca/ca.crt -s https://localhost:9200 >/dev/null; if [[ $$? == 52 ]]; then echo 0; else echo 1; fi
      interval: 30s
      timeout: 10s
      retries: 5

  kib01:
    image: docker.elastic.co/kibana/kibana:${VERSION}
    container_name: kib01
    depends_on: {"es01": {"condition": "service_healthy"}}
    ports:
      - 5601:5601    
    environment:
      SERVERNAME: localhost
      ELASTICSEARCH_URL: https://es01:9200
      ELASTICSEARCH_HOSTS: https://es01:9200
      ELASTICSEARCH_USERNAME: kibana_system
      ELASTICSEARCH_PASSWORD: tJ5Ft7uJWJnHxNhLq54I
      ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES: $CERTS_DIR/ca/ca.crt
      SERVER_SSL_ENABLED: "true"
      SERVER_SSL_KEY: $CERTS_DIR/kib01/kib01.key
      SERVER_SSL_CERTIFICATE: $CERTS_DIR/kib01/kib01.crt
      xpack.security.enabled: "true"
      xpack.fleet.enabled: "true"
      xpack.fleet.agents.kibana.host: https://kib01:5601
      xpack.fleet.agents.elasticsearch.host: https://es01:9200 
      xpack.fleet.agents.tlsCheckDisabled: "true"
      xpack.encryptedSavedObjects.encryptionKey: "5omething_@t_least_32_characters"
    volumes: 
      - certs:$CERTS_DIR
    networks:
      - elastic    
volumes:
  data01:
    driver: local
  certs:
    driver: local

networks: 
  elastic:
    driver: bridge  

Elastic-Agent.yml

id: 4a794f70-2e8e-11eb-96e3-a9b77a29895f
revision: 2
outputs:
  default:
    type: elasticsearch
    hosts:
      - 'https://localhost:9200'
    username: elastic
    password: OOEtrzoc0Wg7ABk9PsIB
agent:
  monitoring:
    enabled: true
    use_output: default
    logs: true
    metrics: true
inputs:
  - id: 6159f050-2e8e-11eb-96e3-a9b77a29895f
    name: system-1
    revision: 1
    type: logfile
    use_output: default
    meta:
      package:
        name: system
        version: 0.9.1
    data_stream:
      namespace: default
    streams:
      - id: logfile-system.auth
        data_stream:
          dataset: system.auth
          type: logs
        paths:
          - /var/log/auth.log*
          - /var/log/secure*
        exclude_files:
          - .gz$
        multiline:
          pattern: ^\s
          match: after
        processors:
          - add_locale: null
          - add_fields:
              target: ''
              fields:
                ecs.version: 1.5.0
      - id: logfile-system.syslog
        data_stream:
          dataset: system.syslog
          type: logs
        paths:
          - /var/log/messages*
          - /var/log/syslog*
        exclude_files:
          - .gz$
        multiline:
          pattern: ^\s
          match: after
        processors:
          - add_locale: null
          - add_fields:
              target: ''
              fields:
                ecs.version: 1.5.0
  - id: 6159f050-2e8e-11eb-96e3-a9b77a29895f
    name: system-1
    revision: 1
    type: system/metrics
    use_output: default
    meta:
      package:
        name: system
        version: 0.9.1
    data_stream:
      namespace: default
    streams:
      - id: system/metrics-system.cpu
        data_stream:
          dataset: system.cpu
          type: metrics
        metricsets:
          - cpu
        cpu.metrics:
          - percentages
          - normalized_percentages
        period: 10s
      - id: system/metrics-system.diskio
        data_stream:
          dataset: system.diskio
          type: metrics
        metricsets:
          - diskio
        diskio.include_devices: null
        period: 10s
      - id: system/metrics-system.filesystem
        data_stream:
          dataset: system.filesystem
          type: metrics
        metricsets:
          - filesystem
        period: 1m
        processors:
          - drop_event.when.regexp:
              system.filesystem.mount_point: ^/(sys|cgroup|proc|dev|etc|host|lib|snap)($|/)
      - id: system/metrics-system.fsstat
        data_stream:
          dataset: system.fsstat
          type: metrics
        metricsets:
          - fsstat
        period: 1m
        processors:
          - drop_event.when.regexp:
              system.fsstat.mount_point: ^/(sys|cgroup|proc|dev|etc|host|lib|snap)($|/)
      - id: system/metrics-system.load
        data_stream:
          dataset: system.load
          type: metrics
        metricsets:
          - load
        period: 10s
      - id: system/metrics-system.memory
        data_stream:
          dataset: system.memory
          type: metrics
        metricsets:
          - memory
        period: 10s
      - id: system/metrics-system.network
        data_stream:
          dataset: system.network
          type: metrics
        metricsets:
          - network
        period: 10s
        network.interfaces: null
      - id: system/metrics-system.process
        data_stream:
          dataset: system.process
          type: metrics
        metricsets:
          - process
        period: 10s
        process.include_top_n.by_cpu: 5
        process.include_top_n.by_memory: 5
        process.cmdline.cache.enabled: true
        process.cgroups.enabled: false
        process.include_cpu_ticks: false
        processes:
          - .*
      - id: system/metrics-system.process_summary
        data_stream:
          dataset: system.process_summary
          type: metrics
        metricsets:
          - process_summary
        period: 10s
      - id: system/metrics-system.socket_summary
        data_stream:
          dataset: system.socket_summary
          type: metrics
        metricsets:
          - socket_summary
        period: 10s
      - id: system/metrics-system.uptime
        data_stream:
          dataset: system.uptime
          type: metrics
        metricsets:
          - uptime
        period: 10s
  - id: 60488e10-2e93-11eb-a99f-61242b1d5693
    name: Windows1
    revision: 1
    type: endpoint
    use_output: default
    meta:
      package:
        name: endpoint
        version: 0.16.1
    data_stream:
      namespace: default
    artifact_manifest:
      manifest_version: 1.0.0
      schema_version: v1
      artifacts:
        endpoint-exceptionlist-macos-v1:
          encryption_algorithm: none
          decoded_sha256: d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658
          decoded_size: 14
          encoded_sha256: f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda
          encoded_size: 22
          relative_url: >-
            /api/endpoint/artifacts/download/endpoint-exceptionlist-macos-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658
          compression_algorithm: zlib
        endpoint-exceptionlist-windows-v1:
          encryption_algorithm: none
          decoded_sha256: d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658
          decoded_size: 14
          encoded_sha256: f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda
          encoded_size: 22
          relative_url: >-
            /api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658
          compression_algorithm: zlib
        endpoint-trustlist-macos-v1:
          encryption_algorithm: none
          decoded_sha256: d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658
          decoded_size: 14
          encoded_sha256: f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda
          encoded_size: 22
          relative_url: >-
            /api/endpoint/artifacts/download/endpoint-trustlist-macos-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658
          compression_algorithm: zlib
        endpoint-trustlist-windows-v1:
          encryption_algorithm: none
          decoded_sha256: d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658
          decoded_size: 14
          encoded_sha256: f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda
          encoded_size: 22
          relative_url: >-
            /api/endpoint/artifacts/download/endpoint-trustlist-windows-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658
          compression_algorithm: zlib
        endpoint-trustlist-linux-v1:
          encryption_algorithm: none
          decoded_sha256: d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658
          decoded_size: 14
          encoded_sha256: f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda
          encoded_size: 22
          relative_url: >-
            /api/endpoint/artifacts/download/endpoint-trustlist-linux-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658
          compression_algorithm: zlib
    policy:
      windows:
        events:
          dll_and_driver_load: true
          dns: true
          file: true
          network: true
          process: true
          registry: true
          security: true
        malware:
          mode: prevent
        logging:
          file: info
      mac:
        events:
          process: true
          file: true
          network: true
        malware:
          mode: prevent
        logging:
          file: info
      linux:
        events:
          process: true
          file: true
          network: true
        logging:
          file: info
    streams: []

Elastic-Agent Log

2020-11-24T13:36:54.510-0700    INFO    [composable.providers.docker]   docker/docker.go:40     Docker provider skipped, unable to connect: protocol not available
2020-11-24T13:36:54.538-0700    DEBUG   [composable.providers.kubernetes]       kubernetes/kubernetes.go:51     Kubernetes provider skipped
2020-11-24T13:36:54.672-0700    INFO    application/local_mode.go:168   Agent is stopped

Elastic Log in full, I ran out of room

2020-11-24T13:36:50.403-0700    INFO    warn/warn.go:18 The Elastic Agent is currently in BETA and should not be used in production
2020-11-24T13:36:50.760-0700    INFO    application/application.go:58   Detecting execution mode
2020-11-24T13:36:50.761-0700    INFO    application/application.go:67   Agent is managed locally
2020-11-24T13:36:52.614-0700    INFO    [composable]    composable/controller.go:44     EXPERIMENTAL - Inputs with variables are currently experimental and should not be used in production
2020-11-24T13:36:52.676-0700    DEBUG   application/emitter.go:146      Supported programs: Endpoint Security, Filebeat, Heartbeat, Metricbeat
2020-11-24T13:36:54.476-0700    DEBUG   [docker]        docker/client.go:48     Docker client will negotiate the API version on the first request.
2020-11-24T13:36:54.510-0700    INFO    [composable.providers.docker]   docker/docker.go:40     Docker provider skipped, unable to connect: protocol not available
2020-11-24T13:36:54.538-0700    DEBUG   [composable.providers.kubernetes]       kubernetes/kubernetes.go:51     Kubernetes provider skipped, unable to connect: unable to build kube config due to error: invalid configuration: no configuration has been provided, try setting KUBERNETES_MASTER environment variable
2020-11-24T13:36:54.554-0700    DEBUG   application/local_mode.go:136   Reloading of configuration is on, frequency is set to 10s
2020-11-24T13:36:54.557-0700    INFO    application/local_mode.go:158   Agent is starting
2020-11-24T13:36:54.672-0700    INFO    application/local_mode.go:168   Agent is stopped
2020-11-24T13:36:54.681-0700    DEBUG   application/periodic.go:60      Adding 1 file to watch
2020-11-24T13:36:54.771-0700    INFO    application/periodic.go:76      Configuration changes detected
2020-11-24T13:36:54.771-0700    DEBUG   application/periodic.go:82      Updated 1 files: C:\Windows\elastic-agent.yml
2020-11-24T13:36:54.906-0700    DEBUG   application/emitter.go:120      Converting single configuration into specific programs configuration
2020-11-24T13:36:55.004-0700    DEBUG   application/router.go:65        Creating stream: default
2020-11-24T13:36:55.014-0700    DEBUG   application/router.go:80        Streams default need to run config with ID wm3zqIvI and programs: Filebeat, Metricbeat, FLEET_MONITORING
2020-11-24T13:36:55.015-0700    INFO    stateresolver/stateresolver.go:47       New State ID is wm3zqIvI
2020-11-24T13:36:55.027-0700    INFO    stateresolver/stateresolver.go:48       Converging state requires execution of 3 step(s)
2020-11-24T13:36:55.028-0700    DEBUG   operation/operator.go:254       operator is looking for filebeat--7.10.0 in app collection: map[]
2020-11-24T13:36:55.034-0700    INFO    operation/operation_fetch.go:61 filebeat.7.10.0 already exists in C:\Windows\data\elastic-agent-1428d5\downloads\filebeat-7.10.0-windows-x86_64.zip. Skipping operation operation-fetch
2020-11-24T13:36:55.041-0700    DEBUG   operation/operator.go:234       running operation 'retryable block: operation-fetch operation-verify' for filebeat.7.10.0
2020-11-24T13:36:55.042-0700    INFO    operation/operation_fetch.go:61 filebeat.7.10.0 already exists in C:\Windows\data\elastic-agent-1428d5\downloads\filebeat-7.10.0-windows-x86_64.zip. Skipping operation operation-fetch
2020-11-24T13:36:55.042-0700    DEBUG   operation/operation_retryable.go:83     running operation 'operation-verify' of the block 'retryable block: operation-fetch operation-verify'
2020-11-24T13:36:57.899-0700    INFO    operation/operator.go:230       operation 'operation-install' skipped for filebeat.7.10.0
2020-11-24T13:36:57.899-0700    DEBUG   operation/operator.go:234       running operation 'operation-start' for filebeat.7.10.0
2020-11-24T13:36:58.862-0700    INFO    log/reporter.go:40      2020-11-24T13:36:58-07:00: type: 'STATE': sub_type: 'STARTING' message: Application: filebeat--7.10.0[c267a1a5-373c-4674-b389-d79c4c231d8e]: State changed to STARTING: Starting
2020-11-24T13:36:58.995-0700    DEBUG   operation/operator.go:234       running operation 'operation-config' for filebeat.7.10.0
2020-11-24T13:36:58.998-0700    DEBUG   operation/operator.go:254       operator is looking for metricbeat--7.10.0 in app collection: map[filebeat--7.10.0:0xc0006999e0]
2020-11-24T13:36:59.024-0700    INFO    operation/operation_fetch.go:61 metricbeat.7.10.0 already exists in C:\Windows\data\elastic-agent-1428d5\downloads\metricbeat-7.10.0-windows-x86_64.zip. Skipping operation operation-fetch
2020-11-24T13:36:59.026-0700    DEBUG   operation/operator.go:234       running operation 'retryable block: operation-fetch operation-verify' for metricbeat.7.10.0
2020-11-24T13:36:59.029-0700    INFO    operation/operation_fetch.go:61 metricbeat.7.10.0 already exists in C:\Windows\data\elastic-agent-1428d5\downloads\metricbeat-7.10.0-windows-x86_64.zip. Skipping operation operation-fetch
2020-11-24T13:36:59.038-0700    DEBUG   operation/operation_retryable.go:83     running operation 'operation-verify' of the block 'retryable block: operation-fetch operation-verify'
2020-11-24T13:37:01.086-0700    INFO    operation/operator.go:230       operation 'operation-install' skipped for metricbeat.7.10.0
2020-11-24T13:37:01.086-0700    DEBUG   operation/operator.go:234       running operation 'operation-start' for metricbeat.7.10.0
2020-11-24T13:37:02.326-0700    INFO    log/reporter.go:40      2020-11-24T13:37:02-07:00: type: 'STATE': sub_type: 'STARTING' message: Application: metricbeat--7.10.0[c267a1a5-373c-4674-b389-d79c4c231d8e]: State changed to STARTING: Starting
2020-11-24T13:37:02.383-0700    DEBUG   operation/operator.go:234       running operation 'operation-config' for metricbeat.7.10.0
2020-11-24T13:37:02.405-0700    DEBUG   operation/monitoring.go:276     monitoring configuration generated for filebeat: map[filebeat:map[inputs:[map[index:logs-elastic_agent-default json:map[keys_under_root:true message_key:message overwrite_keys:true] paths:[C:\Windows\data\elastic-agent-1428d5\logs\elastic-agent-json.log] processors:[map[add_fields:map[fields:map[dataset:elastic_agent namespace:default type:logs] target:data_stream]] map[add_fields:map[fields:map[dataset:elastic_agent] target:event]] map[add_fields:map[fields:map[id:c267a1a5-373c-4674-b389-d79c4c231d8e snapshot:false version:7.10.0] target:elastic_agent]]] type:log] map[index:logs-elastic_agent.filebeat-default json:map[keys_under_root:true message_key:message overwrite_keys:true] paths:[C:\Windows\data\elastic-agent-1428d5\logs\default\filebeat-json.log] processors:[map[add_fields:map[fields:map[dataset:elastic_agent.filebeat namespace:default type:logs] target:data_stream]] map[add_fields:map[fields:map[dataset:elastic_agent.filebeat] target:event]] map[add_fields:map[fields:map[id:c267a1a5-373c-4674-b389-d79c4c231d8e snapshot:false version:7.10.0] target:elastic_agent]]] type:log] map[index:logs-elastic_agent.metricbeat-default json:map[keys_under_root:true message_key:message overwrite_keys:true] paths:[C:\Windows\data\elastic-agent-1428d5\logs\default\metricbeat-json.log] processors:[map[add_fields:map[fields:map[dataset:elastic_agent.metricbeat namespace:default type:logs] target:data_stream]] map[add_fields:map[fields:map[dataset:elastic_agent.metricbeat] target:event]] map[add_fields:map[fields:map[id:c267a1a5-373c-4674-b389-d79c4c231d8e snapshot:false version:7.10.0] target:elastic_agent]]] type:log]]] output:map[elasticsearch:map[hosts:[http://localhost:9200] password:OOEtrzoc0Wg7ABk9PsIB type:elasticsearch username:elastic]]]
2020-11-24T13:37:02.421-0700    DEBUG   operation/monitoring.go:335     monitoring configuration generated for metricbeat: map[metricbeat:map[modules:[map[hosts:[http+npipe:///default-filebeat] index:metrics-elastic_agent.filebeat-default metricsets:[stats state] module:beat period:10s processors:[map[add_fields:map[fields:map[dataset:elastic_agent.filebeat namespace:default type:metrics] target:data_stream]] map[add_fields:map[fields:map[dataset:elastic_agent.filebeat] target:event]] map[add_fields:map[fields:map[id:c267a1a5-373c-4674-b389-d79c4c231d8e snapshot:false version:7.10.0] target:elastic_agent]]]] map[hosts:[http+npipe:///default-metricbeat] index:metrics-elastic_agent.metricbeat-default metricsets:[stats state] module:beat period:10s processors:[map[add_fields:map[fields:map[dataset:elastic_agent.metricbeat namespace:default type:metrics] target:data_stream]] map[add_fields:map[fields:map[dataset:elastic_agent.metricbeat] target:event]] map[add_fields:map[fields:map[id:c267a1a5-373c-4674-b389-d79c4c231d8e snapshot:false version:7.10.0] target:elastic_agent]]]]]] output:map[elasticsearch:map[hosts:[http://localhost:9200] password:OOEtrzoc0Wg7ABk9PsIB type:elasticsearch username:elastic]]]
2020-11-24T13:37:02.428-0700    DEBUG   operation/operator.go:254       operator is looking for filebeat--7.10.0--36643631373035623733363936343635 in app collection: map[filebeat--7.10.0:0xc0006999e0 metricbeat--7.10.0:0xc0002fc5a0]

As your encryption key is red for Kibana, I guess something is not right with your compose setup. Can you have a look at this thread here that had exactly the same issue and got resolved? Kibana unable to configure fleet access Let me know if this doesn't help.

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.