I want to use docker to host Elasticsearch and Kibana. I have that working with xpac. I will include my yml files below. I want to enable the elastic-agent on my host computer to ingest logs. This is just a small dev environment so I have single node working. IF any of my ymls are incorrect please tell me what to fix them with. I enabled the elastic integration for endpoint security. I feel like I followed your user guide, but it's hard to follow. I really appreciate the help. I can't the agent to send any data in.
My agent fleet screen shows:
Green check - Elasticsearch security. Set xpack.security.enabled to true .
Green Check - API key service. Set xpack.security.authc.api_key.enabled to true
Green check - Kibana security. Set xpack.security.enabled to true
Red X - Kibana encryption key. Set xpack.encryptedSavedObjects.encryptionKey
.env file
COMPOSE_PROJECT_NAME=es
CERTS_DIR=/usr/share/elasticsearch/config/certificates
VERSION=7.10.0
instances.yml
instances:
- name: es01
dns:
- es01
- localhost
ip:
- 127.0.0.1
- name: 'kib01'
dns:
- kib01
- localhost
create-certs.yml
version: '2.2'
services:
create_certs:
image: docker.elastic.co/elasticsearch/elasticsearch:${VERSION}
container_name: create_certs
command: >
bash -c '
yum install -y -q -e 0 unzip;
if [[ ! -f /certs/bundle.zip ]]; then
bin/elasticsearch-certutil cert --silent --pem --in config/certificates/instances.yml -out /certs/bundle.zip;
unzip /certs/bundle.zip -d /certs;
fi;
chown -R 1000:0 /certs
'
working_dir: /usr/share/elasticsearch
volumes:
- certs:/certs
- .:/usr/share/elasticsearch/config/certificates
networks:
- elastic
volumes:
certs:
driver: local
networks:
elastic:
driver: bridge
elastic-docker-tls.yml
version: '2.2'
services:
es01:
image: docker.elastic.co/elasticsearch/elasticsearch:${VERSION}
container_name: es01
environment:
- node.name=es01
- discovery.type=single-node
- bootstrap.memory_lock=true
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
- xpack.license.self_generated.type=trial # <1>
- xpack.security.enabled=true
- xpack.security.http.ssl.enabled=true # <2>
- xpack.security.http.ssl.key=$CERTS_DIR/es01/es01.key
- xpack.security.http.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt
- xpack.security.http.ssl.certificate=$CERTS_DIR/es01/es01.crt
- xpack.security.transport.ssl.enabled=true # <3>
- xpack.security.transport.ssl.verification_mode=certificate # <4>
- xpack.security.transport.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt
- xpack.security.transport.ssl.certificate=$CERTS_DIR/es01/es01.crt
- xpack.security.transport.ssl.key=$CERTS_DIR/es01/es01.key
- xpack.security.authc.api_key.enabled=true
ulimits:
memlock:
soft: -1
hard: -1
volumes:
- data01:/usr/share/elasticsearch/data
- certs:$CERTS_DIR
ports:
- 9200:9200
- 9300:9300
networks:
- elastic
healthcheck:
test: curl --cacert $CERTS_DIR/ca/ca.crt -s https://localhost:9200 >/dev/null; if [[ $$? == 52 ]]; then echo 0; else echo 1; fi
interval: 30s
timeout: 10s
retries: 5
kib01:
image: docker.elastic.co/kibana/kibana:${VERSION}
container_name: kib01
depends_on: {"es01": {"condition": "service_healthy"}}
ports:
- 5601:5601
environment:
SERVERNAME: localhost
ELASTICSEARCH_URL: https://es01:9200
ELASTICSEARCH_HOSTS: https://es01:9200
ELASTICSEARCH_USERNAME: kibana_system
ELASTICSEARCH_PASSWORD: tJ5Ft7uJWJnHxNhLq54I
ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES: $CERTS_DIR/ca/ca.crt
SERVER_SSL_ENABLED: "true"
SERVER_SSL_KEY: $CERTS_DIR/kib01/kib01.key
SERVER_SSL_CERTIFICATE: $CERTS_DIR/kib01/kib01.crt
xpack.security.enabled: "true"
xpack.fleet.enabled: "true"
xpack.fleet.agents.kibana.host: https://kib01:5601
xpack.fleet.agents.elasticsearch.host: https://es01:9200
xpack.fleet.agents.tlsCheckDisabled: "true"
xpack.encryptedSavedObjects.encryptionKey: "5omething_@t_least_32_characters"
volumes:
- certs:$CERTS_DIR
networks:
- elastic
volumes:
data01:
driver: local
certs:
driver: local
networks:
elastic:
driver: bridge
Elastic-Agent.yml
id: 4a794f70-2e8e-11eb-96e3-a9b77a29895f
revision: 2
outputs:
default:
type: elasticsearch
hosts:
- 'https://localhost:9200'
username: elastic
password: OOEtrzoc0Wg7ABk9PsIB
agent:
monitoring:
enabled: true
use_output: default
logs: true
metrics: true
inputs:
- id: 6159f050-2e8e-11eb-96e3-a9b77a29895f
name: system-1
revision: 1
type: logfile
use_output: default
meta:
package:
name: system
version: 0.9.1
data_stream:
namespace: default
streams:
- id: logfile-system.auth
data_stream:
dataset: system.auth
type: logs
paths:
- /var/log/auth.log*
- /var/log/secure*
exclude_files:
- .gz$
multiline:
pattern: ^\s
match: after
processors:
- add_locale: null
- add_fields:
target: ''
fields:
ecs.version: 1.5.0
- id: logfile-system.syslog
data_stream:
dataset: system.syslog
type: logs
paths:
- /var/log/messages*
- /var/log/syslog*
exclude_files:
- .gz$
multiline:
pattern: ^\s
match: after
processors:
- add_locale: null
- add_fields:
target: ''
fields:
ecs.version: 1.5.0
- id: 6159f050-2e8e-11eb-96e3-a9b77a29895f
name: system-1
revision: 1
type: system/metrics
use_output: default
meta:
package:
name: system
version: 0.9.1
data_stream:
namespace: default
streams:
- id: system/metrics-system.cpu
data_stream:
dataset: system.cpu
type: metrics
metricsets:
- cpu
cpu.metrics:
- percentages
- normalized_percentages
period: 10s
- id: system/metrics-system.diskio
data_stream:
dataset: system.diskio
type: metrics
metricsets:
- diskio
diskio.include_devices: null
period: 10s
- id: system/metrics-system.filesystem
data_stream:
dataset: system.filesystem
type: metrics
metricsets:
- filesystem
period: 1m
processors:
- drop_event.when.regexp:
system.filesystem.mount_point: ^/(sys|cgroup|proc|dev|etc|host|lib|snap)($|/)
- id: system/metrics-system.fsstat
data_stream:
dataset: system.fsstat
type: metrics
metricsets:
- fsstat
period: 1m
processors:
- drop_event.when.regexp:
system.fsstat.mount_point: ^/(sys|cgroup|proc|dev|etc|host|lib|snap)($|/)
- id: system/metrics-system.load
data_stream:
dataset: system.load
type: metrics
metricsets:
- load
period: 10s
- id: system/metrics-system.memory
data_stream:
dataset: system.memory
type: metrics
metricsets:
- memory
period: 10s
- id: system/metrics-system.network
data_stream:
dataset: system.network
type: metrics
metricsets:
- network
period: 10s
network.interfaces: null
- id: system/metrics-system.process
data_stream:
dataset: system.process
type: metrics
metricsets:
- process
period: 10s
process.include_top_n.by_cpu: 5
process.include_top_n.by_memory: 5
process.cmdline.cache.enabled: true
process.cgroups.enabled: false
process.include_cpu_ticks: false
processes:
- .*
- id: system/metrics-system.process_summary
data_stream:
dataset: system.process_summary
type: metrics
metricsets:
- process_summary
period: 10s
- id: system/metrics-system.socket_summary
data_stream:
dataset: system.socket_summary
type: metrics
metricsets:
- socket_summary
period: 10s
- id: system/metrics-system.uptime
data_stream:
dataset: system.uptime
type: metrics
metricsets:
- uptime
period: 10s
- id: 60488e10-2e93-11eb-a99f-61242b1d5693
name: Windows1
revision: 1
type: endpoint
use_output: default
meta:
package:
name: endpoint
version: 0.16.1
data_stream:
namespace: default
artifact_manifest:
manifest_version: 1.0.0
schema_version: v1
artifacts:
endpoint-exceptionlist-macos-v1:
encryption_algorithm: none
decoded_sha256: d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658
decoded_size: 14
encoded_sha256: f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda
encoded_size: 22
relative_url: >-
/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658
compression_algorithm: zlib
endpoint-exceptionlist-windows-v1:
encryption_algorithm: none
decoded_sha256: d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658
decoded_size: 14
encoded_sha256: f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda
encoded_size: 22
relative_url: >-
/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658
compression_algorithm: zlib
endpoint-trustlist-macos-v1:
encryption_algorithm: none
decoded_sha256: d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658
decoded_size: 14
encoded_sha256: f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda
encoded_size: 22
relative_url: >-
/api/endpoint/artifacts/download/endpoint-trustlist-macos-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658
compression_algorithm: zlib
endpoint-trustlist-windows-v1:
encryption_algorithm: none
decoded_sha256: d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658
decoded_size: 14
encoded_sha256: f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda
encoded_size: 22
relative_url: >-
/api/endpoint/artifacts/download/endpoint-trustlist-windows-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658
compression_algorithm: zlib
endpoint-trustlist-linux-v1:
encryption_algorithm: none
decoded_sha256: d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658
decoded_size: 14
encoded_sha256: f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda
encoded_size: 22
relative_url: >-
/api/endpoint/artifacts/download/endpoint-trustlist-linux-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658
compression_algorithm: zlib
policy:
windows:
events:
dll_and_driver_load: true
dns: true
file: true
network: true
process: true
registry: true
security: true
malware:
mode: prevent
logging:
file: info
mac:
events:
process: true
file: true
network: true
malware:
mode: prevent
logging:
file: info
linux:
events:
process: true
file: true
network: true
logging:
file: info
streams: []
Elastic-Agent Log
2020-11-24T13:36:54.510-0700 INFO [composable.providers.docker] docker/docker.go:40 Docker provider skipped, unable to connect: protocol not available
2020-11-24T13:36:54.538-0700 DEBUG [composable.providers.kubernetes] kubernetes/kubernetes.go:51 Kubernetes provider skipped
2020-11-24T13:36:54.672-0700 INFO application/local_mode.go:168 Agent is stopped