Good afternoon, we are attempting to deploy a Docker Container Based Elastic Stack, with a provided publicly signed CA certificate but on each enroll we are coming up with the following:
BlockquoteError: enrollment failed: exit status 1
For help, please see our troubleshooting guide at Troubleshoot common problems | Fleet and Elastic Agent Guide [8.4] | Elastic
Policy selected for enrollment: fleet-server-policy
Error: certificate private key is required when certificate provided
For help, please see our troubleshooting guide at Troubleshoot common problems | Fleet and Elastic Agent Guide [8.4] | Elastic
Error: enrollment failed: exit status 1
For help, please see our troubleshooting guide at Troubleshoot common problems | Fleet and Elastic Agent Guide [8.4] | Elastic
Policy selected for enrollment: fleet-server-policy
Error: certificate private key is required when certificate provided
For help, please see our troubleshooting guide at Troubleshoot common problems | Fleet and Elastic Agent Guide [8.4] | Elastic
Error: enrollment failed: exit status 1
For help, please see our troubleshooting guide at Troubleshoot common problems | Fleet and Elastic Agent Guide [8.4] | Elastic
Policy selected for enrollment: fleet-server-policy
Error: certificate private key is required when certificate provided
For help, please see our troubleshooting guide at Troubleshoot common problems | Fleet and Elastic Agent Guide [8.4] | Elastic
Error: enrollment failed: exit status 1
Blockquote
The Docker compose files that was used for Fleet
version: "3"
services:
fleet_container_name:
build: .
container_name: fleet_container_name
restart: always
user: root
environment:
- FLEET_SERVER_POLICY_ID=fleet-server-policy
# - FLEET_ENROLL=1
# - FLEET_URL= container_name: fleet_container_name
- FLEET_SERVER_ENABLE=true
- FLEET_SERVER_ELASTICSEARCH_HOST=https://elasticsearch_container_name:9200
- ELASTICSEARCH_CA=/usr/share/elastic-agent/certs/ca.crt
- FLEET_SERVER_CERT=/usr/share/elastic-agent/certs/fleet/wildcard/wildcard.crt
- FLEET_SEVER_CERT_KEY=/usr/share/elastic-agent/certs/fleet/wildcard/wildcard.key
- FLEET_SERVER_SERVICE_TOKEN=AAEAAWVsYXN0aWMvZmxlZXQtc2VydmVyL3Rva2VuLTE2NjY5NTEyNTcxNDE6Wk11V1k1N3BUdm1BaWgtRHpzTVFhdw
extra_hosts:
- "elasticsearch_hostname:ipv6_address"
networks:
ipv6:
ipv4_address: ipv4address
ipv6_address: ipv6address
volumes:
prod_deployment-elastic_certs:
external: true
networks:
ipv6:
external: true
Docker Compose File user for Elasticsearch + Kibana
version: "2.2"
services:
setup:
image: image_registry:5000/elasticsearch:${STACK_VERSION}
extra_hosts:
- "es01:ipv6_address"
networks:
ipv6:
ipv4_address: ipv4_address
ipv6_address: ipv6_address
user: "0"
command: >
bash -c '
if [ x${ELASTIC_PASSWORD} == x ]; then
echo "Set the ELASTIC_PASSWORD environment variable in the .env file";
exit 1;
elif [ x${KIBANA_PASSWORD} == x ]; then
echo "Set the KIBANA_PASSWORD environment variable in the .env file";
exit 1;
fi;
if [ ! -f config/certs/ca.zip ]; then
echo "Creating CA";
bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip;
unzip config/certs/ca.zip -d config/certs;
fi;
if [ ! -f config/certs/certs.zip ]; then
echo "Creating certs";
echo -ne \
"instances:\n"\
" - name: es01\n"\
" dns:\n"\
" - es01\n"\
" - localhost\n"\
" ip:\n"\
" - 127.0.0.1\n"\
> config/certs/instances.yml;
bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key;
unzip config/certs/certs.zip -d config/certs;
fi;
echo "Setting file permissions"
chown -R root:root config/certs;
find . -type d -exec chmod 750 \{\} \;;
find . -type f -exec chmod 640 \{\} \;;
echo "Waiting for Elasticsearch availability";
until curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done;
echo "Setting kibana_system password";
until curl -s -X POST --cacert config/certs/ca/ca.crt -u elastic:${ELASTIC_PASSWORD} -H "Content-Type: application/json" https://es01:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done;
echo "All done!";
'
# healthcheck:
# test: ["CMD-SHELL", "[ -f config/certs/es01/es01.crt ]"]
# interval: 1s
# timeout: 5s
# retries: 120
elasticsearch_hostname:
# depends_on:
# setup:
# condition: service_healthy
build: .
container_name: elasticsearch_ container_name
volumes:
- esdata01:/usr/share/elasticsearch/data
networks:
ipv6:
ipv4_address: ipv4address
ipv6_address: ipv6address
environment:
- node.name=node_name
- cluster.name=${CLUSTER_NAME}
- cluster.initial_master_nodes=node_name
- ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
- SERVER_HOST="::"
- bootstrap.memory_lock=true
- xpack.security.enabled=true
- xpack.security.http.ssl.enabled=true
- xpack.security.http.ssl.key=certs/wildcard/wildcard.key
- xpack.security.http.ssl.certificate=certs/wildcard/wildcard.crt
- xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
- xpack.security.http.ssl.verification_mode=certificate
- xpack.security.transport.ssl.enabled=true
- xpack.security.transport.ssl.key=certs/wildcard/wildcard.key
- xpack.security.transport.ssl.certificate=certs/wildcard/wildcard.crt
- xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
- xpack.security.transport.ssl.verification_mode=certificate
- xpack.license.self_generated.type=${LICENSE}
# mem_limit: ${MEM_LIMIT}
ulimits:
memlock:
soft: -1
hard: -1
# healthcheck:
# test:
# [
# "CMD-SHELL",
# "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication #credentials'",
# ]
# interval: 10s
# timeout: 10s
# retries: 120
kibana01_hostaname:
# depends_on:
# es01:
# condition: service_healthy
build: kibana/.
container_name: kibana_container_name
# image: image_registry_name:5000/kibana:${STACK_VERSION}
volumes:
- kibanadata:/usr/share/kibana/data
extra_hosts:
- "elasticsearch_hostname:ipv6address"
networks:
ipv6:
ipv4_address: ipv4address
ipv6_address: ipv6address
environment:
- ELASTICSEARCH_HOSTS=https://elasticsearch_hostname:9200
- SERVER_HOST="::"
- ELASTICSEARCH_USERNAME=kibana_system
- ELASTICSEARCH_PASSWORD=${KIBANA_PASSWORD}
- ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt
- SERVER_PUBLICBASEURL=https://elasticsearch_hostaname
# mem_limit: ${MEM_LIMIT}
# healthcheck:
# test:
# [
# "CMD-SHELL",
# "curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'",
# ]
# interval: 10s
# timeout: 10s
# retries: 120
volumes:
certs:
driver: local
esdata01:
driver: local
kibanadata:
driver: local
networks:
ipv6:
external: true
We were able to perform an insecure enrollment with self-signed certificates and managed to pass the data trough to elastic by turning off SSL verification, but that is not suitable for a production deployment. Any advice would be very much appreciated. As an additional note, yes the certificates used are wildcard, however both elasticsearch and kibana seem to properly recognize and accept a wildcard certificate however in case of fleet is keeps asking for a key when a certificate is provided but as it can be seen from the compose file the key has been provided in the deployment on a mounted volume.