Hi,
I have deployed 3 node Elastic & Kibana on a 3 Node Docker Swarm.
I could see the Cluster is up & running with 3 node elastic & 1 Kibana.
Now, I'm trying to Provide Elasticsearch & Kibana with TLS/SSL authentication.
I have generated the certs using below commands:
bin/elasticsearch-certutil ca &
bin/elasticsearch-certutil cert --ca elastic-stack-ca.p12
bin/elasticsearch-certutil cert --ca config/certs/elastic-stack-ca.p12 -name "CN=something,OU=Consulting Team,DC=mydomain,DC=com"
openssl pkcs12 -in client.p12 -nocerts -nodes > client.key
openssl pkcs12 -in client.p12 -clcerts -nokeys > client.cer
openssl pkcs12 -in client.p12 -cacerts -nokeys -chain > client-ca.cer
I have copied all these certificates to my local VM & built a docker-compose file as below:
version: "3.7"
services:
es01:
image: docker.elastic.co/elasticsearch/elasticsearch:7.5.0
container_name: es01
environment:
- node.name=es01
- cluster.name=elk-cluster
- discovery.seed_hosts=es02,es03
- cluster.initial_master_nodes=es01,es02,es03
- bootstrap.memory_lock=false
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
- xpack.security.enabled=true
- xpack.security.transport.ssl.enabled=true
- xpack.security.transport.ssl.keystore.type=PKCS12
- xpack.security.transport.ssl.verification_mode=certificate
- xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/certs/elastic-stack-ca.p12
- xpack.security.transport.ssl.truststore.path=/usr/share/elasticsearch/config/certs/elastic-stack-ca.p12
- xpack.security.transport.ssl.truststore.type=PKCS12
- xpack.security.http.ssl.enabled=true
- xpack.security.http.ssl.keystore.path=/usr/share/elasticsearch/config/certs/elastic-certificates.p12
- xpack.security.http.ssl.truststore.path=/usr/share/elasticsearch/config/certs/elastic-certificates.p12
- xpack.security.http.ssl.client_authentication=optional
networks:
- dockerelk
ports:
- "9200:9200"
- "9300:9300"
volumes:
- ./elasticsearch/config/certs/elastic-stack-ca.p12:/usr/share/elasticsearch/config/certs/elastic-stack-ca.p12
- ./elasticsearch/config/certs/elastic-certificates.p12:/usr/share/elasticsearch/config/certs/elastic-certificates.p12
- x2es01:/usr/share/elasticsearch/data
extra_hosts:
- "host1:192.168.x.x"
- "host2:192.168.x.x"
- "host3:192.168.x.x"
deploy:
replicas: 1
es02:
image: docker.elastic.co/elasticsearch/elasticsearch:7.5.0
container_name: es02
environment:
- node.name=es02
- cluster.name=elk-cluster
- discovery.seed_hosts=es01,es03
- cluster.initial_master_nodes=es01,es02,es03
- bootstrap.memory_lock=false
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
- xpack.security.enabled=true
- xpack.security.transport.ssl.enabled=true
- xpack.security.transport.ssl.keystore.type=PKCS12
- xpack.security.transport.ssl.verification_mode=certificate
- xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/certs/elastic-stack-ca.p12
- xpack.security.transport.ssl.truststore.path=/usr/share/elasticsearch/config/certs/elastic-stack-ca.p12
- xpack.security.transport.ssl.truststore.type=PKCS12
- xpack.security.http.ssl.enabled=true
- xpack.security.http.ssl.keystore.path=/usr/share/elasticsearch/config/certs/elastic-certificates.p12
- xpack.security.http.ssl.truststore.path=/usr/share/elasticsearch/config/certs/elastic-certificates.p12
- xpack.security.http.ssl.client_authentication=optional
networks:
- dockerelk
ports:
- "9201:9200"
- "9301:9300"
volumes:
- ./elasticsearch/config/certs/elastic-stack-ca.p12:/usr/share/elasticsearch/config/certs/elastic-stack-ca.p12
- ./elasticsearch/config/certs/elastic-certificates.p12:/usr/share/elasticsearch/config/certs/elastic-certificates.p12
- x2es02:/usr/share/elasticsearch/data
extra_hosts:
- "host1:192.168.x.x"
- "host2:192.168.x.x"
- "host3:192.168.x.x"
deploy:
replicas: 1
es03:
image: docker.elastic.co/elasticsearch/elasticsearch:7.5.0
container_name: es03
environment:
- node.name=es03
- cluster.name=elk-cluster
- discovery.seed_hosts=es01,es02
- cluster.initial_master_nodes=es01,es02,es03
- bootstrap.memory_lock=false
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
- xpack.security.enabled=true
- xpack.security.transport.ssl.enabled=true
- xpack.security.transport.ssl.keystore.type=PKCS12
- xpack.security.transport.ssl.verification_mode=certificate
- xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/certs/elastic-stack-ca.p12
- xpack.security.transport.ssl.truststore.path=/usr/share/elasticsearch/config/certs/elastic-stack-ca.p12
- xpack.security.transport.ssl.truststore.type=PKCS12
- xpack.security.http.ssl.enabled=true
- xpack.security.http.ssl.keystore.path=/usr/share/elasticsearch/config/certs/elastic-certificates.p12
- xpack.security.http.ssl.truststore.path=/usr/share/elasticsearch/config/certs/elastic-certificates.p12
- xpack.security.http.ssl.client_authentication=optional
networks:
- dockerelk
ports:
- "9202:9200"
- "9302:9300"
volumes:
- ./elasticsearch/config/certs/elastic-stack-ca.p12:/usr/share/elasticsearch/config/certs/elastic-stack-ca.p12
- ./elasticsearch/config/certs/elastic-certificates.p12:/usr/share/elasticsearch/config/certs/elastic-certificates.p12
- x2es03:/usr/share/elasticsearch/data
extra_hosts:
- "host1:192.168.x.x"
- "host2:192.168.x.x"
- "host3:192.168.x.x"
deploy:
replicas: 1
kibana01:
image: docker.elastic.co/kibana/kibana:7.5.0
container_name: kib01
environment:
- ELASTICSEARCH_URL=https://192.168.x.x:9200
- ELASTICSEARCH_HOSTS=https://192.168.x.x:9200
- xpack.security.enabled=true
- ELASTICSEARCH_USERNAME=kibana
- ELASTICSEARCH_PASSWORD=kibana
- elasticsearch.ssl.certificate=/usr/share/kibana/config/certs/client.cer
- elasticsearch.ssl.key=/usr/share/kibana/config/certs/client.key
- elasticsearch.ssl.certificateAuthorities=/usr/share/kibana/config/certs/client-ca.cer
- elasticsearch.ssl.verificationMode=certificate
networks:
- dockerelk
ports:
- "5601:5601"
volumes:
- ./kibana/config/certs/client.cer:/usr/share/kibana/config/certs/client.cer
- ./kibana/config/certs/client-ca.cer:/usr/share/kibana/config/certs/client-ca.cer
- ./kibana/config/certs/client.key:/usr/share/kibana/config/certs/client.key
- x2kibgit:/usr/share/kibana
networks:
dockerelk:
external: true
volumes:
x2es01:
x2es02:
x2es03:
x2kibgit:
Also, I have created the dockernet network prior running the compose file.
As, I was unable to set passwords using ./elasticsearch-setup-passwords interactive
So I created a custom user & password using below command.
bin/elasticsearch-users useradd elkuser -p elkpass -r superuser.
So, Now I'am able to login lo all my 3 ES instances using https://192.168.x.x:9200 with elkuser & elkpass command.
Also, I see my cluster ES is healthy Now.
But, not sure why Kibana is not up & running when using TLS/SSL connection.
In my Kibana docker container logs I see that following Warnings:
{"type":"log","@timestamp":"2020-05-22T14:37:19Z","tags":["warning","elasticsearch","admin"],"pid":6,"message":"No living connections"}
{"type":"log","@timestamp":"2020-05-22T14:37:20Z","tags":["warning","elasticsearch","data"],"pid":6,"message":"Unable to revive connection: https://192.168.x.x:9200/"}
{"type":"log","@timestamp":"2020-05-22T14:37:20Z","tags":["warning","elasticsearch","data"],"pid":6,"message":"No living connections"}
{"type":"log","@timestamp":"2020-05-22T14:37:20Z","tags":["warning","plugins","licensing"],"pid":6,"message":"License information could not be obtained from Elasticsearch for the [data] cluster. Error: No Living connections"}
{"type":"log","@timestamp":"2020-05-22T14:37:21Z","tags":["warning","elasticsearch","admin"],"pid":6,"message":"Unable to revive connection: https://192.168.x.x:9200/"}
{"type":"log","@timestamp":"2020-05-22T14:37:21Z","tags":["warning","elasticsearch","admin"],"pid":6,"message":"No living connections"}
{"type":"log","@timestamp":"2020-05-22T14:37:21Z","tags":["warning","elasticsearch","admin"],"pid":6,"message":"Unable to revive connection: https://192.168.x.x:9200/"}
Could someone of you help me in understanding the root cause of this issue & the meaning for these logs.. if either it's related to docker network or swarm setup or it's issue with certificates.
Any help would be appreciated.
Thanks in Advance