Kibana TLS to Elastic cluster in docker-compose: self signed certificate in certificate chain

Elastic Stack Kibana
1

Hi there, I need some help with this docker-compose I am creating, with which I'd like to bring up an Elastic Stack with the security features set.

  1. docker-compose.yml
version: '3.5'

services:
  elasticsearch_certificates:
    container_name: elasticsearch_certificates
    image: docker.elastic.co/elasticsearch/elasticsearch:$ELASTIC_VERSION
    environment:
      CA_PASSWORD: $CA_PASSWORD
      ELASTICSEARCH01_CERT_PASSWORD: $ELASTICSEARCH01_CERT_PASSWORD
      ELASTICSEARCH02_CERT_PASSWORD: $ELASTICSEARCH02_CERT_PASSWORD
      ELASTICSEARCH03_CERT_PASSWORD: $ELASTICSEARCH03_CERT_PASSWORD
      KIBANA_CERT_PASSWORD: $KIBANA_CERT_PASSWORD
    command: >
      bash -c '
        if [[ ! -f /certs/elasticsearch01.p12 || \
              ! -f /certs/elasticsearch02.p12 || \
              ! -f /certs/elasticsearch03.p12 || \
              ! -f /certs/kibana.zip ]]; then
          echo "Removing certificates" &&
          rm -rf /certs/* &&
          
          echo "Generating CA" &&
          bin/elasticsearch-certutil ca --silent --pass ${CA_PASSWORD} --pem --out /certs/ca.zip &&
          unzip /certs/ca.zip -d /certs &&
          
          echo "Generating certificate for Elasticsearch01" &&
          bin/elasticsearch-certutil cert --silent --ca-cert /certs/ca/ca.crt --ca-key /certs/ca/ca.key --ca-pass ${CA_PASSWORD} --pass ${ELASTICSEARCH01_CERT_PASSWORD} --dns elasticsearch01 --out /certs/elasticsearch01.p12 &&
          
          echo "Generating certificate for Elasticsearch02" &&
          bin/elasticsearch-certutil cert --silent --ca-cert /certs/ca/ca.crt --ca-key /certs/ca/ca.key --ca-pass ${CA_PASSWORD} --pass ${ELASTICSEARCH02_CERT_PASSWORD} --dns elasticsearch02 --out /certs/elasticsearch02.p12 &&
          
          echo "Generating certificate for Elasticsearch03" &&
          bin/elasticsearch-certutil cert --silent --ca-cert /certs/ca/ca.crt --ca-key /certs/ca/ca.key --ca-pass ${CA_PASSWORD} --pass ${ELASTICSEARCH02_CERT_PASSWORD} --dns elasticsearch03 --out /certs/elasticsearch03.p12 &&
          
          echo "Generating certificate for Kibana" &&
          bin/elasticsearch-certutil cert --silent --ca-cert /certs/ca/ca.crt --ca-key /certs/ca/ca.key --ca-pass ${CA_PASSWORD} --pass ${KIBANA_CERT_PASSWORD} --pem --dns kibana --out /certs/kibana.zip &&
          unzip /certs/kibana.zip -d /certs &&
          mv /certs/instance/instance.crt /certs/kibana.crt &&
          mv /certs/instance/instance.key /certs/kibana.key &&
          rm -rf /certs/instance &&
          
          chown -R 1000:0 /certs
        fi;
      '
    user: "0"
    working_dir: /usr/share/elasticsearch
    volumes: 
      - certs:/certs

  elasticsearch01:
    container_name: elasticsearch01_$ELASTIC_VERSION
    image: docker.elastic.co/elasticsearch/elasticsearch:$ELASTIC_VERSION
    volumes:
      - certs:/usr/share/elasticsearch/config/certs/:ro
      - ./elasticsearch/config/analysis:/usr/share/elasticsearch/config/analysis:ro
      - elasticsearch01_data:/usr/share/elasticsearch/data
    ports:
      - 9200:9200
    environment:
      - "ES_JAVA_OPTS=-Xmx1g -Xms1g"
      - ELASTIC_PASSWORD=$ELASTIC_PASSWORD
      - node.name=elasticsearch01
      - cluster.name=elasticsearch-cluster
      - network.host=_site_
      - discovery.seed_hosts=elasticsearch02,elasticsearch03
      - cluster.initial_master_nodes=elasticsearch01,elasticsearch02,elasticsearch03
      - bootstrap.memory_lock=true
      - xpack.security.enabled=true
      - xpack.security.authc.token.enabled=true
      - xpack.security.audit.enabled=true
      - xpack.security.authc.realms.file.file1.order=0
      - xpack.security.authc.realms.native.native1.order=1
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.keystore.password=$ELASTICSEARCH01_CERT_PASSWORD
      - xpack.security.transport.ssl.truststore.password=$ELASTICSEARCH01_CERT_PASSWORD
      - xpack.security.transport.ssl.keystore.path=certs/elasticsearch01.p12
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.security.http.ssl.enabled=true
      - xpack.security.http.ssl.keystore.password=$ELASTICSEARCH01_CERT_PASSWORD
      - xpack.security.http.ssl.truststore.password=$ELASTICSEARCH01_CERT_PASSWORD
      - xpack.security.http.ssl.keystore.path=certs/elasticsearch01.p12    
      - xpack.security.http.ssl.verification_mode=certificate
      - xpack.license.self_generated.type=trial
    ulimits:
      memlock:
        soft: -1
        hard: -1      
    networks:
      - elastic
    healthcheck:
      test: [
        "CMD", "test", 
        "-f", "/certs/elasticsearch01.p12", "-a",
        "-f", "/certs/elasticsearch02.p12", "-a",
        "-f", "/certs/elasticsearch03.p12", "-a",
        "-f", "/certs/kibana.crt", "-a",
        "-f", "/certs/kibana.key"
      ]
      interval: 15s
      timeout: 10s
      retries: 10

  elasticsearch02:
    container_name: elasticsearch02_$ELASTIC_VERSION
    image: docker.elastic.co/elasticsearch/elasticsearch:$ELASTIC_VERSION
    volumes:
      - certs:/usr/share/elasticsearch/config/certs/:ro
      - ./elasticsearch/config/analysis:/usr/share/elasticsearch/config/analysis:ro
      - elasticsearch02_data:/usr/share/elasticsearch/data
    environment:
      - "ES_JAVA_OPTS=-Xmx1g -Xms1g"
      - ELASTIC_PASSWORD=$ELASTIC_PASSWORD
      - node.name=elasticsearch02
      - cluster.name=elasticsearch-cluster
      - network.host=_site_
      - discovery.seed_hosts=elasticsearch01,elasticsearch03
      - cluster.initial_master_nodes=elasticsearch01,elasticsearch02,elasticsearch03
      - bootstrap.memory_lock=true
      - xpack.security.enabled=true
      - xpack.security.authc.token.enabled=true
      - xpack.security.audit.enabled=true
      - xpack.security.authc.realms.file.file1.order=0
      - xpack.security.authc.realms.native.native1.order=1
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.keystore.password=$ELASTICSEARCH02_CERT_PASSWORD
      - xpack.security.transport.ssl.truststore.password=$ELASTICSEARCH02_CERT_PASSWORD
      - xpack.security.transport.ssl.keystore.path=certs/elasticsearch01.p12
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.security.http.ssl.enabled=true
      - xpack.security.http.ssl.keystore.password=$ELASTICSEARCH02_CERT_PASSWORD
      - xpack.security.http.ssl.truststore.password=$ELASTICSEARCH02_CERT_PASSWORD
      - xpack.security.http.ssl.keystore.path=certs/elasticsearch01.p12
      - xpack.security.http.ssl.verification_mode=certificate
      - xpack.license.self_generated.type=trial
    ulimits:
      memlock:
        soft: -1
        hard: -1      
    networks:
      - elastic
    depends_on:
      - elasticsearch01

  elasticsearch03:
    container_name: elasticsearch03_$ELASTIC_VERSION
    image: docker.elastic.co/elasticsearch/elasticsearch:$ELASTIC_VERSION
    volumes:
      - certs:/usr/share/elasticsearch/config/certs/:ro
      - ./elasticsearch/config/analysis:/usr/share/elasticsearch/config/analysis:ro
      - elasticsearch03_data:/usr/share/elasticsearch/data
    environment:
      - "ES_JAVA_OPTS=-Xmx1g -Xms1g"
      - ELASTIC_PASSWORD=$ELASTIC_PASSWORD
      - node.name=elasticsearch03
      - cluster.name=elasticsearch-cluster
      - network.host=_site_
      - discovery.seed_hosts=elasticsearch01,elasticsearch02
      - cluster.initial_master_nodes=elasticsearch01,elasticsearch02,elasticsearch03
      - bootstrap.memory_lock=true
      - xpack.security.enabled=true
      - xpack.security.authc.token.enabled=true
      - xpack.security.audit.enabled=true
      - xpack.security.authc.realms.file.file1.order=0
      - xpack.security.authc.realms.native.native1.order=1
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.keystore.password=$ELASTICSEARCH03_CERT_PASSWORD
      - xpack.security.transport.ssl.truststore.password=$ELASTICSEARCH03_CERT_PASSWORD
      - xpack.security.transport.ssl.keystore.path=certs/elasticsearch01.p12
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.security.http.ssl.enabled=true
      - xpack.security.http.ssl.keystore.password=$ELASTICSEARCH03_CERT_PASSWORD
      - xpack.security.http.ssl.truststore.password=$ELASTICSEARCH03_CERT_PASSWORD
      - xpack.security.http.ssl.keystore.path=certs/elasticsearch01.p12
      - xpack.security.http.ssl.verification_mode=certificate
      - xpack.license.self_generated.type=trial
    ulimits:
      memlock:
        soft: -1
        hard: -1      
    networks:
      - elastic
    depends_on:
      - elasticsearch01

  kibana:
    container_name: kibana_$ELASTIC_VERSION
    image: docker.elastic.co/kibana/kibana:$ELASTIC_VERSION
    volumes:
      - certs:/usr/share/kibana/config/certs/:ro
    ports:
      - 5601:5601
    environment:
      - SERVER_NAME=kibana
      - ELASTICSEARCH_HOSTS=["https://elasticsearch01:9200","https://elasticsearch02:9200","https://elasticsearch03:9200"]
      - server.host=_site_
      - xpack.security.enabled=true
      - elasticsearch.username=elastic
      - elasticsearch.password=$ELASTIC_PASSWORD
      - elasticsearch.ssl.certificateAuthorities=config/certs/ca/ca.crt
      - elasticsearch.ssl.verificationMode=certificate
      #- server.ssl.enabled=true
      #- server.ssl.key=/usr/share/kibana/config/certs/kibana.key
      #- server.ssl.certificate=/usr/share/kibana/config/certs/kibana.crt
      #- server.ssl.password=${KIBANA_CERT_PASSWORD}
      #- xpack.monitoring.elasticsearch.ssl.verificationMode=certificate
    networks:
      - elastic
    depends_on:
      - elasticsearch01

networks:
  elastic:

volumes:
  certs:

  elasticsearch01_data:
    name: elasticsearch01_data_$ELASTIC_VERSION
  
  elasticsearch02_data:
    name: elasticsearch02_data_$ELASTIC_VERSION
  
  elasticsearch03_data:
    name: elasticsearch03_data_$ELASTIC_VERSION
  1. What works
  • Looks like the three nodes are TLSing with each other
  • I can HTTPS to the cluster from the kibana container:

curl -XGET https://elasticsearch01:9200 -k -u elastic

  • I can even bring a Kibana instance up outside the docker infrastructure and make it connect to the cluster using the generated ca.crt, using this the kibana.yml:
xpack.security.enabled: true
elasticsearch.hosts: https://localhost:9200
elasticsearch.username: elastic
elasticsearch.password: password
elasticsearch.ssl.certificateAuthorities: config/certs/ca/ca.crt
elasticsearch.ssl.verificationMode: certificate
  1. What doesn't works
  • The Kibana container keeps complaining about self signed certificate in certificate chain:
{"type":"log","@timestamp":"2020-03-21T21:28:54Z","tags":["info","savedobjects-service"],"pid":6,"message":"Waiting until all Elasticsearch nodes are compatible with Kibana before starting saved objects migrations..."}
{"type":"log","@timestamp":"2020-03-21T21:28:54Z","tags":["error","elasticsearch","data"],"pid":6,"message":"Request error, retrying\nHEAD https://elasticsearch03:9200/.apm-agent-configuration => self signed certificate in certificate chain"}
{"type":"log","@timestamp":"2020-03-21T21:28:54Z","tags":["error","elasticsearch","admin"],"pid":6,"message":"Request error, retrying\nGET https://elasticsearch01:9200/_nodes?filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip => self signed certificate in certificate chain"}
{"type":"log","@timestamp":"2020-03-21T21:28:54Z","tags":["error","elasticsearch","data"],"pid":6,"message":"Request error, retrying\nGET https://elasticsearch02:9200/_xpack => self signed certificate in certificate chain"}
{"type":"log","@timestamp":"2020-03-21T21:28:54Z","tags":["error","elasticsearch","admin"],"pid":6,"message":"Request error, retrying\nGET https://elasticsearch03:9200/_nodes?filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip => self signed certificate in certificate chain"}
{"type":"log","@timestamp":"2020-03-21T21:28:54Z","tags":["error","elasticsearch","data"],"pid":6,"message":"Request error, retrying\nGET https://elasticsearch01:9200/_xpack => self signed certificate in certificate chain"}
{"type":"log","@timestamp":"2020-03-21T21:28:54Z","tags":["error","elasticsearch","data"],"pid":6,"message":"Request error, retrying\nHEAD https://elasticsearch01:9200/.apm-agent-configuration => self signed certificate in certificate chain"}
{"type":"log","@timestamp":"2020-03-21T21:28:54Z","tags":["error","elasticsearch","admin"],"pid":6,"message":"Request error, retrying\nGET https://elasticsearch02:9200/_nodes?filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip => self signed certificate in certificate chain"}
{"type":"log","@timestamp":"2020-03-21T21:28:54Z","tags":["warning","elasticsearch","data"],"pid":6,"message":"Unable to revive connection: https://elasticsearch03:9200/"}

Any help will be much appreciated!

2

I notice that if I take another approach, not setting env variables, but creating a kibana.yml file and registering it as a volme of the container, it works...

3

Never mind.... It looks like when setting env variables for the Kibana docker container you have to translate them from YAML-like to Shell-like:

https://www.elastic.co/guide/en/kibana/7.6/docker.html#environment-variable-config

So I changed my docker-compose to this and it worked.

  kibana:
    container_name: kibana_$ELASTIC_VERSION
    image: docker.elastic.co/kibana/kibana:$ELASTIC_VERSION
    volumes:
      - certs:/usr/share/kibana/config/certs/:ro
    ports:
      - 5601:5601
    environment:
      - ELASTICSEARCH_HOSTS=["https://elasticsearch01:9200","https://elasticsearch02:9200","https://elasticsearch03:9200"]
      - XPACK_SECURITY_ENABLED=true
      - ELASTICSEARCH_USERNAME=elastic
      - ELASTICSEARCH_PASSWORD=$ELASTIC_PASSWORD
      - ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt
      - ELASTICSEARCH_SSL_VERIFICATIONMODE=certificate
      - SERVER_SSL_ENABLED=true
      - SERVER_SSL_KEY=config/certs/kibana.key
      - SERVER_SSL_CERTIFICATE=config/certs/kibana.crt
      - SERVER_SSL_PASSWORD=${KIBANA_CERT_PASSWORD}
      #- xpack.monitoring.elasticsearch.ssl.verificationMode=certificate
    networks:
      - elastic
    depends_on:
      - elasticsearch01
3 Likes
4

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.