Kibana does not work with https on Docker Swarm cluster

I have a Docker Swarm cluster which consists of one manager and two virtual machines inside as workers. I have successfully deployed Elasticsearch with basic security and https and connected it with Kibana. However, I was not able to set https between Kibana and browser, I tried to generate the certificates in various ways: by using openssl x509, by explicitly creating elasticsearch-ca.pem and putting it inside the bound volume folder, nothing works. There is no a single error in logs in both elasticsearch and kibana, it says that Kibana was degraded and working on https://kibana:5601 but obviously it is not. Maybe I am doing something wrong and that is why any help would be appreciated.

version: "3.9"
services:
  setup:
    image: docker.elastic.co/elasticsearch/elasticsearch:8.6.2
    user: "0"
    deploy:
      restart_policy:
        condition: none
      placement:
        constraints:
          - node.hostname == ubuntuvm
    command: >
        bash -c '
          if [ ! -f config/certs/ca.zip ]; then
            echo "Creating CA";
            bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip;
            unzip config/certs/ca.zip -d config/certs;
          fi;
          if [ ! -f config/certs/certs.zip ]; then
            echo "Creating certs";
            echo -ne \
            "instances:\n"\
            "  - name: es01\n"\
            "    dns:\n"\
            "      - es01\n"\
            "      - localhost\n"\
            "    ip:\n"\
            "      - 127.0.0.1\n"\
            "  - name: kibana01\n"\
            "    dns:\n"\
            "      - kibana01\n"\
            "      - localhost\n"\
            "    ip:\n"\
            "      - 127.0.0.1\n"\
            > config/certs/instances.yml;
            bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key;
            unzip config/certs/certs.zip -d config/certs;
          fi;
          echo "Setting file permissions"
          chown -R root:root config/certs;
          echo "Setting kibana password"
          until curl -s -X POST --cacert config/certs/ca/ca.crt -u "elastic:elastic" -H "Content-Type: application/json" https://es01:9200/_security/user/kibana_system/_password -d "{\"password\":\"kibana\"}" | grep -q "^{}"; do sleep 10; done;
        '
    healthcheck:
      test: ["CMD-SHELL", "[ -f config/certs/es01/es01.crt ]"]
      interval: 1s
      timeout: 5s
      retries: 120
    volumes:
      - /home/user/certs:/usr/share/elasticsearch/config/certs
  es01:
    image: docker.elastic.co/elasticsearch/elasticsearch:8.6.2
    deploy:
      resources:
        limits:
          memory: 2G
      placement:
        constraints:
          - node.hostname == ubuntuvm
    environment:
      node.name: es01
      discovery.type: single-node
      ELASTIC_PASSWORD: elastic
      cluster.name: docker-cluster
      bootstrap.memory_lock: "true"
      xpack.security.enabled: "true"
      xpack.security.http.ssl.enabled: "true"
      xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certs/es01/es01.key
      xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certs/es01/es01.crt
      xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certs/ca/ca.crt
      xpack.security.transport.ssl.enabled: "true"
      xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certs/es01/es01.key
      xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certs/es01/es01.crt
      xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certs/ca/ca.crt
      xpack.security.transport.ssl.verification_mode: certificate
      xpack.license.self_generated.type: basic
      xpack.security.enrollment.enabled: "true"
    ports:
      - 9200:9200
    volumes:
      - sshfscerts:/usr/share/elasticsearch/config/certs
      - esdata01:/usr/share/elasticsearch/data
    ulimits:
      memlock:
        soft: -1
        hard: -1
    healthcheck:
      test:
        [
          "CMD-SHELL",
          "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
        ]
      interval: 10s
      timeout: 10s
      retries: 120

  kibana:
    image: docker.elastic.co/kibana/kibana:8.6.2
    deploy:
      resources:
        limits:
          memory: 2G
      placement:
        constraints:
          - node.hostname == ubuntuvm2
    hostname: kibana01
    environment:
      SERVERNAME: kibana01
      ELASTICSEARCH_HOSTS: https://es01:9200
      ELASTICSEARCH_USERNAME: kibana_system
      ELASTICSEARCH_PASSWORD: kibana
      ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES: /usr/share/kibana/config/certs/ca/elasticsearch-ca.pem
      ELASTICSEARCH_SSL_VERIFICATIONMODE: "none"
      XPACK_SECURITY_ENABLED: "true"
      XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY: 145723f968410ce2acb49688ab8064a1
      XPACK_REPORTING_ENCRYPTIONKEY: 7096df88ea08c14d2aa1e3fa103854c8
      XPACK_SECURITY_ENCRYPTIONKEY: 927f13902b53ec212a0cd08afdcf3b16
      SERVER_SSL_ENABLED: "true"
      SERVER_SSL_CERTIFICATE: /usr/share/kibana/config/certs/ca/ca.crt
      SERVER_SSL_KEY: /usr/share/kibana/config/certs/ca/ca.key
        #SERVER_SSL_CERTIFICATE: /usr/share/kibana/config/certs/kibana01/kibana01.crt
        #SERVER_SSL_KEY: /usr/share/kibana/config/certs/kibana01/kibana01.key
        #SERVER_SSL_CERTIFICATEAUTHORITIES: /usr/share/kibana/config/certs/ca/ca.crt
    ports:
      - 5601:5601
    volumes:
      - sshfscerts:/usr/share/kibana/config/certs
      - kibanadata:/usr/share/kibana/data
    healthcheck:
      test:
        [
          "CMD-SHELL",
          "curl -s -I https://localhost:5601 | grep -q 'HTTP/1.1 302 Found'",
        ]
      interval: 10s
      timeout: 10s
      retries: 120

volumes:
  certs:
    driver: local
  esdata01:
    driver: local
  kibanadata:
    driver: local
  thehivedata:
    driver: local
  thehiveindex:
    driver: local
  thehive_conf:
    driver: local
  cortex-data:
    driver: local
  sshfscerts:
    driver: vieux/sshfs
    driver_opts:
      sshcmd: "user@192.168.122.231:/home/user/certs"
      password: user
      allow_other: ""

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.