ES, kibana both having ca.crt issues?

I'm not able to start up either the ES or kibana containers and I suspect the root cause has to do with ca-cert issues. i'm attaching my compose file below for reference

  1. with ES, the log shows this error:

"@timestamp":"2026-08-17T21:38:47.019Z","log.level": "WARN","message":"caught exception while handling client http traffic, closing connection Netty4HttpChannel{localAddress=/172.20.0.2:9200, remoteAddress=/192.168.65.1:58936}", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[es01][transport_worker][T#3]","log.logger":"org.elasticsearch.http.AbstractHttpServerTransport","elasticsearch.cluster.uuid":"95GZRlGZRlmu9m-yXVTy-g","elasticsearch.node.id":"D9rUdaBgR6aMHwW4AwADAA","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"es-cluster","error.type":"io.netty.handler.codec.DecoderException","error.message":"javax.net.ssl.SSLHandshakeException: (unknown_ca) Received fatal alert: unknown_ca","error.stack_trace":"io.netty.handler.codec.DecoderException: javax.net.ssl.SSLHandshakeException: (unknown_ca) Received fatal alert: unknown_ca

via curl -k https://127.0.0.1:9200

	{"error":{"root_cause":[{"type":"security_exception","reason":"missing authentication credentials for REST request [/]","header":{"WWW-Authenticate":["Basic realm=\"security\", charset=\"UTF-8\"","Bearer realm=\"security\"","ApiKey"]}}],"type":"security_exception","reason":"missing authentication credentials for REST request [/]","header":{"WWW-Authenticate":["Basic realm=\"security\", charset=\"UTF-8\"","Bearer realm=\"security\"","ApiKey"]}},"status":401}
  1. with kibana the log shows

Unable to retrieve version information from Elasticsearch nodes. getaddrinfo ENOTFOUND elasticsearch

and curl -k https://127.0.0.1:5601 returns

	curl: (35) LibreSSL/3.3.6: error:1404B42E:SSL routines:ST_CONNECT:tlsv1 alert protocol version

my compose file is below. thanks very much for suggestions!

name: ocd4oak

volumes:
certs:
driver: local
node_modules:
driver: local
postgres_data:
driver: local
es01_data:
driver: local
kibana_data:
driver: local

networks:
default:
name: elasticsearch
external: false

services:
setup:
image: docker.elastic.co/elasticsearch/elasticsearch:${ES_VERSION}
volumes:
- certs:/usr/share/elasticsearch/config/certs
user: "0"
command: >
bash -c '
if [ x${ES_PASSWORD} == x ]; then
echo "Set the ES_PASSWORD environment variable in the .env file";
exit 1;
elif [ x${KIBANA_PASSWORD} == x ]; then
echo "Set the KIBANA_PASSWORD environment variable in the .env file";
exit 1;
fi;
if [ ! -f config/certs/ca.zip ]; then
echo "Creating CA";
bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip;
unzip config/certs/ca.zip -d config/certs;
fi;
if [ ! -f config/certs/certs.zip ]; then
echo "Creating certs";
echo -ne 
"instances:\n"
"  - name: es01\n"
"    dns:\n"
"      - es01\n"
"      - localhost\n"
"    ip:\n"
"      - 127.0.0.1\n"
"  - name: kibana\n"
"    dns:\n"
"      - kibana\n"
"      - localhost\n"
"    ip:\n"
"      - 127.0.0.1\n"
> config/certs/instances.yml;
bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key;
unzip config/certs/certs.zip -d config/certs;
fi;
echo "Setting file permissions"
chown -R root:root config/certs;
find . -type d -exec chmod 750 {} ;;
find . -type f -exec chmod 640 {} ;;
echo "Waiting for Elasticsearch availability";
until curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done;
echo "Setting kibana_system password";
until curl -s -X POST --cacert config/certs/ca/ca.crt -u "elastic:${ES_PASSWORD}" -H "Content-Type: application/json" https://es01:9200/_security/user/kibana_system/_password -d "{"password":"${KIBANA_PASSWORD}"}" | grep -q "^{}"; do sleep 10; done;
echo "All done!";
'
healthcheck:
test: ["CMD-SHELL", "[ -f config/certs/es01/es01.crt ]"]
interval: 1s
timeout: 5s
retries: 120

es01:
depends_on:
setup:
condition: service_healthy
image: docker.elastic.co/elasticsearch/elasticsearch:${ES_VERSION}
container_name: es01
ports:
- ${ES_PORT}:9200
volumes:
- certs:/usr/share/elasticsearch/config/certs
- es01_data:/usr/share/elasticsearch/data
environment:
- node.name=es01
- cluster.name=${ES_CLUSTER_NAME}
- discovery.type=single-node
- ELASTIC_PASSWORD=${ES_PASSWORD}
- bootstrap.memory_lock=true
- xpack.security.enabled=true
- xpack.security.http.ssl.enabled=true
- xpack.security.http.ssl.key=certs/es01/es01.key
- xpack.security.http.ssl.certificate=certs/es01/es01.crt
- xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
- xpack.security.transport.ssl.enabled=true
- xpack.security.transport.ssl.key=certs/es01/es01.key
- xpack.security.transport.ssl.certificate=certs/es01/es01.crt
- xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
- xpack.security.transport.ssl.verification_mode=certificate
- xpack.license.self_generated.type=${ES_LICENSE}
mem_limit: ${ES_MEM_LIMIT}
ulimits:
memlock:
soft: -1
hard: -1
healthcheck:
test:
[
"CMD-SHELL",
"curl -s --cacert config/certs/ca/ca.crt http://elasticsearch:9200 | grep -q 'missing authentication credentials'",
]
interval: 10s
timeout: 10s
retries: 120

kibana:
depends_on:
es01:
condition: service_healthy
image: docker.elastic.co/kibana/kibana:${ES_VERSION}
container_name: ${KIBANA_CONTAINER_NAME}
volumes:
- certs:/usr/share/kibana/config/certs
- kibana_data:/usr/share/kibana/data
ports:
- 127.0.0.1:${KIBANA_PORT}:5601
extra_hosts:
- host.docker.internal:host-gateway
- model-runner.docker.internal:host-gateway
environment:
- SERVER_NAME=kibana
- ELASTICSEARCH_HOSTS=http://elasticsearch:9200
- ELASTICSEARCH_USERNAME=kibana_system
- ELASTICSEARCH_PASSWORD=${KIBANA_PASSWORD}
- ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt
# - XPACK_SECURITY_ENCRYPTIONKEY=${ENCRYPTION_KEY}
# - XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY=${ENCRYPTION_KEY}
# - XPACK_REPORTING_ENCRYPTIONKEY=${ENCRYPTION_KEY}
healthcheck:
test:
[
"CMD-SHELL",
"curl -s -I http://elasticsearch:5601 | grep -q 'HTTP/1.1 302 Found'",
]
interval: 10s
timeout: 10s
retries: 30

Rather than me trying to troubleshoot your syntax, can I suggest starting from the official docs?

and/or

In latter case populating the .env file appropriately and maybe removing the es02/es03 sections/references if you only want a single node cluster for now.

Your file is similar, but with names a little bit different, and different quoting in places and ...

hi @RainTown , that's EXACTLY the source of my compose and .env files! i've done a few things like merge the setup and kibana services into the same file, changed some env variable names (STACK_VERSION --> ES_VERSION, LICENSE--> ES_LICENSE, etc) but tried to stay as close as possible. I can appreciate that you don't have time to help me debug the result, but moving forward from the docs is just what I'm trying to do.

EG, I just did a diff of the files (again) to look for things, and noticed that the doc's files used localhost vs elasticsearch in the health checks, and tried again; same errors.

hi @rik

Couple things.

  1. During the debug cycle you have to completely delete the volume mounts for the cert, because if they were created wrong they will not be re-created. And thus will fail repeatedly

  2. I am not sure exactly what you are copying but your health check is checking http not https

http plus several other places in yours

"CMD-SHELL",
"curl -s --cacert config/certs/ca/ca.crt http://elasticsearch:9200 | grep -q 'missing authentication credentials'",
]

I just worked from this from the example

This works comes up fine first time



services:
  setup:
    image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
    volumes:
      - certs:/usr/share/elasticsearch/config/certs
    user: "0"
    command: >
      bash -c '
        if [ x${ELASTIC_PASSWORD} == x ]; then
          echo "Set the ELASTIC_PASSWORD environment variable in the .env file";
          exit 1;
        elif [ x${KIBANA_PASSWORD} == x ]; then
          echo "Set the KIBANA_PASSWORD environment variable in the .env file";
          exit 1;
        fi;
        if [ ! -f config/certs/ca.zip ]; then
          echo "Creating CA";
          bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip;
          unzip config/certs/ca.zip -d config/certs;
        fi;
        if [ ! -f config/certs/certs.zip ]; then
          echo "Creating certs";
          echo -ne \
          "instances:\n"\
          "  - name: es01\n"\
          "    dns:\n"\
          "      - es01\n"\
          "      - localhost\n"\
          "    ip:\n"\
          "      - 127.0.0.1\n"\
          > config/certs/instances.yml;
          bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key;
          unzip config/certs/certs.zip -d config/certs;
        fi;
        echo "Setting file permissions"
        chown -R root:root config/certs;
        find . -type d -exec chmod 750 \{\} \;;
        find . -type f -exec chmod 640 \{\} \;;
        echo "Waiting for Elasticsearch availability";
        until curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done;
        echo "Setting kibana_system password";
        until curl -s -X POST --cacert config/certs/ca/ca.crt -u "elastic:${ELASTIC_PASSWORD}" -H "Content-Type: application/json" https://es01:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done;
        echo "All done!";
      '
    healthcheck:
      test: ["CMD-SHELL", "[ -f config/certs/es01/es01.crt ]"]
      interval: 1s
      timeout: 5s
      retries: 120

  es01:
    depends_on:
      setup:
        condition: service_healthy
    image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
    volumes:
      - certs:/usr/share/elasticsearch/config/certs
      - esdata01:/usr/share/elasticsearch/data
    ports:
      - ${ES_PORT}:9200
    environment:
      - node.name=es01
      - cluster.name=${CLUSTER_NAME}
      - discovery.type=single-node
      #- cluster.initial_master_nodes=es01,es02,es03
      #- discovery.seed_hosts=es02,es03
      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
      - bootstrap.memory_lock=true
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=true
      - xpack.security.http.ssl.key=certs/es01/es01.key
      - xpack.security.http.ssl.certificate=certs/es01/es01.crt
      - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.key=certs/es01/es01.key
      - xpack.security.transport.ssl.certificate=certs/es01/es01.crt
      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.license.self_generated.type=${LICENSE}
      - xpack.ml.use_auto_machine_memory_percent=true
    mem_limit: ${MEM_LIMIT}
    ulimits:
      memlock:
        soft: -1
        hard: -1
    healthcheck:
      test:
        [
          "CMD-SHELL",
          "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
        ]
      interval: 10s
      timeout: 10s
      retries: 120

  kibana:
    depends_on:
      es01:
        condition: service_healthy
    image: docker.elastic.co/kibana/kibana:${STACK_VERSION}
    volumes:
      - certs:/usr/share/kibana/config/certs
      - kibanadata:/usr/share/kibana/data
    ports:
      - ${KIBANA_PORT}:5601
    environment:
      - SERVERNAME=kibana
      - ELASTICSEARCH_HOSTS=https://es01:9200
      - ELASTICSEARCH_USERNAME=kibana_system
      - ELASTICSEARCH_PASSWORD=${KIBANA_PASSWORD}
      - ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt
    mem_limit: ${MEM_LIMIT}
    healthcheck:
      test:
        [
          "CMD-SHELL",
          "curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'",
        ]
      interval: 10s
      timeout: 10s
      retries: 120

volumes:
  certs:
    driver: local
  esdata01:
    driver: local
  kibanadata:
    driver: local

Yeah, but respectfully, you started with a working config and somehow broke it, right? I also don't see what you are trying to do, nor do I really know your level of linux/docker/elasticsearch experience. The configs within the docs do work OOTB. If you start changing things, then change one thing at a time, and check you didn't break it at each and every step. Even things like quoting can make a difference, http vs https as was pointed out by @stephenb, cleaning up after failed attempts, etc.

Example, this is from your config:

if [ ! -f config/certs/certs.zip ]; then
echo "Creating certs";
echo -ne 
"instances:\n"
"  - name: es01\n"
"    dns:\n"
"      - es01\n"
"      - localhost\n"

This is from the docs/github config:

        if [ ! -f config/certs/certs.zip ]; then
          echo "Creating certs";
          echo -ne \
          "instances:\n"\
          "  - name: es01\n"\
          "    dns:\n"\
          "      - es01\n"\
          "      - localhost\n"\

Your version is missing the trailing \ from every line, including the echo -ne . Is this maybe important? :thinking:

Thanks to both thanks to @stephenb and @RainTown for your attention.
Your responses have got me farther!

  1. @stephenb: you have to completely delete the volume mounts

    • I think this was a big part of my confusion! Tests I'd do after making a change had no effect. Thanks for the heads-up!
  2. @stephenb: http vs https

  3. host naming

    • As the es01 vs localhost variations used in the examples above, I'm also
      unclear about host naming. When do you use/not use localhost?
    • Getting started with the Elastic Stack and Docker-Compose | Elastic Blog recommends creating the elasticsearch network. My compose included it, while the reference compose file does not? Which is right?
    • That page also says:

    Notice in our environment section that we’re specifying
    ELASTICSEARCH_HOSTS=https://es01:9200 We’re able to specify the
    container name here for our ES01 Elasticsearch container since
    we’re utilizing the Docker default networking. All containers that
    are using the “elastic” network that was specified at the
    beginning of our docker-compose.yml file will be able to properly
    resolve other container names and communicate with each other.

and so maybe that is part of my issue, too?

Did you simply try the code I provided? It works. I ran it.

Yes, Kibana runs on HTTP, All the ES run on HTTPS

The host / container names matter because they're what the certificates are built off of and then when the SSL does certificate validation the host names of the checks need to match the host names are in the certificates otherwise they will fail.
This is pretty common SSL stuff

localhost is confusing with docker within a container localhost is only local to that container not to the guest host i.e. your laptop or server.

The setup container which is NOT the es container using the container name to check not localhost since it is checking a different container.

Withing the ES container to check itself it can use localhost

From Kibana to check it self it can check localhost

All those checks could actually use proper container names which would probably be less confusing

and not/specifying the network, elasticsearch or otherwise? Does any of the ES stack ASSUME definition of a elasticsearch network?

PS: @RainTown my compose file has the backslashed end-of-lines, not sure why my cut+paste quote of it dropped them? (there seems no way to attach files to discuss items?) sorry for the confusion.

No the compose I am using does not use that.
You can see it's not defined in my compose which worked...
Is there a reason you're resisting starting from a working code and then you can play with it?
I'm a bit confused...

default networks
docker network ls                                                    
NETWORK ID     NAME      DRIVER    SCOPE
5a4d5b1ff408   bridge    bridge    local
c32509034a27   host      host      local
fb756c2c15cf   none      null      local

After I run working compose
docker network ls   
NETWORK ID     NAME             DRIVER    SCOPE
5a4d5b1ff408   bridge           bridge    local
851bba5a95ed   docker_default   bridge    local << Created by compose 
c32509034a27   host             host      local
fb756c2c15cf   none             null      local

Again, if you've half set things up using the one approach and other half other approach, it's not clear to me what you actually using...

@stephenb definitely not trying to resist your help! but blindly copying your code, or anyone else's, leaves me blind. as you know there are 10^6 versions of advice around about the ES stack, and I'm trying to make my own sense of it. thanks again for your help.

Right totally understand.
But to me you take a working piece of code and you go through it line by line to understand it

Most of what it feels like you're struggling with is certs Creation and verification and docker networking and name resolution ...not really elastic stuff

And by the way, I'm a Distinguished Elastic Engineer so my advice is best LOL (and of course someone else will tell you something different :slight_smile: )

I referenced you to the official documentation. I appreciate you are trying to understand things, laudable indeed, but one needs to be proportionate. Your posts are now a mix between Qs about Elastic and more general docker/networking/SSL/... questions.

Yes, but the current official docs hopefully have a little more weight? As does advice from a Distinguished Elastic Engineer :slight_smile:

If some cut and paste errors slip into what you share here, maybe they also slip in on your own systems? Anyways, if you share some code, we need to assume that really is your code, not an approximation.

Of course its up to you how you invest your time, and me mine, and earlier in the week I started from the official version, and tried to "adapt" as you had done, with a few changes in names, order, etc. I got as far as the 2 files below, then moved on to other things. Maybe it helps you, maybe not. No warranty given. Now there's 10^6 + 1.

Good luck!

$ cat .env
ES_PASSWORD=banana1234
KIBANA_PASSWORD=banana5678
ES_VERSION=9.5.1
ES_CLUSTER_NAME=discuss-cluster
ES_LICENSE=basic
ES_PORT=9200
KIBANA_PORT=5061
MEM_LIMIT=1073741824

$ cat docker-compose.yml
version: "2.2"

volumes:
  certs:
    driver: local
  es01_data:
    driver: local
  kibana_data:
    driver: local

services:
  setup:
    image: docker.elastic.co/elasticsearch/elasticsearch:${ES_VERSION}
    volumes:
      - certs:/usr/share/elasticsearch/config/certs
    user: "0"
    command: >
      bash -c '
        if [ x${ES_PASSWORD} == x ]; then
          echo "Set the ES_PASSWORD environment variable in the .env file";
          exit 1;
        elif [ x${KIBANA_PASSWORD} == x ]; then
          echo "Set the KIBANA_PASSWORD environment variable in the .env file";
          exit 1;
        fi;
        if [ ! -f config/certs/ca.zip ]; then
          echo "Creating CA";
          bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip;
          unzip config/certs/ca.zip -d config/certs;
        fi;
        if [ ! -f config/certs/certs.zip ]; then
          echo "Creating certs";
          echo -ne \
          "instances:\n"\
          "  - name: es01\n"\
          "    dns:\n"\
          "      - es01\n"\
          "      - localhost\n"\
          "    ip:\n"\
          "      - 127.0.0.1\n"\
          > config/certs/instances.yml;
          bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key;
          unzip config/certs/certs.zip -d config/certs;
        fi;
        echo "Setting file permissions"
        chown -R root:root config/certs;
        find . -type d -exec chmod 750 \{\} \;;
        find . -type f -exec chmod 640 \{\} \;;
        echo "Waiting for Elasticsearch availability";
        until curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done;
        echo "Setting kibana_system password";
        until curl -s -X POST --cacert config/certs/ca/ca.crt -u "elastic:${ES_PASSWORD}" -H "Content-Type: application/json" https://es01:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done;
        echo "All done!";
      '
    healthcheck:
      test: ["CMD-SHELL", "[ -f config/certs/es01/es01.crt ]"]
      interval: 1s
      timeout: 5s
      retries: 120

  es01:
    depends_on:
      setup:
        condition: service_healthy
    image: docker.elastic.co/elasticsearch/elasticsearch:${ES_VERSION}
    container_name: es01
    ports:
      - ${ES_PORT}:9200
    volumes:
      - certs:/usr/share/elasticsearch/config/certs
      - es01_data:/usr/share/elasticsearch/data
    environment:
      - node.name=es01
      - cluster.name=${ES_CLUSTER_NAME}
      - discovery.type=single-node
      - ELASTIC_PASSWORD=${ES_PASSWORD}
      - bootstrap.memory_lock=true
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=true
      - xpack.security.http.ssl.key=certs/es01/es01.key
      - xpack.security.http.ssl.certificate=certs/es01/es01.crt
      - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.key=certs/es01/es01.key
      - xpack.security.transport.ssl.certificate=certs/es01/es01.crt
      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.license.self_generated.type=${ES_LICENSE}
      - xpack.ml.use_auto_machine_memory_percent=true
    mem_limit: ${MEM_LIMIT}
    ulimits:
      memlock:
        soft: -1
        hard: -1
    healthcheck:
      test:
        [
          "CMD-SHELL",
          "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
        ]
      interval: 10s
      timeout: 10s
      retries: 120

  kibana:
    depends_on:
      es01:
        condition: service_healthy
    image: docker.elastic.co/kibana/kibana:${ES_VERSION}
    volumes:
      - certs:/usr/share/kibana/config/certs
      - kibana_data:/usr/share/kibana/data
    ports:
      - ${KIBANA_PORT}:5601
    environment:
      - SERVER_NAME=kibana
      - ELASTICSEARCH_HOSTS=https://es01:9200
      - ELASTICSEARCH_USERNAME=kibana_system
      - ELASTICSEARCH_PASSWORD=${KIBANA_PASSWORD}
      - ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt
    mem_limit: ${MEM_LIMIT}
    healthcheck:
      test:
        [
          "CMD-SHELL",
          "curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'",
        ]
      interval: 10s
      timeout: 10s
      retries: 120