Authentication of [elastic] was terminated by realm [reserved] - failed to authenticate user [elastic]

Elastic Security
1

Hi everyone,

I’m following this guide: Getting Started with the Elastic Stack and Docker Compose and I’m using Elastic Stack version 8.19.4.

I have a setup container in my Docker Compose configuration, but it gets stuck at this step:

echo "Setting kibana_system password";

until curl -s -X POST --cacert config/certs/ca/ca.crt -u elastic:${ELASTIC_PASSWORD} -H "Content-Type: application/json" https://elasticsearch:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done;

Meanwhile, the Elasticsearch logs show the following:

{
  "@timestamp": "2025-10-15T15:24:06.516Z",
  "log.level": "INFO",
  "message": "Authentication of [elastic] was terminated by realm [reserved] - failed to authenticate user [elastic]",
  "ecs.version": "1.2.0",
  "service.name": "ES_ECS",
  "event.dataset": "elasticsearch.server",
  "process.thread.name": "elasticsearch[es01][get][T#5]",
  "log.logger": "org.elasticsearch.xpack.security.authc.RealmsAuthenticator",
  "elasticsearch.cluster.uuid": "UddDp46-S1SRY1XRbuN8nQ",
  "elasticsearch.node.id": "tcR-pjH7TK61ZyyJkMbIhg",
  "elasticsearch.node.name": "es01",
  "elasticsearch.cluster.name": "docker-cluster"
}

Has anyone encountered this issue with Elastic Stack 8.19.x? Any advice on how to fix the password setup for kibana_system in Docker Compose?

Thanks in advance!

2

Welcome to the community @onitnas

pls share you complete configuration (obfuscate/withhold passwords)

The setup container is just to wait til Elasticsearch is up and then set things like the kibana user password, as here. But it's dependent on other things being setup correctly.

3

Hi Kevin, thanks!

docker-compose.yml

services:

#------------------------#

# Elastic | Kibana setup #

#------------------------#

  setup:

    image: docker.elastic.co/elasticsearch/elasticsearch:8.19.4
    container_name: elasticsearch-setup
    user: "0"
    volumes:
      - certs:/usr/share/elasticsearch/config/certs
    command: >
      bash -c '
        if [ x${ELASTIC_PASSWORD} == x ]; then
          echo "Set the ELASTIC_PASSWORD environment variable in the .env file";
          exit 1;
        elif [ x${KIBANA_PASSWORD} == x ]; then
          echo "Set the KIBANA_PASSWORD environment variable in the .env file";
        exit 1;
        fi;

      if [ ! -f config/certs/ca.zip ]; then
        echo "Creating CA";
        bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip;
        unzip config/certs/ca.zip -d config/certs;
    fi;

    if [ ! -f config/certs/certs.zip ]; then

      echo "Creating certs";

      echo -ne \
      "instances:\n"\
      "  - name: elasticsearch\n"\
      "    dns:\n"\
      "      - elasticsearch\n"\
      "      - localhost\n"\
      "    ip:\n"\
      "      - 127.0.0.1\n"\
      "  - name: kibana\n"\
      "    dns:\n"\
      "      - kibana\n"\
      "      - localhost\n"\
      "    ip:\n"\
      "      - 127.0.0.1\n"\
      > config/certs/instances.yml;

      bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key;

      unzip config/certs/certs.zip -d config/certs;
    fi;

    echo "Setting file permissions"

    chown -R root:root config/certs;
    find . -type d -exec chmod 750 \{\} \;;
    find . -type f -exec chmod 640 \{\} \;;
    echo "Waiting for Elasticsearch availability";

    until curl -s --cacert config/certs/ca/ca.crt https://elasticsearch:9200 | grep -q "missing authentication credentials"; do sleep 30; done;

    echo "Setting kibana_system password";

    until curl -s -X POST --cacert config/certs/ca/ca.crt -u elastic:${ELASTIC_PASSWORD} -H "Content-Type: application/json" https://elasticsearch:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done;

    echo "All done!";'
  healthcheck:
    test: ["CMD-SHELL", "[ -f config/certs/elasticsearch/elasticsearch.crt ]"]
    interval: 1s
    timeout: 5s
    retries: 120
  networks:
    - tmf-network
  mem_limit: 256m

#-----------------------------#

# Elasticsearch with security #

#-----------------------------#
  elasticsearch:
    depends_on:
      setup:
        condition: service_healthy
    image: docker.elastic.co/elasticsearch/elasticsearch:8.19.4
    ports:
      - "9200:9200"
    volumes:
      - esdata:/usr/share/elasticsearch/data
      - certs:/usr/share/elasticsearch/config/certs
    hostname: elasticsearch
    container_name: elasticsearch
    environment:
      - node.name=es01
      - discovery.type=single-node
      - xpack.security.enabled=true
      - xpack.security.authc.api_key.enabled=true
      - ELASTIC_PASSWORD='rootlibero'
      - ES_JAVA_OPTS=-Xms512m -Xmx512m #HEAP SIZE
     #- bootstrap.memory_lock=true
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=true
      - xpack.security.http.ssl.key=certs/elasticsearch/elasticsearch.key
      -xpack.security.http.ssl.certificate=certs/elasticsearch/elasticsearch.crt
      - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.key=certs/elasticsearch/elasticsearch.key
      - xpack.security.transport.ssl.certificate=certs/elasticsearch/elasticsearch.crt
      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.license.self_generated.type=basic

    healthcheck:
      test:
        ["CMD-SHELL", "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'"]
      interval: 10s
      timeout: 10s
      retries: 120

    networks:

      - tmf-network

    mem_limit: 2g

#--------#

# Kibana #

#--------#

  kibana:
    depends_on:
      elasticsearch:
        condition: service_healthy
    image: docker.elastic.co/kibana/kibana:8.19.4
    container_name: kibana
    hostname: kibana
    ports:

      - "5601:5601"

    environment:
      ELASTICSEARCH_HOSTS: 'https://elasticsearch:9200'
      ELASTICSEARCH_USERNAME: 'kibana_system'
      ELASTICSEARCH_PASSWORD: 'rootlibero'
      ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES: config/certs/ca/ca.crt

    healthcheck:
      test: ["CMD-SHELL", "curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'"]
      interval: 10s
      timeout: 10s
      retries: 120

    volumes:
      - kibanadata:/usr/share/kibana/data
      - certs:/usr/share/kibana/config/certs
      - ./dashboard/kibana.yml:/usr/share/kibana/config/kibana.yml

networks:
      - tmf-network


volumes:
  esdata:
    driver: local

  kibanadata:
    driver: local

  certs:
    driver: local

networks:
  tmf-network:

.env

ELASTIC_PASSWORD = "rootlibero"
KIBANA_PASSWORD = "rootlibero"
ENCRYPTION_KEY=c34d38b3a14956121ff2170e5030b471551370178f43e5626eec58b04a30fae2
4

Hi @onitnas try taking the spaces out

ELASTIC_PASSWORD=rootlibero
KIBANA_PASSWORD=rootlibero

Clean up everything and try again...

5

@onitnas

Yeah, there were some issues that meant I could not get docker-compose to start up without some editing.

      - ./dashboard/kibana.yml:/usr/share/kibana/config/kibana.yml

I have no idea what was in your kibana.yml file, so just deleted the setting

After some editing, I ended up with below, the spaces in .env were OK for me:

% cat docker-compose.yml
services:
  setup:
    image: docker.elastic.co/elasticsearch/elasticsearch:8.19.4
    container_name: elasticsearch-setup
    user: "0"
    volumes:
      - certs:/usr/share/elasticsearch/config/certs
    command: >
      bash -c '
        if [ x${ELASTIC_PASSWORD} == x ]; then
          echo "Set the ELASTIC_PASSWORD environment variable in the .env file";
          exit 1;
        elif [ x${KIBANA_PASSWORD} == x ]; then
          echo "Set the KIBANA_PASSWORD environment variable in the .env file";
        exit 1;
        fi;
        if [ ! -f config/certs/ca.zip ]; then
          echo "Creating CA";
          bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip;
          unzip config/certs/ca.zip -d config/certs;
      fi;
      if [ ! -f config/certs/certs.zip ]; then
        echo "Creating certs";
        echo -ne \
        "instances:\n"\
        "  - name: elasticsearch\n"\
        "    dns:\n"\
        "      - elasticsearch\n"\
        "      - localhost\n"\
        "    ip:\n"\
        "      - 127.0.0.1\n"\
        "  - name: kibana\n"\
        "    dns:\n"\
        "      - kibana\n"\
        "      - localhost\n"\
        "    ip:\n"\
        "      - 127.0.0.1\n"\
        > config/certs/instances.yml;
        bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key;
        unzip config/certs/certs.zip -d config/certs;
      fi;
      echo "Setting file permissions"
      chown -R root:root config/certs;
      find . -type d -exec chmod 750 \{\} \;;
      find . -type f -exec chmod 640 \{\} \;;
      echo "Waiting for Elasticsearch availability";
      until curl -s --cacert config/certs/ca/ca.crt https://elasticsearch:9200 | grep -q "missing authentication credentials"; do sleep 30; done;
      echo "Setting kibana_system password";
      until curl -s -X POST --cacert config/certs/ca/ca.crt -u elastic:${ELASTIC_PASSWORD} -H "Content-Type: application/json" https://elasticsearch:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done;
      echo "All done!";'
    healthcheck:
      test: ["CMD-SHELL", "[ -f config/certs/elasticsearch/elasticsearch.crt ]"]
      interval: 1s
      timeout: 5s
      retries: 120
    networks:
      - tmf-network
    mem_limit: 256m

  elasticsearch:
    depends_on:
      setup:
        condition: service_healthy
    image: docker.elastic.co/elasticsearch/elasticsearch:8.19.4
    ports:
      - "9200:9200"
    volumes:
      - esdata:/usr/share/elasticsearch/data
      - certs:/usr/share/elasticsearch/config/certs
    hostname: elasticsearch
    container_name: elasticsearch
    environment:
      - node.name=es01
      - discovery.type=single-node
      - xpack.security.enabled=true
      - xpack.security.authc.api_key.enabled=true
      - ELASTIC_PASSWORD=rootlibero1
      - ES_JAVA_OPTS=-Xms512m -Xmx512m
      - xpack.security.http.ssl.enabled=true
      - xpack.security.http.ssl.key=certs/elasticsearch/elasticsearch.key
      - xpack.security.http.ssl.certificate=certs/elasticsearch/elasticsearch.crt
      - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.key=certs/elasticsearch/elasticsearch.key
      - xpack.security.transport.ssl.certificate=certs/elasticsearch/elasticsearch.crt
      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.license.self_generated.type=basic
    healthcheck:
      test:
        ["CMD-SHELL", "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'"]
      interval: 10s
      timeout: 10s
      retries: 120
    networks:
      - tmf-network
    mem_limit: 2g

  kibana:
    depends_on:
      elasticsearch:
        condition: service_healthy
    image: docker.elastic.co/kibana/kibana:8.19.4
    container_name: kibana
    hostname: kibana
    ports:
      - "5601:5601"
    environment:
      ELASTICSEARCH_HOSTS: 'https://elasticsearch:9200'
      ELASTICSEARCH_USERNAME: 'kibana_system'
      ELASTICSEARCH_PASSWORD: 'rootlibero2'
      ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES: config/certs/ca/ca.crt
    healthcheck:
      test: ["CMD-SHELL", "curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'"]
      interval: 10s
      timeout: 10s
      retries: 120
    volumes:
      - kibanadata:/usr/share/kibana/data
      - certs:/usr/share/kibana/config/certs
    networks:
      - tmf-network

volumes:
  esdata:
    driver: local
  kibanadata:
    driver: local
  certs:
    driver: local

networks:
  tmf-network:

% cat .env 
ELASTIC_PASSWORD = "rootlibero1"
KIBANA_PASSWORD = "rootlibero2"
ENCRYPTION_KEY=c34d38b3a14956121ff2170e5030b471551370178f43e5626eec58b04a30fae2

which "worked for me".

% EUSER=elastic EPASS=rootlibero1 EHOST=localhost EPORT=9200
% escurl /
{
  "name" : "es01",
  "cluster_name" : "docker-cluster",
  "cluster_uuid" : "uEtT8DGPQNGDbT-Jix0piw",
  "version" : {
    "number" : "8.19.4",
    "build_flavor" : "default",
    "build_type" : "docker",
    "build_hash" : "aa0a7826e719b392e7782716b323c4fb8fa3b392",
    "build_date" : "2025-09-16T22:06:03.940754111Z",
    "build_snapshot" : false,
    "lucene_version" : "9.12.2",
    "minimum_wire_compatibility_version" : "7.17.0",
    "minimum_index_compatibility_version" : "7.0.0"
  },
  "tagline" : "You Know, for Search"
}
% 
% EUSER=kibana_system EPASS=rootlibero2 EHOST=localhost EPORT=9200
% escurl /
{
  "name" : "es01",
  "cluster_name" : "docker-cluster",
  "cluster_uuid" : "uEtT8DGPQNGDbT-Jix0piw",
  "version" : {
    "number" : "8.19.4",
    "build_flavor" : "default",
    "build_type" : "docker",
    "build_hash" : "aa0a7826e719b392e7782716b323c4fb8fa3b392",
    "build_date" : "2025-09-16T22:06:03.940754111Z",
    "build_snapshot" : false,
    "lucene_version" : "9.12.2",
    "minimum_wire_compatibility_version" : "7.17.0",
    "minimum_index_compatibility_version" : "7.0.0"
  },
  "tagline" : "You Know, for Search"
}
6

If you are starting from scratch, then you can use the start-local script to deploy latest release into a docker env, or a specific older release with command line options, see this README.

7

The issue was caused by the way the environment variable was defined.

Originally, it was written as:

environment:
  - ELASTIC_PASSWORD='rootlibero'

It should be corrected to:

environment:
  - ELASTIC_PASSWORD=rootlibero

Thanks🙏

1 Like