Problem with PEM Certificate when Adding New Node in Elasticsearch Docker Cluster

Elastic Stack Elasticsearch
,
1

Following this guide, I am trying to add a second node (es02) to my elasticsearch cluster that has been working so far. The docker-compose.yml is the following:

version: "2.2"

services:
  setup:
    image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
    volumes:
      - certs:/usr/share/elasticsearch/config/certs
    user: "0"
    command: >
      bash -c '
        if [ x${ELASTIC_PASSWORD} == x ]; then
          echo "Set the ELASTIC_PASSWORD environment variable in the .env file";
          exit 1;
        elif [ x${KIBANA_PASSWORD} == x ]; then
          echo "Set the KIBANA_PASSWORD environment variable in the .env file";
          exit 1;
        fi;
        if [ ! -f config/certs/ca.zip ]; then
          echo "Creating CA";
          bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip;
          unzip config/certs/ca.zip -d config/certs;
        fi;
        if [ ! -f config/certs/certs.zip ]; then
          echo "Creating certs";
          echo -ne \
          "instances:\n"\
          "  - name: es01\n"\
          "    dns:\n"\
          "      - es01\n"\
          "      - localhost\n"\
          "    ip:\n"\
          "      - 127.0.0.1\n"\
          "  - name: es02\n"\
          "    dns:\n"\
          "      - es02\n"\
          "      - localhost\n"\
          "    ip:\n"\
          "      - 127.0.0.1\n"\
          > config/certs/instances.yml;
          bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key;
          unzip config/certs/certs.zip -d config/certs;
        fi;
        echo "Setting file permissions"
        chown -R root:root config/certs;
        find . -type d -exec chmod 750 \{\} \;;
        find . -type f -exec chmod 640 \{\} \;;
        echo "Setup: Waiting for Elasticsearch availability";
        until curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done;
        echo "Setting kibana_system password";
        until curl -s -X POST --cacert config/certs/ca/ca.crt -u elastic:${ELASTIC_PASSWORD} -H "Content-Type: application/json" https://es01:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done;
        echo "All done!";
      '
    healthcheck:
      test: ["CMD-SHELL", "[ -f config/certs/es01/es01.crt ]"]
      interval: 1s
      timeout: 5s
      retries: 120

  es01:
    depends_on:
      setup:
        condition: service_healthy
    image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
    volumes:
      - certs:/usr/share/elasticsearch/config/certs
      - esdata01:/usr/share/elasticsearch/data
    ports:
      - ${ES_PORT}:9200
    environment:
      - node.name=es01
      - cluster.name=${CLUSTER_NAME}
      - cluster.initial_master_nodes=es01,es02
      - discovery.seed_hosts=es02
      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
      - bootstrap.memory_lock=true
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=true
      - xpack.security.http.ssl.key=certs/es01/es01.key
      - xpack.security.http.ssl.certificate=certs/es01/es01.crt
      - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.http.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.key=certs/es01/es01.key
      - xpack.security.transport.ssl.certificate=certs/es01/es01.crt
      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.license.self_generated.type=${LICENSE}
    mem_limit: 4000000000
    ulimits:
      memlock:
        soft: -1
        hard: -1
    healthcheck:
      test:
        [
          "CMD-SHELL",
          "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
        ]
      interval: 10s
      timeout: 10s
      retries: 120

  es02:
    depends_on:
      - es01
    image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
    volumes:
      - certs:/usr/share/elasticsearch/config/certs
      - esdata02:/usr/share/elasticsearch/data
    environment:
      - node.name=es02
      - cluster.name=${CLUSTER_NAME}
      - cluster.initial_master_nodes=es01,es02
      - discovery.seed_hosts=es01
      - bootstrap.memory_lock=true
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=true
      - xpack.security.http.ssl.key=certs/es02/es02.key
      - xpack.security.http.ssl.certificate=certs/es02/es02.crt
      - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.http.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.key=certs/es02/es02.key
      - xpack.security.transport.ssl.certificate=certs/es02/es02.crt
      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.license.self_generated.type=${LICENSE}
    mem_limit: 4000000000
    ulimits:
      memlock:
        soft: -1
        hard: -1
    healthcheck:
      test:
        [
          "CMD-SHELL",
          "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
        ]
      interval: 10s
      timeout: 10s
      retries: 120

  kibana:
    depends_on:
      es01:
        condition: service_healthy
    image: docker.elastic.co/kibana/kibana:${STACK_VERSION}
    volumes:
      - certs:/usr/share/kibana/config/certs
      - kibanadata:/usr/share/kibana/data
      - ./kibana/config/kibana-ca.crt:/usr/share/kibana/config/kibana-ca.crt
      - ./kibana/config/kibana.crt:/usr/share/kibana/config/kibana.crt
      - ./kibana/config/kibana.key:/usr/share/kibana/config/kibana.key
    ports:
      - ${KIBANA_PORT}:5601
    environment:
      - SERVERNAME=kibana
      - ELASTICSEARCH_HOSTS=https://es01:9200
      - ELASTICSEARCH_USERNAME=kibana_system
      - ELASTICSEARCH_PASSWORD=${KIBANA_PASSWORD}
      - ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt
      - SERVER_SSL_ENABLED=true
      - SERVER_SSL_CERTIFICATE=/usr/share/kibana/config/kibana.crt
      - SERVER_SSL_KEY=/usr/share/kibana/config/kibana.key
      - SERVER_SSL_CERTIFICATEAUTHORITIES=/usr/share/kibana/config/kibana-ca.crt
    mem_limit: 4000000000
    healthcheck:
      test:
        [
          "CMD-SHELL",
          "curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'",
        ]
      interval: 5s
      timeout: 10s
      retries: 10

  logstash:
    build:
      context: ./logstash
      dockerfile: Dockerfile
    depends_on:
      es01:
        condition: service_healthy

    image: docker.elastic.co/logstash/logstash:${STACK_VERSION}
    volumes:
      - ./logstash/config/certs:/usr/share/logstash/config/certs
      - ./logstash/pipelines:/usr/share/logstash/pipelines
      - ./logstash/templates:/usr/share/logstash/templates
      - ./logstash/queries:/usr/share/logstash/queries
      - ./logstash/logs:/usr/share/logstash/logs
    environment:
      - LS_JAVA_OPTS=-Xms16g -Xmx24g
    ports:
      - 4000:4000
      - 9600:9600
    mem_limit: ${MEM}
    healthcheck:
      test:
        [
          "CMD-SHELL",
          "curl -s -XGET localhost:9600 | grep green",
        ]

volumes:
  certs:
    driver: local
  esdata01:
    driver: local
  esdata02:
    driver: local
  kibanadata:
    driver: local

I run in the following problem:

es02_1      | {"@timestamp":"2024-04-17T20:26:25.494Z", "log.level":"ERROR", "message":"fatal exception while booting Elasticsearch", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.bootstrap.Elasticsearch","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster","error.type":"org.elasticsearch.ElasticsearchSecurityException","error.message":"failed to load SSL configuration [xpack.security.transport.ssl] - Error parsing Private Key [/usr/share/elasticsearch/config/certs/es02/es02.key], file is empty","error.stack_trace":"org.elasticsearch.ElasticsearchSecurityException: failed to load SSL configuration [xpack.security.transport.ssl] - Error parsing Private Key [/usr/share/elasticsearch/config/certs/es02/es02.key], file is empty\n\tat org.elasticsearch.xcore@8.6.1/org.elasticsearch.xpack.core.ssl.SSLService.lambda$loadSslConfigurations$11(SSLService.java:605)\n\tat java.base/java.util.HashMap.forEach(HashMap.java:1429)\n\tat java.base/java.util.Collections$UnmodifiableMap.forEach(Collections.java:1553)\n\tat org.elasticsearch.xcore@8.6.1/org.elasticsearch.xpack.core.ssl.SSLService.loadSslConfigurations(SSLService.java:601)\n\tat org.elasticsearch.xcore@8.6.1/org.elasticsearch.xpack.core.ssl.SSLService.<init>(SSLService.java:156)\n\tat org.elasticsearch.xcore@8.6.1/org.elasticsearch.xpack.core.XPackPlugin.createSSLService(XPackPlugin.java:465)\n\tat org.elasticsearch.xcore@8.6.1/org.elasticsearch.xpack.core.XPackPlugin.createComponents(XPackPlugin.java:314)\n\tat org.elasticsearch.server@8.6.1/org.elasticsearch.node.Node.lambda$new$16(Node.java:721)\n\tat org.elasticsearch.server@8.6.1/org.elasticsearch.plugins.PluginsService.lambda$flatMap$0(PluginsService.java:252)\n\tat java.base/java.util.stream.ReferencePipeline$7$1.accept(ReferencePipeline.java:273)\n\tat java.base/java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:197)\n\tat java.base/java.util.AbstractList$RandomAccessSpliterator.forEachRemaining(AbstractList.java:722)\n\tat java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:509)\n\tat java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:499)\n\tat java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:575)\n\tat java.base/java.util.stream.AbstractPipeline.evaluateToArrayNode(AbstractPipeline.java:260)\n\tat java.base/java.util.stream.ReferencePipeline.toArray(ReferencePipeline.java:616)\n\tat java.base/java.util.stream.ReferencePipeline.toArray(ReferencePipeline.java:622)\n\tat java.base/java.util.stream.ReferencePipeline.toList(ReferencePipeline.java:627)\n\tat org.elasticsearch.server@8.6.1/org.elasticsearch.node.Node.<init>(Node.java:736)\n\tat org.elasticsearch.server@8.6.1/org.elasticsearch.node.Node.<init>(Node.java:322)\n\tat org.elasticsearch.server@8.6.1/org.elasticsearch.bootstrap.Elasticsearch$2.<init>(Elasticsearch.java:214)\n\tat org.elasticsearch.server@8.6.1/org.elasticsearch.bootstrap.Elasticsearch.initPhase3(Elasticsearch.java:214)\n\tat org.elasticsearch.server@8.6.1/org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:67)\nCaused by: org.elasticsearch.common.ssl.SslConfigException: Error parsing Private Key [/usr/share/elasticsearch/config/certs/es02/es02.key], file is empty\n\tat org.elasticsearch.sslconfig@8.6.1/org.elasticsearch.common.ssl.PemUtils.parsePrivateKey(PemUtils.java:136)\n\tat org.elasticsearch.sslconfig@8.6.1/org.elasticsearch.common.ssl.PemKeyConfig.getPrivateKey(PemKeyConfig.java:122)\n\tat org.elasticsearch.sslconfig@8.6.1/org.elasticsearch.common.ssl.PemKeyConfig.createKeyManager(PemKeyConfig.java:88)\n\tat org.elasticsearch.xcore@8.6.1/org.elasticsearch.xpack.core.ssl.SSLService.createSslContext(SSLService.java:474)\n\tat java.base/java.util.HashMap.computeIfAbsent(HashMap.java:1228)\n\tat org.elasticsearch.xcore@8.6.1/org.elasticsearch.xpack.core.ssl.SSLService.lambda$loadSslConfigurations$11(SSLService.java:603)\n\t... 23 more\n"}
es02_1      | ERROR: Elasticsearch did not exit normally - check the logs at /usr/share/elasticsearch/logs/docker-cluster.log
es02_1      |
es02_1      | ERROR: Elasticsearch exited unexpectedly
elastic-docker_es02_1 exited with code 1

The new node's container is failing to map the .key and .crt files, and it is not clear to me if they should be manually fetched and mapped. Could anybody please share a thought on how this works?

2

You going to need to give us some hint as to what the logs say...

3

Unfortunately this log path is not present neither on the host, nor on the container.
I can only provide the log retrieved with docker logs:

2024-04-17T23:44:51.833835000Z {"@timestamp":"2024-04-17T23:44:51.823Z", "log.level": "INFO", "message":"version[8.6.1], pid[71], build[docker/180c9830da956993e59e2cd70eb32b5e383ea42c/2023-01-24T21:35:11.506992272Z], OS[Linux/3.10.0-1160.105.1.el7.x86_64/amd64], JVM[Oracle Corporation/OpenJDK 64-Bit Server VM/19.0.1/19.0.1+10-21]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.node.Node","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:51.837253000Z {"@timestamp":"2024-04-17T23:44:51.827Z", "log.level": "INFO", "message":"JVM home [/usr/share/elasticsearch/jdk], using bundled JDK [true]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.node.Node","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:51.838785000Z {"@timestamp":"2024-04-17T23:44:51.828Z", "log.level": "INFO", "message":"JVM arguments [-Des.networkaddress.cache.ttl=60, -Des.networkaddress.cache.negative.ttl=10, -Djava.security.manager=allow, -XX:+AlwaysPreTouch, -Xss1m, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djna.nosys=true, -XX:-OmitStackTraceInFastThrow, -Dio.netty.noUnsafe=true, -Dio.netty.noKeySetOptimization=true, -Dio.netty.recycler.maxCapacityPerThread=0, -Dlog4j.shutdownHookEnabled=false, -Dlog4j2.disable.jmx=true, -Dlog4j2.formatMsgNoLookups=true, -Djava.locale.providers=SPI,COMPAT, --add-opens=java.base/java.io=ALL-UNNAMED, -Des.cgroups.hierarchy.override=/, -XX:+UseG1GC, -Djava.io.tmpdir=/tmp/elasticsearch-10088485150002837676, -XX:+HeapDumpOnOutOfMemoryError, -XX:+ExitOnOutOfMemoryError, -XX:HeapDumpPath=data, -XX:ErrorFile=logs/hs_err_pid%p.log, -Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m, -Xms1907m, -Xmx1907m, -XX:MaxDirectMemorySize=1000341504, -XX:G1HeapRegionSize=4m, -XX:InitiatingHeapOccupancyPercent=30, -XX:G1ReservePercent=15, -Des.distribution.type=docker, --module-path=/usr/share/elasticsearch/lib, --add-modules=jdk.net, -Djdk.module.main=org.elasticsearch.server]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.node.Node","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:53.500819000Z {"@timestamp":"2024-04-17T23:44:53.498Z", "log.level": "INFO", "message":"Package versions: jackson-annotations=2.13.2, jackson-core=2.13.2, jackson-databind=2.13.2.2, jackson-dataformat-xml=2.13.2, jackson-datatype-jsr310=2.13.2, azure-core=1.27.0, Troubleshooting version conflicts: https://aka.ms/azsdk/java/dependency/troubleshoot", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"com.azure.core.implementation.jackson.JacksonVersion","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.730337000Z {"@timestamp":"2024-04-17T23:44:54.723Z", "log.level": "INFO", "message":"loaded module [aggregations]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.731547000Z {"@timestamp":"2024-04-17T23:44:54.724Z", "log.level": "INFO", "message":"loaded module [aggs-matrix-stats]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.732756000Z {"@timestamp":"2024-04-17T23:44:54.724Z", "log.level": "INFO", "message":"loaded module [analysis-common]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.733901000Z {"@timestamp":"2024-04-17T23:44:54.724Z", "log.level": "INFO", "message":"loaded module [apm]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.735019000Z {"@timestamp":"2024-04-17T23:44:54.724Z", "log.level": "INFO", "message":"loaded module [constant-keyword]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.736099000Z {"@timestamp":"2024-04-17T23:44:54.724Z", "log.level": "INFO", "message":"loaded module [data-streams]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.737189000Z {"@timestamp":"2024-04-17T23:44:54.724Z", "log.level": "INFO", "message":"loaded module [frozen-indices]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.738260000Z {"@timestamp":"2024-04-17T23:44:54.724Z", "log.level": "INFO", "message":"loaded module [ingest-attachment]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.739397000Z {"@timestamp":"2024-04-17T23:44:54.725Z", "log.level": "INFO", "message":"loaded module [ingest-common]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.740582000Z {"@timestamp":"2024-04-17T23:44:54.725Z", "log.level": "INFO", "message":"loaded module [ingest-geoip]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.741708000Z {"@timestamp":"2024-04-17T23:44:54.725Z", "log.level": "INFO", "message":"loaded module [ingest-user-agent]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.742813000Z {"@timestamp":"2024-04-17T23:44:54.725Z", "log.level": "INFO", "message":"loaded module [kibana]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.743914000Z {"@timestamp":"2024-04-17T23:44:54.725Z", "log.level": "INFO", "message":"loaded module [lang-expression]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.745046000Z {"@timestamp":"2024-04-17T23:44:54.725Z", "log.level": "INFO", "message":"loaded module [lang-mustache]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.746138000Z {"@timestamp":"2024-04-17T23:44:54.725Z", "log.level": "INFO", "message":"loaded module [lang-painless]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.747227000Z {"@timestamp":"2024-04-17T23:44:54.725Z", "log.level": "INFO", "message":"loaded module [legacy-geo]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.748304000Z {"@timestamp":"2024-04-17T23:44:54.725Z", "log.level": "INFO", "message":"loaded module [mapper-extras]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.749413000Z {"@timestamp":"2024-04-17T23:44:54.725Z", "log.level": "INFO", "message":"loaded module [mapper-version]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.750551000Z {"@timestamp":"2024-04-17T23:44:54.725Z", "log.level": "INFO", "message":"loaded module [old-lucene-versions]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.751665000Z {"@timestamp":"2024-04-17T23:44:54.726Z", "log.level": "INFO", "message":"loaded module [parent-join]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.752775000Z {"@timestamp":"2024-04-17T23:44:54.726Z", "log.level": "INFO", "message":"loaded module [percolator]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.753880000Z {"@timestamp":"2024-04-17T23:44:54.726Z", "log.level": "INFO", "message":"loaded module [rank-eval]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.755007000Z {"@timestamp":"2024-04-17T23:44:54.726Z", "log.level": "INFO", "message":"loaded module [reindex]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.756096000Z {"@timestamp":"2024-04-17T23:44:54.726Z", "log.level": "INFO", "message":"loaded module [repositories-metering-api]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.757212000Z {"@timestamp":"2024-04-17T23:44:54.726Z", "log.level": "INFO", "message":"loaded module [repository-azure]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.758314000Z {"@timestamp":"2024-04-17T23:44:54.726Z", "log.level": "INFO", "message":"loaded module [repository-encrypted]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.759400000Z {"@timestamp":"2024-04-17T23:44:54.726Z", "log.level": "INFO", "message":"loaded module [repository-gcs]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.760552000Z {"@timestamp":"2024-04-17T23:44:54.726Z", "log.level": "INFO", "message":"loaded module [repository-s3]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.761687000Z {"@timestamp":"2024-04-17T23:44:54.727Z", "log.level": "INFO", "message":"loaded module [repository-url]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.762825000Z {"@timestamp":"2024-04-17T23:44:54.727Z", "log.level": "INFO", "message":"loaded module [runtime-fields-common]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.763949000Z {"@timestamp":"2024-04-17T23:44:54.727Z", "log.level": "INFO", "message":"loaded module [search-business-rules]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.765092000Z {"@timestamp":"2024-04-17T23:44:54.727Z", "log.level": "INFO", "message":"loaded module [searchable-snapshots]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.766186000Z {"@timestamp":"2024-04-17T23:44:54.727Z", "log.level": "INFO", "message":"loaded module [snapshot-based-recoveries]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.767271000Z {"@timestamp":"2024-04-17T23:44:54.727Z", "log.level": "INFO", "message":"loaded module [snapshot-repo-test-kit]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.768360000Z {"@timestamp":"2024-04-17T23:44:54.727Z", "log.level": "INFO", "message":"loaded module [spatial]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.769442000Z {"@timestamp":"2024-04-17T23:44:54.727Z", "log.level": "INFO", "message":"loaded module [transform]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.770534000Z {"@timestamp":"2024-04-17T23:44:54.727Z", "log.level": "INFO", "message":"loaded module [transport-netty4]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.771674000Z {"@timestamp":"2024-04-17T23:44:54.727Z", "log.level": "INFO", "message":"loaded module [unsigned-long]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.772815000Z {"@timestamp":"2024-04-17T23:44:54.728Z", "log.level": "INFO", "message":"loaded module [vector-tile]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.773933000Z {"@timestamp":"2024-04-17T23:44:54.728Z", "log.level": "INFO", "message":"loaded module [wildcard]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.775077000Z {"@timestamp":"2024-04-17T23:44:54.728Z", "log.level": "INFO", "message":"loaded module [x-pack-aggregate-metric]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.776181000Z {"@timestamp":"2024-04-17T23:44:54.728Z", "log.level": "INFO", "message":"loaded module [x-pack-analytics]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.777314000Z {"@timestamp":"2024-04-17T23:44:54.728Z", "log.level": "INFO", "message":"loaded module [x-pack-async]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.778435000Z {"@timestamp":"2024-04-17T23:44:54.728Z", "log.level": "INFO", "message":"loaded module [x-pack-async-search]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.779545000Z {"@timestamp":"2024-04-17T23:44:54.728Z", "log.level": "INFO", "message":"loaded module [x-pack-autoscaling]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.780640000Z {"@timestamp":"2024-04-17T23:44:54.728Z", "log.level": "INFO", "message":"loaded module [x-pack-ccr]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.781984000Z {"@timestamp":"2024-04-17T23:44:54.728Z", "log.level": "INFO", "message":"loaded module [x-pack-core]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.783107000Z {"@timestamp":"2024-04-17T23:44:54.728Z", "log.level": "INFO", "message":"loaded module [x-pack-deprecation]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.784273000Z {"@timestamp":"2024-04-17T23:44:54.729Z", "log.level": "INFO", "message":"loaded module [x-pack-enrich]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.785392000Z {"@timestamp":"2024-04-17T23:44:54.729Z", "log.level": "INFO", "message":"loaded module [x-pack-eql]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.786531000Z {"@timestamp":"2024-04-17T23:44:54.729Z", "log.level": "INFO", "message":"loaded module [x-pack-fleet]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.787640000Z {"@timestamp":"2024-04-17T23:44:54.729Z", "log.level": "INFO", "message":"loaded module [x-pack-graph]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.788726000Z {"@timestamp":"2024-04-17T23:44:54.729Z", "log.level": "INFO", "message":"loaded module [x-pack-identity-provider]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.789824000Z {"@timestamp":"2024-04-17T23:44:54.729Z", "log.level": "INFO", "message":"loaded module [x-pack-ilm]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.790923000Z {"@timestamp":"2024-04-17T23:44:54.729Z", "log.level": "INFO", "message":"loaded module [x-pack-logstash]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.792035000Z {"@timestamp":"2024-04-17T23:44:54.729Z", "log.level": "INFO", "message":"loaded module [x-pack-ml]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.793126000Z {"@timestamp":"2024-04-17T23:44:54.729Z", "log.level": "INFO", "message":"loaded module [x-pack-monitoring]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.794255000Z {"@timestamp":"2024-04-17T23:44:54.729Z", "log.level": "INFO", "message":"loaded module [x-pack-ql]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.795369000Z {"@timestamp":"2024-04-17T23:44:54.729Z", "log.level": "INFO", "message":"loaded module [x-pack-rollup]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.796512000Z {"@timestamp":"2024-04-17T23:44:54.730Z", "log.level": "INFO", "message":"loaded module [x-pack-security]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.797612000Z {"@timestamp":"2024-04-17T23:44:54.730Z", "log.level": "INFO", "message":"loaded module [x-pack-shutdown]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.798724000Z {"@timestamp":"2024-04-17T23:44:54.730Z", "log.level": "INFO", "message":"loaded module [x-pack-sql]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.799818000Z {"@timestamp":"2024-04-17T23:44:54.730Z", "log.level": "INFO", "message":"loaded module [x-pack-stack]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.800924000Z {"@timestamp":"2024-04-17T23:44:54.730Z", "log.level": "INFO", "message":"loaded module [x-pack-text-structure]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.802027000Z {"@timestamp":"2024-04-17T23:44:54.730Z", "log.level": "INFO", "message":"loaded module [x-pack-voting-only-node]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.803109000Z {"@timestamp":"2024-04-17T23:44:54.730Z", "log.level": "INFO", "message":"loaded module [x-pack-watcher]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.804209000Z {"@timestamp":"2024-04-17T23:44:54.730Z", "log.level": "INFO", "message":"loaded module [x-pack-write-load-forecaster]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:54.805337000Z {"@timestamp":"2024-04-17T23:44:54.730Z", "log.level": "INFO", "message":"no plugins loaded", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:57.228830000Z {"@timestamp":"2024-04-17T23:44:57.224Z", "log.level": "INFO", "message":"using [1] data paths, mounts [[/usr/share/elasticsearch/data (/dev/mapper/elkvg-elkdata)]], net usable_space [3.6tb], net total_space [3.8tb], types [ext4]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.env.NodeEnvironment","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:57.230103000Z {"@timestamp":"2024-04-17T23:44:57.225Z", "log.level": "INFO", "message":"heap size [1.8gb], compressed ordinary object pointers [true]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.env.NodeEnvironment","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:57.277467000Z {"@timestamp":"2024-04-17T23:44:57.275Z", "log.level": "INFO", "message":"node name [es02], node ID [rR2pK1arTJKaERUiF-mtZA], cluster name [docker-cluster], roles [data_content, transform, data_hot, ml, data_frozen, ingest, data_cold, data, remote_cluster_client, master, data_warm]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.node.Node","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster"}
2024-04-17T23:44:59.624928000Z ERROR: Elasticsearch did not exit normally - check the logs at /usr/share/elasticsearch/logs/docker-cluster.log
2024-04-17T23:44:59.626364000Z {"@timestamp":"2024-04-17T23:44:59.620Z", "log.level":"ERROR", "message":"fatal exception while booting Elasticsearch", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.bootstrap.Elasticsearch","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"docker-cluster","error.type":"org.elasticsearch.ElasticsearchSecurityException","error.message":"failed to load SSL configuration [xpack.security.transport.ssl] - Error parsing Private Key [/usr/share/elasticsearch/config/certs/es02/es02.key], file is empty","error.stack_trace":"org.elasticsearch.ElasticsearchSecurityException: failed to load SSL configuration [xpack.security.transport.ssl] - Error parsing Private Key [/usr/share/elasticsearch/config/certs/es02/es02.key], file is empty\n\tat org.elasticsearch.xcore@8.6.1/org.elasticsearch.xpack.core.ssl.SSLService.lambda$loadSslConfigurations$11(SSLService.java:605)\n\tat java.base/java.util.HashMap.forEach(HashMap.java:1429)\n\tat java.base/java.util.Collections$UnmodifiableMap.forEach(Collections.java:1553)\n\tat org.elasticsearch.xcore@8.6.1/org.elasticsearch.xpack.core.ssl.SSLService.loadSslConfigurations(SSLService.java:601)\n\tat org.elasticsearch.xcore@8.6.1/org.elasticsearch.xpack.core.ssl.SSLService.<init>(SSLService.java:156)\n\tat org.elasticsearch.xcore@8.6.1/org.elasticsearch.xpack.core.XPackPlugin.createSSLService(XPackPlugin.java:465)\n\tat org.elasticsearch.xcore@8.6.1/org.elasticsearch.xpack.core.XPackPlugin.createComponents(XPackPlugin.java:314)\n\tat org.elasticsearch.server@8.6.1/org.elasticsearch.node.Node.lambda$new$16(Node.java:721)\n\tat org.elasticsearch.server@8.6.1/org.elasticsearch.plugins.PluginsService.lambda$flatMap$0(PluginsService.java:252)\n\tat java.base/java.util.stream.ReferencePipeline$7$1.accept(ReferencePipeline.java:273)\n\tat java.base/java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:197)\n\tat java.base/java.util.AbstractList$RandomAccessSpliterator.forEachRemaining(AbstractList.java:722)\n\tat java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:509)\n\tat java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:499)\n\tat java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:575)\n\tat java.base/java.util.stream.AbstractPipeline.evaluateToArrayNode(AbstractPipeline.java:260)\n\tat java.base/java.util.stream.ReferencePipeline.toArray(ReferencePipeline.java:616)\n\tat java.base/java.util.stream.ReferencePipeline.toArray(ReferencePipeline.java:622)\n\tat java.base/java.util.stream.ReferencePipeline.toList(ReferencePipeline.java:627)\n\tat org.elasticsearch.server@8.6.1/org.elasticsearch.node.Node.<init>(Node.java:736)\n\tat org.elasticsearch.server@8.6.1/org.elasticsearch.node.Node.<init>(Node.java:322)\n\tat org.elasticsearch.server@8.6.1/org.elasticsearch.bootstrap.Elasticsearch$2.<init>(Elasticsearch.java:214)\n\tat org.elasticsearch.server@8.6.1/org.elasticsearch.bootstrap.Elasticsearch.initPhase3(Elasticsearch.java:214)\n\tat org.elasticsearch.server@8.6.1/org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:67)\nCaused by: org.elasticsearch.common.ssl.SslConfigException: Error parsing Private Key [/usr/share/elasticsearch/config/certs/es02/es02.key], file is empty\n\tat org.elasticsearch.sslconfig@8.6.1/org.elasticsearch.common.ssl.PemUtils.parsePrivateKey(PemUtils.java:136)\n\tat org.elasticsearch.sslconfig@8.6.1/org.elasticsearch.common.ssl.PemKeyConfig.getPrivateKey(PemKeyConfig.java:122)\n\tat org.elasticsearch.sslconfig@8.6.1/org.elasticsearch.common.ssl.PemKeyConfig.createKeyManager(PemKeyConfig.java:88)\n\tat org.elasticsearch.xcore@8.6.1/org.elasticsearch.xpack.core.ssl.SSLService.createSslContext(SSLService.java:474)\n\tat java.base/java.util.HashMap.computeIfAbsent(HashMap.java:1228)\n\tat org.elasticsearch.xcore@8.6.1/org.elasticsearch.xpack.core.ssl.SSLService.lambda$loadSslConfigurations$11(SSLService.java:603)\n\t... 23 more\n"}
2024-04-17T23:45:00.060800000Z ERROR: Elasticsearch exited unexpectedly
4

Error parsing Private Key [/usr/share/elasticsearch/config/certs/es02/es02.key], file is empty

5

The .crt and .key certificates are indeed empty files on the es02 node. I don't know how to track the process of creating and transferring to the node of these files.
Can anybody please explain how this works?

6

I have realized that Docker mechanism of generating and transferring the certificate files has been preventing certificate updates once they were in place on setup container.
I have managed to work around this constraint by adding the following block in setup, right before the certs.zip checking:

        # Remove existing certs.zip if it exists
        if [ -f config/certs/certs.zip ]; then
          rm config/certs/certs.zip;
        fi;
        if [ `ls -1 config/certs/es02/es* 2>/dev/null | wc -l` -gt 0 ]; then
          rm config/certs/es02/es*;
        fi;

I have included a way of removing the empty certificates (still to determine the way they got to be saved this way in the first place).
The new node is now part of the cluster.